From f90b1ae6a42e7e32bc3788b8e035620c975da5ea Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 30 Sep 2013 18:07:02 -0400 Subject: [PATCH 1/2] Update the genPOI to uses multiple processes --- overviewer_core/aux_files/genPOI.py | 50 ++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index bc71b5b..1ce19d8 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -18,6 +18,13 @@ import os import logging import json import sys +import Queue +import multiprocessing + +from multiprocessing import Process +from multiprocessing import Pool + +from functools import partial from optparse import OptionParser from overviewer_core import logger @@ -32,6 +39,27 @@ def replaceBads(s): x = x.replace(bad,"_") return x +def parseBucketChunks(bucket, rset): + pid = multiprocessing.current_process().pid + pois = dict(TileEntities=[], Entities=[]); + + i = 0 + cnt = 0 + l = len(bucket) + for b in bucket: + data = rset.get_chunk(b[0],b[1]) + pois['TileEntities'] += data['TileEntities'] + pois['Entities'] += data['Entities'] + + # Perhaps only on verbose ? + i = i + 1 + if i == 250: + i = 0 + cnt = 250 + cnt + logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt); + + return pois + def handleEntities(rset, outputdir, render, rname): # if we're already handled the POIs for this region regionset, do nothing @@ -43,8 +71,28 @@ def handleEntities(rset, outputdir, render, rname): filters = render['markers'] rset._pois = dict(TileEntities=[], Entities=[]) + numbuckets = 8; + buckets = [[] for i in range(numbuckets)]; + for (x,z,mtime) in rset.iterate_chunks(): - data = rset.get_chunk(x,z) + i = x / 32 + z / 32 + i = i % numbuckets + buckets[i].append([x,z]) + + for b in buckets: + logging.info("Buckets has %d entries", len(b)); + + # create the partial to lock the rset + dest = partial(parseBucketChunks, rset=rset) + + # Create a pool of processes and run all the functions + pool = Pool(processes=numbuckets) + results = pool.map(dest, buckets) + + logging.info("All the threads completed") + + # Fix up all the quests in the reset + for data in results: rset._pois['TileEntities'] += data['TileEntities'] rset._pois['Entities'] += data['Entities'] From 55f7183c4d5ef16ed73c5f70794abbf93d5fee30 Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 1 Oct 2013 13:13:14 -0400 Subject: [PATCH 2/2] Cleanup even more --- overviewer_core/aux_files/genPOI.py | 60 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 1ce19d8..218a2f7 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -60,7 +60,7 @@ def parseBucketChunks(bucket, rset): return pois -def handleEntities(rset, outputdir, render, rname): +def handleEntities(rset, outputdir, render, rname, config): # if we're already handled the POIs for this region regionset, do nothing if hasattr(rset, "_pois"): @@ -71,30 +71,40 @@ def handleEntities(rset, outputdir, render, rname): filters = render['markers'] rset._pois = dict(TileEntities=[], Entities=[]) - numbuckets = 8; - buckets = [[] for i in range(numbuckets)]; + numbuckets = config['processes']; + if numbuckets < 0: + numbuckets = multiprocessing.cpu_count() - for (x,z,mtime) in rset.iterate_chunks(): - i = x / 32 + z / 32 - i = i % numbuckets - buckets[i].append([x,z]) - - for b in buckets: - logging.info("Buckets has %d entries", len(b)); - - # create the partial to lock the rset - dest = partial(parseBucketChunks, rset=rset) - - # Create a pool of processes and run all the functions - pool = Pool(processes=numbuckets) - results = pool.map(dest, buckets) - - logging.info("All the threads completed") - - # Fix up all the quests in the reset - for data in results: - rset._pois['TileEntities'] += data['TileEntities'] - rset._pois['Entities'] += data['Entities'] + if numbuckets == 1: + for (x,z,mtime) in rset.iterate_chunks(): + data = rset.get_chunk(x,z) + rset._pois['TileEntities'] += data['TileEntities'] + rset._pois['Entities'] += data['Entities'] + + else: + buckets = [[] for i in range(numbuckets)]; + + for (x,z,mtime) in rset.iterate_chunks(): + i = x / 32 + z / 32 + i = i % numbuckets + buckets[i].append([x,z]) + + for b in buckets: + logging.info("Buckets has %d entries", len(b)); + + # create the partial to lock the rset + dest = partial(parseBucketChunks, rset=rset) + + # Create a pool of processes and run all the functions + pool = Pool(processes=numbuckets) + results = pool.map(dest, buckets) + + logging.info("All the threads completed") + + # Fix up all the quests in the reset + for data in results: + rset._pois['TileEntities'] += data['TileEntities'] + rset._pois['Entities'] += data['Entities'] logging.info("Done.") @@ -233,7 +243,7 @@ def main(): markers[rname] = [to_append] if not options.skipscan: - handleEntities(rset, os.path.join(destdir, rname), render, rname) + handleEntities(rset, os.path.join(destdir, rname), render, rname, config) handlePlayers(rset, render, worldpath) handleManual(rset, render['manualpois'])