diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index bc71b5b..1ce19d8 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -18,6 +18,13 @@ import os import logging import json import sys +import Queue +import multiprocessing + +from multiprocessing import Process +from multiprocessing import Pool + +from functools import partial from optparse import OptionParser from overviewer_core import logger @@ -32,6 +39,27 @@ def replaceBads(s): x = x.replace(bad,"_") return x +def parseBucketChunks(bucket, rset): + pid = multiprocessing.current_process().pid + pois = dict(TileEntities=[], Entities=[]); + + i = 0 + cnt = 0 + l = len(bucket) + for b in bucket: + data = rset.get_chunk(b[0],b[1]) + pois['TileEntities'] += data['TileEntities'] + pois['Entities'] += data['Entities'] + + # Perhaps only on verbose ? + i = i + 1 + if i == 250: + i = 0 + cnt = 250 + cnt + logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt); + + return pois + def handleEntities(rset, outputdir, render, rname): # if we're already handled the POIs for this region regionset, do nothing @@ -43,8 +71,28 @@ def handleEntities(rset, outputdir, render, rname): filters = render['markers'] rset._pois = dict(TileEntities=[], Entities=[]) + numbuckets = 8; + buckets = [[] for i in range(numbuckets)]; + for (x,z,mtime) in rset.iterate_chunks(): - data = rset.get_chunk(x,z) + i = x / 32 + z / 32 + i = i % numbuckets + buckets[i].append([x,z]) + + for b in buckets: + logging.info("Buckets has %d entries", len(b)); + + # create the partial to lock the rset + dest = partial(parseBucketChunks, rset=rset) + + # Create a pool of processes and run all the functions + pool = Pool(processes=numbuckets) + results = pool.map(dest, buckets) + + logging.info("All the threads completed") + + # Fix up all the quests in the reset + for data in results: rset._pois['TileEntities'] += data['TileEntities'] rset._pois['Entities'] += data['Entities']