Update the genPOI to uses multiple processes

overviewer_core/aux_files/genPOI.py

@@ -18,6 +18,13 @@ import os
 import logging
 import json
 import sys
+import Queue
+import multiprocessing
+
+from multiprocessing import Process
+from multiprocessing import Pool
+
+from functools import partial
 from optparse import OptionParser
 
 from overviewer_core import logger
@@ -32,6 +39,27 @@ def replaceBads(s):
         x = x.replace(bad,"_")
     return x
 
+def parseBucketChunks(bucket, rset):
+    pid = multiprocessing.current_process().pid
+    pois = dict(TileEntities=[], Entities=[]);
+  
+    i = 0
+    cnt = 0
+    l = len(bucket)
+    for b in bucket:
+        data = rset.get_chunk(b[0],b[1])
+        pois['TileEntities'] += data['TileEntities']
+        pois['Entities']     += data['Entities']
+
+        # Perhaps only on verbose ?
+        i = i + 1
+        if i == 250:
+            i = 0
+            cnt = 250 + cnt
+            logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt);
+
+    return pois
+
 def handleEntities(rset, outputdir, render, rname):
 
     # if we're already handled the POIs for this region regionset, do nothing
@@ -43,8 +71,28 @@ def handleEntities(rset, outputdir, render, rname):
     filters = render['markers']
     rset._pois = dict(TileEntities=[], Entities=[])
 
+    numbuckets = 8;
+    buckets = [[] for i in range(numbuckets)];
+
     for (x,z,mtime) in rset.iterate_chunks():
-        data = rset.get_chunk(x,z)
+        i = x / 32 + z / 32
+        i = i % numbuckets 
+        buckets[i].append([x,z])
+
+    for b in buckets:
+        logging.info("Buckets has %d entries", len(b));
+
+    # create the partial to lock the rset
+    dest = partial(parseBucketChunks, rset=rset)
+
+    # Create a pool of processes and run all the functions
+    pool = Pool(processes=numbuckets)
+    results = pool.map(dest, buckets)
+
+    logging.info("All the threads completed")
+
+    # Fix up all the quests in the reset
+    for data in results:
         rset._pois['TileEntities'] += data['TileEntities']
         rset._pois['Entities']     += data['Entities']