chunk scans are now done in parallel for multiple rendermodes

overviewer_core/quadtree.py

@@ -503,7 +503,7 @@ class QuadtreeGen(object):
 
         dirty = DirtyTiles(depth)
 
-        logging.debug("	Scanning chunks for tiles that need rendering...")
+        #logging.debug("	Scanning chunks for tiles that need rendering...")
         chunkcount = 0
         stime = time.time()
 
@@ -516,8 +516,8 @@ class QuadtreeGen(object):
         # circuit checking mtimes of all chunks in a region
         for chunkx, chunky, chunkmtime in self.world.iterate_chunk_metadata():
             chunkcount += 1
-            if chunkcount % 10000 == 0:
-                logging.info("	%s chunks scanned", chunkcount)
+            #if chunkcount % 10000 == 0:
+            #    logging.info("	%s chunks scanned", chunkcount)
 
             chunkcol, chunkrow = self.world.convert_coords(chunkx, chunky)
             #logging.debug("Looking at chunk %s,%s", chunkcol, chunkrow)
@@ -588,17 +588,18 @@ class QuadtreeGen(object):
                         #logging.debug("	Setting tile as dirty. Will render.")
 
         t = int(time.time()-stime)
-        logging.debug("	Done. %s chunks scanned in %s second%s", chunkcount, t,
+        logging.debug("Done with scan for '%s'. %s chunks scanned in %s second%s", 
+                self.rendermode, chunkcount, t,
                 "s" if t != 1 else "")
 
-        if logging.getLogger().isEnabledFor(logging.DEBUG):
-            logging.debug("	Counting tiles that need rendering...")
-            tilecount = 0
-            stime = time.time()
-            for _ in dirty.iterate_dirty():
-                tilecount += 1
-            logging.debug("	Done. %s tiles need to be rendered. (count took %s seconds)",
-                    tilecount, int(time.time()-stime))
+        #if logging.getLogger().isEnabledFor(logging.DEBUG):
+        #    logging.debug("	Counting tiles that need rendering...")
+        #    tilecount = 0
+        #    stime = time.time()
+        #    for _ in dirty.iterate_dirty():
+        #        tilecount += 1
+        #    logging.debug("	Done. %s tiles need to be rendered. (count took %s seconds)",
+        #            tilecount, int(time.time()-stime))
         
         return dirty
 

overviewer_core/rendernode.py

@@ -159,8 +159,14 @@ class RenderNode(object):
     def go(self, procs):
         """Renders all tiles"""
         
-        logging.debug("Parent process {0}".format(os.getpid()))
+        # Signal to the quadtrees to scan the chunks and their respective tile
+        # directories to find what needs to be rendered. We get from this the
+        # total tiles that need to be rendered (at the highest level across all
+        # quadtrees) as well as a list of [qtree, DirtyTiles object]
+        total_rendertiles, dirty_list = self._get_dirty_tiles(procs)
+
         # Create a pool
+        logging.debug("Parent process {0}".format(os.getpid()))
         if procs == 1:
             pool = FakePool()
             pool_initializer(self)
@@ -188,12 +194,6 @@ class RenderNode(object):
                 max_p = q.p
         self.max_p = max_p
 
-        # Signal to the quadtrees to scan the chunks and their respective tile
-        # directories to find what needs to be rendered. We get from this the
-        # total tiles that need to be rendered (at the highest level across all
-        # quadtrees) as well as a list of [qtree, DirtyTiles object]
-        total_rendertiles, dirty_list = self._get_dirty_tiles()
-
         if total_rendertiles == 0:
             logging.info(r"There is no work to do, your map is up to date! \o/")
             return
@@ -212,6 +212,8 @@ class RenderNode(object):
         logging.info("")
         logging.info("Rendering highest zoom level of tiles now.")
         logging.info("Rendering {0} rendermode{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' ))
+        logging.info("Started {0} worker process{1}".format(
+            procs, "es" if procs != 1 else ""))
         logging.info("There are {0} tiles to render at this level".format(total_rendertiles))        
         logging.info("There are {0} total levels".format(self.max_p))
 
@@ -375,7 +377,7 @@ class RenderNode(object):
         for q in quadtrees:
             q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
 
-    def _get_dirty_tiles(self):
+    def _get_dirty_tiles(self, procs):
         """Returns two items:
         1) The total number of tiles needing rendering
         2) a list of (qtree, DirtyTiles) objects holding which tiles in the
@@ -384,17 +386,40 @@ class RenderNode(object):
         """
         all_dirty = []
         total = 0
+        numqtrees = len(self.quadtrees)
+        procs = min(procs, numqtrees)
 
-        logging.info("Scanning for tiles to update. This shouldn't take too long...")
-        for i, q in enumerate(self.quadtrees):
-            logging.info("Scanning for tiles in rendermode %s", q.rendermode)
-            dirty = q.scan_chunks()
+        if procs == 1:
+            pool = FakePool()
+        else:
+            pool = multiprocessing.Pool(processes=procs)
 
-            total += dirty.count()
+        logging.info("Scanning chunks and determining tiles to update for each rendermode requested.")
+        logging.info("Doing %s scan%s in %s worker process%s",
+                numqtrees, "s" if numqtrees != 1 else "",
+                procs, "es" if procs != 1 else "",
+                )
 
+        # Push all scan jobs to the workers
+        results = []
+        for q in self.quadtrees:
+            r = pool.apply_async(scan_quadtree_chunks, (q,))
+            results.append(r)
+        pool.close()
+
+        # Wait for workers to finish
+        for q, r in zip(self.quadtrees, results):
+            dirty, numtiles = r.get()
+            total += numtiles
             all_dirty.append((q, dirty))
+        pool.join() # ought to be redundant
 
-        logging.info("Scan finished. %s total tiles need to be rendered at the highest level", total)
+        logging.info("%s finished. %s %s to be rendered at the highest level",
+                "All scans" if numqtrees != 1 else "Scan",
+                total,
+                # Probably won't happen, but just in case:
+                "total tiles need" if total != 1 else "tile needs",
+                )
         return total, all_dirty
 
     def _apply_render_worldtiles(self, tileset, pool,batch_size):
@@ -468,6 +493,10 @@ class RenderNode(object):
         if jobcount > 0:
             yield pool.apply_async(func=render_innertile_batch, args= [batch])    
             
+
+########################################################################################
+# The following three functions are entry points for workers in the multiprocessing pool
+
 @catch_keyboardinterrupt
 def render_worldtile_batch(batch):
     """Main entry point for workers processing a render-tile (also called a
@@ -502,6 +531,19 @@ def render_innertile_batch(batch):
         dest = quadtree.full_tiledir+os.sep+job[1]
         quadtree.render_innertile(dest=dest,name=job[2])
     return count
+
+@catch_keyboardinterrupt
+def scan_quadtree_chunks(qtree):
+    """The entry point for workers when scanning chunks for tiles needing
+    updating. Builds and returns a dirtytiles tree.
+
+    Returns two things: the dirtytree from qtree.scan_chunks(), and the total
+    from the tree.count() method
+
+    """
+    logging.debug("Scanning chunks for rendermode '%s'", qtree.rendermode)
+    tree = qtree.scan_chunks()
+    return tree, tree.count()
     
 class FakeResult(object):
     def __init__(self, res):