From cd7b9456a956accc9c8ff6429af78d88eaecf2b7 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 23:36:26 +0800 Subject: [PATCH] Fixed worker processes being passed the full quadtree object. Caused massive performance regressions when caching stuff in quadtree.world Offloaded self._get_chunks_in_range into worker process. --- quadtree.py | 61 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/quadtree.py b/quadtree.py index d577325..452b481 100644 --- a/quadtree.py +++ b/quadtree.py @@ -82,6 +82,8 @@ def catch_keyboardinterrupt(func): raise return newfunc +child_quadtree = None + class QuadtreeGen(object): def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False): """Generates a quadtree from the world given into the @@ -308,24 +310,17 @@ class QuadtreeGen(object): # This image is rendered at: dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path)) #logging.debug("this is rendered at %s", dest) - - # And uses these chunks - tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, - rowend) - #logging.debug(" tilechunks: %r", tilechunks) - # Put this in the batch to be submited to the pool - # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - batch.append((tilechunks, colstart, colend, rowstart, rowend, dest)) + # Put this in the batch to be submited to the pool + batch.append((colstart, colend, rowstart, rowend, dest)) tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) batch = [] if tiles > 0: - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= (batch,)) def _apply_render_inntertile(self, pool, zoom,batch_size): @@ -344,12 +339,15 @@ class QuadtreeGen(object): tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + yield pool.apply_async(func=render_innertile_batch, args= [batch]) batch = [] - if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + if tiles > 0: + yield pool.apply_async(func=render_innertile_batch, args= [batch]) + def pool_initializer(args): + logging.debug("Child process {0}".format(os.getpid())) + def go(self, procs): """Renders all tiles""" @@ -365,12 +363,18 @@ class QuadtreeGen(object): for _ in xrange(curdepth - self.p): self._decrease_depth() + logging.debug("Parent process {0}".format(os.getpid())) + #stash the quadtree object so child process's can + global child_quadtree + child_quadtree = self # Create a pool if procs == 1: pool = FakePool() else: - pool = multiprocessing.Pool(processes=procs) - + pool = multiprocessing.Pool(processes=procs,initializer=self.pool_initializer,initargs=()) + #warm up the pool so it reports all the worker id's + pool.map(bool,xrange(multiprocessing.cpu_count()),1) + # Render the highest level of tiles from the chunks results = collections.deque() complete = 0 @@ -381,7 +385,7 @@ class QuadtreeGen(object): logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") count = 0 - batch_size = 50 + batch_size = 10 for result in self._apply_render_worldtiles(pool,batch_size): results.append(result) if len(results) > (10000/batch_size): @@ -393,7 +397,6 @@ class QuadtreeGen(object): # Wait for the rest of the results while len(results) > 0: - complete += results.popleft().get() self.print_statusline(complete, total, 1) @@ -468,7 +471,7 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile_batch(quadtree, batch): +def render_innertile_batch(batch): count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: @@ -540,12 +543,28 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt -def render_worldtile_batch(quadtree, batch): +def render_worldtile_batch(batch): + global child_quadtree + return render_worldtile_batch_(child_quadtree, batch) + +def render_worldtile_batch_(quadtree, batch): count = 0 + _get_chunks_in_range = quadtree._get_chunks_in_range #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5]) + colstart = job[0] + colend = job[1] + rowstart = job[2] + rowend = job[3] + path = job[4] + # (even if tilechunks is empty, render_worldtile will delete + # existing images if appropriate) + # And uses these chunks + tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) + #logging.debug(" tilechunks: %r", tilechunks) + + render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path) return count def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):