Merged in dtt-c-render

Conflicts:
	src/overviewer.h

README.rst

@@ -246,6 +246,15 @@ Options
 --night
     This option enables --lighting, and renders the world at night.
 
+--web-assets-hook=HOOK
+    This option lets you specify a script to run after the web assets have been
+    copied into the output directory, but before any tile rendering takes
+    place. This is an ideal time to do any custom postprocessing for markers.js
+    or other web assets.
+    
+    The script should be executable, and it should accept one argument:
+    the path to the output directory.
+
 Viewing the Results
 -------------------
 Within the output directory you will find two things: an index.html file, and a

chunk.py

@@ -46,15 +46,20 @@ image
 # alpha_over extension, BUT this extension may fall back to PIL's
 # paste(), which DOES need the workaround.)
 
-def get_lvldata(world,filename, x, y):
+def get_lvldata(world, filename, x, y, retries=2):
     """Takes a filename and chunkcoords and returns the Level struct, which contains all the
     level info"""
     
     try:
         d =  world.load_from_region(filename, x, y)
     except Exception, e:
-        logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e)
-        raise ChunkCorrupt(str(e))
+        if retries > 0:
+            # wait a little bit, and try again (up to `retries` times)
+            time.sleep(1)
+            return get_lvldata(world, filename, x, y, retries=retries-1)
+        else:
+            logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e)
+            raise ChunkCorrupt(str(e))
     
     if not d: raise NoSuchChunk(x,y)
     return d[1]['Level']
@@ -569,7 +574,7 @@ class ChunkRenderer(object):
 
         # check to see if there are any signs in the persistentData list that are from this chunk.
         # if so, remove them from the persistentData list (since they're have been added to the world.POI
-        # list above
+        # list above.
         if self.queue:
             self.queue.put(['removePOI', (self.chunkX, self.chunkY)])
 

composite.py

@@ -23,12 +23,7 @@ Overviewer. It defaults to the PIL paste function when the custom
 alpha-over extension cannot be found.
 """
 
-extension_alpha_over = None
-try:
-    from c_overviewer import alpha_over as _extension_alpha_over
-    extension_alpha_over = _extension_alpha_over
-except ImportError:
-    pass
+from c_overviewer import alpha_over as extension_alpha_over
 
 def alpha_over(dest, src, pos_or_rect=(0, 0), mask=None):
     """Composite src over dest, using mask as the alpha channel (if
@@ -40,12 +35,4 @@ def alpha_over(dest, src, pos_or_rect=(0, 0), mask=None):
         mask = src
     
     global extension_alpha_over
-    if extension_alpha_over is not None:
-        # extension ALWAYS expects rects, so convert if needed
-        if len(pos_or_rect) == 2:
-            pos_or_rect = (pos_or_rect[0], pos_or_rect[1], src.size[0], src.size[1])
-        extension_alpha_over(dest, src, pos_or_rect, mask)
-    else:
-        # fallback
-        dest.paste(src, pos_or_rect, mask)
-
+    return extension_alpha_over(dest, src, pos_or_rect, mask)

configParser.py

@@ -0,0 +1,151 @@
+from optparse import OptionParser
+import sys
+import os.path
+
+class OptionsResults(object):
+    pass
+
+class ConfigOptionParser(object):
+    def __init__(self, **kwargs):
+        self.cmdParser = OptionParser(usage=kwargs.get("usage",""))
+        self.configFile = kwargs.get("config","settings.py")
+        self.configVars = []
+
+        # these are arguments not understood by OptionParser, so they must be removed
+        # in add_option before being passed to the OptionParser
+        # note that default is a valid OptionParser argument, but we remove it
+        # because we want to do our default value handling
+        self.customArgs = ["required", "commandLineOnly", "default"]
+
+        self.requiredArgs = []
+
+    def display_config(self):
+        for x in self.configVars:
+            n = x['dest']
+            print  "%s: %r" % (n, self.configResults.__dict__[n])
+
+    def add_option(self, *args, **kwargs):
+
+        if kwargs.get("configFileOnly", False) and kwargs.get("commandLineOnly", False):
+            raise Exception(args, "configFileOnly and commandLineOnly are mututally exclusive")
+
+        self.configVars.append(kwargs.copy())
+
+        if not kwargs.get("configFileOnly", False):
+            for arg in self.customArgs:
+                if arg in kwargs.keys(): del kwargs[arg]
+
+            self.cmdParser.add_option(*args, **kwargs)
+
+    def print_help(self):
+        self.cmdParser.print_help()
+
+    def parse_args(self):
+
+        # first, load the results from the command line:
+        options, args = self.cmdParser.parse_args()
+
+        # second, use these values to seed the locals dict
+        l = dict()
+        g = dict()
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('configFileOnly', False): continue
+            if a.get('commandLineOnly', False): continue
+            v = getattr(options, n)
+            if v != None:
+                #print "seeding %s with %s" % (n, v)
+                l[n] = v
+            else:
+                # if this has a default, use that to seed the globals dict
+                if a.get("default", None): g[n] = a['default']
+        g['args'] = args
+
+        try:
+            if os.path.exists(self.configFile):
+                execfile(self.configFile, g, l)
+        except NameError, ex:
+            import traceback
+            traceback.print_exc()
+            print "\nError parsing %s.  Please check the trackback above" % self.configFile
+            sys.exit(1)
+        except SyntaxError, ex:
+            import traceback
+            traceback.print_exc()
+            tb = sys.exc_info()[2]
+            #print tb.tb_frame.f_code.co_filename
+            print "\nError parsing %s.  Please check the trackback above" % self.configFile
+            sys.exit(1)
+
+        #print l.keys()
+
+        configResults = OptionsResults()
+        # third, load the results from the config file:
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('commandLineOnly', False):
+                if n in l.keys():
+                    print "Error: %s can only be specified on the command line.  It is not valid in the config file" % n
+                    sys.exit(1)
+
+            configResults.__dict__[n] = l.get(n)
+
+
+        
+        # third, merge options into configReslts (with options overwriting anything in configResults)
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('configFileOnly', False): continue
+            if getattr(options, n) != None:
+                configResults.__dict__[n] = getattr(options, n)
+
+        # forth, set defaults for any empty values
+        for a in self.configVars:
+            n = a['dest']
+            if (n not in configResults.__dict__.keys() or configResults.__dict__[n] == None) and 'default' in a.keys():
+                configResults.__dict__[n] = a['default']
+
+        # fifth, check required args:
+        for a in self.configVars:
+            n = a['dest']
+            if configResults.__dict__[n] == None and a.get('required',False):
+                raise Exception("%s is required" % n)
+
+        # sixth, check types
+        for a in self.configVars:
+            n = a['dest']
+            if 'type' in a.keys() and configResults.__dict__[n] != None:
+                try:
+                    # switch on type.  there are only 6 types that can be used with optparse
+                    if a['type'] == "int":
+                        configResults.__dict__[n] = int(configResults.__dict__[n])
+                    elif a['type'] == "string":
+                        configResults.__dict__[n] = str(configResults.__dict__[n])
+                    elif a['type'] == "long":
+                        configResults.__dict__[n] = long(configResults.__dict__[n])
+                    elif a['type'] == "choice":
+                        if configResults.__dict__[n] not in a['choices']:
+                            print "The value '%s' is not valid for config parameter '%s'" % (configResults.__dict__[n], n)
+                            sys.exit(1)
+                    elif a['type'] == "float":
+                        configResults.__dict__[n] = long(configResults.__dict__[n])
+                    elif a['type'] == "complex":
+                        configResults.__dict__[n] = complex(configResults.__dict__[n])
+                    elif a['type'] == "function":
+                        if not callable(configResults.__dict__[n]):
+                            raise ValueError("Not callable")
+                    else:
+                        print "Unknown type!"
+                        sys.exit(1)
+                except ValueError, ex:
+                    print "There was a problem converting the value '%s' to type %s for config parameter '%s'" % (configResults.__dict__[n], a['type'], n)
+                    import traceback
+                    #traceback.print_exc()
+                    sys.exit(1)
+
+
+
+        self.configResults = configResults
+
+        return configResults, args
+

contrib/benchmark.py

@@ -0,0 +1,51 @@
+import chunk
+import world
+import tempfile
+import glob
+import time
+import cProfile
+import os
+import sys
+import shutil
+
+# Simple Benchmarking script.  Usage and example:
+
+#  $ python contrib/benchmark.py World4/
+#  Rendering 50 chunks...
+#  Took 20.290062 seconds or 0.405801 seconds per chunk, or 2.464261 chunks per second
+
+
+# create a new, empty, cache dir
+cachedir = tempfile.mkdtemp(prefix="benchmark_cache", dir=".")
+if os.path.exists("benchmark.prof"): os.unlink("benchmark.prof")
+
+w = world.WorldRenderer("World4", cachedir)
+
+numchunks = 50
+chunklist = w._find_chunkfiles()[:numchunks]
+
+print "Rendering %d chunks..." % (numchunks)
+def go():
+    for f in chunklist:
+        chunk.render_and_save(f[2], w.cachedir, w, (None,None), None)
+start = time.time()
+if "-profile" in sys.argv:
+    cProfile.run("go()", 'benchmark.prof')
+else:
+    go()
+stop = time.time()
+
+delta = stop - start
+
+print "Took %f seconds or %f seconds per chunk, or %f chunks per second" % (delta, delta/numchunks, numchunks/delta)
+
+if "-profile" in sys.argv:
+    print "Profile is below:\n----\n"
+
+    import pstats
+    p = pstats.Stats('benchmark.prof')
+
+    p.strip_dirs().sort_stats("cumulative").print_stats(20)
+
+
+shutil.rmtree(cachedir)

contrib/validateRegionFile.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import os.path
+import sys
+overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
+sys.path.insert(0, overviewer_dir)
+import nbt
+
+def check_region(region_filename):
+    chunk_errors = []
+    if not os.path.exists(region_filename):
+        raise Exception('Region file not found: %s' % region_filename)
+    try:
+        region = nbt.load_region(region_filename)
+    except IOError, e:
+        raise Exception('Error loading region (%s): %s' % (region_filename, e))
+    try:
+        chunks = region.get_chunk_info(False)
+    except IOError, e:
+        raise Exception('Error reading region header (%s): %s' % (region_filename, e))
+    except Exception, e:
+        raise Exception('Error reading region (%s): %s' % (region_filename, e))
+    for x,y in chunks:
+        try:
+            check_chunk(region, x, y)
+        except Exception, e:
+            chunk_errors.append(e)
+    return (chunk_errors, len(chunks))
+    
+def check_chunk(region, x, y):
+    try:
+        data = region.load_chunk(x ,y)
+    except Exception, e:
+        raise Exception('Error reading chunk (%i, %i): %s' % (x, y, e))
+    if data is None:
+        raise Exception('Chunk (%i, %i) is unexpectedly empty' % (x, y))
+    else:
+        try:
+            processed_data = data.read_all()
+        except Exception, e:
+            raise Exception('Error reading chunk (%i, %i) data: %s' % (x, y, e))
+        if processed_data == []:
+            raise Exception('Chunk (%i, %i) is an unexpectedly empty set' % (x, y))
+
+if __name__ == '__main__':
+    try:
+        from optparse import OptionParser
+
+        parser = OptionParser(usage='python contrib/%prog [OPTIONS] <path/to/regions|path/to/regions/*.mcr|regionfile1.mcr regionfile2.mcr ...>',
+                              description='This script will valide a minecraft region file for errors.')
+        parser.add_option('-v', dest='verbose', action='store_true', help='Print additional information.')
+        opts, args = parser.parse_args()
+        
+        region_files = []
+        for path in args:
+            if os.path.isdir(path):
+                for dirpath, dirnames, filenames in os.walk(path, True):
+                    for filename in filenames:
+                        if filename.startswith('r.') and filename.endswith('.mcr'):
+                            if filename not in region_files:
+                                region_files.append(os.path.join(dirpath, filename))
+                        elif opts.verbose:
+                            print('Ignoring non-region file: %s' % os.path.join(dirpath, filename))
+            elif os.path.isfile(path):
+                dirpath,filename = os.path.split(path)
+                if filename.startswith('r.') and filename.endswith('.mcr'):
+                    if path not in region_files:
+                        region_files.append(path)
+                else:
+                    print('Ignoring non-region file: %s' % path)
+            else:
+                if opts.verbose:
+                    print('Ignoring arg: %s' % path)
+        if len(region_files) < 1:
+            print 'You must list at least one region file.'
+            parser.print_help()
+            sys.exit(1)
+        else:
+            overall_chunk_total = 0
+            bad_chunk_total = 0
+            bad_region_total = 0
+            for region_file in region_files:
+                try:
+                    (chunk_errors, region_chunks) = check_region(region_file)
+                    bad_chunk_total += len(chunk_errors)
+                    overall_chunk_total += region_chunks
+                except Exception, e:
+                    bad_region_total += 1
+                    print('FAILED(%s): %s' % (region_file, e))
+                else:
+                    if len(chunk_errors) is not 0:
+                        print('WARNING(%s) Chunks: %i/%' % (region_file, region_chunks - len(chunk_errors), region_chunks))
+                        if opts.verbose:
+                            for error in chunk_errors:
+                                print(error)
+                    elif opts.verbose:
+                            print ('PASSED(%s) Chunks: %i/%i' % (region_file, region_chunks - len(chunk_errors), region_chunks))
+            if opts.verbose:
+                print 'REGIONS: %i/%i' % (len(region_files) - bad_region_total, len(region_files))
+                print 'CHUNKS: %i/%i' % (overall_chunk_total - bad_chunk_total, overall_chunk_total)
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except Exception, e:
+        print('ERROR: %s' % e)
+

gmap.py

@@ -22,16 +22,25 @@ if not (sys.version_info[0] == 2 and sys.version_info[1] >= 6):
 
 import os
 import os.path
-from optparse import OptionParser
+from configParser import ConfigOptionParser
 import re
+import subprocess
 import multiprocessing
 import time
 import logging
-import optimizeimages
-import composite
 
 logging.basicConfig(level=logging.INFO,format="%(asctime)s [%(levelname)s] %(message)s")
 
+# make sure the c_overviewer extension is available
+try:
+    import c_overviewer
+except ImportError:
+    print "You need to compile the c_overviewer module to run Minecraft Overviewer."
+    print "Run `python setup.py build`, or see the README for details."
+    sys.exit(1)
+
+import optimizeimages
+import composite
 import world
 import quadtree
 
@@ -44,19 +53,21 @@ def main():
         cpus = multiprocessing.cpu_count()
     except NotImplementedError:
         cpus = 1
-    parser = OptionParser(usage=helptext)
+
+    parser = ConfigOptionParser(usage=helptext, config="settings.py")
     parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int")
-    parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int")
-    parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true")
+    parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int", configFileOnly=True)
+    parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True)
     parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.")
-    parser.add_option("--lighting", dest="lighting", help="Renders shadows using light data from each chunk.", action="store_true")
-    parser.add_option("--night", dest="night", help="Renders shadows using light data from each chunk, as if it were night. Implies --lighting.", action="store_true")
-    parser.add_option("--spawn", dest="spawn", help="Renders shadows using light data from each chunk, as if it were night, while also highlighting areas that are dark enough to spawn mobs. Implies --lighting and --night.", action="store_true")
-    parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg.")
-    parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%")
+    parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal")
+    parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True )
+    parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True)
+    parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this function after the web assets have been copied, but before actual tile rendering begins. It should accept a QuadtreeGen object as its only argument.", action="store", metavar="SCRIPT", type="function", configFileOnly=True)
     parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.")
     parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.")
     parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js")
+    parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map.  Requires all required options to be specified", commandLineOnly=True)
+    #parser.add_option("--write-config", dest="write_config", action="store_true", help="Writes out a sample config file", commandLineOnly=True)
 
     options, args = parser.parse_args()
 
@@ -101,6 +112,11 @@ def main():
         parser.error("Where do you want to save the tiles?")
 
     destdir = args[1]
+    if options.display_config:
+        # just display the config file and exit
+        parser.display_config()
+        sys.exit(0)
+
 
     if options.delete:
         return delete_all(worlddir, destdir)
@@ -145,7 +161,7 @@ def main():
 
     # Now generate the tiles
     # TODO chunklist
-    q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, lighting=options.lighting, night=options.night, spawn=options.spawn)
+    q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode, web_assets_hook=options.web_assets_hook)
     q.write_html(options.skipjs)
     q.go(options.procs)
 

quadtree.py

@@ -82,8 +82,15 @@ def catch_keyboardinterrupt(func):
             raise
     return newfunc
 
+child_quadtree = None
+def pool_initializer(quadtree):
+    logging.debug("Child process {0}".format(os.getpid()))
+    #stash the quadtree object in a global variable after fork() for windows compat.
+    global child_quadtree
+    child_quadtree = quadtree    
+    
 class QuadtreeGen(object):
-    def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False):
+    def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, rendermode="normal", web_assets_hook=None):
         """Generates a quadtree from the world given into the
         given dest directory
 
@@ -96,10 +103,11 @@ class QuadtreeGen(object):
         assert(imgformat)
         self.imgformat = imgformat
         self.optimizeimg = optimizeimg
+        self.web_assets_hook = web_assets_hook
         
-        self.lighting = lighting
-        self.night = night
-        self.spawn = spawn
+        self.lighting = rendermode in ("lighting", "night", "spawn")
+        self.night = rendermode in ("night", "spawn")
+        self.spawn = rendermode in ("spawn",)
 
         # Make the destination dir
         if not os.path.exists(destdir):
@@ -189,6 +197,8 @@ class QuadtreeGen(object):
             output.write(index)
 
         if skipjs:
+            if self.web_assets_hook:
+                self.web_assets_hook(self)
             return
 
         # since we will only discover PointsOfInterest in chunks that need to be 
@@ -217,6 +227,9 @@ class QuadtreeGen(object):
             output.write('  // ]},\n')
             output.write('];')
         
+        if self.web_assets_hook:
+            self.web_assets_hook(self)
+        
     def _get_cur_depth(self):
         """How deep is the quadtree currently in the destdir? This glances in
         config.js to see what maxZoom is set to.
@@ -292,10 +305,13 @@ class QuadtreeGen(object):
             shutil.rmtree(getpath("3"))
             os.rename(getpath("new3"), getpath("3"))
 
-    def _apply_render_worldtiles(self, pool):
+    def _apply_render_worldtiles(self, pool,batch_size):
         """Returns an iterator over result objects. Each time a new result is
         requested, a new task is added to the pool and a result returned.
         """
+             
+        batch = []
+        tiles = 0
         for path in iterate_base4(self.p):
             # Get the range for this tile
             colstart, rowstart = self._get_range_by_path(path)
@@ -306,28 +322,39 @@ class QuadtreeGen(object):
             dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path))
             #logging.debug("this is rendered at %s", dest)
             
-            # And uses these chunks
-            tilechunks = self._get_chunks_in_range(colstart, colend, rowstart,
-                    rowend)
-            #logging.debug(" tilechunks: %r", tilechunks)
+            # Put this in the batch to be submited to the pool                        
+            batch.append((colstart, colend, rowstart, rowend, dest))
+            tiles += 1
+            if tiles >= batch_size:
+                tiles = 0
+                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
+                batch = []
 
-            # Put this in the pool
-            # (even if tilechunks is empty, render_worldtile will delete
-            # existing images if appropriate)
-            yield pool.apply_async(func=render_worldtile, args= (self,
-                tilechunks, colstart, colend, rowstart, rowend, dest))
+        if tiles > 0:
+            yield pool.apply_async(func=render_worldtile_batch, args= (batch,))
 
-    def _apply_render_inntertile(self, pool, zoom):
+
+    def _apply_render_inntertile(self, pool, zoom,batch_size):
         """Same as _apply_render_worltiles but for the inntertile routine.
         Returns an iterator that yields result objects from tasks that have
         been applied to the pool.
         """
+        batch = []
+        tiles = 0        
         for path in iterate_base4(zoom):
             # This image is rendered at:
             dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1]))
             name = str(path[-1])
             
-            yield pool.apply_async(func=render_innertile, args= (dest, name, self.imgformat, self.optimizeimg))
+            batch.append((dest, name, self.imgformat, self.optimizeimg))
+            tiles += 1
+            if tiles >= batch_size:
+                tiles = 0
+                yield pool.apply_async(func=render_innertile_batch, args= [batch])
+                batch = []
+            
+        if tiles > 0:       
+            yield pool.apply_async(func=render_innertile_batch, args= [batch])
         
     def go(self, procs):
         """Renders all tiles"""
@@ -344,11 +371,16 @@ class QuadtreeGen(object):
                 for _ in xrange(curdepth - self.p):
                     self._decrease_depth()
 
+        logging.debug("Parent process {0}".format(os.getpid()))
         # Create a pool
         if procs == 1:
             pool = FakePool()
+            global child_quadtree
+            child_quadtree = self
         else:
-            pool = multiprocessing.Pool(processes=procs)
+            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
+            #warm up the pool so it reports all the worker id's
+            pool.map(bool,xrange(multiprocessing.cpu_count()),1)
         
         # Render the highest level of tiles from the chunks
         results = collections.deque()
@@ -359,20 +391,20 @@ class QuadtreeGen(object):
         logging.info("There are {0} total levels to render".format(self.p))
         logging.info("Don't worry, each level has only 25% as many tiles as the last.")
         logging.info("The others will go faster")
-        for result in self._apply_render_worldtiles(pool):
+        count = 0
+        batch_size = 10
+        for result in self._apply_render_worldtiles(pool,batch_size):
             results.append(result)
-            if len(results) > 10000:
+            if len(results) > (10000/batch_size):
                 # Empty the queue before adding any more, so that memory
                 # required has an upper bound
-                while len(results) > 500:
-                    results.popleft().get()
-                    complete += 1
+                while len(results) > (500/batch_size):
+                    complete += results.popleft().get()
                     self.print_statusline(complete, total, 1)
 
         # Wait for the rest of the results
         while len(results) > 0:
-            results.popleft().get()
-            complete += 1
+            complete += results.popleft().get()
             self.print_statusline(complete, total, 1)
 
         self.print_statusline(complete, total, 1, True)
@@ -384,17 +416,15 @@ class QuadtreeGen(object):
             complete = 0
             total = 4**zoom
             logging.info("Starting level {0}".format(level))
-            for result in self._apply_render_inntertile(pool, zoom):
+            for result in self._apply_render_inntertile(pool, zoom,batch_size):
                 results.append(result)
-                if len(results) > 10000:
-                    while len(results) > 500:
-                        results.popleft().get()
-                        complete += 1
+                if len(results) > (10000/batch_size):
+                    while len(results) > (500/batch_size):
+                        complete += results.popleft().get()
                         self.print_statusline(complete, total, level)
             # Empty the queue
             while len(results) > 0:
-                results.popleft().get()
-                complete += 1
+                complete += results.popleft().get()
                 self.print_statusline(complete, total, level)
 
             self.print_statusline(complete, total, level, True)
@@ -448,6 +478,14 @@ class QuadtreeGen(object):
         return chunklist
 
 @catch_keyboardinterrupt
+def render_innertile_batch(batch):    
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1
+        render_innertile(job[0],job[1],job[2],job[3])
+    return count
+    
 def render_innertile(dest, name, imgformat, optimizeimg):
     """
     Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
@@ -512,6 +550,30 @@ def render_innertile(dest, name, imgformat, optimizeimg):
             optimize_image(imgpath, imgformat, optimizeimg)
 
 @catch_keyboardinterrupt
+def render_worldtile_batch(batch):  
+    global child_quadtree
+    return render_worldtile_batch_(child_quadtree, batch)
+
+def render_worldtile_batch_(quadtree, batch):   
+    count = 0
+    _get_chunks_in_range = quadtree._get_chunks_in_range
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1
+        colstart = job[0]
+        colend = job[1]
+        rowstart = job[2]
+        rowend = job[3]
+        path = job[4]
+        # (even if tilechunks is empty, render_worldtile will delete
+        # existing images if appropriate)    
+        # And uses these chunks
+        tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend)
+        #logging.debug(" tilechunks: %r", tilechunks)
+            
+        render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path)
+    return count
+
 def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):
     """Renders just the specified chunks into a tile and save it. Unlike usual
     python conventions, rowend and colend are inclusive. Additionally, the
@@ -597,16 +659,10 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
     
     # check chunk mtimes to see if they are newer
     try:
-        #tile_mtime = os.path.getmtime(imgpath)
-        regionMtimes = {}
         needs_rerender = False
         for col, row, chunkx, chunky, regionfile in chunks:
             # check region file mtime first. 
-            # Note: we cache the value since it's actually very likely we will have multipule chunks in the same region, and syscalls are expensive
-            regionMtime = regionMtimes.get(regionfile,None)
-            if  regionMtime is None:
-                regionMtime = os.path.getmtime(regionfile)  
-                regionMtimes[regionfile] = regionMtime 
+            regionMtime = world.get_region_mtime(regionfile)  
             if regionMtime <= tile_mtime:
                 continue
             

src/composite.c

@@ -253,6 +253,7 @@ alpha_over_wrap(PyObject *self, PyObject *args)
     /* destination position read */
     if (!PyArg_ParseTuple(pos, "iiii", &dx, &dy, &xsize, &ysize)) {
         /* try again, but this time try to read a point */
+        PyErr_Clear();
         xsize = 0;
         ysize = 0;
         if (!PyArg_ParseTuple(pos, "ii", &dx, &dy)) {

src/iterate.c

@@ -22,6 +22,40 @@
 /* macro for getting blockID from a chunk of memory */
 #define getBlock(blockThing, x,y,z) (*(unsigned char *)(PyArray_GETPTR3(blockThing, (x), (y), (z))))
 
+static PyObject *textures = NULL;
+static PyObject *chunk_mod = NULL;
+static PyObject *blockmap = NULL;
+static PyObject *special_blocks = NULL;
+static PyObject *specialblockmap = NULL;
+static PyObject *transparent_blocks = NULL;
+
+int init_chunk_render(void) {
+   
+    /* if blockmap (or any of these) is not NULL, then that means that we've 
+     * somehow called this function twice.  error out so we can notice this
+     * */
+    if (blockmap) return 1;
+
+    textures = PyImport_ImportModule("textures");
+    chunk_mod = PyImport_ImportModule("chunk");
+
+    blockmap = PyObject_GetAttrString(textures, "blockmap");
+    special_blocks = PyObject_GetAttrString(textures, "special_blocks");
+    specialblockmap = PyObject_GetAttrString(textures, "specialblockmap");
+    transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks");
+    
+    /* ensure none of these pointers are NULL */    
+    if ((!transparent_blocks) || (!blockmap) || (!special_blocks) || (!specialblockmap)){
+        fprintf(stderr, "\ninit_chunk_render failed\n");
+        return 1;
+    }
+
+    Py_DECREF(textures);
+    Py_DECREF(chunk_mod);
+    return 0;
+
+}
+
 static inline int isTransparent(PyObject* tup, unsigned char b) {
     PyObject *block = PyInt_FromLong(b);
     int ret = PySequence_Contains(tup, block);
@@ -302,8 +336,6 @@ chunk_render(PyObject *self, PyObject *args) {
     
     PyObject *blocks_py;
     
-    PyObject *textures, *blockmap, *special_blocks, *specialblockmap, *chunk_mod, *transparent_blocks;
-    
     int imgx, imgy;
     int x, y, z;
 
@@ -331,23 +363,6 @@ chunk_render(PyObject *self, PyObject *args) {
     PyObject *right_blocks = PyObject_GetAttrString(chunk, "right_blocks");
     */
     
-    textures = PyImport_ImportModule("textures");
-    chunk_mod = PyImport_ImportModule("chunk");
-
-    /* TODO can these be global static?  these don't change during program execution */
-    blockmap = PyObject_GetAttrString(textures, "blockmap");
-    special_blocks = PyObject_GetAttrString(textures, "special_blocks");
-    specialblockmap = PyObject_GetAttrString(textures, "specialblockmap");
-    transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks");
-    if (transparent_blocks == NULL) {
-        PyErr_SetString(PyExc_ValueError,
-                        "transparent_blocks is NULL");
-        return NULL;
-    }
-        
-
-    Py_DECREF(textures);
-    Py_DECREF(chunk_mod);
     
     for (x = 15; x > -1; x--) {
         for (y = 0; y < 16; y++) {
@@ -427,9 +442,6 @@ chunk_render(PyObject *self, PyObject *args) {
     } 
 
     Py_DECREF(blocks_py);
-    Py_DECREF(blockmap);
-    Py_DECREF(special_blocks);
-    Py_DECREF(specialblockmap);
 
     return Py_BuildValue("i",2);
 }

src/main.c

@@ -35,4 +35,10 @@ initc_overviewer(void)
     (void)Py_InitModule("c_overviewer", COverviewerMethods);
     /* for numpy */
     import_array();
+
+    /* initialize some required variables in iterage.c */
+    if (init_chunk_render()) {
+        fprintf(stderr, "failed to init_chunk_render\n");
+        exit(1); // TODO better way to indicate error?
+    }
 }

src/overviewer.h

@@ -38,5 +38,6 @@ PyObject *brightness(PyObject *img, double factor);
 /* in iterate.c */
 PyObject *chunk_render(PyObject *self, PyObject *args);
 PyObject *chunk_render_lighting(PyObject *self, PyObject *args);
+int init_chunk_render(void);
 
 #endif /* __OVERVIEWER_H_INCLUDED__ */

world.py

@@ -79,7 +79,7 @@ class World(object):
         for x, y, regionfile in self._iterate_regionfiles():            
             mcr = nbt.MCRFileReader(regionfile)
             mcr.get_chunk_info()
-            regions[regionfile] = mcr
+            regions[regionfile] = (mcr,os.path.getmtime(regionfile))
             regionfiles[(x,y)]	= (x,y,regionfile)
         self.regionfiles = regionfiles	
         self.regions = regions
@@ -118,8 +118,6 @@ class World(object):
         _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
         return regionfile
     
-    
-    
     def load_from_region(self,filename, x, y):
         nbt = self.load_region(filename).load_chunk(x, y)
         if nbt is None:
@@ -128,11 +126,15 @@ class World(object):
         return nbt.read_all()
       
       
-    #filo region cache
-    def load_region(self,filename):                
-        #return nbt.MCRFileReader(filename)    
-        return self.regions[filename]
+    #used to reload a changed region
+    def reload_region(self,filename):
+        self.regions[filename] = (nbt.MCRFileReader(filename),os.path.getmtime(regionfile))
         
+    def load_region(self,filename):                  
+        return self.regions[filename][0]
+        
+    def get_region_mtime(self,filename):                  
+        return self.regions[filename][1]        
         
     def convert_coords(self, chunkx, chunky):
         """Takes a coordinate (chunkx, chunky) where chunkx and chunky are