Merge commit 'xon/dtt-c-render' into dtt-c-render

Conflicts:
	nbt.py
	overviewer.py

chunk.py

@@ -73,10 +73,10 @@ def get_blockarray(level):
     Block array, which just contains all the block ids"""
     return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
 
-def get_blockarray_fromfile(world,filename):
-    """Same as get_blockarray except takes a filename and uses get_lvldata to
-    open it. This is a shortcut"""
-    level = get_lvldata(world,filename)
+def get_blockarray_fromfile(filename):
+    """Same as get_blockarray except takes a filename. This is a shortcut"""
+    d =  nbt.load_from_region(filename, x, y)
+    level = d[1]['Level']
     return get_blockarray(level)
 
 def get_skylight_array(level):

contrib/findSigns.py

@@ -33,16 +33,20 @@ if os.path.exists(worlddir):
 else:
     sys.exit("Bad WorldDir")
 
-matcher = re.compile(r"^c\..*\.dat$")
+matcher = re.compile(r"^r\..*\.mcr$")
 
 POI = []
 
 for dirpath, dirnames, filenames in os.walk(worlddir):
     for f in filenames:
         if matcher.match(f):
+            print f
             full = os.path.join(dirpath, f)
-            #print "inspecting %s" % full
-            data = nbt.load(full)[1]['Level']['TileEntities']
+            r = nbt.load_region(full)
+            chunks = r.get_chunks()
+            for x,y in chunks:
+                chunk = r.load_chunk(x,y).read_all()                
+                data = chunk[1]['Level']['TileEntities']
                 for entity in data:
                     if entity['id'] == 'Sign':
                         msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])

contrib/rerenderBlocks.py

@@ -7,8 +7,8 @@ blockID.  The output is a chunklist file that is suitable to use with the
 
 Example:
 
-python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> chunklist.txt
-    python overviewer.py --chunklist=chunklist.txt world/ output_dir/
+python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> regionlist.txt
+    python overviewer.py --regionlist=regionlist.txt world/ output_dir/
 
 This will rerender any chunks that contain either TNT (46), Ice (79), or 
 a Jack-O-Lantern (91)
@@ -42,15 +42,20 @@ ids = map(lambda x: int(x),options.ids.split(","))
 sys.stderr.write("Searching for these blocks: %r...\n" % ids)
 
 
-matcher = re.compile(r"^c\..*\.dat$")
+matcher = re.compile(r"^r\..*\.mcr$")
 
 for dirpath, dirnames, filenames in os.walk(options.world):
     for f in filenames:
         if matcher.match(f):
             full = os.path.join(dirpath, f)
-            blocks = get_blockarray_fromfile(full)
+            r = nbt.load_region(full)
+            chunks = r.get_chunks()
+            for x,y in chunks:
+                chunk = r.load_chunk(x,y).read_all()                
+                blocks = get_blockarray(chunk[1]['Level'])
                 for i in ids:
                     if i in blocks:
                         print full
                         break
 
+

googlemap.py

@@ -54,7 +54,8 @@ class MapGen(object):
     def __init__(self, quadtrees, skipjs=False, web_assets_hook=None):
         """Generates a Google Maps interface for the given list of
         quadtrees. All of the quadtrees must have the same destdir,
-        image format, and world."""
+        image format, and world. 
+        Note:tiledir for each quadtree should be unique. By default the tiledir is determined by the rendermode"""
         
         self.skipjs = skipjs
         self.web_assets_hook = web_assets_hook

nbt.py

@@ -26,7 +26,7 @@ def _file_loader(func):
                return None
 
             # Is actually a filename
-            fileobj = open(fileobj, 'rb')
+            fileobj = open(fileobj, 'rb',4096)
         return func(fileobj, *args)
     return wrapper
 
@@ -44,6 +44,20 @@ def load_from_region(filename, x, y):
 def load_region(filename):            
     return MCRFileReader(filename)
   
+  
+# compile the unpacker's into a classes
+_byte   = struct.Struct("b")
+_short  = struct.Struct(">h")
+_int    = struct.Struct(">i")
+_long   = struct.Struct(">q")
+_float  = struct.Struct(">f")
+_double = struct.Struct(">d") 
+
+_24bit_int = struct.Struct("B B B")
+_unsigned_byte = struct.Struct("B")
+_unsigned_int = struct.Struct(">I")
+_chunk_header = struct.Struct(">I B")
+ 
 class NBTFileReader(object):
     def __init__(self, fileobj, is_gzip=True):
         if is_gzip:
@@ -61,27 +75,32 @@ class NBTFileReader(object):
 
     def _read_tag_byte(self):
         byte = self._file.read(1)
-        return struct.unpack("b", byte)[0]
+        return _byte.unpack(byte)[0]
     
     def _read_tag_short(self):
         bytes = self._file.read(2)
-        return struct.unpack(">h", bytes)[0]
+        global _short
+        return _short.unpack(bytes)[0]
 
     def _read_tag_int(self):
         bytes = self._file.read(4)
-        return struct.unpack(">i", bytes)[0]
+        global _int
+        return _int.unpack(bytes)[0]
 
     def _read_tag_long(self):
         bytes = self._file.read(8)
-        return struct.unpack(">q", bytes)[0]
+        global _long
+        return _long.unpack(bytes)[0]
 
     def _read_tag_float(self):
         bytes = self._file.read(4)
-        return struct.unpack(">f", bytes)[0]
+        global _float
+        return _float.unpack(bytes)[0]
 
     def _read_tag_double(self):
         bytes = self._file.read(8)
-        return struct.unpack(">d", bytes)[0]
+        global _double
+        return _double.unpack(bytes)[0]
 
     def _read_tag_byte_array(self):
         length = self._read_tag_int()
@@ -194,9 +213,11 @@ class MCRFileReader(object):
         
         ret = 0
         bytes = self._file.read(3)
+        global _24bit_int
+        bytes = _24bit_int.unpack(bytes)
         for i in xrange(3):
             ret = ret << 8
-            ret += struct.unpack("B", bytes[i])[0]
+            ret += bytes[i]
         
         return ret
     
@@ -222,8 +243,9 @@ class MCRFileReader(object):
             offset_sectors = self._read_24bit_int()
             
             # 1-byte length in 4KiB sectors, rounded up
+            global _unsigned_byte
             byte = self._file.read(1)
-            length_sectors = struct.unpack("B", byte)[0]
+            length_sectors = _unsigned_byte.unpack(byte)[0]
         except (IndexError, struct.error):
             # got a problem somewhere
             return None
@@ -253,24 +275,39 @@ class MCRFileReader(object):
         
         try:
             bytes = self._file.read(4)
-            timestamp = struct.unpack(">I", bytes)[0]
+            global _unsigned_int
+            timestamp = _unsigned_int.unpack(bytes)[0]
         except (IndexError, struct.error):
             return 0
         
         return timestamp
     
-    def get_chunk_info(self,closeFile = True):
+    def get_chunks(self):    
         """Return a list of all chunks contained in this region file,
         as a list of (x, y) coordinate tuples. To load these chunks,
         provide these coordinates to load_chunk()."""
         
-        if self._chunks:
+        if self._chunks is not None:
+            return self._chunks
+        if self._locations is None:
+            self.get_chunk_info()       
+        self._chunks = [] 
+        for x in xrange(32): 
+            for y in xrange(32): 
+                if self._locations[x + y * 32] is not None:
+                    self._chunks.append((x,y))
         return self._chunks
         
-        if self._file is None:
-            self._file = open(self._filename,'rb');
+    def get_chunk_info(self,closeFile = True):
+        """Preloads region header information."""
         
-        self._chunks = []
+        if self._locations:
+            return
+        
+        if self._file is None:
+            self._file = open(self._filename,'rb')
+
+        self._chunks = None
         self._locations = []
         self._timestamps = []
         
@@ -278,24 +315,20 @@ class MCRFileReader(object):
         self._file.seek(0)        
         
         # read chunk location table
-        for y in xrange(32):
-            for x in xrange(32):
-                location = self._read_chunk_location()
-                self._locations.append(location)
-                if location:
-                    self._chunks.append((x, y))
+        locations_append = self._locations.append
+        for _ in xrange(32*32): 
+            locations_append(self._read_chunk_location())
         
         # read chunk timestamp table
-        for y in xrange(32):
-            for x in xrange(32):
-                timestamp = self._read_chunk_timestamp()
-                self._timestamps.append(timestamp)
+        timestamp_append = self._timestamps.append
+        for _ in xrange(32*32): 
+            timestamp_append(self._read_chunk_timestamp())
  
         if closeFile:        
             #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
             self._file.close()
             self._file =  None
-        return self._chunks
+        return
     
     def get_chunk_timestamp(self, x, y):
         """Return the given chunk's modification time. If the given
@@ -339,10 +372,8 @@ class MCRFileReader(object):
         self._file.seek(location[0])
         
         # read in the chunk data header
-        bytes = self._file.read(4)
-        data_length = struct.unpack(">I", bytes)[0]
-        bytes = self._file.read(1)
-        compression = struct.unpack("B", bytes)[0]
+        bytes = self._file.read(5)        
+        data_length,compression =  _chunk_header.unpack(bytes)
         
         # figure out the compression
         is_gzip = True

overviewer.py

@@ -45,6 +45,7 @@ import composite
 import world
 import quadtree
 import googlemap
+import rendernode
 
 helptext = """
 %prog [OPTIONS] <World # / Name / Path to World> <tiles dest dir>
@@ -179,15 +180,25 @@ def main():
 
     # create the quadtrees
     # TODO chunklist
-    # NOTE: options.rendermode is now a list of rendermodes.  for now, always use the first one
-    q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode[0])
+    q = []
+    qtree_args = {'depth' : options.zoom, 'imgformat' : imgformat, 'optimizeimg' : optimizeimg}
+    for rendermode in options.rendermode:
+        if rendermode == 'normal':
+            qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, tiledir='tiles', **qtree_args)
+        else:
+            qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, **qtree_args)
+        q.append(qtree)
+
+    #create the distributed render
+    r = rendernode.RenderNode(q)
     
     # write out the map and web assets
-    m = googlemap.MapGen([q,], skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
+    m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
     m.go(options.procs)
     
     # render the tiles!
-    q.go(options.procs)
+    r.go(options.procs)
+
 
 def delete_all(worlddir, tiledir):
     # TODO should we delete tiledir here too?

quadtree.py

@@ -27,6 +27,7 @@ import util
 import cPickle
 import stat
 import errno 
+import time
 from time import gmtime, strftime, sleep
 
 from PIL import Image
@@ -46,31 +47,8 @@ def iterate_base4(d):
     """Iterates over a base 4 number with d digits"""
     return itertools.product(xrange(4), repeat=d)
    
-def catch_keyboardinterrupt(func):
-    """Decorator that catches a keyboardinterrupt and raises a real exception
-    so that multiprocessing will propagate it properly"""
-    @functools.wraps(func)
-    def newfunc(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except KeyboardInterrupt:
-            logging.error("Ctrl-C caught!")
-            raise Exception("Exiting")
-        except:
-            import traceback
-            traceback.print_exc()
-            raise
-    return newfunc
-
-child_quadtree = None
-def pool_initializer(quadtree):
-    logging.debug("Child process {0}".format(os.getpid()))
-    #stash the quadtree object in a global variable after fork() for windows compat.
-    global child_quadtree
-    child_quadtree = quadtree    
-    
 class QuadtreeGen(object):
-    def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, rendermode="normal"):
+    def __init__(self, worldobj, destdir, depth=None, tiledir=None, imgformat=None, optimizeimg=None, rendermode="normal"):
         """Generates a quadtree from the world given into the
         given dest directory
 
@@ -84,11 +62,16 @@ class QuadtreeGen(object):
         self.imgformat = imgformat
         self.optimizeimg = optimizeimg
         
+        self.lighting = rendermode in ("lighting", "night", "spawn")
+        self.night = rendermode in ("night", "spawn")
+        self.spawn = rendermode in ("spawn",)
         self.rendermode = rendermode
 
         # Make the destination dir
         if not os.path.exists(destdir):
             os.mkdir(destdir)
+        if tiledir is None:
+            tiledir = rendermode
         self.tiledir = tiledir        
         
         if depth is None:
@@ -122,21 +105,7 @@ class QuadtreeGen(object):
 
         self.world = worldobj
         self.destdir = destdir
-
-    def print_statusline(self, complete, total, level, unconditional=False):
-        if unconditional:
-            pass
-        elif complete < 100:
-            if not complete % 25 == 0:
-                return
-        elif complete < 1000:
-            if not complete % 100 == 0:
-                return
-        else:
-            if not complete % 1000 == 0:
-                return
-        logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
-                complete, total, level, self.p))
+        self.full_tiledir = os.path.join(destdir, tiledir)
         
     def _get_cur_depth(self):
         """How deep is the quadtree currently in the destdir? This glances in
@@ -213,59 +182,8 @@ class QuadtreeGen(object):
             shutil.rmtree(getpath("3"))
             os.rename(getpath("new3"), getpath("3"))
         
-    def _apply_render_worldtiles(self, pool,batch_size):
-        """Returns an iterator over result objects. Each time a new result is
-        requested, a new task is added to the pool and a result returned.
-        """
-             
-        batch = []
-        tiles = 0
-        for path in iterate_base4(self.p):
-            # Get the range for this tile
-            colstart, rowstart = self._get_range_by_path(path)
-            colend = colstart + 2
-            rowend = rowstart + 4
-
-            # This image is rendered at:
-            dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path))
-            #logging.debug("this is rendered at %s", dest)
-            
-            # Put this in the batch to be submited to the pool                        
-            batch.append((colstart, colend, rowstart, rowend, dest))
-            tiles += 1
-            if tiles >= batch_size:
-                tiles = 0
-                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
-                batch = []
-
-        if tiles > 0:
-            yield pool.apply_async(func=render_worldtile_batch, args= (batch,))
-
-
-    def _apply_render_inntertile(self, pool, zoom,batch_size):
-        """Same as _apply_render_worltiles but for the inntertile routine.
-        Returns an iterator that yields result objects from tasks that have
-        been applied to the pool.
-        """
-        batch = []
-        tiles = 0        
-        for path in iterate_base4(zoom):
-            # This image is rendered at:
-            dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1]))
-            name = str(path[-1])
-            
-            batch.append((dest, name, self.imgformat, self.optimizeimg))
-            tiles += 1
-            if tiles >= batch_size:
-                tiles = 0
-                yield pool.apply_async(func=render_innertile_batch, args= [batch])
-                batch = []
-            
-        if tiles > 0:       
-            yield pool.apply_async(func=render_innertile_batch, args= [batch])
-        
     def go(self, procs):
-        """Renders all tiles"""
+        """Processing before tile rendering"""
 
         curdepth = self._get_cur_depth()
         if curdepth != -1:
@@ -279,71 +197,6 @@ class QuadtreeGen(object):
                 for _ in xrange(curdepth - self.p):
                     self._decrease_depth()
     
-        logging.debug("Parent process {0}".format(os.getpid()))
-        # Create a pool
-        if procs == 1:
-            pool = FakePool()
-            global child_quadtree
-            child_quadtree = self
-        else:
-            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
-            #warm up the pool so it reports all the worker id's
-            pool.map(bool,xrange(multiprocessing.cpu_count()),1)
-        
-        # Render the highest level of tiles from the chunks
-        results = collections.deque()
-        complete = 0
-        total = 4**self.p
-        logging.info("Rendering highest zoom level of tiles now.")
-        logging.info("There are {0} tiles to render".format(total))
-        logging.info("There are {0} total levels to render".format(self.p))
-        logging.info("Don't worry, each level has only 25% as many tiles as the last.")
-        logging.info("The others will go faster")
-        count = 0
-        batch_size = 10
-        for result in self._apply_render_worldtiles(pool,batch_size):
-            results.append(result)
-            if len(results) > (10000/batch_size):
-                # Empty the queue before adding any more, so that memory
-                # required has an upper bound
-                while len(results) > (500/batch_size):
-                    complete += results.popleft().get()
-                    self.print_statusline(complete, total, 1)
-
-        # Wait for the rest of the results
-        while len(results) > 0:
-            complete += results.popleft().get()
-            self.print_statusline(complete, total, 1)
-
-        self.print_statusline(complete, total, 1, True)
-
-        # Now do the other layers
-        for zoom in xrange(self.p-1, 0, -1):
-            level = self.p - zoom + 1
-            assert len(results) == 0
-            complete = 0
-            total = 4**zoom
-            logging.info("Starting level {0}".format(level))
-            for result in self._apply_render_inntertile(pool, zoom,batch_size):
-                results.append(result)
-                if len(results) > (10000/batch_size):
-                    while len(results) > (500/batch_size):
-                        complete += results.popleft().get()
-                        self.print_statusline(complete, total, level)
-            # Empty the queue
-            while len(results) > 0:
-                complete += results.popleft().get()
-                self.print_statusline(complete, total, level)
-
-            self.print_statusline(complete, total, level, True)
-
-            logging.info("Done")
-
-        pool.close()
-        pool.join()
-
-        # Do the final one right here:
-        render_innertile(os.path.join(self.destdir, self.tiledir), "base", self.imgformat, self.optimizeimg)
     
     def _get_range_by_path(self, path):
         """Returns the x, y chunk coordinates of this tile"""
@@ -362,7 +215,7 @@ class QuadtreeGen(object):
 
         return x, y
         
-    def _get_chunks_in_range(self, colstart, colend, rowstart, rowend):
+    def get_chunks_in_range(self, colstart, colend, rowstart, rowend):
         """Get chunks that are relevant to the tile rendering function that's
         rendering that range"""
         chunklist = []
@@ -380,25 +233,45 @@ class QuadtreeGen(object):
                 # return (col, row, chunkx, chunky, regionpath)
                 chunkx, chunky = unconvert_coords(col, row)
                 #c = get_region_path(chunkx, chunky)
-                _, _, c = get_region((chunkx//32, chunky//32),(None,None,None));
-                if c is not None:
+                _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None));
+                if c is not None and mcr.chunkExists(chunkx,chunky):                  
                     chunklist.append((col, row, chunkx, chunky, c))
         return chunklist   
         
-@catch_keyboardinterrupt
-def render_innertile_batch(batch):    
-    count = 0
-    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
-    for job in batch:
-        count += 1
-        render_innertile(job[0],job[1],job[2],job[3])
-    return count
+    def get_worldtiles(self):
+        """Returns an iterator over the tiles of the most detailed layer
+        """
+        for path in iterate_base4(self.p):
+            # Get the range for this tile
+            colstart, rowstart = self._get_range_by_path(path)
+            colend = colstart + 2
+            rowend = rowstart + 4   
             
-def render_innertile(dest, name, imgformat, optimizeimg):
+            # This image is rendered at(relative to the worker's destdir):
+            tilepath = [str(x) for x in path]
+            tilepath = os.sep.join(tilepath)
+            #logging.debug("this is rendered at %s", dest)
+        
+            # Put this in the batch to be submited to the pool     
+            yield [self,colstart, colend, rowstart, rowend, tilepath]
+        
+    def get_innertiles(self,zoom):
+        """Same as get_worldtiles but for the inntertile routine.
+        """    
+        for path in iterate_base4(zoom):
+            # This image is rendered at(relative to the worker's destdir):
+            tilepath = [str(x) for x in path[:-1]]
+            tilepath = os.sep.join(tilepath)
+            name = str(path[-1])
+  
+            yield [self,tilepath, name]
+    
+    def render_innertile(self, dest, name):
         """
         Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
         os.path.join(dest, name, "{0,1,2,3}.png")
         """
+        imgformat = self.imgformat 
         imgpath = os.path.join(dest, name) + "." + imgformat
 
         if name == "base":
@@ -450,39 +323,17 @@ def render_innertile(dest, name, imgformat, optimizeimg):
                 logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
 
         # Save it
-    if imgformat == 'jpg':
+        if self.imgformat == 'jpg':
             img.save(imgpath, quality=95, subsampling=0)
         else: # png
             img.save(imgpath)
-        if optimizeimg:
-            optimize_image(imgpath, imgformat, optimizeimg)
             
-@catch_keyboardinterrupt
-def render_worldtile_batch(batch):  
-    global child_quadtree
-    return render_worldtile_batch_(child_quadtree, batch)
+        if self.optimizeimg:
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)
 
-def render_worldtile_batch_(quadtree, batch):   
-    count = 0
-    _get_chunks_in_range = quadtree._get_chunks_in_range
-    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
-    for job in batch:
-        count += 1
-        colstart = job[0]
-        colend = job[1]
-        rowstart = job[2]
-        rowend = job[3]
-        path = job[4]
-        # (even if tilechunks is empty, render_worldtile will delete
-        # existing images if appropriate)    
-        # And uses these chunks
-        tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend)
-        #logging.debug(" tilechunks: %r", tilechunks)
 
-        render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path)
-    return count
 
-def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):
+    def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path):
         """Renders just the specified chunks into a tile and save it. Unlike usual
         python conventions, rowend and colend are inclusive. Additionally, the
         chunks around the edges are half-way cut off (so that neighboring tiles
@@ -491,7 +342,7 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
         chunks is a list of (col, row, chunkx, chunky, filename) of chunk
         images that are relevant to this call (with their associated regions)
 
-    The image is saved to path+"."+quadtree.imgformat
+        The image is saved to path+"."+self.imgformat
 
         If there are no chunks, this tile is not saved (if it already exists, it is
         deleted)
@@ -528,17 +379,8 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
         # way above this one are possibly visible in this tile). Render them
         # anyways just in case). "chunks" should include up to rowstart-16
 
-    imgpath = path + "." + quadtree.imgformat
-    
-    world = quadtree.world
-    # first, remove chunks from `chunks` that don't actually exist in
-    # their region files
-    def chunk_exists(chunk):
-        _, _, chunkx, chunky, region = chunk
-        r = world.load_region(region)
-        return r.chunkExists(chunkx, chunky)            
-    chunks = filter(chunk_exists, chunks)
-
+        imgpath = path + "." + self.imgformat
+        world = self.world
         #stat the file, we need to know if it exists or it's mtime
         try:    
             tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
@@ -570,12 +412,11 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
             needs_rerender = False
             for col, row, chunkx, chunky, regionfile in chunks:
                 # check region file mtime first. 
-            regionMtime = world.get_region_mtime(regionfile)  
+                region,regionMtime = world.get_region_mtime(regionfile)  
                 if regionMtime <= tile_mtime:
                     continue
                 
                 # checking chunk mtime
-            region = world.load_region(regionfile)
                 if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
                     needs_rerender = True
                     break
@@ -600,28 +441,10 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
 
             # draw the chunk!
             # TODO POI queue
-        chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), quadtree, False, None)
+            chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None)
 
         # Save them
         tileimg.save(imgpath)
 
-    if quadtree.optimizeimg:
-        optimize_image(imgpath, quadtree.imgformat, quadtree.optimizeimg)
-
-class FakeResult(object):
-    def __init__(self, res):
-        self.res = res
-    def get(self):
-        return self.res
-class FakePool(object):
-    """A fake pool used to render things in sync. Implements a subset of
-    multiprocessing.Pool"""
-    def apply_async(self, func, args=(), kwargs=None):
-        if not kwargs:
-            kwargs = {}
-        result = func(*args, **kwargs)
-        return FakeResult(result)
-    def close(self):
-        pass
-    def join(self):
-        pass
+        if self.optimizeimg:
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)

rendernode.py

@@ -0,0 +1,318 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import multiprocessing
+import itertools
+from itertools import cycle, islice
+import os
+import os.path
+import functools
+import re
+import shutil
+import collections
+import json
+import logging
+import util
+import cPickle
+import stat
+import errno 
+import time
+from time import gmtime, strftime, sleep
+
+
+"""
+This module has routines related to distributing the render job to multipule nodes
+
+"""
+
+def catch_keyboardinterrupt(func):
+    """Decorator that catches a keyboardinterrupt and raises a real exception
+    so that multiprocessing will propagate it properly"""
+    @functools.wraps(func)
+    def newfunc(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except KeyboardInterrupt:
+            logging.error("Ctrl-C caught!")
+            raise Exception("Exiting")
+        except:
+            import traceback
+            traceback.print_exc()
+            raise
+    return newfunc
+    
+child_rendernode = None
+def pool_initializer(rendernode):
+    logging.debug("Child process {0}".format(os.getpid()))
+    #stash the quadtree objects in a global variable after fork() for windows compat.
+    global child_rendernode
+    child_rendernode = rendernode  
+    
+#http://docs.python.org/library/itertools.html    
+def roundrobin(iterables):
+    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+    # Recipe credited to George Sakkis
+    pending = len(iterables)
+    nexts = cycle(iter(it).next for it in iterables)
+    while pending:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            pending -= 1
+            nexts = cycle(islice(nexts, pending))
+
+            
+class RenderNode(object):
+    def __init__(self, quadtrees):
+        """Distributes the rendering of a list of quadtrees."""
+
+        if not len(quadtrees) > 0:
+            raise ValueError("there must be at least one quadtree to work on")    
+
+        self.quadtrees = quadtrees
+        #bind an index value to the quadtree so we can find it again
+        i = 0
+        for q in quadtrees:
+            q._render_index = i
+            i += 1       
+
+    def print_statusline(self, complete, total, level, unconditional=False):
+        if unconditional:
+            pass
+        elif complete < 100:
+            if not complete % 25 == 0:
+                return
+        elif complete < 1000:
+            if not complete % 100 == 0:
+                return
+        else:
+            if not complete % 1000 == 0:
+                return
+        logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
+                complete, total, level, self.max_p))
+                
+    def go(self, procs):
+        """Renders all tiles"""
+        
+        logging.debug("Parent process {0}".format(os.getpid()))
+        # Create a pool
+        if procs == 1:
+            pool = FakePool()
+            global child_rendernode
+            child_rendernode = self
+        else:
+            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
+            #warm up the pool so it reports all the worker id's
+            pool.map(bool,xrange(multiprocessing.cpu_count()),1)        
+                
+        quadtrees = self.quadtrees
+        
+        # do per-quadtree init.
+        max_p = 0        
+        total = 0
+        for q in quadtrees:
+            total += 4**q.p
+            if q.p > max_p:
+                max_p = q.p
+            q.go(procs) 
+        self.max_p = max_p
+        # Render the highest level of tiles from the chunks
+        results = collections.deque()
+        complete = 0
+        logging.info("Rendering highest zoom level of tiles now.")
+        logging.info("Rendering {0} layer{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' ))
+        logging.info("There are {0} tiles to render".format(total))        
+        logging.info("There are {0} total levels to render".format(self.max_p))
+        logging.info("Don't worry, each level has only 25% as many tiles as the last.")
+        logging.info("The others will go faster")
+        count = 0
+        batch_size = 4*len(quadtrees)
+        while batch_size < 10:
+            batch_size *= 2
+        timestamp = time.time()
+        for result in self._apply_render_worldtiles(pool,batch_size):
+            results.append(result)               
+            # every second drain some of the queue
+            timestamp2 = time.time()
+            if timestamp2 >= timestamp + 1:
+                timestamp = timestamp2                
+                count_to_remove = (1000//batch_size)
+                if count_to_remove < len(results):
+                    while count_to_remove > 0:
+                        count_to_remove -= 1
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, 1)  
+            if len(results) > (10000//batch_size):
+                # Empty the queue before adding any more, so that memory
+                # required has an upper bound
+                while len(results) > (500//batch_size):
+                    complete += results.popleft().get()
+                    self.print_statusline(complete, total, 1)
+
+        # Wait for the rest of the results
+        while len(results) > 0:
+            complete += results.popleft().get()
+            self.print_statusline(complete, total, 1)
+
+        self.print_statusline(complete, total, 1, True)
+
+        # Now do the other layers
+        for zoom in xrange(self.max_p-1, 0, -1):
+            level = self.max_p - zoom + 1
+            assert len(results) == 0
+            complete = 0
+            total = 0
+            for q in quadtrees:
+                if zoom <= q.p:
+                    total += 4**zoom
+            logging.info("Starting level {0}".format(level))
+            timestamp = time.time()
+            for result in self._apply_render_inntertile(pool, zoom,batch_size):
+                results.append(result)
+                # every second drain some of the queue
+                timestamp2 = time.time()
+                if timestamp2 >= timestamp + 1:
+                    timestamp = timestamp2                
+                    count_to_remove = (1000//batch_size)
+                    if count_to_remove < len(results):
+                        while count_to_remove > 0:
+                            count_to_remove -= 1
+                            complete += results.popleft().get()
+                            self.print_statusline(complete, total, 1)    
+                if len(results) > (10000/batch_size):
+                    while len(results) > (500/batch_size):
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, level)
+            # Empty the queue
+            while len(results) > 0:
+                complete += results.popleft().get()
+                self.print_statusline(complete, total, level)
+
+            self.print_statusline(complete, total, level, True)
+
+            logging.info("Done")
+
+        pool.close()
+        pool.join()
+
+        # Do the final one right here:
+        for q in quadtrees:
+            q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
+        
+    def _apply_render_worldtiles(self, pool,batch_size):
+        """Returns an iterator over result objects. Each time a new result is
+        requested, a new task is added to the pool and a result returned.
+        """
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)         
+        batch = []
+        jobcount = 0       
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_worldtiles() for q in self.quadtrees]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is                 
+            job[0] = job[0]._render_index      
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0        
+                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
+                batch = []              
+        if jobcount > 0:
+            yield pool.apply_async(func=render_worldtile_batch, args= [batch])         
+
+    def _apply_render_inntertile(self, pool, zoom,batch_size):
+        """Same as _apply_render_worltiles but for the inntertile routine.
+        Returns an iterator that yields result objects from tasks that have
+        been applied to the pool.
+        """
+        
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)
+        batch = []
+        jobcount = 0
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is  
+            job[0] = job[0]._render_index
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0
+                yield pool.apply_async(func=render_innertile_batch, args= [batch])
+                batch = []
+                
+        if jobcount > 0:
+            yield pool.apply_async(func=render_innertile_batch, args= [batch])    
+            
+@catch_keyboardinterrupt
+def render_worldtile_batch(batch):   
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))        
+    for job in batch:
+        count += 1    
+        quadtree = rendernode.quadtrees[job[0]]
+        colstart = job[1]
+        colend = job[2]
+        rowstart = job[3]
+        rowend = job[4]
+        path = job[5]
+        path = quadtree.full_tiledir+os.sep+path        
+        # (even if tilechunks is empty, render_worldtile will delete
+        # existing images if appropriate)    
+        # And uses these chunks
+        tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend)
+        #logging.debug(" tilechunks: %r", tilechunks)
+        
+        quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path)      
+    return count
+
+@catch_keyboardinterrupt
+def render_innertile_batch(batch):    
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0   
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1        
+        quadtree = rendernode.quadtrees[job[0]]               
+        dest = quadtree.full_tiledir+os.sep+job[1]
+        quadtree.render_innertile(dest=dest,name=job[2])
+    return count
+    
+class FakeResult(object):
+    def __init__(self, res):
+        self.res = res
+    def get(self):
+        return self.res
+class FakePool(object):
+    """A fake pool used to render things in sync. Implements a subset of
+    multiprocessing.Pool"""
+    def apply_async(self, func, args=(), kwargs=None):
+        if not kwargs:
+            kwargs = {}
+        result = func(*args, **kwargs)
+        return FakeResult(result)
+    def close(self):
+        pass
+    def join(self):
+        pass
+    

world.py

@@ -73,15 +73,17 @@ class World(object):
                 
         #find region files, or load the region list
         #this also caches all the region file header info
+        logging.info("Scanning regions")
         regionfiles = {}
         regions = {}
         for x, y, regionfile in self._iterate_regionfiles():            
             mcr = nbt.MCRFileReader(regionfile)
             mcr.get_chunk_info()
             regions[regionfile] = (mcr,os.path.getmtime(regionfile))
-            regionfiles[(x,y)]	= (x,y,regionfile)
+            regionfiles[(x,y)]	= (x,y,regionfile,mcr)
         self.regionfiles = regionfiles	
         self.regions = regions
+        logging.debug("Done scanning regions")
         
         # figure out chunk format is in use
         # if not mcregion, error out early
@@ -114,7 +116,7 @@ class World(object):
     def get_region_path(self, chunkX, chunkY):
         """Returns the path to the region that contains chunk (chunkX, chunkY)
         """
-        _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
+        _, _, regionfile,_ = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None,None));
         return regionfile
     
     def load_from_region(self,filename, x, y):
@@ -133,7 +135,7 @@ class World(object):
         return self.regions[filename][0]
         
     def get_region_mtime(self,filename):                  
-        return self.regions[filename][1]        
+        return self.regions[filename]        
         
     def convert_coords(self, chunkx, chunky):
         """Takes a coordinate (chunkx, chunky) where chunkx and chunky are