Merge commit 'xon/dtt-c-render' into dtt-c-render

Conflicts:
	nbt.py
	overviewer.py

chunk.py

@@ -73,10 +73,10 @@ def get_blockarray(level):
     Block array, which just contains all the block ids"""
     return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
 
-def get_blockarray_fromfile(world,filename):
+def get_blockarray_fromfile(filename):
-    """Same as get_blockarray except takes a filename and uses get_lvldata to
+    """Same as get_blockarray except takes a filename. This is a shortcut"""
-    open it. This is a shortcut"""
+    d =  nbt.load_from_region(filename, x, y)
-    level = get_lvldata(world,filename)
+    level = d[1]['Level']
     return get_blockarray(level)
 
 def get_skylight_array(level):

contrib/findSigns.py

@@ -33,30 +33,34 @@ if os.path.exists(worlddir):
 else:
     sys.exit("Bad WorldDir")
 
-matcher = re.compile(r"^c\..*\.dat$")
+matcher = re.compile(r"^r\..*\.mcr$")
 
 POI = []
 
 for dirpath, dirnames, filenames in os.walk(worlddir):
     for f in filenames:
         if matcher.match(f):
+            print f
             full = os.path.join(dirpath, f)
-            #print "inspecting %s" % full
+            r = nbt.load_region(full)
-            data = nbt.load(full)[1]['Level']['TileEntities']
+            chunks = r.get_chunks()
-            for entity in data:
+            for x,y in chunks:
-                if entity['id'] == 'Sign':
+                chunk = r.load_chunk(x,y).read_all()                
-                    msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])
+                data = chunk[1]['Level']['TileEntities']
-                    #print "checking -->%s<--" % msg.strip()
+                for entity in data:
-                    if msg.strip():
+                    if entity['id'] == 'Sign':
-                        newPOI = dict(type="sign",
+                        msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])
-                                        x= entity['x'],
+                        #print "checking -->%s<--" % msg.strip()
-                                        y= entity['y'],
+                        if msg.strip():
-                                        z= entity['z'],
+                            newPOI = dict(type="sign",
-                                        msg=msg,
+                                            x= entity['x'],
-                                        chunk= (entity['x']/16, entity['z']/16),
+                                            y= entity['y'],
-                                       )
+                                            z= entity['z'],
-                        POI.append(newPOI)
+                                            msg=msg,
-                        print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg'])
+                                            chunk= (entity['x']/16, entity['z']/16),
+                                           )
+                            POI.append(newPOI)
+                            print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg'])
 
 
 

contrib/rerenderBlocks.py

@@ -7,8 +7,8 @@ blockID.  The output is a chunklist file that is suitable to use with the
 
 Example:
 
-python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> chunklist.txt
+python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> regionlist.txt
-    python overviewer.py --chunklist=chunklist.txt world/ output_dir/
+    python overviewer.py --regionlist=regionlist.txt world/ output_dir/
 
 This will rerender any chunks that contain either TNT (46), Ice (79), or 
 a Jack-O-Lantern (91)
@@ -42,15 +42,20 @@ ids = map(lambda x: int(x),options.ids.split(","))
 sys.stderr.write("Searching for these blocks: %r...\n" % ids)
 
 
-matcher = re.compile(r"^c\..*\.dat$")
+matcher = re.compile(r"^r\..*\.mcr$")
 
 for dirpath, dirnames, filenames in os.walk(options.world):
     for f in filenames:
         if matcher.match(f):
             full = os.path.join(dirpath, f)
-            blocks = get_blockarray_fromfile(full)
+            r = nbt.load_region(full)
-            for i in ids:
+            chunks = r.get_chunks()
-                if i in blocks:
+            for x,y in chunks:
-                    print full
+                chunk = r.load_chunk(x,y).read_all()                
-                    break
+                blocks = get_blockarray(chunk[1]['Level'])
+                for i in ids:
+                    if i in blocks:
+                        print full
+                        break
+
 

googlemap.py

@@ -54,7 +54,8 @@ class MapGen(object):
     def __init__(self, quadtrees, skipjs=False, web_assets_hook=None):
         """Generates a Google Maps interface for the given list of
         quadtrees. All of the quadtrees must have the same destdir,
-        image format, and world."""
+        image format, and world. 
+        Note:tiledir for each quadtree should be unique. By default the tiledir is determined by the rendermode"""
         
         self.skipjs = skipjs
         self.web_assets_hook = web_assets_hook

nbt.py

@@ -26,7 +26,7 @@ def _file_loader(func):
                return None
 
             # Is actually a filename
-            fileobj = open(fileobj, 'rb')
+            fileobj = open(fileobj, 'rb',4096)
         return func(fileobj, *args)
     return wrapper
 
@@ -44,6 +44,20 @@ def load_from_region(filename, x, y):
 def load_region(filename):            
     return MCRFileReader(filename)
   
+  
+# compile the unpacker's into a classes
+_byte   = struct.Struct("b")
+_short  = struct.Struct(">h")
+_int    = struct.Struct(">i")
+_long   = struct.Struct(">q")
+_float  = struct.Struct(">f")
+_double = struct.Struct(">d") 
+
+_24bit_int = struct.Struct("B B B")
+_unsigned_byte = struct.Struct("B")
+_unsigned_int = struct.Struct(">I")
+_chunk_header = struct.Struct(">I B")
+ 
 class NBTFileReader(object):
     def __init__(self, fileobj, is_gzip=True):
         if is_gzip:
@@ -61,27 +75,32 @@ class NBTFileReader(object):
 
     def _read_tag_byte(self):
         byte = self._file.read(1)
-        return struct.unpack("b", byte)[0]
+        return _byte.unpack(byte)[0]
     
     def _read_tag_short(self):
         bytes = self._file.read(2)
-        return struct.unpack(">h", bytes)[0]
+        global _short
+        return _short.unpack(bytes)[0]
 
     def _read_tag_int(self):
         bytes = self._file.read(4)
-        return struct.unpack(">i", bytes)[0]
+        global _int
+        return _int.unpack(bytes)[0]
 
     def _read_tag_long(self):
         bytes = self._file.read(8)
-        return struct.unpack(">q", bytes)[0]
+        global _long
+        return _long.unpack(bytes)[0]
 
     def _read_tag_float(self):
         bytes = self._file.read(4)
-        return struct.unpack(">f", bytes)[0]
+        global _float
+        return _float.unpack(bytes)[0]
 
     def _read_tag_double(self):
         bytes = self._file.read(8)
-        return struct.unpack(">d", bytes)[0]
+        global _double
+        return _double.unpack(bytes)[0]
 
     def _read_tag_byte_array(self):
         length = self._read_tag_int()
@@ -194,9 +213,11 @@ class MCRFileReader(object):
         
         ret = 0
         bytes = self._file.read(3)
+        global _24bit_int
+        bytes = _24bit_int.unpack(bytes)
         for i in xrange(3):
             ret = ret << 8
-            ret += struct.unpack("B", bytes[i])[0]
+            ret += bytes[i]
         
         return ret
     
@@ -222,8 +243,9 @@ class MCRFileReader(object):
             offset_sectors = self._read_24bit_int()
             
             # 1-byte length in 4KiB sectors, rounded up
+            global _unsigned_byte
             byte = self._file.read(1)
-            length_sectors = struct.unpack("B", byte)[0]
+            length_sectors = _unsigned_byte.unpack(byte)[0]
         except (IndexError, struct.error):
             # got a problem somewhere
             return None
@@ -253,49 +275,60 @@ class MCRFileReader(object):
         
         try:
             bytes = self._file.read(4)
-            timestamp = struct.unpack(">I", bytes)[0]
+            global _unsigned_int
+            timestamp = _unsigned_int.unpack(bytes)[0]
         except (IndexError, struct.error):
             return 0
         
         return timestamp
     
-    def get_chunk_info(self,closeFile = True):
+    def get_chunks(self):    
         """Return a list of all chunks contained in this region file,
         as a list of (x, y) coordinate tuples. To load these chunks,
         provide these coordinates to load_chunk()."""
         
-        if self._chunks:
+        if self._chunks is not None:
             return self._chunks
+        if self._locations is None:
+            self.get_chunk_info()       
+        self._chunks = [] 
+        for x in xrange(32): 
+            for y in xrange(32): 
+                if self._locations[x + y * 32] is not None:
+                    self._chunks.append((x,y))
+        return self._chunks
+        
+    def get_chunk_info(self,closeFile = True):
+        """Preloads region header information."""
+        
+        if self._locations:
+            return
         
         if self._file is None:
-            self._file = open(self._filename,'rb');
+            self._file = open(self._filename,'rb')
 
-        self._chunks = []
+        self._chunks = None
         self._locations = []
         self._timestamps = []
         
         # go to the beginning of the file
-        self._file.seek(0)
+        self._file.seek(0)        
         
         # read chunk location table
-        for y in xrange(32):
+        locations_append = self._locations.append
-            for x in xrange(32):
+        for _ in xrange(32*32): 
-                location = self._read_chunk_location()
+            locations_append(self._read_chunk_location())
-                self._locations.append(location)
-                if location:
-                    self._chunks.append((x, y))
         
         # read chunk timestamp table
-        for y in xrange(32):
+        timestamp_append = self._timestamps.append
-            for x in xrange(32):
+        for _ in xrange(32*32): 
-                timestamp = self._read_chunk_timestamp()
+            timestamp_append(self._read_chunk_timestamp())
-                self._timestamps.append(timestamp)
+ 
-
         if closeFile:        
             #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
             self._file.close()
             self._file =  None
-        return self._chunks
+        return
     
     def get_chunk_timestamp(self, x, y):
         """Return the given chunk's modification time. If the given
@@ -339,10 +372,8 @@ class MCRFileReader(object):
         self._file.seek(location[0])
         
         # read in the chunk data header
-        bytes = self._file.read(4)
+        bytes = self._file.read(5)        
-        data_length = struct.unpack(">I", bytes)[0]
+        data_length,compression =  _chunk_header.unpack(bytes)
-        bytes = self._file.read(1)
-        compression = struct.unpack("B", bytes)[0]
         
         # figure out the compression
         is_gzip = True

overviewer.py

@@ -45,6 +45,7 @@ import composite
 import world
 import quadtree
 import googlemap
+import rendernode
 
 helptext = """
 %prog [OPTIONS] <World # / Name / Path to World> <tiles dest dir>
@@ -172,22 +173,32 @@ def main():
     useBiomeData = os.path.exists(os.path.join(worlddir, 'biomes'))
     if not useBiomeData:
         logging.info("Notice: Not using biome data for tinting")
-
+    
     # First do world-level preprocessing
     w = world.World(worlddir, useBiomeData=useBiomeData)
     w.go(options.procs)
 
     # create the quadtrees
     # TODO chunklist
-    # NOTE: options.rendermode is now a list of rendermodes.  for now, always use the first one
+    q = []
-    q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode[0])
+    qtree_args = {'depth' : options.zoom, 'imgformat' : imgformat, 'optimizeimg' : optimizeimg}
+    for rendermode in options.rendermode:
+        if rendermode == 'normal':
+            qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, tiledir='tiles', **qtree_args)
+        else:
+            qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, **qtree_args)
+        q.append(qtree)
+
+    #create the distributed render
+    r = rendernode.RenderNode(q)
     
     # write out the map and web assets
-    m = googlemap.MapGen([q,], skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
+    m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
     m.go(options.procs)
     
     # render the tiles!
-    q.go(options.procs)
+    r.go(options.procs)
+
 
 def delete_all(worlddir, tiledir):
     # TODO should we delete tiledir here too?

quadtree.py

@@ -27,6 +27,7 @@ import util
 import cPickle
 import stat
 import errno 
+import time
 from time import gmtime, strftime, sleep
 
 from PIL import Image
@@ -45,32 +46,9 @@ This module has routines related to generating a quadtree of tiles
 def iterate_base4(d):
     """Iterates over a base 4 number with d digits"""
     return itertools.product(xrange(4), repeat=d)
-
+   
-def catch_keyboardinterrupt(func):
-    """Decorator that catches a keyboardinterrupt and raises a real exception
-    so that multiprocessing will propagate it properly"""
-    @functools.wraps(func)
-    def newfunc(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except KeyboardInterrupt:
-            logging.error("Ctrl-C caught!")
-            raise Exception("Exiting")
-        except:
-            import traceback
-            traceback.print_exc()
-            raise
-    return newfunc
-
-child_quadtree = None
-def pool_initializer(quadtree):
-    logging.debug("Child process {0}".format(os.getpid()))
-    #stash the quadtree object in a global variable after fork() for windows compat.
-    global child_quadtree
-    child_quadtree = quadtree    
-    
 class QuadtreeGen(object):
-    def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, rendermode="normal"):
+    def __init__(self, worldobj, destdir, depth=None, tiledir=None, imgformat=None, optimizeimg=None, rendermode="normal"):
         """Generates a quadtree from the world given into the
         given dest directory
 
@@ -84,13 +62,18 @@ class QuadtreeGen(object):
         self.imgformat = imgformat
         self.optimizeimg = optimizeimg
         
+        self.lighting = rendermode in ("lighting", "night", "spawn")
+        self.night = rendermode in ("night", "spawn")
+        self.spawn = rendermode in ("spawn",)
         self.rendermode = rendermode
 
         # Make the destination dir
         if not os.path.exists(destdir):
             os.mkdir(destdir)
-        self.tiledir = tiledir
+        if tiledir is None:
-
+            tiledir = rendermode
+        self.tiledir = tiledir        
+        
         if depth is None:
             # Determine quadtree depth (midpoint is always 0,0)
             for p in xrange(15):
@@ -122,21 +105,7 @@ class QuadtreeGen(object):
 
         self.world = worldobj
         self.destdir = destdir
-
+        self.full_tiledir = os.path.join(destdir, tiledir)
-    def print_statusline(self, complete, total, level, unconditional=False):
-        if unconditional:
-            pass
-        elif complete < 100:
-            if not complete % 25 == 0:
-                return
-        elif complete < 1000:
-            if not complete % 100 == 0:
-                return
-        else:
-            if not complete % 1000 == 0:
-                return
-        logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
-                complete, total, level, self.p))
         
     def _get_cur_depth(self):
         """How deep is the quadtree currently in the destdir? This glances in
@@ -212,60 +181,9 @@ class QuadtreeGen(object):
             os.rename(getpath("3", "0"), getpath("new3"))
             shutil.rmtree(getpath("3"))
             os.rename(getpath("new3"), getpath("3"))
-
-    def _apply_render_worldtiles(self, pool,batch_size):
-        """Returns an iterator over result objects. Each time a new result is
-        requested, a new task is added to the pool and a result returned.
-        """
-             
-        batch = []
-        tiles = 0
-        for path in iterate_base4(self.p):
-            # Get the range for this tile
-            colstart, rowstart = self._get_range_by_path(path)
-            colend = colstart + 2
-            rowend = rowstart + 4
-
-            # This image is rendered at:
-            dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path))
-            #logging.debug("this is rendered at %s", dest)
-            
-            # Put this in the batch to be submited to the pool                        
-            batch.append((colstart, colend, rowstart, rowend, dest))
-            tiles += 1
-            if tiles >= batch_size:
-                tiles = 0
-                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
-                batch = []
-
-        if tiles > 0:
-            yield pool.apply_async(func=render_worldtile_batch, args= (batch,))
-
-
-    def _apply_render_inntertile(self, pool, zoom,batch_size):
-        """Same as _apply_render_worltiles but for the inntertile routine.
-        Returns an iterator that yields result objects from tasks that have
-        been applied to the pool.
-        """
-        batch = []
-        tiles = 0        
-        for path in iterate_base4(zoom):
-            # This image is rendered at:
-            dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1]))
-            name = str(path[-1])
-            
-            batch.append((dest, name, self.imgformat, self.optimizeimg))
-            tiles += 1
-            if tiles >= batch_size:
-                tiles = 0
-                yield pool.apply_async(func=render_innertile_batch, args= [batch])
-                batch = []
-            
-        if tiles > 0:       
-            yield pool.apply_async(func=render_innertile_batch, args= [batch])
         
     def go(self, procs):
-        """Renders all tiles"""
+        """Processing before tile rendering"""
 
         curdepth = self._get_cur_depth()
         if curdepth != -1:
@@ -278,73 +196,8 @@ class QuadtreeGen(object):
                 logging.warning("Your map seems to have shrunk. Re-arranging tiles, just a sec...")
                 for _ in xrange(curdepth - self.p):
                     self._decrease_depth()
-
+    
-        logging.debug("Parent process {0}".format(os.getpid()))
+    
-        # Create a pool
-        if procs == 1:
-            pool = FakePool()
-            global child_quadtree
-            child_quadtree = self
-        else:
-            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
-            #warm up the pool so it reports all the worker id's
-            pool.map(bool,xrange(multiprocessing.cpu_count()),1)
-        
-        # Render the highest level of tiles from the chunks
-        results = collections.deque()
-        complete = 0
-        total = 4**self.p
-        logging.info("Rendering highest zoom level of tiles now.")
-        logging.info("There are {0} tiles to render".format(total))
-        logging.info("There are {0} total levels to render".format(self.p))
-        logging.info("Don't worry, each level has only 25% as many tiles as the last.")
-        logging.info("The others will go faster")
-        count = 0
-        batch_size = 10
-        for result in self._apply_render_worldtiles(pool,batch_size):
-            results.append(result)
-            if len(results) > (10000/batch_size):
-                # Empty the queue before adding any more, so that memory
-                # required has an upper bound
-                while len(results) > (500/batch_size):
-                    complete += results.popleft().get()
-                    self.print_statusline(complete, total, 1)
-
-        # Wait for the rest of the results
-        while len(results) > 0:
-            complete += results.popleft().get()
-            self.print_statusline(complete, total, 1)
-
-        self.print_statusline(complete, total, 1, True)
-
-        # Now do the other layers
-        for zoom in xrange(self.p-1, 0, -1):
-            level = self.p - zoom + 1
-            assert len(results) == 0
-            complete = 0
-            total = 4**zoom
-            logging.info("Starting level {0}".format(level))
-            for result in self._apply_render_inntertile(pool, zoom,batch_size):
-                results.append(result)
-                if len(results) > (10000/batch_size):
-                    while len(results) > (500/batch_size):
-                        complete += results.popleft().get()
-                        self.print_statusline(complete, total, level)
-            # Empty the queue
-            while len(results) > 0:
-                complete += results.popleft().get()
-                self.print_statusline(complete, total, level)
-
-            self.print_statusline(complete, total, level, True)
-
-            logging.info("Done")
-
-        pool.close()
-        pool.join()
-
-        # Do the final one right here:
-        render_innertile(os.path.join(self.destdir, self.tiledir), "base", self.imgformat, self.optimizeimg)
-
     def _get_range_by_path(self, path):
         """Returns the x, y chunk coordinates of this tile"""
         x, y = self.mincol, self.minrow
@@ -361,8 +214,8 @@ class QuadtreeGen(object):
             ysize //= 2
 
         return x, y
-
+        
-    def _get_chunks_in_range(self, colstart, colend, rowstart, rowend):
+    def get_chunks_in_range(self, colstart, colend, rowstart, rowend):
         """Get chunks that are relevant to the tile rendering function that's
         rendering that range"""
         chunklist = []
@@ -380,248 +233,218 @@ class QuadtreeGen(object):
                 # return (col, row, chunkx, chunky, regionpath)
                 chunkx, chunky = unconvert_coords(col, row)
                 #c = get_region_path(chunkx, chunky)
-                _, _, c = get_region((chunkx//32, chunky//32),(None,None,None));
+                _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None));
-                if c is not None:
+                if c is not None and mcr.chunkExists(chunkx,chunky):                  
                     chunklist.append((col, row, chunkx, chunky, c))
-        return chunklist
+        return chunklist   
-
-@catch_keyboardinterrupt
-def render_innertile_batch(batch):    
-    count = 0
-    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
-    for job in batch:
-        count += 1
-        render_innertile(job[0],job[1],job[2],job[3])
-    return count
-    
-def render_innertile(dest, name, imgformat, optimizeimg):
-    """
-    Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
-    os.path.join(dest, name, "{0,1,2,3}.png")
-    """
-    imgpath = os.path.join(dest, name) + "." + imgformat
-
-    if name == "base":
-        quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]]
-    else:
-        quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]]    
-   
-    #stat the tile, we need to know if it exists or it's mtime
-    try:    
-        tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
-    except OSError, e:
-        if e.errno != errno.ENOENT:
-            raise
-        tile_mtime = None
         
-    #check mtimes on each part of the quad, this also checks if they exist
+    def get_worldtiles(self):
-    needs_rerender = tile_mtime is None
+        """Returns an iterator over the tiles of the most detailed layer
-    quadPath_filtered = []
+        """
-    for path in quadPath:
+        for path in iterate_base4(self.p):
-        try:
+            # Get the range for this tile
-            quad_mtime = os.stat(path[1])[stat.ST_MTIME]; 
+            colstart, rowstart = self._get_range_by_path(path)
-            quadPath_filtered.append(path)
+            colend = colstart + 2
-            if quad_mtime > tile_mtime:     
+            rowend = rowstart + 4   
-                needs_rerender = True            
-        except OSError:
-            # We need to stat all the quad files, so keep looping
-            pass      
-    # do they all not exist?
-    if quadPath_filtered == []:
-        if tile_mtime is not None:
-            os.unlink(imgpath)
-        return
-    # quit now if we don't need rerender            
-    if not needs_rerender:
-        return    
-    #logging.debug("writing out innertile {0}".format(imgpath))
-
-    # Create the actual image now
-    img = Image.new("RGBA", (384, 384), (38,92,255,0))
-    
-    # we'll use paste (NOT alpha_over) for quadtree generation because
-    # this is just straight image stitching, not alpha blending
-    
-    for path in quadPath_filtered:
-        try:
-            quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
-            img.paste(quad, path[0])
-        except Exception, e:
-            logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
-
-    # Save it
-    if imgformat == 'jpg':
-        img.save(imgpath, quality=95, subsampling=0)
-    else: # png
-        img.save(imgpath)
-        if optimizeimg:
-            optimize_image(imgpath, imgformat, optimizeimg)
-
-@catch_keyboardinterrupt
-def render_worldtile_batch(batch):  
-    global child_quadtree
-    return render_worldtile_batch_(child_quadtree, batch)
-
-def render_worldtile_batch_(quadtree, batch):   
-    count = 0
-    _get_chunks_in_range = quadtree._get_chunks_in_range
-    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
-    for job in batch:
-        count += 1
-        colstart = job[0]
-        colend = job[1]
-        rowstart = job[2]
-        rowend = job[3]
-        path = job[4]
-        # (even if tilechunks is empty, render_worldtile will delete
-        # existing images if appropriate)    
-        # And uses these chunks
-        tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend)
-        #logging.debug(" tilechunks: %r", tilechunks)
             
-        render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path)
+            # This image is rendered at(relative to the worker's destdir):
-    return count
+            tilepath = [str(x) for x in path]
-
+            tilepath = os.sep.join(tilepath)
-def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):
+            #logging.debug("this is rendered at %s", dest)
-    """Renders just the specified chunks into a tile and save it. Unlike usual
-    python conventions, rowend and colend are inclusive. Additionally, the
-    chunks around the edges are half-way cut off (so that neighboring tiles
-    will render the other half)
-
-    chunks is a list of (col, row, chunkx, chunky, filename) of chunk
-    images that are relevant to this call (with their associated regions)
-
-    The image is saved to path+"."+quadtree.imgformat
-
-    If there are no chunks, this tile is not saved (if it already exists, it is
-    deleted)
-
-    Standard tile size has colend-colstart=2 and rowend-rowstart=4
-
-    There is no return value
-    """    
-    
-    # width of one chunk is 384. Each column is half a chunk wide. The total
-    # width is (384 + 192*(numcols-1)) since the first column contributes full
-    # width, and each additional one contributes half since they're staggered.
-    # However, since we want to cut off half a chunk at each end (384 less
-    # pixels) and since (colend - colstart + 1) is the number of columns
-    # inclusive, the equation simplifies to:
-    width = 192 * (colend - colstart)
-    # Same deal with height
-    height = 96 * (rowend - rowstart)
-
-    # The standard tile size is 3 columns by 5 rows, which works out to 384x384
-    # pixels for 8 total chunks. (Since the chunks are staggered but the grid
-    # is not, some grid coordinates do not address chunks) The two chunks on
-    # the middle column are shown in full, the two chunks in the middle row are
-    # half cut off, and the four remaining chunks are one quarter shown.
-    # The above example with cols 0-3 and rows 0-4 has the chunks arranged like this:
-    #   0,0         2,0
-    #         1,1
-    #   0,2         2,2
-    #         1,3
-    #   0,4         2,4
-
-    # Due to how the tiles fit together, we may need to render chunks way above
-    # this (since very few chunks actually touch the top of the sky, some tiles
-    # way above this one are possibly visible in this tile). Render them
-    # anyways just in case). "chunks" should include up to rowstart-16
-
-    imgpath = path + "." + quadtree.imgformat
-    
-    world = quadtree.world
-    # first, remove chunks from `chunks` that don't actually exist in
-    # their region files
-    def chunk_exists(chunk):
-        _, _, chunkx, chunky, region = chunk
-        r = world.load_region(region)
-        return r.chunkExists(chunkx, chunky)            
-    chunks = filter(chunk_exists, chunks)
-
-    #stat the file, we need to know if it exists or it's mtime
-    try:    
-        tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
-    except OSError, e:
-        if e.errno != errno.ENOENT:
-            raise
-        tile_mtime = None
         
-    if not chunks:
+            # Put this in the batch to be submited to the pool     
-        # No chunks were found in this tile
+            yield [self,colstart, colend, rowstart, rowend, tilepath]
-        if tile_mtime is not None:
+        
-            os.unlink(imgpath)
+    def get_innertiles(self,zoom):
-        return None
+        """Same as get_worldtiles but for the inntertile routine.
+        """    
+        for path in iterate_base4(zoom):
+            # This image is rendered at(relative to the worker's destdir):
+            tilepath = [str(x) for x in path[:-1]]
+            tilepath = os.sep.join(tilepath)
+            name = str(path[-1])
+  
+            yield [self,tilepath, name]
+    
+    def render_innertile(self, dest, name):
+        """
+        Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
+        os.path.join(dest, name, "{0,1,2,3}.png")
+        """
+        imgformat = self.imgformat 
+        imgpath = os.path.join(dest, name) + "." + imgformat
 
-    # Create the directory if not exists
+        if name == "base":
-    dirdest = os.path.dirname(path)
+            quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]]
-    if not os.path.exists(dirdest):
+        else:
-        try:
+            quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]]    
-            os.makedirs(dirdest)
+       
+        #stat the tile, we need to know if it exists or it's mtime
+        try:    
+            tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
         except OSError, e:
-            # Ignore errno EEXIST: file exists. Since this is multithreaded,
+            if e.errno != errno.ENOENT:
-            # two processes could conceivably try and create the same directory
-            # at the same time.            
-            if e.errno != errno.EEXIST:
                 raise
-    
+            tile_mtime = None
-    # check chunk mtimes to see if they are newer
-    try:
-        needs_rerender = False
-        for col, row, chunkx, chunky, regionfile in chunks:
-            # check region file mtime first. 
-            regionMtime = world.get_region_mtime(regionfile)  
-            if regionMtime <= tile_mtime:
-                continue
             
-            # checking chunk mtime
+        #check mtimes on each part of the quad, this also checks if they exist
-            region = world.load_region(regionfile)
+        needs_rerender = tile_mtime is None
-            if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
+        quadPath_filtered = []
-                needs_rerender = True
+        for path in quadPath:
-                break
+            try:
-        
+                quad_mtime = os.stat(path[1])[stat.ST_MTIME]; 
-        # if after all that, we don't need a rerender, return
+                quadPath_filtered.append(path)
+                if quad_mtime > tile_mtime:     
+                    needs_rerender = True            
+            except OSError:
+                # We need to stat all the quad files, so keep looping
+                pass      
+        # do they all not exist?
+        if quadPath_filtered == []:
+            if tile_mtime is not None:
+                os.unlink(imgpath)
+            return
+        # quit now if we don't need rerender            
         if not needs_rerender:
+            return    
+        #logging.debug("writing out innertile {0}".format(imgpath))
+
+        # Create the actual image now
+        img = Image.new("RGBA", (384, 384), (38,92,255,0))
+        
+        # we'll use paste (NOT alpha_over) for quadtree generation because
+        # this is just straight image stitching, not alpha blending
+        
+        for path in quadPath_filtered:
+            try:
+                quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
+                img.paste(quad, path[0])
+            except Exception, e:
+                logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
+
+        # Save it
+        if self.imgformat == 'jpg':
+            img.save(imgpath, quality=95, subsampling=0)
+        else: # png
+            img.save(imgpath)
+            
+        if self.optimizeimg:
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)
+
+
+
+    def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path):
+        """Renders just the specified chunks into a tile and save it. Unlike usual
+        python conventions, rowend and colend are inclusive. Additionally, the
+        chunks around the edges are half-way cut off (so that neighboring tiles
+        will render the other half)
+
+        chunks is a list of (col, row, chunkx, chunky, filename) of chunk
+        images that are relevant to this call (with their associated regions)
+
+        The image is saved to path+"."+self.imgformat
+
+        If there are no chunks, this tile is not saved (if it already exists, it is
+        deleted)
+
+        Standard tile size has colend-colstart=2 and rowend-rowstart=4
+
+        There is no return value
+        """    
+        
+        # width of one chunk is 384. Each column is half a chunk wide. The total
+        # width is (384 + 192*(numcols-1)) since the first column contributes full
+        # width, and each additional one contributes half since they're staggered.
+        # However, since we want to cut off half a chunk at each end (384 less
+        # pixels) and since (colend - colstart + 1) is the number of columns
+        # inclusive, the equation simplifies to:
+        width = 192 * (colend - colstart)
+        # Same deal with height
+        height = 96 * (rowend - rowstart)
+
+        # The standard tile size is 3 columns by 5 rows, which works out to 384x384
+        # pixels for 8 total chunks. (Since the chunks are staggered but the grid
+        # is not, some grid coordinates do not address chunks) The two chunks on
+        # the middle column are shown in full, the two chunks in the middle row are
+        # half cut off, and the four remaining chunks are one quarter shown.
+        # The above example with cols 0-3 and rows 0-4 has the chunks arranged like this:
+        #   0,0         2,0
+        #         1,1
+        #   0,2         2,2
+        #         1,3
+        #   0,4         2,4
+
+        # Due to how the tiles fit together, we may need to render chunks way above
+        # this (since very few chunks actually touch the top of the sky, some tiles
+        # way above this one are possibly visible in this tile). Render them
+        # anyways just in case). "chunks" should include up to rowstart-16
+
+        imgpath = path + "." + self.imgformat
+        world = self.world
+        #stat the file, we need to know if it exists or it's mtime
+        try:    
+            tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+            tile_mtime = None
+            
+        if not chunks:
+            # No chunks were found in this tile
+            if tile_mtime is not None:
+                os.unlink(imgpath)
             return None
-    except OSError:
-        # couldn't get tile mtime, skip check
-        pass
-    
-    #logging.debug("writing out worldtile {0}".format(imgpath))
 
-    # Compile this image
+        # Create the directory if not exists
-    tileimg = Image.new("RGBA", (width, height), (38,92,255,0))
+        dirdest = os.path.dirname(path)
+        if not os.path.exists(dirdest):
+            try:
+                os.makedirs(dirdest)
+            except OSError, e:
+                # Ignore errno EEXIST: file exists. Since this is multithreaded,
+                # two processes could conceivably try and create the same directory
+                # at the same time.            
+                if e.errno != errno.EEXIST:
+                    raise
+        
+        # check chunk mtimes to see if they are newer
+        try:
+            needs_rerender = False
+            for col, row, chunkx, chunky, regionfile in chunks:
+                # check region file mtime first. 
+                region,regionMtime = world.get_region_mtime(regionfile)  
+                if regionMtime <= tile_mtime:
+                    continue
+                
+                # checking chunk mtime
+                if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
+                    needs_rerender = True
+                    break
+            
+            # if after all that, we don't need a rerender, return
+            if not needs_rerender:
+                return None
+        except OSError:
+            # couldn't get tile mtime, skip check
+            pass
+        
+        #logging.debug("writing out worldtile {0}".format(imgpath))
 
-    # col colstart will get drawn on the image starting at x coordinates -(384/2)
+        # Compile this image
-    # row rowstart will get drawn on the image starting at y coordinates -(192/2)
+        tileimg = Image.new("RGBA", (width, height), (38,92,255,0))
-    for col, row, chunkx, chunky, regionfile in chunks:
-        xpos = -192 + (col-colstart)*192
-        ypos = -96 + (row-rowstart)*96
 
-        # draw the chunk!
+        # col colstart will get drawn on the image starting at x coordinates -(384/2)
-        # TODO POI queue
+        # row rowstart will get drawn on the image starting at y coordinates -(192/2)
-        chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), quadtree, False, None)
+        for col, row, chunkx, chunky, regionfile in chunks:
+            xpos = -192 + (col-colstart)*192
+            ypos = -96 + (row-rowstart)*96
 
-    # Save them
+            # draw the chunk!
-    tileimg.save(imgpath)
+            # TODO POI queue
+            chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None)
 
-    if quadtree.optimizeimg:
+        # Save them
-        optimize_image(imgpath, quadtree.imgformat, quadtree.optimizeimg)
+        tileimg.save(imgpath)
 
-class FakeResult(object):
+        if self.optimizeimg:
-    def __init__(self, res):
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)
-        self.res = res
-    def get(self):
-        return self.res
-class FakePool(object):
-    """A fake pool used to render things in sync. Implements a subset of
-    multiprocessing.Pool"""
-    def apply_async(self, func, args=(), kwargs=None):
-        if not kwargs:
-            kwargs = {}
-        result = func(*args, **kwargs)
-        return FakeResult(result)
-    def close(self):
-        pass
-    def join(self):
-        pass

rendernode.py

@@ -0,0 +1,318 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import multiprocessing
+import itertools
+from itertools import cycle, islice
+import os
+import os.path
+import functools
+import re
+import shutil
+import collections
+import json
+import logging
+import util
+import cPickle
+import stat
+import errno 
+import time
+from time import gmtime, strftime, sleep
+
+
+"""
+This module has routines related to distributing the render job to multipule nodes
+
+"""
+
+def catch_keyboardinterrupt(func):
+    """Decorator that catches a keyboardinterrupt and raises a real exception
+    so that multiprocessing will propagate it properly"""
+    @functools.wraps(func)
+    def newfunc(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except KeyboardInterrupt:
+            logging.error("Ctrl-C caught!")
+            raise Exception("Exiting")
+        except:
+            import traceback
+            traceback.print_exc()
+            raise
+    return newfunc
+    
+child_rendernode = None
+def pool_initializer(rendernode):
+    logging.debug("Child process {0}".format(os.getpid()))
+    #stash the quadtree objects in a global variable after fork() for windows compat.
+    global child_rendernode
+    child_rendernode = rendernode  
+    
+#http://docs.python.org/library/itertools.html    
+def roundrobin(iterables):
+    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+    # Recipe credited to George Sakkis
+    pending = len(iterables)
+    nexts = cycle(iter(it).next for it in iterables)
+    while pending:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            pending -= 1
+            nexts = cycle(islice(nexts, pending))
+
+            
+class RenderNode(object):
+    def __init__(self, quadtrees):
+        """Distributes the rendering of a list of quadtrees."""
+
+        if not len(quadtrees) > 0:
+            raise ValueError("there must be at least one quadtree to work on")    
+
+        self.quadtrees = quadtrees
+        #bind an index value to the quadtree so we can find it again
+        i = 0
+        for q in quadtrees:
+            q._render_index = i
+            i += 1       
+
+    def print_statusline(self, complete, total, level, unconditional=False):
+        if unconditional:
+            pass
+        elif complete < 100:
+            if not complete % 25 == 0:
+                return
+        elif complete < 1000:
+            if not complete % 100 == 0:
+                return
+        else:
+            if not complete % 1000 == 0:
+                return
+        logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
+                complete, total, level, self.max_p))
+                
+    def go(self, procs):
+        """Renders all tiles"""
+        
+        logging.debug("Parent process {0}".format(os.getpid()))
+        # Create a pool
+        if procs == 1:
+            pool = FakePool()
+            global child_rendernode
+            child_rendernode = self
+        else:
+            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
+            #warm up the pool so it reports all the worker id's
+            pool.map(bool,xrange(multiprocessing.cpu_count()),1)        
+                
+        quadtrees = self.quadtrees
+        
+        # do per-quadtree init.
+        max_p = 0        
+        total = 0
+        for q in quadtrees:
+            total += 4**q.p
+            if q.p > max_p:
+                max_p = q.p
+            q.go(procs) 
+        self.max_p = max_p
+        # Render the highest level of tiles from the chunks
+        results = collections.deque()
+        complete = 0
+        logging.info("Rendering highest zoom level of tiles now.")
+        logging.info("Rendering {0} layer{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' ))
+        logging.info("There are {0} tiles to render".format(total))        
+        logging.info("There are {0} total levels to render".format(self.max_p))
+        logging.info("Don't worry, each level has only 25% as many tiles as the last.")
+        logging.info("The others will go faster")
+        count = 0
+        batch_size = 4*len(quadtrees)
+        while batch_size < 10:
+            batch_size *= 2
+        timestamp = time.time()
+        for result in self._apply_render_worldtiles(pool,batch_size):
+            results.append(result)               
+            # every second drain some of the queue
+            timestamp2 = time.time()
+            if timestamp2 >= timestamp + 1:
+                timestamp = timestamp2                
+                count_to_remove = (1000//batch_size)
+                if count_to_remove < len(results):
+                    while count_to_remove > 0:
+                        count_to_remove -= 1
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, 1)  
+            if len(results) > (10000//batch_size):
+                # Empty the queue before adding any more, so that memory
+                # required has an upper bound
+                while len(results) > (500//batch_size):
+                    complete += results.popleft().get()
+                    self.print_statusline(complete, total, 1)
+
+        # Wait for the rest of the results
+        while len(results) > 0:
+            complete += results.popleft().get()
+            self.print_statusline(complete, total, 1)
+
+        self.print_statusline(complete, total, 1, True)
+
+        # Now do the other layers
+        for zoom in xrange(self.max_p-1, 0, -1):
+            level = self.max_p - zoom + 1
+            assert len(results) == 0
+            complete = 0
+            total = 0
+            for q in quadtrees:
+                if zoom <= q.p:
+                    total += 4**zoom
+            logging.info("Starting level {0}".format(level))
+            timestamp = time.time()
+            for result in self._apply_render_inntertile(pool, zoom,batch_size):
+                results.append(result)
+                # every second drain some of the queue
+                timestamp2 = time.time()
+                if timestamp2 >= timestamp + 1:
+                    timestamp = timestamp2                
+                    count_to_remove = (1000//batch_size)
+                    if count_to_remove < len(results):
+                        while count_to_remove > 0:
+                            count_to_remove -= 1
+                            complete += results.popleft().get()
+                            self.print_statusline(complete, total, 1)    
+                if len(results) > (10000/batch_size):
+                    while len(results) > (500/batch_size):
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, level)
+            # Empty the queue
+            while len(results) > 0:
+                complete += results.popleft().get()
+                self.print_statusline(complete, total, level)
+
+            self.print_statusline(complete, total, level, True)
+
+            logging.info("Done")
+
+        pool.close()
+        pool.join()
+
+        # Do the final one right here:
+        for q in quadtrees:
+            q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
+        
+    def _apply_render_worldtiles(self, pool,batch_size):
+        """Returns an iterator over result objects. Each time a new result is
+        requested, a new task is added to the pool and a result returned.
+        """
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)         
+        batch = []
+        jobcount = 0       
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_worldtiles() for q in self.quadtrees]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is                 
+            job[0] = job[0]._render_index      
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0        
+                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
+                batch = []              
+        if jobcount > 0:
+            yield pool.apply_async(func=render_worldtile_batch, args= [batch])         
+
+    def _apply_render_inntertile(self, pool, zoom,batch_size):
+        """Same as _apply_render_worltiles but for the inntertile routine.
+        Returns an iterator that yields result objects from tasks that have
+        been applied to the pool.
+        """
+        
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)
+        batch = []
+        jobcount = 0
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is  
+            job[0] = job[0]._render_index
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0
+                yield pool.apply_async(func=render_innertile_batch, args= [batch])
+                batch = []
+                
+        if jobcount > 0:
+            yield pool.apply_async(func=render_innertile_batch, args= [batch])    
+            
+@catch_keyboardinterrupt
+def render_worldtile_batch(batch):   
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))        
+    for job in batch:
+        count += 1    
+        quadtree = rendernode.quadtrees[job[0]]
+        colstart = job[1]
+        colend = job[2]
+        rowstart = job[3]
+        rowend = job[4]
+        path = job[5]
+        path = quadtree.full_tiledir+os.sep+path        
+        # (even if tilechunks is empty, render_worldtile will delete
+        # existing images if appropriate)    
+        # And uses these chunks
+        tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend)
+        #logging.debug(" tilechunks: %r", tilechunks)
+        
+        quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path)      
+    return count
+
+@catch_keyboardinterrupt
+def render_innertile_batch(batch):    
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0   
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1        
+        quadtree = rendernode.quadtrees[job[0]]               
+        dest = quadtree.full_tiledir+os.sep+job[1]
+        quadtree.render_innertile(dest=dest,name=job[2])
+    return count
+    
+class FakeResult(object):
+    def __init__(self, res):
+        self.res = res
+    def get(self):
+        return self.res
+class FakePool(object):
+    """A fake pool used to render things in sync. Implements a subset of
+    multiprocessing.Pool"""
+    def apply_async(self, func, args=(), kwargs=None):
+        if not kwargs:
+            kwargs = {}
+        result = func(*args, **kwargs)
+        return FakeResult(result)
+    def close(self):
+        pass
+    def join(self):
+        pass
+    

world.py

@@ -70,18 +70,20 @@ class World(object):
     def __init__(self, worlddir, useBiomeData=False,regionlist=None):
         self.worlddir = worlddir
         self.useBiomeData = useBiomeData
-
+                
         #find region files, or load the region list
         #this also caches all the region file header info
+        logging.info("Scanning regions")
         regionfiles = {}
         regions = {}
         for x, y, regionfile in self._iterate_regionfiles():            
             mcr = nbt.MCRFileReader(regionfile)
             mcr.get_chunk_info()
             regions[regionfile] = (mcr,os.path.getmtime(regionfile))
-            regionfiles[(x,y)]	= (x,y,regionfile)
+            regionfiles[(x,y)]	= (x,y,regionfile,mcr)
         self.regionfiles = regionfiles	
         self.regions = regions
+        logging.debug("Done scanning regions")
         
         # figure out chunk format is in use
         # if not mcregion, error out early
@@ -114,7 +116,7 @@ class World(object):
     def get_region_path(self, chunkX, chunkY):
         """Returns the path to the region that contains chunk (chunkX, chunkY)
         """
-        _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
+        _, _, regionfile,_ = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None,None));
         return regionfile
     
     def load_from_region(self,filename, x, y):
@@ -133,7 +135,7 @@ class World(object):
         return self.regions[filename][0]
         
     def get_region_mtime(self,filename):                  
-        return self.regions[filename][1]        
+        return self.regions[filename]        
         
     def convert_coords(self, chunkx, chunky):
         """Takes a coordinate (chunkx, chunky) where chunkx and chunky are