initial work at reorganizing source tree

overviewer/chunk.py

@@ -0,0 +1,505 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import numpy
+from PIL import Image, ImageDraw, ImageEnhance, ImageOps
+import os.path
+import logging
+import time
+import math
+import sys
+
+import nbt
+import textures
+import world
+import composite
+import c_overviewer
+
+"""
+This module has routines related to rendering one particular chunk into an
+image
+
+"""
+
+# General note about pasting transparent image objects onto an image with an
+# alpha channel:
+# If you use the image as its own mask, it will work fine only if the alpha
+# channel is binary. If there's any translucent parts, then the alpha channel
+# of the dest image will have its alpha channel modified. To prevent this:
+# first use im.split() and take the third item which is the alpha channel and
+# use that as the mask. Then take the image and use im.convert("RGB") to strip
+# the image from its alpha channel, and use that as the source to alpha_over()
+
+# (note that this workaround is NOT technically needed when using the
+# alpha_over extension, BUT this extension may fall back to PIL's
+# paste(), which DOES need the workaround.)
+
+def get_lvldata(world, filename, x, y, retries=2):
+    """Takes a filename and chunkcoords and returns the Level struct, which contains all the
+    level info"""
+    
+    # non existent region file doesn't mean corrupt chunk.
+    if filename == None:
+        raise NoSuchChunk
+    
+    try:
+        d =  world.load_from_region(filename, x, y)
+    except Exception, e:
+        if retries > 0:
+            # wait a little bit, and try again (up to `retries` times)
+            time.sleep(1)
+            #make sure we reload region info
+            world.reload_region(filename)
+            return get_lvldata(world, filename, x, y, retries=retries-1)
+        else:
+            logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e)
+            raise ChunkCorrupt(str(e))
+    
+    if not d: raise NoSuchChunk(x,y)
+    return d
+
+def get_blockarray(level):
+    """Takes the level struct as returned from get_lvldata, and returns the
+    Block array, which just contains all the block ids"""
+    return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
+
+def get_blockarray_fromfile(filename):
+    """Same as get_blockarray except takes a filename. This is a shortcut"""
+    d =  nbt.load_from_region(filename, x, y)
+    level = d[1]['Level']
+    return get_blockarray(level)
+
+def get_skylight_array(level):
+    """Returns the skylight array. This is 4 bits per block, but it is
+    expanded for you so you may index it normally."""
+    skylight = numpy.frombuffer(level['SkyLight'], dtype=numpy.uint8).reshape((16,16,64))
+    # this array is 2 blocks per byte, so expand it
+    skylight_expanded = numpy.empty((16,16,128), dtype=numpy.uint8)
+    # Even elements get the lower 4 bits
+    skylight_expanded[:,:,::2] = skylight & 0x0F
+    # Odd elements get the upper 4 bits
+    skylight_expanded[:,:,1::2] = (skylight & 0xF0) >> 4
+    return skylight_expanded
+
+def get_blocklight_array(level):
+    """Returns the blocklight array. This is 4 bits per block, but it
+    is expanded for you so you may index it normally."""
+    # expand just like get_skylight_array()
+    blocklight = numpy.frombuffer(level['BlockLight'], dtype=numpy.uint8).reshape((16,16,64))
+    blocklight_expanded = numpy.empty((16,16,128), dtype=numpy.uint8)
+    blocklight_expanded[:,:,::2] = blocklight & 0x0F
+    blocklight_expanded[:,:,1::2] = (blocklight & 0xF0) >> 4
+    return blocklight_expanded
+
+def get_blockdata_array(level):
+    """Returns the ancillary data from the 'Data' byte array.  Data is packed
+    in a similar manner to skylight data"""
+    return numpy.frombuffer(level['Data'], dtype=numpy.uint8).reshape((16,16,64))
+
+def get_tileentity_data(level):
+    """Returns the TileEntities TAG_List from chunk dat file"""
+    data = level['TileEntities']
+    return data
+
+# This set holds blocks ids that can be seen through, for occlusion calculations
+transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 44, 50, 51, 52, 53, 55,
+    59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 83, 85, 92])
+
+# This set holds block ids that are solid blocks
+solid_blocks = set([1, 2, 3, 4, 5, 7, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+    23, 24, 25, 35, 41, 42, 43, 44, 45, 46, 47, 48, 49, 53, 54, 56, 57, 58, 60, 
+    61, 62, 64, 65, 66, 67, 71, 73, 74, 78, 79, 80, 81, 82, 84, 86, 87, 88, 89, 91, 92])
+
+# This set holds block ids that are fluid blocks
+fluid_blocks = set([8,9,10,11])
+
+# This set holds block ids that are not candidates for spawning mobs on
+# (glass, half blocks)
+nospawn_blocks = set([20,44])
+
+# chunkcoords should be the coordinates of a possible chunk.  it may not exist
+def render_to_image(chunkcoords, img, imgcoords, quadtreeobj, cave=False, queue=None):
+    """Used to render a chunk to a tile in quadtree.py.
+
+    chunkcoords is a tuple:  (chunkX, chunkY)
+    
+    imgcoords is as well: (imgX, imgY), which represents the "origin"
+    to use for drawing.
+
+    If the chunk doesn't exist, return False.
+    Else, returns True."""
+    a = ChunkRenderer(chunkcoords, quadtreeobj.world, quadtreeobj.rendermode, queue)
+    try:
+        a.chunk_render(img, imgcoords[0], imgcoords[1], cave)
+        return True
+    except ChunkCorrupt:
+        # This should be non-fatal, but should print a warning
+        pass
+    except Exception, e:
+        import traceback
+        traceback.print_exc()
+        raise
+    except KeyboardInterrupt:
+        print
+        print "You pressed Ctrl-C. Exiting..."
+        # Raise an exception that is an instance of Exception. Unlike
+        # KeyboardInterrupt, this will re-raise in the parent, killing the
+        # entire program, instead of this process dying and the parent waiting
+        # forever for it to finish.
+        raise Exception()
+    return False
+
+class ChunkCorrupt(Exception):
+    pass
+
+class NoSuchChunk(Exception):
+    pass
+
+class ChunkRenderer(object):
+    def __init__(self, chunkcoords, worldobj, rendermode, queue):
+        """Make a new chunk renderer for the given chunk coordinates.
+        chunkcoors should be a tuple: (chunkX, chunkY)
+        
+        cachedir is a directory to save the resulting chunk images to
+        """
+        self.queue = queue
+        
+        self.regionfile = worldobj.get_region_path(*chunkcoords)    
+        #if not os.path.exists(self.regionfile):
+        #    raise ValueError("Could not find regionfile: %s" % self.regionfile)
+
+        ## TODO TODO all of this class
+
+        #destdir, filename = os.path.split(self.chunkfile)
+        #filename_split = filename.split(".")
+        #chunkcoords = filename_split[1:3]
+        
+        #self.coords = map(world.base36decode, chunkcoords)
+        #self.blockid = "%d.%d" % chunkcoords
+
+        # chunk coordinates (useful to converting local block coords to 
+        # global block coords)
+        self.chunkX = chunkcoords[0]
+        self.chunkY = chunkcoords[1]
+
+        self.world = worldobj
+        self.rendermode = rendermode
+
+    def _load_level(self):
+        """Loads and returns the level structure"""
+        if not hasattr(self, "_level"):
+            try:
+                self._level = get_lvldata(self.world,self.regionfile, self.chunkX, self.chunkY)
+            except NoSuchChunk, e:
+                logging.debug("Skipping non-existant chunk")
+                raise
+        return self._level
+    level = property(_load_level)
+        
+    def _load_blocks(self):
+        """Loads and returns the block array"""
+        if not hasattr(self, "_blocks"):
+            self._blocks = get_blockarray(self._load_level())
+        return self._blocks
+    blocks = property(_load_blocks)
+    
+    def _load_skylight(self):
+        """Loads and returns skylight array"""
+        if not hasattr(self, "_skylight"):
+            self._skylight = get_skylight_array(self.level)
+        return self._skylight
+    skylight = property(_load_skylight)
+
+    def _load_blocklight(self):
+        """Loads and returns blocklight array"""
+        if not hasattr(self, "_blocklight"):
+            self._blocklight = get_blocklight_array(self.level)
+        return self._blocklight
+    blocklight = property(_load_blocklight)
+    
+    def _load_left(self):
+        """Loads and sets data from lower-left chunk"""
+        chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY)
+        try:
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX - 1, self.chunkY)
+            self._left_skylight = get_skylight_array(chunk_data)
+            self._left_blocklight = get_blocklight_array(chunk_data)
+            self._left_blocks = get_blockarray(chunk_data)
+        except NoSuchChunk:
+            self._left_skylight = None
+            self._left_blocklight = None
+            self._left_blocks = None
+    
+    def _load_left_blocks(self):
+        """Loads and returns lower-left block array"""
+        if not hasattr(self, "_left_blocks"):
+            self._load_left()
+        return self._left_blocks
+    left_blocks = property(_load_left_blocks)
+
+    def _load_left_skylight(self):
+        """Loads and returns lower-left skylight array"""
+        if not hasattr(self, "_left_skylight"):
+            self._load_left()
+        return self._left_skylight
+    left_skylight = property(_load_left_skylight)
+
+    def _load_left_blocklight(self):
+        """Loads and returns lower-left blocklight array"""
+        if not hasattr(self, "_left_blocklight"):
+            self._load_left()
+        return self._left_blocklight
+    left_blocklight = property(_load_left_blocklight)
+
+    def _load_right(self):
+        """Loads and sets data from lower-right chunk"""
+        chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1)
+        try:
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY + 1)
+            self._right_skylight = get_skylight_array(chunk_data)
+            self._right_blocklight = get_blocklight_array(chunk_data)
+            self._right_blocks = get_blockarray(chunk_data)
+        except NoSuchChunk:
+            self._right_skylight = None
+            self._right_blocklight = None
+            self._right_blocks = None
+    
+    def _load_right_blocks(self):
+        """Loads and returns lower-right block array"""
+        if not hasattr(self, "_right_blocks"):
+            self._load_right()
+        return self._right_blocks
+    right_blocks = property(_load_right_blocks)
+
+    def _load_right_skylight(self):
+        """Loads and returns lower-right skylight array"""
+        if not hasattr(self, "_right_skylight"):
+            self._load_right()
+        return self._right_skylight
+    right_skylight = property(_load_right_skylight)
+
+    def _load_right_blocklight(self):
+        """Loads and returns lower-right blocklight array"""
+        if not hasattr(self, "_right_blocklight"):
+            self._load_right()
+        return self._right_blocklight
+    right_blocklight = property(_load_right_blocklight)
+
+    def _load_up_right(self):
+        """Loads and sets data from upper-right chunk"""
+        chunk_path = self.world.get_region_path(self.chunkX + 1, self.chunkY)
+        try:
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX + 1, self.chunkY)
+            self._up_right_skylight = get_skylight_array(chunk_data)
+            self._up_right_blocklight = get_blocklight_array(chunk_data)
+            self._up_right_blocks = get_blockarray(chunk_data)
+        except NoSuchChunk:
+            self._up_right_skylight = None
+            self._up_right_blocklight = None
+            self._up_right_blocks = None
+    
+    def _load_up_right_blocks(self):
+        """Loads and returns upper-right block array"""
+        if not hasattr(self, "_up_right_blocks"):
+            self._load_up_right()
+        return self._up_right_blocks
+    up_right_blocks = property(_load_up_right_blocks)
+    
+    def _load_up_left(self):
+        """Loads and sets data from upper-left chunk"""
+        chunk_path = self.world.get_region_path(self.chunkX, self.chunkY - 1)
+        try:
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY - 1)
+            self._up_left_skylight = get_skylight_array(chunk_data)
+            self._up_left_blocklight = get_blocklight_array(chunk_data)
+            self._up_left_blocks = get_blockarray(chunk_data)
+        except NoSuchChunk:
+            self._up_left_skylight = None
+            self._up_left_blocklight = None
+            self._up_left_blocks = None
+    
+    def _load_up_left_blocks(self):
+        """Loads and returns lower-left block array"""
+        if not hasattr(self, "_up_left_blocks"):
+            self._load_up_left()
+        return self._up_left_blocks
+    up_left_blocks = property(_load_up_left_blocks)
+
+    def generate_pseudo_ancildata(self,x,y,z,blockid, north_position = 0 ):
+        """ Generates a pseudo ancillary data for blocks that depend of 
+        what are surrounded and don't have ancillary data
+        
+        This uses a binary number of 4 digits to encode the info. 
+        The encode is:
+        
+        Bit:   1   2   3   4
+        Side:  x   y  -x  -y
+        Values: bit = 0 -> The corresponding side block has different blockid
+                bit = 1 -> The corresponding side block has same blockid
+        Example: if the bit1 is 1 that means that there is a block with 
+        blockid in the side of the +x direction.
+        
+        You can rotate the pseudo data multiplying by 2 and
+        if it is > 15 subtracting 15 and adding 1. (moving bits
+        in the left direction is like rotate 90 degree in anticlockwise
+        direction). In this way can be used for maps with other
+        north orientation.
+        
+        North position can have the values 0, 1, 2, 3, corresponding to 
+        north in bottom-left, bottom-right, top-right and top-left of
+        the screen.
+        
+        The rotation feature is not used anywhere yet.
+        """
+        
+        blocks = self.blocks
+        up_left_blocks = self.up_left_blocks 
+        up_right_blocks = self.up_right_blocks
+        left_blocks = self.left_blocks
+        right_blocks = self.right_blocks
+
+        pseudo_data = 0
+
+        # first check if we are in the border of a chunk, next check for chunks adjacent to this 
+        # and finally check for a block with same blockid. I we aren't in the border of a chunk, 
+        # check for the block having the sme blockid.
+        
+        if (up_right_blocks is not None and up_right_blocks[0,y,z] == blockid) if x == 15 else blocks[x+1,y,z] == blockid:
+            pseudo_data = pseudo_data | 0b1000
+        
+        if (right_blocks is not None and right_blocks[x,0,z] == blockid) if y == 15 else blocks[x,y + 1,z] == blockid:
+            pseudo_data = pseudo_data | 0b0100
+        
+        if (left_blocks is not None and left_blocks[15,y,z] == blockid) if x == 0 else blocks[x - 1,y,z] == blockid:
+            pseudo_data = pseudo_data | 0b0010
+            
+        if (up_left_blocks is not None and up_left_blocks[x,15,z] == blockid) if y == 0 else blocks[x,y - 1,z] == blockid:
+            pseudo_data = pseudo_data | 0b0001
+
+        # rotate the bits for other north orientations
+        while north_position > 0:
+            pseudo_data *= 2
+            if pseudo_data > 15:
+                pseudo_data -= 16
+                pseudo_data +=1
+            north_position -= 1
+            
+        return pseudo_data
+        
+    def chunk_render(self, img=None, xoff=0, yoff=0, cave=False):
+        """Renders a chunk with the given parameters, and returns the image.
+        If img is given, the chunk is rendered to that image object. Otherwise,
+        a new one is created. xoff and yoff are offsets in the image.
+        
+        For cave mode, all blocks that have any direct sunlight are not
+        rendered, and blocks are drawn with a color tint depending on their
+        depth."""
+        
+        blockData = get_blockdata_array(self.level)
+        blockData_expanded = numpy.empty((16,16,128), dtype=numpy.uint8)
+        # Even elements get the lower 4 bits
+        blockData_expanded[:,:,::2] = blockData & 0x0F
+        # Odd elements get the upper 4 bits
+        blockData_expanded[:,:,1::2] = blockData >> 4
+
+
+        # Each block is 24x24
+        # The next block on the X axis adds 12px to x and subtracts 6px from y in the image
+        # The next block on the Y axis adds 12px to x and adds 6px to y in the image
+        # The next block up on the Z axis subtracts 12 from y axis in the image
+
+        # Since there are 16x16x128 blocks in a chunk, the image will be 384x1728
+        # (height is 128*12 high, plus the size of the horizontal plane: 16*12)
+        if not img:
+            img = Image.new("RGBA", (384, 1728), (38,92,255,0))
+
+        c_overviewer.render_loop(self, img, xoff, yoff, blockData_expanded)
+
+        tileEntities = get_tileentity_data(self.level)
+        for entity in tileEntities:
+            if entity['id'] == 'Sign':
+                msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])
+                if msg.strip():
+                    # convert the blockID coordinates from local chunk
+                    # coordinates to global world coordinates
+                    newPOI = dict(type="sign",
+                                    x= entity['x'],
+                                    y= entity['y'],
+                                    z= entity['z'],
+                                    msg=msg,
+                                    chunk= (self.chunkX, self.chunkY),
+                                  )
+                    if self.queue:
+                        self.queue.put(["newpoi", newPOI])
+
+
+        # check to see if there are any signs in the persistentData list that are from this chunk.
+        # if so, remove them from the persistentData list (since they're have been added to the world.POI
+        # list above.
+        if self.queue:
+            self.queue.put(['removePOI', (self.chunkX, self.chunkY)])
+
+        return img
+
+# Render 3 blending masks for lighting
+# first is top (+Z), second is left (-X), third is right (+Y)
+def generate_facemasks():
+    white = Image.new("L", (24,24), 255)
+    
+    top = Image.new("L", (24,24), 0)
+    left = Image.new("L", (24,24), 0)
+    whole = Image.new("L", (24,24), 0)
+    
+    toppart = textures.transform_image(white)
+    leftpart = textures.transform_image_side(white)
+    
+    # using the real PIL paste here (not alpha_over) because there is
+    # no alpha channel (and it's mode "L")
+    top.paste(toppart, (0,0))
+    left.paste(leftpart, (0,6))
+    right = left.transpose(Image.FLIP_LEFT_RIGHT)
+    
+    # Manually touch up 6 pixels that leave a gap, like in
+    # textures._build_block()
+    for x,y in [(13,23), (17,21), (21,19)]:
+        right.putpixel((x,y), 255)
+    for x,y in [(3,4), (7,2), (11,0)]:
+        top.putpixel((x,y), 255)
+    
+    return (top, left, right)
+facemasks = generate_facemasks()
+black_color = Image.new("RGB", (24,24), (0,0,0))
+red_color = Image.new("RGB", (24,24), (229,36,38))
+
+# Render 128 different color images for color coded depth blending in cave mode
+def generate_depthcolors():
+    depth_colors = []
+    r = 255
+    g = 0
+    b = 0
+    for z in range(128):
+        img = Image.new("RGB", (24,24), (r,g,b))
+        depth_colors.append(img)
+        if z < 32:
+            g += 7
+        elif z < 64:
+            r -= 7
+        elif z < 96:
+            b += 7
+        else:
+            g -= 7
+
+    return depth_colors
+depth_colors = generate_depthcolors()

overviewer/composite.py

@@ -0,0 +1,38 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+from PIL import Image
+
+"""
+This module has an alpha-over function that is used throughout
+Overviewer. It defaults to the PIL paste function when the custom
+alpha-over extension cannot be found.
+"""
+
+from c_overviewer import alpha_over as extension_alpha_over
+
+def alpha_over(dest, src, pos_or_rect=(0, 0), mask=None):
+    """Composite src over dest, using mask as the alpha channel (if
+    given), otherwise using src's alpha channel. pos_or_rect can
+    either be a position or a rectangle, specifying where on dest to
+    put src. Falls back to dest.paste() if the alpha_over extension
+    can't be found."""
+    if mask is None:
+        mask = src
+    
+    global extension_alpha_over
+    return extension_alpha_over(dest, src, pos_or_rect, mask)

overviewer/configParser.py

@@ -0,0 +1,168 @@
+from optparse import OptionParser
+import sys
+import os.path
+
+class OptionsResults(object):
+    pass
+
+class ConfigOptionParser(object):
+    def __init__(self, **kwargs):
+        self.cmdParser = OptionParser(usage=kwargs.get("usage",""))
+        self.configFile = kwargs.get("config","settings.py")
+        self.configVars = []
+
+        # these are arguments not understood by OptionParser, so they must be removed
+        # in add_option before being passed to the OptionParser
+
+        # note that default is a valid OptionParser argument, but we remove it
+        # because we want to do our default value handling
+
+        self.customArgs = ["required", "commandLineOnly", "default", "listify", "listdelim", "choices"]
+
+        self.requiredArgs = []
+
+    def display_config(self):
+        for x in self.configVars:
+            n = x['dest']
+            print  "%s: %r" % (n, self.configResults.__dict__[n])
+
+    def add_option(self, *args, **kwargs):
+
+        if kwargs.get("configFileOnly", False) and kwargs.get("commandLineOnly", False):
+            raise Exception(args, "configFileOnly and commandLineOnly are mututally exclusive")
+
+        self.configVars.append(kwargs.copy())
+
+        if not kwargs.get("configFileOnly", False):
+            for arg in self.customArgs:
+                if arg in kwargs.keys(): del kwargs[arg]
+
+            if kwargs.get("type", None):
+                kwargs['type'] = 'string' # we'll do our own converting later
+            self.cmdParser.add_option(*args, **kwargs)
+
+    def print_help(self):
+        self.cmdParser.print_help()
+
+    def parse_args(self):
+
+        # first, load the results from the command line:
+        options, args = self.cmdParser.parse_args()
+
+        # second, use these values to seed the locals dict
+        l = dict()
+        g = dict()
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('configFileOnly', False): continue
+            if a.get('commandLineOnly', False): continue
+            v = getattr(options, n)
+            if v != None:
+                #print "seeding %s with %s" % (n, v)
+                l[n] = v
+            else:
+                # if this has a default, use that to seed the globals dict
+                if a.get("default", None): g[n] = a['default']
+        g['args'] = args
+
+        try:
+            if os.path.exists(self.configFile):
+                execfile(self.configFile, g, l)
+        except NameError, ex:
+            import traceback
+            traceback.print_exc()
+            print "\nError parsing %s.  Please check the trackback above" % self.configFile
+            sys.exit(1)
+        except SyntaxError, ex:
+            import traceback
+            traceback.print_exc()
+            tb = sys.exc_info()[2]
+            #print tb.tb_frame.f_code.co_filename
+            print "\nError parsing %s.  Please check the trackback above" % self.configFile
+            sys.exit(1)
+
+        #print l.keys()
+
+        configResults = OptionsResults()
+        # third, load the results from the config file:
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('commandLineOnly', False):
+                if n in l.keys():
+                    print "Error: %s can only be specified on the command line.  It is not valid in the config file" % n
+                    sys.exit(1)
+
+            configResults.__dict__[n] = l.get(n)
+
+
+        
+        # third, merge options into configReslts (with options overwriting anything in configResults)
+        for a in self.configVars:
+            n = a['dest']
+            if a.get('configFileOnly', False): continue
+            if getattr(options, n) != None:
+                configResults.__dict__[n] = getattr(options, n)
+
+        # forth, set defaults for any empty values
+        for a in self.configVars:
+            n = a['dest']
+            if (n not in configResults.__dict__.keys() or configResults.__dict__[n] == None) and 'default' in a.keys():
+                configResults.__dict__[n] = a['default']
+
+        # fifth, check required args:
+        for a in self.configVars:
+            n = a['dest']
+            if configResults.__dict__[n] == None and a.get('required',False):
+                raise Exception("%s is required" % n)
+
+        # sixth, check types
+        for a in self.configVars:
+            n = a['dest']
+            if 'listify' in a.keys():
+                # this thing may be a list!
+                if configResults.__dict__[n] != None and type(configResults.__dict__[n]) == str:
+                    configResults.__dict__[n] = configResults.__dict__[n].split(a.get("listdelim",","))
+                elif type(configResults.__dict__[n]) != list:
+                    configResults.__dict__[n] = [configResults.__dict__[n]]
+            if 'type' in a.keys() and configResults.__dict__[n] != None:
+                try:
+                    configResults.__dict__[n] = self.checkType(configResults.__dict__[n], a)
+                except ValueError, ex:
+                    print "There was a problem converting the value '%s' to type %s for config parameter '%s'" % (configResults.__dict__[n], a['type'], n)
+                    import traceback
+                    #traceback.print_exc()
+                    sys.exit(1)
+
+
+
+        self.configResults = configResults
+
+        return configResults, args
+
+    def checkType(self, value, a):
+
+        if type(value) == list:
+            return map(lambda x: self.checkType(x, a), value)
+
+        # switch on type.  there are only 7 types that can be used with optparse
+        if a['type'] == "int":
+            return int(value)
+        elif a['type'] == "string":
+            return str(value)
+        elif a['type'] == "long":
+            return long(value)
+        elif a['type'] == "choice":
+            if value not in a['choices']:
+                print "The value '%s' is not valid for config parameter '%s'" % (value, a['dest'])
+                sys.exit(1)
+            return value
+        elif a['type'] == "float":
+            return long(value)
+        elif a['type'] == "complex":
+            return complex(value)
+        elif a['type'] == "function":
+            if not callable(value):
+                raise ValueError("Not callable")
+        else:
+            print "Unknown type!"
+            sys.exit(1)

overviewer/googlemap.py

@@ -0,0 +1,163 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import os.path
+import stat
+import cPickle
+import Image
+import shutil
+from time import strftime, gmtime
+import json
+
+import util
+
+"""
+This module has routines related to generating a Google Maps-based
+interface out of a set of tiles.
+
+"""
+
+def mirror_dir(src, dst, entities=None):
+    '''copies all of the entities from src to dst'''
+    if not os.path.exists(dst):
+        os.mkdir(dst)
+    if entities and type(entities) != list: raise Exception("Expected a list, got a %r instead" % type(entities))
+    
+    # files which are problematic and should not be copied
+    # usually, generated by the OS
+    skip_files = ['Thumbs.db', '.DS_Store']
+    
+    for entry in os.listdir(src):
+        if entry in skip_files:
+            continue
+        if entities and entry not in entities:
+            continue
+        
+        if os.path.isdir(os.path.join(src,entry)):
+            mirror_dir(os.path.join(src, entry), os.path.join(dst, entry))
+        elif os.path.isfile(os.path.join(src,entry)):
+            try:
+                shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
+            except IOError:
+                # maybe permission problems?
+                os.chmod(os.path.join(src, entry), stat.S_IRUSR)
+                os.chmod(os.path.join(dst, entry), stat.S_IWUSR)
+                shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
+                # if this stills throws an error, let it propagate up
+
+class MapGen(object):
+    def __init__(self, quadtrees, skipjs=False, web_assets_hook=None):
+        """Generates a Google Maps interface for the given list of
+        quadtrees. All of the quadtrees must have the same destdir,
+        image format, and world. 
+        Note:tiledir for each quadtree should be unique. By default the tiledir is determined by the rendermode"""
+        
+        self.skipjs = skipjs
+        self.web_assets_hook = web_assets_hook
+        
+        if not len(quadtrees) > 0:
+            raise ValueError("there must be at least one quadtree to work on")
+        
+        self.destdir = quadtrees[0].destdir
+        self.imgformat = quadtrees[0].imgformat
+        self.world = quadtrees[0].world
+        self.p = quadtrees[0].p
+        for i in quadtrees:
+            if i.destdir != self.destdir or i.imgformat != self.imgformat or i.world != self.world:
+                raise ValueError("all the given quadtrees must have the same destdir")
+        
+        self.quadtrees = quadtrees
+        
+    def go(self, procs):
+        """Writes out config.js, marker.js, and region.js
+        Copies web assets into the destdir"""
+        zoomlevel = self.p
+        imgformat = self.imgformat
+        configpath = os.path.join(util.get_program_path(), "config.js")
+
+        config = open(configpath, 'r').read()
+        config = config.replace(
+                "{maxzoom}", str(zoomlevel))
+        config = config.replace(
+                "{imgformat}", str(imgformat))
+        
+        config = config.replace("{spawn_coords}",
+                                json.dumps(list(self.world.spawn)))
+        
+        # create generated map type data, from given quadtrees
+        maptypedata = map(lambda q: {'label' : q.rendermode.capitalize(),
+                                     'path' : q.tiledir}, self.quadtrees)
+        config = config.replace("{maptypedata}", json.dumps(maptypedata))
+        
+        with open(os.path.join(self.destdir, "config.js"), 'w') as output:
+            output.write(config)
+
+        # Write a blank image
+        for quadtree in self.quadtrees:
+            blank = Image.new("RGBA", (1,1))
+            tileDir = os.path.join(self.destdir, quadtree.tiledir)
+            if not os.path.exists(tileDir): os.mkdir(tileDir)
+            blank.save(os.path.join(tileDir, "blank."+self.imgformat))
+
+        # copy web assets into destdir:
+        mirror_dir(os.path.join(util.get_program_path(), "web_assets"), self.destdir)
+
+        # Add time in index.html
+        indexpath = os.path.join(self.destdir, "index.html")
+
+        index = open(indexpath, 'r').read()
+        index = index.replace(
+                "{time}", str(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())))
+
+        with open(os.path.join(self.destdir, "index.html"), 'w') as output:
+            output.write(index)
+
+        if self.skipjs:
+            if self.web_assets_hook:
+                self.web_assets_hook(self)
+            return
+
+
+    def finalize(self):
+        if self.skipjs:
+            return
+
+        # since we will only discover PointsOfInterest in chunks that need to be 
+        # [re]rendered, POIs like signs in unchanged chunks will not be listed
+        # in self.world.POI.  To make sure we don't remove these from markers.js
+        # we need to merge self.world.POI with the persistant data in world.PersistentData
+
+        self.world.POI += filter(lambda x: x['type'] != 'spawn', self.world.persistentData['POI'])
+
+        # write out the default marker table
+        with open(os.path.join(self.destdir, "markers.js"), 'w') as output:
+            output.write("var markerData=%s" % json.dumps(self.world.POI))
+        
+        # save persistent data
+        self.world.persistentData['POI'] = self.world.POI
+        with open(self.world.pickleFile,"wb") as f:
+            cPickle.dump(self.world.persistentData,f)
+
+        # write out the default (empty, but documented) region table
+        with open(os.path.join(self.destdir, "regions.js"), 'w') as output:
+            output.write('var regionData=[\n')
+            output.write('  // {"color": "#FFAA00", "opacity": 0.5, "closed": true, "path": [\n')
+            output.write('  //   {"x": 0, "y": 0, "z": 0},\n')
+            output.write('  //   {"x": 0, "y": 10, "z": 0},\n')
+            output.write('  //   {"x": 0, "y": 0, "z": 10}\n')
+            output.write('  // ]},\n')
+            output.write('];')
+        

overviewer/nbt.py

@@ -0,0 +1,404 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import gzip, zlib
+import struct
+import StringIO
+import os
+
+# decorator to handle filename or object as first parameter
+def _file_loader(func):
+    def wrapper(fileobj, *args):
+        if isinstance(fileobj, basestring):
+            if not os.path.isfile(fileobj):
+               return None
+
+            # Is actually a filename
+            fileobj = open(fileobj, 'rb',4096)
+        return func(fileobj, *args)
+    return wrapper
+
+@_file_loader
+def load(fileobj):
+    return NBTFileReader(fileobj).read_all()
+
+def load_from_region(filename, x, y):
+    nbt = load_region(filename).load_chunk(x, y)
+    if nbt is None:
+        return None ## return none.  I think this is who we should indicate missing chunks
+        #raise IOError("No such chunk in region: (%i, %i)" % (x, y))     
+    return nbt.read_all()
+  
+def load_region(filename):            
+    return MCRFileReader(filename)
+  
+  
+# compile the unpacker's into a classes
+_byte   = struct.Struct("b")
+_short  = struct.Struct(">h")
+_int    = struct.Struct(">i")
+_long   = struct.Struct(">q")
+_float  = struct.Struct(">f")
+_double = struct.Struct(">d") 
+
+_24bit_int = struct.Struct("B B B")
+_unsigned_byte = struct.Struct("B")
+_unsigned_int = struct.Struct(">I")
+_chunk_header = struct.Struct(">I B")
+ 
+class NBTFileReader(object):
+    def __init__(self, fileobj, is_gzip=True):
+        if is_gzip:
+            self._file = gzip.GzipFile(fileobj=fileobj, mode='rb')
+        else:
+            # pure zlib stream -- maybe later replace this with
+            # a custom zlib file object?
+            data = zlib.decompress(fileobj.read())
+            self._file = StringIO.StringIO(data)
+
+    # These private methods read the payload only of the following types
+    def _read_tag_end(self):
+        # Nothing to read
+        return 0
+
+    def _read_tag_byte(self):
+        byte = self._file.read(1)
+        return _byte.unpack(byte)[0]
+    
+    def _read_tag_short(self):
+        bytes = self._file.read(2)
+        global _short
+        return _short.unpack(bytes)[0]
+
+    def _read_tag_int(self):
+        bytes = self._file.read(4)
+        global _int
+        return _int.unpack(bytes)[0]
+
+    def _read_tag_long(self):
+        bytes = self._file.read(8)
+        global _long
+        return _long.unpack(bytes)[0]
+
+    def _read_tag_float(self):
+        bytes = self._file.read(4)
+        global _float
+        return _float.unpack(bytes)[0]
+
+    def _read_tag_double(self):
+        bytes = self._file.read(8)
+        global _double
+        return _double.unpack(bytes)[0]
+
+    def _read_tag_byte_array(self):
+        length = self._read_tag_int()
+        bytes = self._file.read(length)
+        return bytes
+
+    def _read_tag_string(self):
+        length = self._read_tag_short()
+
+        # Read the string
+        string = self._file.read(length)
+
+        # decode it and return
+        return string.decode("UTF-8")
+
+    def _read_tag_list(self):
+        tagid = self._read_tag_byte()
+        length = self._read_tag_int()
+
+        read_tagmap = {
+                0: self._read_tag_end,
+                1: self._read_tag_byte,
+                2: self._read_tag_short,
+                3: self._read_tag_int,
+                4: self._read_tag_long,
+                5: self._read_tag_float,
+                6: self._read_tag_double,
+                7: self._read_tag_byte_array,
+                8: self._read_tag_string,
+                9: self._read_tag_list,
+                10:self._read_tag_compound,
+                }
+
+        read_method = read_tagmap[tagid]
+        l = []
+        for _ in xrange(length):
+            l.append(read_method())
+        return l
+
+    def _read_tag_compound(self):
+        # Build a dictionary of all the tag names mapping to their payloads
+        tags = {}
+        while True:
+            # Read a tag
+            tagtype = ord(self._file.read(1))
+
+            if tagtype == 0:
+                break
+
+            name = self._read_tag_string()
+            read_tagmap = {
+                    0: self._read_tag_end,
+                    1: self._read_tag_byte,
+                    2: self._read_tag_short,
+                    3: self._read_tag_int,
+                    4: self._read_tag_long,
+                    5: self._read_tag_float,
+                    6: self._read_tag_double,
+                    7: self._read_tag_byte_array,
+                    8: self._read_tag_string,
+                    9: self._read_tag_list,
+                    10:self._read_tag_compound,
+                    }
+            payload = read_tagmap[tagtype]()
+            
+            tags[name] = payload
+
+        return tags
+
+
+
+    def read_all(self):
+        """Reads the entire file and returns (name, payload)
+        name is the name of the root tag, and payload is a dictionary mapping
+        names to their payloads
+
+        """
+        # Read tag type
+        tagtype = ord(self._file.read(1))
+        if tagtype != 10:
+            raise Exception("Expected a tag compound")
+
+        # Read the tag name
+        name = self._read_tag_string()
+
+        payload = self._read_tag_compound()
+
+        return name, payload
+
+
+# For reference, the MCR format is outlined at
+# <http://www.minecraftwiki.net/wiki/Beta_Level_Format>
+class MCRFileReader(object):
+    """A class for reading chunk region files, as introduced in the
+    Beta 1.3 update. It provides functions for opening individual
+    chunks (as instances of NBTFileReader), getting chunk timestamps,
+    and for listing chunks contained in the file."""
+    
+    def __init__(self, filename):
+        self._file = None
+        self._filename = filename
+        # cache used when the entire header tables are read in get_chunks()
+        self._locations = None
+        self._timestamps = None
+        self._chunks = None
+    
+    def _read_24bit_int(self):
+        """Read in a 24-bit, big-endian int, used in the chunk
+        location table."""
+        
+        ret = 0
+        bytes = self._file.read(3)
+        global _24bit_int
+        bytes = _24bit_int.unpack(bytes)
+        for i in xrange(3):
+            ret = ret << 8
+            ret += bytes[i]
+        
+        return ret
+    
+    def _read_chunk_location(self, x=None, y=None):
+        """Read and return the (offset, length) of the given chunk
+        coordinate, or None if the requested chunk doesn't exist. x
+        and y must be between 0 and 31, or None. If they are None,
+        then there will be no file seek before doing the read."""
+        
+        if x is not None and y is not None:
+            if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32):
+                raise ValueError("Chunk location out of range.")
+            
+            # check for a cached value
+            if self._locations:
+                return self._locations[x + y * 32]
+            
+            # go to the correct entry in the chunk location table
+            self._file.seek(4 * (x + y * 32))
+        
+        try:
+            # 3-byte offset in 4KiB sectors
+            offset_sectors = self._read_24bit_int()
+            
+            # 1-byte length in 4KiB sectors, rounded up
+            global _unsigned_byte
+            byte = self._file.read(1)
+            length_sectors = _unsigned_byte.unpack(byte)[0]
+        except (IndexError, struct.error):
+            # got a problem somewhere
+            return None
+        
+        # check for empty chunks
+        if offset_sectors == 0 or length_sectors == 0:
+            return None
+        
+        return (offset_sectors * 4096, length_sectors * 4096)
+    
+    def _read_chunk_timestamp(self, x=None, y=None):
+        """Read and return the last modification time of the given
+        chunk coordinate. x and y must be between 0 and 31, or
+        None. If they are, None, then there will be no file seek
+        before doing the read."""
+        
+        if x is not None and y is not None:
+            if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32):
+                raise ValueError("Chunk location out of range.")
+            
+            # check for a cached value
+            if self._timestamps:
+                return self._timestamps[x + y * 32]
+            
+            # go to the correct entry in the chunk timestamp table
+            self._file.seek(4 * (x + y * 32) + 4096)
+        
+        try:
+            bytes = self._file.read(4)
+            global _unsigned_int
+            timestamp = _unsigned_int.unpack(bytes)[0]
+        except (IndexError, struct.error):
+            return 0
+        
+        return timestamp
+    
+    def openfile(self):
+        #make sure we clean up
+        if self._file is None:
+            self._file = open(self._filename,'rb')   
+
+    def closefile(self):
+        #make sure we clean up
+        if self._file is not None:
+            self._file.close()
+            self._file =  None
+
+    def get_chunks(self):    
+        """Return a list of all chunks contained in this region file,
+        as a list of (x, y) coordinate tuples. To load these chunks,
+        provide these coordinates to load_chunk()."""
+        
+        if self._chunks is not None:
+            return self._chunks
+        if self._locations is None:
+            self.get_chunk_info()       
+        self._chunks = [] 
+        for x in xrange(32): 
+            for y in xrange(32): 
+                if self._locations[x + y * 32] is not None:
+                    self._chunks.append((x,y))
+        return self._chunks
+        
+    def get_chunk_info(self,closeFile = True):
+        """Preloads region header information."""
+        
+        if self._locations:
+            return
+        
+        self.openfile()
+
+        self._chunks = None
+        self._locations = []
+        self._timestamps = []
+        
+        # go to the beginning of the file
+        self._file.seek(0)        
+        
+        # read chunk location table
+        locations_append = self._locations.append
+        for _ in xrange(32*32): 
+            locations_append(self._read_chunk_location())
+        
+        # read chunk timestamp table
+        timestamp_append = self._timestamps.append
+        for _ in xrange(32*32): 
+            timestamp_append(self._read_chunk_timestamp())
+ 
+        if closeFile:        
+            self.closefile()
+        return
+    
+    def get_chunk_timestamp(self, x, y):
+        """Return the given chunk's modification time. If the given
+        chunk doesn't exist, this number may be nonsense. Like
+        load_chunk(), this will wrap x and y into the range [0, 31].
+        """
+        x = x % 32
+        y = y % 32        
+        if self._timestamps is None:
+            self.get_chunk_info() 
+        return self._timestamps[x + y * 32]   
+    
+    def chunkExists(self, x, y):
+        """Determines if a chunk exists without triggering loading of the backend data"""
+        x = x % 32
+        y = y % 32
+        if self._locations is None:
+            self.get_chunk_info()
+        location = self._locations[x + y * 32]
+        return location is not None        
+
+    def load_chunk(self, x, y,closeFile=True):
+        """Return a NBTFileReader instance for the given chunk, or
+        None if the given chunk doesn't exist in this region file. If
+        you provide an x or y not between 0 and 31, it will be
+        modulo'd into this range (x % 32, etc.) This is so you can
+        provide chunk coordinates in global coordinates, and still
+        have the chunks load out of regions properly."""
+        x = x % 32
+        y = y % 32
+        if self._locations is None:
+            self.get_chunk_info()   
+                    
+        location = self._locations[x + y * 32]
+        if location is None:
+            return None
+
+        self.openfile()
+        
+        # seek to the data
+        self._file.seek(location[0])
+        
+        # read in the chunk data header
+        bytes = self._file.read(5)        
+        data_length,compression =  _chunk_header.unpack(bytes)
+        
+        # figure out the compression
+        is_gzip = True
+        if compression == 1:
+            # gzip -- not used by the official client, but trivial to support here so...
+            is_gzip = True
+        elif compression == 2:
+            # deflate -- pure zlib stream
+            is_gzip = False
+        else:
+            # unsupported!
+            raise Exception("Unsupported chunk compression type: %i" % (compression))
+        # turn the rest of the data into a StringIO object
+        # (using data_length - 1, as we already read 1 byte for compression)
+        data = self._file.read(data_length - 1)
+        data = StringIO.StringIO(data)
+        
+        if closeFile:        
+            self.closefile()        
+        return NBTFileReader(data, is_gzip=is_gzip)

overviewer/optimizeimages.py

@@ -0,0 +1,49 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import subprocess
+import shlex
+
+pngcrush = "pngcrush"
+optipng = "optipng"
+advdef = "advdef"
+
+def check_programs(level):
+    path = os.environ.get("PATH").split(os.pathsep)
+
+    for prog,l in [(pngcrush,1), (optipng,2), (advdef,2)]:
+        if l <= level:
+            result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path)
+            if len(result) == 0:
+                raise Exception("Optimization prog %s for level %d not found!" % (prog, l))
+
+def optimize_image(imgpath, imgformat, optimizeimg):
+    if imgformat == 'png':
+        if optimizeimg >= 1:
+            # we can't do an atomic replace here because windows is terrible
+            # so instead, we make temp files, delete the old ones, and rename
+            # the temp files. go windows!
+            subprocess.Popen([pngcrush, imgpath, imgpath + ".tmp"],
+                stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
+            os.remove(imgpath)
+            os.rename(imgpath+".tmp", imgpath)
+
+        if optimizeimg >= 2:
+            subprocess.Popen([optipng, imgpath], stderr=subprocess.STDOUT,
+                stdout=subprocess.PIPE).communicate()[0]
+            subprocess.Popen([advdef, "-z4",imgpath], stderr=subprocess.STDOUT,
+                stdout=subprocess.PIPE).communicate()[0]
+

overviewer/quadtree.py

@@ -0,0 +1,455 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import multiprocessing
+import itertools
+import os
+import os.path
+import functools
+import re
+import shutil
+import collections
+import json
+import logging
+import util
+import cPickle
+import stat
+import errno 
+import time
+from time import gmtime, strftime, sleep
+
+from PIL import Image
+
+import nbt
+import chunk
+from optimizeimages import optimize_image
+import composite
+
+
+"""
+This module has routines related to generating a quadtree of tiles
+
+"""
+
+def iterate_base4(d):
+    """Iterates over a base 4 number with d digits"""
+    return itertools.product(xrange(4), repeat=d)
+   
+class QuadtreeGen(object):
+    def __init__(self, worldobj, destdir, depth=None, tiledir=None, imgformat=None, optimizeimg=None, rendermode="normal"):
+        """Generates a quadtree from the world given into the
+        given dest directory
+
+        worldobj is a world.WorldRenderer object that has already been processed
+
+        If depth is given, it overrides the calculated value. Otherwise, the
+        minimum depth that contains all chunks is calculated and used.
+
+        """
+        assert(imgformat)
+        self.imgformat = imgformat
+        self.optimizeimg = optimizeimg
+        
+        self.lighting = rendermode in ("lighting", "night", "spawn")
+        self.night = rendermode in ("night", "spawn")
+        self.spawn = rendermode in ("spawn",)
+        self.rendermode = rendermode
+
+        # Make the destination dir
+        if not os.path.exists(destdir):
+            os.mkdir(destdir)
+        if tiledir is None:
+            tiledir = rendermode
+        self.tiledir = tiledir        
+        
+        if depth is None:
+            # Determine quadtree depth (midpoint is always 0,0)
+            for p in xrange(15):
+                # Will 2^p tiles wide and high suffice?
+
+                # X has twice as many chunks as tiles, then halved since this is a
+                # radius
+                xradius = 2**p
+                # Y has 4 times as many chunks as tiles, then halved since this is
+                # a radius
+                yradius = 2*2**p
+                if xradius >= worldobj.maxcol and -xradius <= worldobj.mincol and \
+                        yradius >= worldobj.maxrow and -yradius <= worldobj.minrow:
+                    break
+            else:
+                raise ValueError("Your map is waaaay too big! Use the '-z' or '--zoom' options.")
+
+            self.p = p
+        else:
+            self.p = depth
+            xradius = 2**depth
+            yradius = 2*2**depth
+
+        # Make new row and column ranges
+        self.mincol = -xradius
+        self.maxcol = xradius
+        self.minrow = -yradius
+        self.maxrow = yradius
+
+        self.world = worldobj
+        self.destdir = destdir
+        self.full_tiledir = os.path.join(destdir, tiledir)
+        
+    def _get_cur_depth(self):
+        """How deep is the quadtree currently in the destdir? This glances in
+        config.js to see what maxZoom is set to.
+        returns -1 if it couldn't be detected, file not found, or nothing in
+        config.js matched
+        """
+        indexfile = os.path.join(self.destdir, "config.js")
+        if not os.path.exists(indexfile):
+            return -1
+        matcher = re.compile(r"maxZoom:\s*(\d+)")
+        p = -1
+        for line in open(indexfile, "r"):
+            res = matcher.search(line)
+            if res:
+                p = int(res.group(1))
+                break
+        return p
+
+    def _increase_depth(self):
+        """Moves existing tiles into place for a larger tree"""
+        getpath = functools.partial(os.path.join, self.destdir, self.tiledir)
+
+        # At top level of the tree:
+        # quadrant 0 is now 0/3
+        # 1 is now 1/2
+        # 2 is now 2/1
+        # 3 is now 3/0
+        # then all that needs to be done is to regenerate the new top level
+        for dirnum in range(4):
+            newnum = (3,2,1,0)[dirnum]
+
+            newdir = "new" + str(dirnum)
+            newdirpath = getpath(newdir)
+
+            files = [str(dirnum)+"."+self.imgformat, str(dirnum)]
+            newfiles = [str(newnum)+"."+self.imgformat, str(newnum)]
+
+            os.mkdir(newdirpath)
+            for f, newf in zip(files, newfiles):
+                p = getpath(f)
+                if os.path.exists(p):
+                    os.rename(p, getpath(newdir, newf))
+            os.rename(newdirpath, getpath(str(dirnum)))
+
+    def _decrease_depth(self):
+        """If the map size decreases, or perhaps the user has a depth override
+        in effect, re-arrange existing tiles for a smaller tree"""
+        getpath = functools.partial(os.path.join, self.destdir, self.tiledir)
+
+        # quadrant 0/3 goes to 0
+        # 1/2 goes to 1
+        # 2/1 goes to 2
+        # 3/0 goes to 3
+        # Just worry about the directories here, the files at the top two
+        # levels are cheap enough to replace
+        if os.path.exists(getpath("0", "3")):
+            os.rename(getpath("0", "3"), getpath("new0"))
+            shutil.rmtree(getpath("0"))
+            os.rename(getpath("new0"), getpath("0"))
+
+        if os.path.exists(getpath("1", "2")):
+            os.rename(getpath("1", "2"), getpath("new1"))
+            shutil.rmtree(getpath("1"))
+            os.rename(getpath("new1"), getpath("1"))
+
+        if os.path.exists(getpath("2", "1")):
+            os.rename(getpath("2", "1"), getpath("new2"))
+            shutil.rmtree(getpath("2"))
+            os.rename(getpath("new2"), getpath("2"))
+
+        if os.path.exists(getpath("3", "0")):
+            os.rename(getpath("3", "0"), getpath("new3"))
+            shutil.rmtree(getpath("3"))
+            os.rename(getpath("new3"), getpath("3"))
+        
+    def go(self, procs):
+        """Processing before tile rendering"""
+
+        curdepth = self._get_cur_depth()
+        if curdepth != -1:
+            if self.p > curdepth:
+                logging.warning("Your map seemes to have expanded beyond its previous bounds.")
+                logging.warning( "Doing some tile re-arrangements... just a sec...")
+                for _ in xrange(self.p-curdepth):
+                    self._increase_depth()
+            elif self.p < curdepth:
+                logging.warning("Your map seems to have shrunk. Re-arranging tiles, just a sec...")
+                for _ in xrange(curdepth - self.p):
+                    self._decrease_depth()
+    
+    
+    def _get_range_by_path(self, path):
+        """Returns the x, y chunk coordinates of this tile"""
+        x, y = self.mincol, self.minrow
+        
+        xsize = self.maxcol
+        ysize = self.maxrow
+
+        for p in path:
+            if p in (1, 3):
+                x += xsize
+            if p in (2, 3):
+                y += ysize
+            xsize //= 2
+            ysize //= 2
+
+        return x, y
+        
+    def get_chunks_in_range(self, colstart, colend, rowstart, rowend):
+        """Get chunks that are relevant to the tile rendering function that's
+        rendering that range"""
+        chunklist = []
+        unconvert_coords = self.world.unconvert_coords
+        #get_region_path = self.world.get_region_path
+        get_region = self.world.regionfiles.get
+        for row in xrange(rowstart-16, rowend+1):
+            for col in xrange(colstart, colend+1):
+                # due to how chunks are arranged, we can only allow
+                # even row, even column or odd row, odd column
+                # otherwise, you end up with duplicates!
+                if row % 2 != col % 2:
+                    continue
+                
+                # return (col, row, chunkx, chunky, regionpath)
+                chunkx, chunky = unconvert_coords(col, row)
+                #c = get_region_path(chunkx, chunky)
+                _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None));
+                if c is not None and mcr.chunkExists(chunkx,chunky):                  
+                    chunklist.append((col, row, chunkx, chunky, c))
+        return chunklist   
+        
+    def get_worldtiles(self):
+        """Returns an iterator over the tiles of the most detailed layer
+        """
+        for path in iterate_base4(self.p):
+            # Get the range for this tile
+            colstart, rowstart = self._get_range_by_path(path)
+            colend = colstart + 2
+            rowend = rowstart + 4   
+            
+            # This image is rendered at(relative to the worker's destdir):
+            tilepath = [str(x) for x in path]
+            tilepath = os.sep.join(tilepath)
+            #logging.debug("this is rendered at %s", dest)
+        
+            # Put this in the batch to be submited to the pool     
+            yield [self,colstart, colend, rowstart, rowend, tilepath]
+        
+    def get_innertiles(self,zoom):
+        """Same as get_worldtiles but for the inntertile routine.
+        """    
+        for path in iterate_base4(zoom):
+            # This image is rendered at(relative to the worker's destdir):
+            tilepath = [str(x) for x in path[:-1]]
+            tilepath = os.sep.join(tilepath)
+            name = str(path[-1])
+  
+            yield [self,tilepath, name]
+    
+    def render_innertile(self, dest, name):
+        """
+        Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
+        os.path.join(dest, name, "{0,1,2,3}.png")
+        """
+        imgformat = self.imgformat 
+        imgpath = os.path.join(dest, name) + "." + imgformat
+
+        if name == "base":
+            quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]]
+        else:
+            quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]]    
+       
+        #stat the tile, we need to know if it exists or it's mtime
+        try:    
+            tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+            tile_mtime = None
+            
+        #check mtimes on each part of the quad, this also checks if they exist
+        needs_rerender = tile_mtime is None
+        quadPath_filtered = []
+        for path in quadPath:
+            try:
+                quad_mtime = os.stat(path[1])[stat.ST_MTIME]; 
+                quadPath_filtered.append(path)
+                if quad_mtime > tile_mtime:     
+                    needs_rerender = True            
+            except OSError:
+                # We need to stat all the quad files, so keep looping
+                pass      
+        # do they all not exist?
+        if quadPath_filtered == []:
+            if tile_mtime is not None:
+                os.unlink(imgpath)
+            return
+        # quit now if we don't need rerender            
+        if not needs_rerender:
+            return    
+        #logging.debug("writing out innertile {0}".format(imgpath))
+
+        # Create the actual image now
+        img = Image.new("RGBA", (384, 384), (38,92,255,0))
+        
+        # we'll use paste (NOT alpha_over) for quadtree generation because
+        # this is just straight image stitching, not alpha blending
+        
+        for path in quadPath_filtered:
+            try:
+                quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
+                img.paste(quad, path[0])
+            except Exception, e:
+                logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
+
+        # Save it
+        if self.imgformat == 'jpg':
+            img.save(imgpath, quality=95, subsampling=0)
+        else: # png
+            img.save(imgpath)
+            
+        if self.optimizeimg:
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)
+
+
+
+    def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path, poi_queue=None):
+        """Renders just the specified chunks into a tile and save it. Unlike usual
+        python conventions, rowend and colend are inclusive. Additionally, the
+        chunks around the edges are half-way cut off (so that neighboring tiles
+        will render the other half)
+
+        chunks is a list of (col, row, chunkx, chunky, filename) of chunk
+        images that are relevant to this call (with their associated regions)
+
+        The image is saved to path+"."+self.imgformat
+
+        If there are no chunks, this tile is not saved (if it already exists, it is
+        deleted)
+
+        Standard tile size has colend-colstart=2 and rowend-rowstart=4
+
+        There is no return value
+        """    
+        
+        # width of one chunk is 384. Each column is half a chunk wide. The total
+        # width is (384 + 192*(numcols-1)) since the first column contributes full
+        # width, and each additional one contributes half since they're staggered.
+        # However, since we want to cut off half a chunk at each end (384 less
+        # pixels) and since (colend - colstart + 1) is the number of columns
+        # inclusive, the equation simplifies to:
+        width = 192 * (colend - colstart)
+        # Same deal with height
+        height = 96 * (rowend - rowstart)
+
+        # The standard tile size is 3 columns by 5 rows, which works out to 384x384
+        # pixels for 8 total chunks. (Since the chunks are staggered but the grid
+        # is not, some grid coordinates do not address chunks) The two chunks on
+        # the middle column are shown in full, the two chunks in the middle row are
+        # half cut off, and the four remaining chunks are one quarter shown.
+        # The above example with cols 0-3 and rows 0-4 has the chunks arranged like this:
+        #   0,0         2,0
+        #         1,1
+        #   0,2         2,2
+        #         1,3
+        #   0,4         2,4
+
+        # Due to how the tiles fit together, we may need to render chunks way above
+        # this (since very few chunks actually touch the top of the sky, some tiles
+        # way above this one are possibly visible in this tile). Render them
+        # anyways just in case). "chunks" should include up to rowstart-16
+
+        imgpath = path + "." + self.imgformat
+        world = self.world
+        #stat the file, we need to know if it exists or it's mtime
+        try:    
+            tile_mtime =  os.stat(imgpath)[stat.ST_MTIME];
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+            tile_mtime = None
+            
+        if not chunks:
+            # No chunks were found in this tile
+            if tile_mtime is not None:
+                os.unlink(imgpath)
+            return None
+
+        # Create the directory if not exists
+        dirdest = os.path.dirname(path)
+        if not os.path.exists(dirdest):
+            try:
+                os.makedirs(dirdest)
+            except OSError, e:
+                # Ignore errno EEXIST: file exists. Since this is multithreaded,
+                # two processes could conceivably try and create the same directory
+                # at the same time.            
+                if e.errno != errno.EEXIST:
+                    raise
+        
+        # check chunk mtimes to see if they are newer
+        try:
+            needs_rerender = False
+            get_region_mtime = world.get_region_mtime
+            for col, row, chunkx, chunky, regionfile in chunks:
+                # check region file mtime first. 
+                region,regionMtime = get_region_mtime(regionfile)  
+                if regionMtime <= tile_mtime:
+                    continue
+                
+                # checking chunk mtime
+                if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
+                    needs_rerender = True
+                    break
+            
+            # if after all that, we don't need a rerender, return
+            if not needs_rerender:
+                return None
+        except OSError:
+            # couldn't get tile mtime, skip check
+            pass
+        
+        #logging.debug("writing out worldtile {0}".format(imgpath))
+
+        # Compile this image
+        tileimg = Image.new("RGBA", (width, height), (38,92,255,0))
+
+        world = self.world
+        rendermode = self.rendermode
+        # col colstart will get drawn on the image starting at x coordinates -(384/2)
+        # row rowstart will get drawn on the image starting at y coordinates -(192/2)
+        for col, row, chunkx, chunky, regionfile in chunks:
+            xpos = -192 + (col-colstart)*192
+            ypos = -96 + (row-rowstart)*96
+
+            # draw the chunk!
+            a = chunk.ChunkRenderer((chunkx, chunky), world, rendermode, poi_queue)
+            a.chunk_render(tileimg, xpos, ypos, None)            
+#            chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None)
+
+
+        # Save them
+        tileimg.save(imgpath)
+
+        if self.optimizeimg:
+            optimize_image(imgpath, self.imgformat, self.optimizeimg)

overviewer/rendernode.py

@@ -0,0 +1,367 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import multiprocessing
+import Queue
+import itertools
+from itertools import cycle, islice
+import os
+import os.path
+import functools
+import re
+import shutil
+import collections
+import json
+import logging
+import util
+import cPickle
+import stat
+import errno 
+import time
+from time import gmtime, strftime, sleep
+
+
+"""
+This module has routines related to distributing the render job to multipule nodes
+
+"""
+
+def catch_keyboardinterrupt(func):
+    """Decorator that catches a keyboardinterrupt and raises a real exception
+    so that multiprocessing will propagate it properly"""
+    @functools.wraps(func)
+    def newfunc(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except KeyboardInterrupt:
+            logging.error("Ctrl-C caught!")
+            raise Exception("Exiting")
+        except:
+            import traceback
+            traceback.print_exc()
+            raise
+    return newfunc
+    
+child_rendernode = None
+def pool_initializer(rendernode):
+    logging.debug("Child process {0}".format(os.getpid()))
+    #stash the quadtree objects in a global variable after fork() for windows compat.
+    global child_rendernode
+    child_rendernode = rendernode  
+    for quadtree in rendernode.quadtrees:
+        if quadtree.world.useBiomeData:
+            import textures
+            # make sure we've at least *tried* to load the color arrays in this process...
+            textures.prepareBiomeData(quadtree.world.worlddir)
+            if not textures.grasscolor or not textures.foliagecolor:
+                raise Exception("Can't find grasscolor.png or foliagecolor.png")
+                    
+#http://docs.python.org/library/itertools.html    
+def roundrobin(iterables):
+    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+    # Recipe credited to George Sakkis
+    pending = len(iterables)
+    nexts = cycle(iter(it).next for it in iterables)
+    while pending:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            pending -= 1
+            nexts = cycle(islice(nexts, pending))
+
+            
+class RenderNode(object):
+    def __init__(self, quadtrees):
+        """Distributes the rendering of a list of quadtrees."""
+
+        if not len(quadtrees) > 0:
+            raise ValueError("there must be at least one quadtree to work on")    
+
+        self.quadtrees = quadtrees
+        #bind an index value to the quadtree so we can find it again
+        #and figure out which worlds are where
+        i = 0
+        self.worlds = []
+        for q in quadtrees:
+            q._render_index = i
+            i += 1   
+            if q.world not in self.worlds:
+                self.worlds.append(q.world)            
+
+        manager = multiprocessing.Manager() 
+        # queue for receiving interesting events from the renderer
+        # (like the discovery of signs!
+        #stash into the world object like we stash an index into the quadtree
+        for world in self.worlds:
+            world.poi_q = manager.Queue() 
+
+
+    def print_statusline(self, complete, total, level, unconditional=False):
+        if unconditional:
+            pass
+        elif complete < 100:
+            if not complete % 25 == 0:
+                return
+        elif complete < 1000:
+            if not complete % 100 == 0:
+                return
+        else:
+            if not complete % 1000 == 0:
+                return
+        logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
+                complete, total, level, self.max_p))
+                
+    def go(self, procs):
+        """Renders all tiles"""
+        
+        logging.debug("Parent process {0}".format(os.getpid()))
+        # Create a pool
+        if procs == 1:
+            pool = FakePool()
+            pool_initializer(self)
+        else:
+            pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
+            #warm up the pool so it reports all the worker id's
+            if logging.getLogger().level >= 10:
+                pool.map(bool,xrange(multiprocessing.cpu_count()),1)
+            else:
+                pool.map_async(bool,xrange(multiprocessing.cpu_count()),1)
+                
+        quadtrees = self.quadtrees
+        
+        # do per-quadtree init.
+        max_p = 0        
+        total = 0
+        for q in quadtrees:
+            total += 4**q.p
+            if q.p > max_p:
+                max_p = q.p
+            q.go(procs) 
+        self.max_p = max_p
+        # Render the highest level of tiles from the chunks
+        results = collections.deque()
+        complete = 0
+        logging.info("Rendering highest zoom level of tiles now.")
+        logging.info("Rendering {0} layer{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' ))
+        logging.info("There are {0} tiles to render".format(total))        
+        logging.info("There are {0} total levels to render".format(self.max_p))
+        logging.info("Don't worry, each level has only 25% as many tiles as the last.")
+        logging.info("The others will go faster")
+        count = 0
+        batch_size = 4*len(quadtrees)
+        while batch_size < 10:
+            batch_size *= 2
+        timestamp = time.time()
+        for result in self._apply_render_worldtiles(pool,batch_size):
+            results.append(result)               
+            # every second drain some of the queue
+            timestamp2 = time.time()
+            if timestamp2 >= timestamp + 1:
+                timestamp = timestamp2                
+                count_to_remove = (1000//batch_size)
+                if count_to_remove < len(results):
+                    for world in self.worlds:
+                        try:
+                            while (1):
+                                # an exception will break us out of this loop
+                                item = world.poi_q.get(block=False)
+                                if item[0] == "newpoi":
+                                    if item[1] not in world.POI:
+                                        #print "got an item from the queue!"
+                                        world.POI.append(item[1])
+                                elif item[0] == "removePOI":
+                                    world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI'])
+                        except Queue.Empty:
+                            pass
+                    while count_to_remove > 0:
+                        count_to_remove -= 1
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, 1)  
+            if len(results) > (10000//batch_size):
+                # Empty the queue before adding any more, so that memory
+                # required has an upper bound
+                while len(results) > (500//batch_size):
+                    complete += results.popleft().get()
+                    self.print_statusline(complete, total, 1)
+
+        # Wait for the rest of the results
+        while len(results) > 0:
+            complete += results.popleft().get()
+            self.print_statusline(complete, total, 1)
+        for world in self.worlds:    
+            try:
+                while (1):
+                    # an exception will break us out of this loop
+                    item = world.poi_q.get(block=False)
+                    if item[0] == "newpoi":
+                        if item[1] not in world.POI:
+                            #print "got an item from the queue!"
+                            world.POI.append(item[1])
+                    elif item[0] == "removePOI":
+                        world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI'])
+            except Queue.Empty:
+                pass
+
+        self.print_statusline(complete, total, 1, True)
+
+        # Now do the other layers
+        for zoom in xrange(self.max_p-1, 0, -1):
+            level = self.max_p - zoom + 1
+            assert len(results) == 0
+            complete = 0
+            total = 0
+            for q in quadtrees:
+                if zoom <= q.p:
+                    total += 4**zoom
+            logging.info("Starting level {0}".format(level))
+            timestamp = time.time()
+            for result in self._apply_render_inntertile(pool, zoom,batch_size):
+                results.append(result)
+                # every second drain some of the queue
+                timestamp2 = time.time()
+                if timestamp2 >= timestamp + 1:
+                    timestamp = timestamp2                
+                    count_to_remove = (1000//batch_size)
+                    if count_to_remove < len(results):
+                        while count_to_remove > 0:
+                            count_to_remove -= 1
+                            complete += results.popleft().get()
+                            self.print_statusline(complete, total, level)
+                if len(results) > (10000/batch_size):
+                    while len(results) > (500/batch_size):
+                        complete += results.popleft().get()
+                        self.print_statusline(complete, total, level)
+            # Empty the queue
+            while len(results) > 0:
+                complete += results.popleft().get()
+                self.print_statusline(complete, total, level)
+
+            self.print_statusline(complete, total, level, True)
+
+            logging.info("Done")
+
+        pool.close()
+        pool.join()
+
+        # Do the final one right here:
+        for q in quadtrees:
+            q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
+        
+    def _apply_render_worldtiles(self, pool,batch_size):
+        """Returns an iterator over result objects. Each time a new result is
+        requested, a new task is added to the pool and a result returned.
+        """
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)         
+        batch = []
+        jobcount = 0       
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_worldtiles() for q in self.quadtrees]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is                 
+            job[0] = job[0]._render_index      
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0        
+                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
+                batch = []              
+        if jobcount > 0:
+            yield pool.apply_async(func=render_worldtile_batch, args= [batch])
+
+    def _apply_render_inntertile(self, pool, zoom,batch_size):
+        """Same as _apply_render_worltiles but for the inntertile routine.
+        Returns an iterator that yields result objects from tasks that have
+        been applied to the pool.
+        """
+        
+        if batch_size < len(self.quadtrees):
+            batch_size = len(self.quadtrees)
+        batch = []
+        jobcount = 0
+        # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
+        iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p]
+        for job in roundrobin(iterables):
+            # fixup so the worker knows which quadtree this is  
+            job[0] = job[0]._render_index
+            # Put this in the batch to be submited to the pool  
+            batch.append(job)
+            jobcount += 1
+            if jobcount >= batch_size:
+                jobcount = 0
+                yield pool.apply_async(func=render_innertile_batch, args= [batch])
+                batch = []
+                
+        if jobcount > 0:
+            yield pool.apply_async(func=render_innertile_batch, args= [batch])    
+            
+@catch_keyboardinterrupt
+def render_worldtile_batch(batch):
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))        
+    for job in batch:
+        count += 1    
+        quadtree = rendernode.quadtrees[job[0]]
+        colstart = job[1]
+        colend = job[2]
+        rowstart = job[3]
+        rowend = job[4]
+        path = job[5]
+        poi_queue = quadtree.world.poi_q
+        path = quadtree.full_tiledir+os.sep+path        
+        # (even if tilechunks is empty, render_worldtile will delete
+        # existing images if appropriate)    
+        # And uses these chunks
+        tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend)
+        #logging.debug(" tilechunks: %r", tilechunks)
+        
+        quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path, poi_queue)
+    return count
+
+@catch_keyboardinterrupt
+def render_innertile_batch(batch):    
+    global child_rendernode
+    rendernode = child_rendernode
+    count = 0   
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1        
+        quadtree = rendernode.quadtrees[job[0]]               
+        dest = quadtree.full_tiledir+os.sep+job[1]
+        quadtree.render_innertile(dest=dest,name=job[2])
+    return count
+    
+class FakeResult(object):
+    def __init__(self, res):
+        self.res = res
+    def get(self):
+        return self.res
+class FakePool(object):
+    """A fake pool used to render things in sync. Implements a subset of
+    multiprocessing.Pool"""
+    def apply_async(self, func, args=(), kwargs=None):
+        if not kwargs:
+            kwargs = {}
+        result = func(*args, **kwargs)
+        return FakeResult(result)
+    def close(self):
+        pass
+    def join(self):
+        pass
+    

overviewer/src/composite.c

@@ -0,0 +1,356 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* 
+ * This file implements a custom alpha_over function for (some) PIL
+ * images. It's designed to be used through composite.py, which
+ * includes a proxy alpha_over function that falls back to the default
+ * PIL paste if this extension is not found.
+ */
+
+#include "overviewer.h"
+
+/* like (a * b + 127) / 255), but much faster on most platforms
+   from PIL's _imaging.c */
+#define MULDIV255(a, b, tmp)								\
+	(tmp = (a) * (b) + 128, ((((tmp) >> 8) + (tmp)) >> 8))
+
+typedef struct {
+    PyObject_HEAD
+    Imaging image;
+} ImagingObject;
+
+inline Imaging
+imaging_python_to_c(PyObject *obj)
+{
+    PyObject *im;
+    Imaging image;
+
+    /* first, get the 'im' attribute */
+    im = PyObject_GetAttrString(obj, "im");
+    if (!im)
+        return NULL;
+
+    /* make sure 'im' is the right type */
+    if (strcmp(im->ob_type->tp_name, "ImagingCore") != 0) {
+        /* it's not -- raise an error and exit */
+        PyErr_SetString(PyExc_TypeError,
+                        "image attribute 'im' is not a core Imaging type");
+        return NULL;
+    }
+
+    image = ((ImagingObject *)im)->image;
+    Py_DECREF(im);
+    return image;
+}
+
+/* helper function to setup s{x,y}, d{x,y}, and {x,y}size variables
+   in these composite functions -- even handles auto-sizing to src! */
+static inline void
+setup_source_destination(Imaging src, Imaging dest,
+                         int *sx, int *sy, int *dx, int *dy, int *xsize, int *ysize)
+{
+   /* handle negative/zero sizes appropriately */
+    if (*xsize <= 0 || *ysize <= 0) {
+        *xsize = src->xsize;
+        *ysize = src->ysize;
+    }
+    
+    /* set up the source position, size and destination position */
+    /* handle negative dest pos */
+    if (*dx < 0) {
+        *sx = -(*dx);
+        *dx = 0;
+    } else {
+        *sx = 0;
+    }
+
+    if (*dy < 0) {
+        *sy = -(*dy);
+        *dy = 0;
+    } else {
+        *sy = 0;
+    }
+
+    /* set up source dimensions */
+    *xsize -= *sx;
+    *ysize -= *sy;
+
+    /* clip dimensions, if needed */
+    if (*dx + *xsize > dest->xsize)
+        *xsize = dest->xsize - *dx;
+    if (*dy + *ysize > dest->ysize)
+        *ysize = dest->ysize - *dy;
+}
+
+/* convenience alpha_over with 1.0 as overall_alpha */
+inline PyObject* alpha_over(PyObject *dest, PyObject *src, PyObject *mask,
+                            int dx, int dy, int xsize, int ysize) {
+    return alpha_over_full(dest, src, mask, 1.0f, dx, dy, xsize, ysize);
+}
+
+/* the full alpha_over function, in a form that can be called from C
+ * overall_alpha is multiplied with the whole mask, useful for lighting...
+ * if xsize, ysize are negative, they are instead set to the size of the image in src
+ * returns NULL on error, dest on success. You do NOT need to decref the return!
+ */
+inline PyObject *
+alpha_over_full(PyObject *dest, PyObject *src, PyObject *mask, float overall_alpha,
+                int dx, int dy, int xsize, int ysize) {
+    /* libImaging handles */
+    Imaging imDest, imSrc, imMask;
+    /* cached blend properties */
+    int src_has_alpha, mask_offset, mask_stride;
+    /* source position */
+    int sx, sy;
+    /* iteration variables */
+    unsigned int x, y, i;
+    /* temporary calculation variables */
+    int tmp1, tmp2, tmp3;
+    /* integer [0, 255] version of overall_alpha */
+    UINT8 overall_alpha_int = 255 * overall_alpha;
+    
+    /* short-circuit this whole thing if overall_alpha is zero */
+    if (overall_alpha_int == 0)
+        return dest;
+
+    imDest = imaging_python_to_c(dest);
+    imSrc = imaging_python_to_c(src);
+    imMask = imaging_python_to_c(mask);
+
+    if (!imDest || !imSrc || !imMask)
+        return NULL;
+
+    /* check the various image modes, make sure they make sense */
+    if (strcmp(imDest->mode, "RGBA") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "given destination image does not have mode \"RGBA\"");
+        return NULL;
+    }
+
+    if (strcmp(imSrc->mode, "RGBA") != 0 && strcmp(imSrc->mode, "RGB") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "given source image does not have mode \"RGBA\" or \"RGB\"");
+        return NULL;
+    }
+
+    if (strcmp(imMask->mode, "RGBA") != 0 && strcmp(imMask->mode, "L") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "given mask image does not have mode \"RGBA\" or \"L\"");
+        return NULL;
+    }
+
+    /* make sure mask size matches src size */
+    if (imSrc->xsize != imMask->xsize || imSrc->ysize != imMask->ysize) {
+        PyErr_SetString(PyExc_ValueError,
+                        "mask and source image sizes do not match");
+        return NULL;
+    }
+
+    /* set up flags for the src/mask type */
+    src_has_alpha = (imSrc->pixelsize == 4 ? 1 : 0);
+    /* how far into image the first alpha byte resides */
+    mask_offset = (imMask->pixelsize == 4 ? 3 : 0);
+    /* how many bytes to skip to get to the next alpha byte */
+    mask_stride = imMask->pixelsize;
+
+    /* setup source & destination vars */
+    setup_source_destination(imSrc, imDest, &sx, &sy, &dx, &dy, &xsize, &ysize);
+
+    /* check that there remains any blending to be done */
+    if (xsize <= 0 || ysize <= 0) {
+        /* nothing to do, return */
+        return dest;
+    }
+
+    for (y = 0; y < ysize; y++) {
+        UINT8 *out = (UINT8 *)imDest->image[dy + y] + dx * 4;
+        UINT8 *outmask = (UINT8 *)imDest->image[dy + y] + dx * 4 + 3;
+        UINT8 *in = (UINT8 *)imSrc->image[sy + y] + sx * (imSrc->pixelsize);
+        UINT8 *inmask = (UINT8 *)imMask->image[sy + y] + sx * mask_stride + mask_offset;
+
+        for (x = 0; x < xsize; x++) {
+            UINT8 in_alpha;
+            
+            /* apply overall_alpha */
+            if (overall_alpha_int != 255 && *inmask != 0) {
+                in_alpha = MULDIV255(*inmask, overall_alpha_int, tmp1);
+            } else {
+                in_alpha = *inmask;
+            }
+            
+            /* special cases */
+            if (in_alpha == 255 || *outmask == 0) {
+                *outmask = in_alpha;
+
+                *out = *in;
+                out++, in++;
+                *out = *in;
+                out++, in++;
+                *out = *in;
+                out++, in++;
+            } else if (in_alpha == 0) {
+                /* do nothing -- source is fully transparent */
+                out += 3;
+                in += 3;
+            } else {
+                /* general case */
+                int alpha = in_alpha + MULDIV255(*outmask, 255 - in_alpha, tmp1);
+                for (i = 0; i < 3; i++) {
+                    /* general case */
+                    *out = MULDIV255(*in, in_alpha, tmp1) +
+                        MULDIV255(MULDIV255(*out, *outmask, tmp2), 255 - in_alpha, tmp3);
+                    
+                    *out = (*out * 255) / alpha;
+                    out++, in++;
+                }
+
+                *outmask = alpha;
+            }
+
+            out++;
+            if (src_has_alpha)
+                in++;
+            outmask += 4;
+            inmask += mask_stride;
+        }
+    }
+
+    return dest;
+}
+
+/* wraps alpha_over so it can be called directly from python */
+/* properly refs the return value when needed: you DO need to decref the return */
+PyObject *
+alpha_over_wrap(PyObject *self, PyObject *args)
+{
+    /* raw input python variables */
+    PyObject *dest, *src, *pos, *mask;
+    /* destination position and size */
+    int dx, dy, xsize, ysize;
+    /* return value: dest image on success */
+    PyObject *ret;
+
+    if (!PyArg_ParseTuple(args, "OOOO", &dest, &src, &pos, &mask))
+        return NULL;
+
+    /* destination position read */
+    if (!PyArg_ParseTuple(pos, "iiii", &dx, &dy, &xsize, &ysize)) {
+        /* try again, but this time try to read a point */
+        PyErr_Clear();
+        xsize = 0;
+        ysize = 0;
+        if (!PyArg_ParseTuple(pos, "ii", &dx, &dy)) {
+            PyErr_SetString(PyExc_TypeError,
+                            "given blend destination rect is not valid");
+            return NULL;
+        }
+    }
+
+    ret = alpha_over(dest, src, mask, dx, dy, xsize, ysize);
+    if (ret == dest) {
+        /* Python needs us to own our return value */
+        Py_INCREF(dest);
+    }
+    return ret;
+}
+
+/* like alpha_over, but instead of src image it takes a source color
+ * also, it multiplies instead of doing an over operation
+ */
+PyObject *
+tint_with_mask(PyObject *dest, unsigned char sr, unsigned char sg, unsigned char sb,
+               PyObject *mask, int dx, int dy, int xsize, int ysize) {
+    /* libImaging handles */
+    Imaging imDest, imMask;
+    /* cached blend properties */
+    int mask_offset, mask_stride;
+    /* source position */
+    int sx, sy;
+    /* iteration variables */
+    unsigned int x, y;
+    /* temporary calculation variables */
+    int tmp1, tmp2;
+
+    imDest = imaging_python_to_c(dest);
+    imMask = imaging_python_to_c(mask);
+
+    if (!imDest || !imMask)
+        return NULL;
+
+    /* check the various image modes, make sure they make sense */
+    if (strcmp(imDest->mode, "RGBA") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "given destination image does not have mode \"RGBA\"");
+        return NULL;
+    }
+
+    if (strcmp(imMask->mode, "RGBA") != 0 && strcmp(imMask->mode, "L") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "given mask image does not have mode \"RGBA\" or \"L\"");
+        return NULL;
+    }
+
+    /* how far into image the first alpha byte resides */
+    mask_offset = (imMask->pixelsize == 4 ? 3 : 0);
+    /* how many bytes to skip to get to the next alpha byte */
+    mask_stride = imMask->pixelsize;
+
+    /* setup source & destination vars */
+    setup_source_destination(imMask, imDest, &sx, &sy, &dx, &dy, &xsize, &ysize);
+
+    /* check that there remains any blending to be done */
+    if (xsize <= 0 || ysize <= 0) {
+        /* nothing to do, return */
+        return dest;
+    }
+
+    for (y = 0; y < ysize; y++) {
+        UINT8 *out = (UINT8 *)imDest->image[dy + y] + dx * 4;
+        UINT8 *inmask = (UINT8 *)imMask->image[sy + y] + sx * mask_stride + mask_offset;
+
+        for (x = 0; x < xsize; x++) {
+            /* special cases */
+            if (*inmask == 255) {
+                *out = MULDIV255(*out, sr, tmp1);
+                out++;
+                *out = MULDIV255(*out, sg, tmp1);
+                out++;
+                *out = MULDIV255(*out, sb, tmp1);
+                out++;
+            } else if (*inmask == 0) {
+                /* do nothing -- source is fully transparent */
+                out += 3;
+            } else {
+                /* general case */
+                
+                /* TODO work out general case */
+                *out = MULDIV255(*out, (255 - *inmask) + MULDIV255(sr, *inmask, tmp1), tmp2);
+                out++;
+                *out = MULDIV255(*out, (255 - *inmask) + MULDIV255(sg, *inmask, tmp1), tmp2);
+                out++;
+                *out = MULDIV255(*out, (255 - *inmask) + MULDIV255(sb, *inmask, tmp1), tmp2);
+                out++;
+            }
+
+            out++;
+            inmask += mask_stride;
+        }
+    }
+
+    return dest;
+}

overviewer/src/endian.c

@@ -0,0 +1,39 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* simple routines for dealing with endian conversion */
+
+#define UNKNOWN_ENDIAN 0
+#define BIG_ENDIAN 1
+#define LITTLE_ENDIAN 2
+
+static int endianness = UNKNOWN_ENDIAN;
+
+void init_endian(void) {
+    /* figure out what our endianness is! */
+    short word = 0x0001;
+    char* byte = (char*)(&word);
+    endianness = byte[0] ? LITTLE_ENDIAN : BIG_ENDIAN;
+}
+
+unsigned short big_endian_ushort(unsigned short in) {
+    return (endianness == LITTLE_ENDIAN) ? ((in >> 8) | (in << 8)) : in;
+}
+
+unsigned int big_endian_uint(unsigned int in) {
+    return (endianness == LITTLE_ENDIAN) ? (((in & 0x000000FF) << 24) + ((in & 0x0000FF00) << 8) + ((in & 0x00FF0000) >> 8) + ((in & 0xFF000000) >> 24)) : in;
+}

overviewer/src/iterate.c

@@ -0,0 +1,379 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+
+static PyObject *textures = NULL;
+static PyObject *chunk_mod = NULL;
+static PyObject *blockmap = NULL;
+static PyObject *special_blocks = NULL;
+static PyObject *specialblockmap = NULL;
+static PyObject *transparent_blocks = NULL;
+
+int init_chunk_render(void) {
+   
+    /* if blockmap (or any of these) is not NULL, then that means that we've 
+     * somehow called this function twice.  error out so we can notice this
+     * */
+    if (blockmap) return 1;
+
+    textures = PyImport_ImportModule("overviewer.textures");
+    /* ensure none of these pointers are NULL */    
+    if ((!textures)) {
+        fprintf(stderr, "\ninit_chunk_render failed to load; textures\n");
+        PyErr_Print();
+        return 1;
+    }
+
+    chunk_mod = PyImport_ImportModule("overviewer.chunk");
+    /* ensure none of these pointers are NULL */    
+    if ((!chunk_mod)) {
+        fprintf(stderr, "\ninit_chunk_render failed to load; chunk\n");
+        PyErr_Print();
+        return 1;
+    }
+    
+    blockmap = PyObject_GetAttrString(textures, "blockmap");
+    special_blocks = PyObject_GetAttrString(textures, "special_blocks");
+    specialblockmap = PyObject_GetAttrString(textures, "specialblockmap");
+    transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks");
+    
+    /* ensure none of these pointers are NULL */    
+    if ((!transparent_blocks) || (!blockmap) || (!special_blocks) || (!specialblockmap)) {
+        fprintf(stderr, "\ninit_chunk_render failed\n");
+        PyErr_Print();
+        return 1;
+    }
+
+    return 0;
+
+}
+
+int
+is_transparent(unsigned char b) {
+    PyObject *block = PyInt_FromLong(b);
+    int ret = PySequence_Contains(transparent_blocks, block);
+    Py_DECREF(block);
+    return ret;
+
+}
+
+
+unsigned char
+    check_adjacent_blocks(RenderState *state, int x,int y,int z, unsigned char blockid) {
+        /*
+         * Generates a pseudo ancillary data for blocks that depend of 
+         * what are surrounded and don't have ancillary data. This 
+         * function is through generate_pseudo_data.
+         *
+         * This uses a binary number of 4 digits to encode the info. 
+         * The encode is:
+         *
+         * 0b1234:
+         * Bit:   1   2   3   4
+         * Side: +x  +y  -x  -y
+         * Values: bit = 0 -> The corresponding side block has different blockid
+         *         bit = 1 -> The corresponding side block has same blockid
+         * Example: if the bit1 is 1 that means that there is a block with 
+         * blockid in the side of the +x direction.
+         */
+        
+        unsigned char pdata=0;
+        
+        if (state->x == 15) { /* +x direction */
+            if (state->up_right_blocks != Py_None) { /* just in case we are in the end of the world */
+                if (getArrayByte3D(state->up_right_blocks, 0, y, z) == blockid) {
+                    pdata = pdata|(1 << 3);
+                }
+            }
+        } else {
+            if (getArrayByte3D(state->blocks, x + 1, y, z) == blockid) {
+                pdata = pdata|(1 << 3);
+            }
+        }
+        
+        if (state->y == 15) { /* +y direction*/
+            if (state->right_blocks != Py_None) {
+                if (getArrayByte3D(state->right_blocks, x, 0, z) == blockid) {
+                    pdata = pdata|(1 << 2);
+                }
+            }
+        } else {
+            if (getArrayByte3D(state->blocks, x, y + 1, z) == blockid) {
+                pdata = pdata|(1 << 2);
+            }
+        }
+        
+        if (state->x == 0) { /* -x direction*/
+            if (state->left_blocks != Py_None) {
+                if (getArrayByte3D(state->left_blocks, 15, y, z) == blockid) {
+                    pdata = pdata|(1 << 1);
+                }
+            }
+        } else {
+            if (getArrayByte3D(state->blocks, x - 1, y, z) == blockid) {
+                pdata = pdata|(1 << 1);
+            }
+        }
+        
+        if (state->y == 0) { /* -y direction */
+            if (state->up_left_blocks != Py_None) {
+                if (getArrayByte3D(state->up_left_blocks, x, 15, z) == blockid) {
+                    pdata = pdata|(1 << 0);
+                }
+            }
+        } else {
+            if (getArrayByte3D(state->blocks, x, y - 1, z) == blockid) {
+                pdata = pdata|(1 << 0);
+            }
+        }
+
+        return pdata;
+}
+
+
+unsigned char
+generate_pseudo_data(RenderState *state, unsigned char ancilData) {
+    /*
+     * Generates a fake ancillary data for blocks that are drawn 
+     * depending on what are surrounded.
+     */
+    int x = state->x, y = state->y, z = state->z;
+    unsigned char data = 0;
+    
+    if (state->block == 9) { /* water */
+        /* an aditional bit for top is added to the 4 bits of check_adjacent_blocks */
+        if ((ancilData == 0) || (ancilData >= 10)) { /* static water, only top, and unkown ancildata values */
+            data = 16;
+            return data; /* = 0b10000 */
+        } else if ((ancilData > 0) && (ancilData < 8)) { /* flowing water */
+            data = (check_adjacent_blocks(state, x, y, z, state->block) ^ 0x0f) | 0x10;
+            return data;
+        } else if ((ancilData == 8) || (ancilData == 9)) { /* falling water */
+            data = (check_adjacent_blocks(state, x, y, z, state->block) ^ 0x0f);
+            return data;
+        }
+
+
+    } else if (state->block == 85) { /* fences */
+        return check_adjacent_blocks(state, x, y, z, state->block);
+
+
+    } else if (state->block == 55) { /* redstone */
+        /* three addiotional bit are added, one for on/off state, and
+         * another two for going-up redstone wire in the same block
+         * (connection with the level z+1) */
+        unsigned char above_level_data = 0, same_level_data = 0, below_level_data = 0, possibly_connected = 0, final_data = 0;
+
+        /* check for air in z+1, no air = no connection with upper level */        
+        if ((z != 127) && (getArrayByte3D(state->left_blocks, x, y, z) == 0)) { 
+            above_level_data = check_adjacent_blocks(state, x, y, z + 1, state->block);
+        }   /* else above_level_data = 0 */
+        
+        /* check connection with same level */
+        same_level_data = check_adjacent_blocks(state, x, y, z, 55);
+        
+        /* check the posibility of connection with z-1 level, check for air */
+        possibly_connected = check_adjacent_blocks(state, x, y, z, 0);
+        
+        /* check connection with z-1 level */
+        if (z != 0) {
+            below_level_data = check_adjacent_blocks(state, x, y, z - 1, state->block);
+        } /* else below_level_data = 0 */
+        
+        final_data = above_level_data | same_level_data | (below_level_data & possibly_connected);
+        
+        /* add the three bits */
+        if (ancilData > 0) { /* powered redstone wire */
+            final_data = final_data | 0x40;
+        }
+        if ((above_level_data & 0x01)) { /* draw top left going up redstonewire */
+            final_data = final_data | 0x20;
+        }
+        if ((above_level_data & 0x08)) { /* draw top right going up redstonewire */
+            final_data = final_data | 0x10;
+        }
+        return final_data;        
+    }
+
+    return 0;
+
+}
+
+
+/* TODO triple check this to make sure reference counting is correct */
+PyObject*
+chunk_render(PyObject *self, PyObject *args) {
+    RenderState state;
+
+    PyObject *blockdata_expanded; 
+    int xoff, yoff;
+    
+    PyObject *imgsize, *imgsize0_py, *imgsize1_py;
+    int imgsize0, imgsize1;
+    
+    PyObject *blocks_py;
+    PyObject *left_blocks_py;
+    PyObject *right_blocks_py;
+    PyObject *up_left_blocks_py;
+    PyObject *up_right_blocks_py;
+
+    RenderModeInterface *rendermode;
+
+    void *rm_data;
+                
+    PyObject *t = NULL;
+    
+    if (!PyArg_ParseTuple(args, "OOiiO",  &state.self, &state.img, &xoff, &yoff, &blockdata_expanded))
+        return Py_BuildValue("i", "-1");
+    
+    /* fill in important modules */
+    state.textures = textures;
+    state.chunk = chunk_mod;
+    
+    /* set up the render mode */
+    rendermode = get_render_mode(&state);
+    rm_data = calloc(1, rendermode->data_size);
+    if (rendermode->start(rm_data, &state)) {
+        free(rm_data);
+        return Py_BuildValue("i", "-1");
+    }
+
+    /* get the image size */
+    imgsize = PyObject_GetAttrString(state.img, "size");
+
+    imgsize0_py = PySequence_GetItem(imgsize, 0);
+    imgsize1_py = PySequence_GetItem(imgsize, 1);
+    Py_DECREF(imgsize);
+
+    imgsize0 = PyInt_AsLong(imgsize0_py);
+    imgsize1 = PyInt_AsLong(imgsize1_py);
+    Py_DECREF(imgsize0_py);
+    Py_DECREF(imgsize1_py);
+
+
+    /* get the block data directly from numpy: */
+    blocks_py = PyObject_GetAttrString(state.self, "blocks");
+    state.blocks = blocks_py;
+
+    left_blocks_py = PyObject_GetAttrString(state.self, "left_blocks");
+    state.left_blocks = left_blocks_py;
+
+    right_blocks_py = PyObject_GetAttrString(state.self, "right_blocks");
+    state.right_blocks = right_blocks_py;
+
+    up_left_blocks_py = PyObject_GetAttrString(state.self, "up_left_blocks");
+    state.up_left_blocks = up_left_blocks_py;
+
+    up_right_blocks_py = PyObject_GetAttrString(state.self, "up_right_blocks");
+    state.up_right_blocks = up_right_blocks_py;
+    
+    for (state.x = 15; state.x > -1; state.x--) {
+        for (state.y = 0; state.y < 16; state.y++) {
+            PyObject *blockid = NULL;
+            
+            /* set up the render coordinates */
+            state.imgx = xoff + state.x*12 + state.y*12;
+            /* 128*12 -- offset for z direction, 15*6 -- offset for x */
+            state.imgy = yoff - state.x*6 + state.y*6 + 128*12 + 15*6;
+            
+            for (state.z = 0; state.z < 128; state.z++) {
+                state.imgy -= 12;
+                
+                /* make sure we're rendering inside the image boundaries */
+                if ((state.imgx >= imgsize0 + 24) || (state.imgx <= -24)) {
+                    continue;
+                }
+                if ((state.imgy >= imgsize1 + 24) || (state.imgy <= -24)) {
+                    continue;
+                }
+
+                /* get blockid */
+                state.block = getArrayByte3D(blocks_py, state.x, state.y, state.z);
+                if (state.block == 0) {
+                    continue;
+                }
+                
+                /* decref'd on replacement *and* at the end of the z for block */
+                if (blockid) {
+                    Py_DECREF(blockid);
+                }
+                blockid = PyInt_FromLong(state.block);
+
+                // check for occlusion
+                if (rendermode->occluded(rm_data, &state)) {
+                    continue;
+                }
+                
+                // everything stored here will be a borrowed ref
+
+                /* get the texture and mask from block type / ancil. data */
+                if (!PySequence_Contains(special_blocks, blockid)) {
+                    /* t = textures.blockmap[blockid] */
+                    t = PyList_GetItem(blockmap, state.block);
+                } else {
+                    PyObject *tmp;
+                    
+                    unsigned char ancilData = getArrayByte3D(blockdata_expanded, state.x, state.y, state.z);
+                    if ((state.block == 85) || (state.block == 9) || (state.block == 55)) {
+                        ancilData = generate_pseudo_data(&state, ancilData);
+                    }
+                    
+                    tmp = PyTuple_New(2);
+
+                    Py_INCREF(blockid); /* because SetItem steals */
+                    PyTuple_SetItem(tmp, 0, blockid);
+                    PyTuple_SetItem(tmp, 1, PyInt_FromLong(ancilData));
+                    
+                    /* this is a borrowed reference. no need to decref */
+                    t = PyDict_GetItem(specialblockmap, tmp);
+                    Py_DECREF(tmp);
+                }
+                
+                /* if we found a proper texture, render it! */
+                if (t != NULL && t != Py_None)
+                {
+                    PyObject *src, *mask;
+                    src = PyTuple_GetItem(t, 0);
+                    mask = PyTuple_GetItem(t, 1);
+                    
+                    if (mask == Py_None)
+                        mask = src;
+                    
+                    rendermode->draw(rm_data, &state, src, mask);
+                }               
+            }
+            
+            if (blockid) {
+                Py_DECREF(blockid);
+                blockid = NULL;
+            }
+        }
+    } 
+
+    /* free up the rendermode info */
+    rendermode->finish(rm_data, &state);
+    free(rm_data);
+    
+    Py_DECREF(blocks_py);
+    Py_XDECREF(left_blocks_py);
+    Py_XDECREF(right_blocks_py);
+    Py_XDECREF(up_left_blocks_py);
+    Py_XDECREF(up_right_blocks_py);
+
+    return Py_BuildValue("i",2);
+}

overviewer/src/main.c

@@ -0,0 +1,46 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+
+static PyMethodDef COverviewerMethods[] = {
+    {"alpha_over", alpha_over_wrap, METH_VARARGS,
+     "alpha over composite function"},
+    {"render_loop", chunk_render, METH_VARARGS,
+     "Renders stuffs"},
+    {"get_render_modes", get_render_modes, METH_VARARGS,
+     "returns available render modes"},
+    {"get_render_mode_info", get_render_mode_info, METH_VARARGS,
+     "returns info for a particular render mode"},
+    {NULL, NULL, 0, NULL}       /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initc_overviewer(void)
+{
+    (void)Py_InitModule("c_overviewer", COverviewerMethods);
+    /* for numpy */
+    import_array();
+
+    /* initialize some required variables in iterage.c */
+    if (init_chunk_render()) {
+        fprintf(stderr, "failed to init_chunk_render\n");
+        exit(1); // TODO better way to indicate error?
+    }
+    
+    init_endian();
+}

overviewer/src/overviewer.h

@@ -0,0 +1,86 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This is a general include file for the Overviewer C extension. It
+ * lists useful, defined functions as well as those that are exported
+ * to python, so all files can use them.
+ */
+
+#ifndef __OVERVIEWER_H_INCLUDED__
+#define __OVERVIEWER_H_INCLUDED__
+
+/* Python PIL, and numpy headers */
+#include <Python.h>
+#include <Imaging.h>
+#include <numpy/arrayobject.h>
+
+/* macro for getting a value out of various numpy arrays */
+#define getArrayByte3D(array, x,y,z) (*(unsigned char *)(PyArray_GETPTR3((array), (x), (y), (z))))
+#define getArrayShort1D(array, x) (*(unsigned short *)(PyArray_GETPTR1((array), (x))))
+
+/* generally useful MAX / MIN macros */
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+/* in composite.c */
+Imaging imaging_python_to_c(PyObject *obj);
+PyObject *alpha_over(PyObject *dest, PyObject *src, PyObject *mask,
+                     int dx, int dy, int xsize, int ysize);
+PyObject *alpha_over_full(PyObject *dest, PyObject *src, PyObject *mask, float overall_alpha,
+                          int dx, int dy, int xsize, int ysize);
+PyObject *alpha_over_wrap(PyObject *self, PyObject *args);
+PyObject *tint_with_mask(PyObject *dest, unsigned char sr, unsigned char sg, unsigned char sb,
+                         PyObject *mask, int dx, int dy, int xsize, int ysize);
+
+/* in iterate.c */
+typedef struct {
+    /* the ChunkRenderer object */
+    PyObject *self;
+    
+    /* important modules, for convenience */
+    PyObject *textures;
+    PyObject *chunk;
+    
+    /* the rest only make sense for occluded() and draw() !! */
+    
+    /* the tile image and destination */
+    PyObject *img;
+    int imgx, imgy;
+    
+    /* the block position and type, and the block array */
+    int x, y, z;
+    unsigned char block;
+    PyObject *blocks;
+    PyObject *up_left_blocks;
+    PyObject *up_right_blocks;
+    PyObject *left_blocks;
+    PyObject *right_blocks;
+} RenderState;
+int init_chunk_render(void);
+int is_transparent(unsigned char b);
+PyObject *chunk_render(PyObject *self, PyObject *args);
+
+/* pull in the rendermode info */
+#include "rendermodes.h"
+
+/* in endian.c */
+void init_endian(void);
+unsigned short big_endian_ushort(unsigned short in);
+unsigned int big_endian_uint(unsigned int in);
+
+#endif /* __OVERVIEWER_H_INCLUDED__ */

overviewer/src/rendermode-lighting.c

@@ -0,0 +1,237 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+#include <math.h>
+
+/* figures out the black_coeff from a given skylight and blocklight,
+   used in lighting calculations */
+static float calculate_darkness(unsigned char skylight, unsigned char blocklight) {
+    return 1.0f - powf(0.8f, 15.0 - MAX(blocklight, skylight));
+}
+
+/* loads the appropriate light data for the given (possibly non-local)
+ * coordinates, and returns a black_coeff this is exposed, so other (derived)
+ * rendermodes can use it
+ *  
+ * authoratative is a return slot for whether or not this lighting calculation
+ * is true, or a guess. If we guessed, *authoratative will be false, but if it
+ * was calculated correctly from available light data, it will be true. You
+ * may (and probably should) pass NULL.
+ */
+inline float
+get_lighting_coefficient(RenderModeLighting *self, RenderState *state,
+                         int x, int y, int z, int *authoratative) {
+    
+    /* placeholders for later data arrays, coordinates */
+    PyObject *blocks = NULL;
+    PyObject *skylight = NULL;
+    PyObject *blocklight = NULL;
+    int local_x = x, local_y = y, local_z = z;
+    unsigned char block, skylevel, blocklevel;
+    
+    /* defaults to "guess" until told otherwise */
+    if (authoratative)
+        *authoratative = 0;
+    
+    /* find out what chunk we're in, and translate accordingly */
+    if (x >= 0 && y < 16) {
+        blocks = state->blocks;
+        skylight = self->skylight;
+        blocklight = self->blocklight;
+    } else if (x < 0) {
+        local_x += 16;        
+        blocks = state->left_blocks;
+        skylight = self->left_skylight;
+        blocklight = self->left_blocklight;
+    } else if (y >= 16) {
+        local_y -= 16;
+        blocks = state->right_blocks;
+        skylight = self->right_skylight;
+        blocklight = self->right_blocklight;
+    }
+    
+    /* make sure we have correctly-ranged coordinates */
+    if (!(local_x >= 0 && local_x < 16 &&
+          local_y >= 0 && local_y < 16 &&
+          local_z >= 0 && local_z < 128)) {
+        
+        return self->calculate_darkness(15, 0);
+    }
+    
+    /* also, make sure we have enough info to correctly calculate lighting */
+    if (blocks == Py_None || blocks == NULL ||
+        skylight == Py_None || skylight == NULL ||
+        blocklight == Py_None || blocklight == NULL) {
+        
+        return self->calculate_darkness(15, 0);
+    }
+    
+    block = getArrayByte3D(blocks, local_x, local_y, local_z);
+    
+    /* if this block is opaque, use a fully-lit coeff instead
+       to prevent stippled lines along chunk boundaries! */
+    if (!is_transparent(block)) {
+        return self->calculate_darkness(15, 0);
+    }
+    
+    /* only do special half-step handling if no authoratative pointer was
+       passed in, which is a sign that we're recursing */
+    if (block == 44 && authoratative == NULL) {
+        float average_gather = 0.0f;
+        unsigned int average_count = 0;
+        int auth;
+        float coeff;
+        
+        /* iterate through all surrounding blocks to take an average */
+        int dx, dy, dz;
+        for (dx = -1; dx <= 1; dx += 2) {
+            for (dy = -1; dy <= 1; dy += 2) {
+                for (dz = -1; dz <= 1; dz += 2) {
+                    coeff = get_lighting_coefficient(self, state, x+dx, y+dy, z+dz, &auth);
+                    if (auth) {
+                        average_gather += coeff;
+                        average_count++;
+                    }
+                }
+            }
+        }
+        
+        /* only return the average if at least one was authoratative */
+        if (average_count > 0)
+            return average_gather / average_count;
+    }
+    
+    if (block == 10 || block == 11) {
+        /* lava blocks should always be lit! */
+        return 0.0f;
+    }
+    
+    skylevel = getArrayByte3D(skylight, local_x, local_y, local_z);
+    blocklevel = getArrayByte3D(blocklight, local_x, local_y, local_z);
+    
+    /* no longer a guess */
+    if (authoratative)
+        *authoratative = 1;
+    
+    return self->calculate_darkness(skylevel, blocklevel);
+}
+
+/* shades the drawn block with the given facemask/black_color, based on the
+   lighting results from (x, y, z) */
+static inline void
+do_shading_with_mask(RenderModeLighting *self, RenderState *state,
+                     int x, int y, int z, PyObject *mask) {
+    float black_coeff;
+
+    /* first, check for occlusion if the block is in the local chunk */
+    if (x >= 0 && x < 16 && y >= 0 && y < 16 && z >= 0 && z < 128) {
+        unsigned char block = getArrayByte3D(state->blocks, x, y, z);
+        if (!is_transparent(block)) {
+            /* this face isn't visible, so don't draw anything */
+            return;
+        }
+    }
+    
+    black_coeff = get_lighting_coefficient(self, state, x, y, z, NULL);
+    alpha_over_full(state->img, self->black_color, mask, black_coeff, state->imgx, state->imgy, 0, 0);
+}
+
+static int
+rendermode_lighting_start(void *data, RenderState *state) {
+    RenderModeLighting* self;
+
+    /* first, chain up */
+    int ret = rendermode_normal.start(data, state);
+    if (ret != 0)
+        return ret;
+    
+    self = (RenderModeLighting *)data;
+    
+    self->black_color = PyObject_GetAttrString(state->chunk, "black_color");
+    self->facemasks_py = PyObject_GetAttrString(state->chunk, "facemasks");
+    // borrowed references, don't need to be decref'd
+    self->facemasks[0] = PyTuple_GetItem(self->facemasks_py, 0);
+    self->facemasks[1] = PyTuple_GetItem(self->facemasks_py, 1);
+    self->facemasks[2] = PyTuple_GetItem(self->facemasks_py, 2);
+    
+    self->skylight = PyObject_GetAttrString(state->self, "skylight");
+    self->blocklight = PyObject_GetAttrString(state->self, "blocklight");
+    self->left_skylight = PyObject_GetAttrString(state->self, "left_skylight");
+    self->left_blocklight = PyObject_GetAttrString(state->self, "left_blocklight");
+    self->right_skylight = PyObject_GetAttrString(state->self, "right_skylight");
+    self->right_blocklight = PyObject_GetAttrString(state->self, "right_blocklight");
+    
+    self->calculate_darkness = calculate_darkness;
+    
+    return 0;
+}
+
+static void
+rendermode_lighting_finish(void *data, RenderState *state) {
+    RenderModeLighting *self = (RenderModeLighting *)data;
+    
+    Py_DECREF(self->black_color);
+    Py_DECREF(self->facemasks_py);
+    
+    Py_DECREF(self->skylight);
+    Py_DECREF(self->blocklight);
+    Py_DECREF(self->left_skylight);
+    Py_DECREF(self->left_blocklight);
+    Py_DECREF(self->right_skylight);
+    Py_DECREF(self->right_blocklight);
+    
+    /* now chain up */
+    rendermode_normal.finish(data, state);
+}
+
+static int
+rendermode_lighting_occluded(void *data, RenderState *state) {
+    /* no special occlusion here */
+    return rendermode_normal.occluded(data, state);
+}
+
+static void
+rendermode_lighting_draw(void *data, RenderState *state, PyObject *src, PyObject *mask) {
+    RenderModeLighting* self;
+    int x, y, z;
+
+    /* first, chain up */
+    rendermode_normal.draw(data, state, src, mask);
+    
+    self = (RenderModeLighting *)data;
+    x = state->x, y = state->y, z = state->z;
+    
+    if (is_transparent(state->block)) {
+        /* transparent: do shading on whole block */
+        do_shading_with_mask(self, state, x, y, z, mask);
+    } else {
+        /* opaque: do per-face shading */
+        do_shading_with_mask(self, state, x, y, z+1, self->facemasks[0]);
+        do_shading_with_mask(self, state, x-1, y, z, self->facemasks[1]);
+        do_shading_with_mask(self, state, x, y+1, z, self->facemasks[2]);
+    }
+}
+
+RenderModeInterface rendermode_lighting = {
+    "lighting", "draw shadows from the lighting data",
+    sizeof(RenderModeLighting),
+    rendermode_lighting_start,
+    rendermode_lighting_finish,
+    rendermode_lighting_occluded,
+    rendermode_lighting_draw,
+};

overviewer/src/rendermode-night.c

@@ -0,0 +1,69 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+#include <math.h>
+
+/* figures out the black_coeff from a given skylight and blocklight, used in
+   lighting calculations -- note this is *different* from the one in
+   rendermode-lighting.c (the "skylight - 11" part) */
+static float calculate_darkness(unsigned char skylight, unsigned char blocklight) {
+    return 1.0f - powf(0.8f, 15.0 - MAX(blocklight, skylight - 11));
+}
+
+static int
+rendermode_night_start(void *data, RenderState *state) {
+    RenderModeNight* self;    
+
+    /* first, chain up */
+    int ret = rendermode_lighting.start(data, state);
+    if (ret != 0)
+        return ret;
+    
+    /* override the darkness function with our night version! */
+    self = (RenderModeNight *)data;    
+    self->parent.calculate_darkness = calculate_darkness;
+    
+    return 0;
+}
+
+static void
+rendermode_night_finish(void *data, RenderState *state) {
+    /* nothing special to do */
+    rendermode_lighting.finish(data, state);
+}
+
+static int
+rendermode_night_occluded(void *data, RenderState *state) {
+    /* no special occlusion here */
+    return rendermode_lighting.occluded(data, state);
+}
+
+static void
+rendermode_night_draw(void *data, RenderState *state, PyObject *src, PyObject *mask) {
+    /* nothing special to do */
+    rendermode_lighting.draw(data, state, src, mask);
+}
+
+RenderModeInterface rendermode_night = {
+    "night", "like \"lighting\", except at night",
+    sizeof(RenderModeNight),
+    rendermode_night_start,
+    rendermode_night_finish,
+    rendermode_night_occluded,
+    rendermode_night_draw,
+};

overviewer/src/rendermode-normal.c

@@ -0,0 +1,173 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+
+static int
+rendermode_normal_start(void *data, RenderState *state) {
+    PyObject *chunk_x_py, *chunk_y_py, *world, *use_biomes, *worlddir;
+    RenderModeNormal *self = (RenderModeNormal *)data;
+    
+    chunk_x_py = PyObject_GetAttrString(state->self, "chunkX");
+    chunk_y_py = PyObject_GetAttrString(state->self, "chunkY");
+    
+    /* careful now -- C's % operator works differently from python's
+       we can't just do x % 32 like we did before */
+    self->chunk_x = PyInt_AsLong(chunk_x_py);
+    self->chunk_y = PyInt_AsLong(chunk_y_py);
+    
+    while (self->chunk_x < 0)
+        self->chunk_x += 32;
+    while (self->chunk_y < 0)
+        self->chunk_y += 32;
+    
+    self->chunk_x %= 32;
+    self->chunk_y %= 32;
+    
+    /* fetch the biome data from textures.py, if needed */
+    world = PyObject_GetAttrString(state->self, "world");
+    worlddir = PyObject_GetAttrString(world, "worlddir");
+    use_biomes = PyObject_GetAttrString(world, "useBiomeData");
+    Py_DECREF(world);
+    
+    if (PyObject_IsTrue(use_biomes)) {
+        PyObject *facemasks_py;
+        
+        self->biome_data = PyObject_CallMethod(state->textures, "getBiomeData", "OOO",
+                                               worlddir, chunk_x_py, chunk_y_py);
+        self->foliagecolor = PyObject_GetAttrString(state->textures, "foliagecolor");
+        self->grasscolor = PyObject_GetAttrString(state->textures, "grasscolor");
+        
+        self->leaf_texture = PyObject_GetAttrString(state->textures, "biome_leaf_texture");
+        self->grass_texture = PyObject_GetAttrString(state->textures, "biome_grass_texture");
+        
+        facemasks_py = PyObject_GetAttrString(state->chunk, "facemasks");
+        /* borrowed reference, needs to be incref'd if we keep it */
+        self->facemask_top = PyTuple_GetItem(facemasks_py, 0);
+        Py_INCREF(self->facemask_top);
+        Py_DECREF(facemasks_py);
+    } else {
+        self->biome_data = NULL;
+        self->foliagecolor = NULL;
+        self->grasscolor = NULL;
+        
+        self->leaf_texture = NULL;
+        self->grass_texture = NULL;
+        self->facemask_top = NULL;
+    }
+    
+    Py_DECREF(use_biomes);
+    Py_DECREF(worlddir);
+    Py_DECREF(chunk_x_py);
+    Py_DECREF(chunk_y_py);
+    
+    return 0;
+}
+
+static void
+rendermode_normal_finish(void *data, RenderState *state) {
+    RenderModeNormal *self = (RenderModeNormal *)data;
+    
+    Py_XDECREF(self->biome_data);
+    Py_XDECREF(self->foliagecolor);
+    Py_XDECREF(self->grasscolor);
+    Py_XDECREF(self->leaf_texture);
+    Py_XDECREF(self->grass_texture);
+    Py_XDECREF(self->facemask_top);
+}
+
+static int
+rendermode_normal_occluded(void *data, RenderState *state) {
+    int x = state->x, y = state->y, z = state->z;
+    
+    if ( (x != 0) && (y != 15) && (z != 127) &&
+         !is_transparent(getArrayByte3D(state->blocks, x-1, y, z)) &&
+         !is_transparent(getArrayByte3D(state->blocks, x, y, z+1)) &&
+         !is_transparent(getArrayByte3D(state->blocks, x, y+1, z))) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static void
+rendermode_normal_draw(void *data, RenderState *state, PyObject *src, PyObject *mask) {
+    RenderModeNormal *self = (RenderModeNormal *)data;
+    
+    /* first, check to see if we should use biome-compatible src, mask */
+    if (self->biome_data) {
+        switch (state->block) {
+        case 2:
+            src = mask = self->grass_texture;
+            break;
+        case 18:
+            src = mask = self->leaf_texture;
+            break;
+        default:
+            break;
+        };
+    }
+    
+    /* draw the block! */
+    alpha_over(state->img, src, mask, state->imgx, state->imgy, 0, 0);
+    
+    if (self->biome_data) {
+        /* do the biome stuff! */
+        unsigned int index;
+        PyObject *color = NULL, *facemask = NULL;
+        unsigned char r, g, b;
+        
+        index = ((self->chunk_y * 16) + state->y) * 16 * 32 + (self->chunk_x * 16) + state->x;
+        index = big_endian_ushort(getArrayShort1D(self->biome_data, index));
+        
+        switch (state->block) {
+        case 2:
+            /* grass */
+            color = PySequence_GetItem(self->grasscolor, index);
+            facemask = self->facemask_top;
+            break;
+        case 18:
+            /* leaves */
+            color = PySequence_GetItem(self->foliagecolor, index);
+            facemask = mask;
+            break;
+        default:
+            break;
+        };
+        
+        if (color)
+        {
+            /* we've got work to do */
+            
+            r = PyInt_AsLong(PyTuple_GET_ITEM(color, 0));
+            g = PyInt_AsLong(PyTuple_GET_ITEM(color, 1));
+            b = PyInt_AsLong(PyTuple_GET_ITEM(color, 2));
+            Py_DECREF(color);
+
+            tint_with_mask(state->img, r, g, b, facemask, state->imgx, state->imgy, 0, 0);
+        }
+    }
+}
+
+RenderModeInterface rendermode_normal = {
+    "normal", "nothing special, just render the blocks",
+    sizeof(RenderModeNormal),
+    rendermode_normal_start,
+    rendermode_normal_finish,
+    rendermode_normal_occluded,
+    rendermode_normal_draw,
+};

overviewer/src/rendermode-spawn.c

@@ -0,0 +1,127 @@
+/*
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+#include <math.h>
+
+static int
+rendermode_spawn_start(void *data, RenderState *state) {
+    RenderModeSpawn* self;
+
+    /* first, chain up */
+    int ret = rendermode_night.start(data, state);
+    if (ret != 0)
+        return ret;
+    
+    /* now do custom initializations */
+    self = (RenderModeSpawn *)data;
+    self->solid_blocks = PyObject_GetAttrString(state->chunk, "solid_blocks");
+    self->nospawn_blocks = PyObject_GetAttrString(state->chunk, "nospawn_blocks");
+    self->fluid_blocks = PyObject_GetAttrString(state->chunk, "fluid_blocks");
+    self->red_color = PyObject_GetAttrString(state->chunk, "red_color");
+    
+    return 0;
+}
+
+static void
+rendermode_spawn_finish(void *data, RenderState *state) {
+    /* first free all *our* stuff */
+    RenderModeSpawn* self = (RenderModeSpawn *)data;    
+    
+    Py_DECREF(self->solid_blocks);
+    Py_DECREF(self->nospawn_blocks);
+    Py_DECREF(self->fluid_blocks);
+    
+    /* now, chain up */
+    rendermode_night.finish(data, state);
+}
+
+static int
+rendermode_spawn_occluded(void *data, RenderState *state) {
+    /* no special occlusion here */
+    return rendermode_night.occluded(data, state);
+}
+
+static void
+rendermode_spawn_draw(void *data, RenderState *state, PyObject *src, PyObject *mask) {
+    /* different versions of self (spawn, lighting) */
+    RenderModeSpawn* self = (RenderModeSpawn *)data;
+    RenderModeLighting *lighting = (RenderModeLighting *)self;
+    
+    int x = state->x, y = state->y, z = state->z;
+    PyObject *old_black_color = NULL;
+    
+    /* figure out the appropriate darkness:
+       this block for transparents, the block above for non-transparent */
+    float darkness = 0.0;
+    if (is_transparent(state->block)) {
+        darkness = get_lighting_coefficient((RenderModeLighting *)self, state, x, y, z, NULL);
+    } else {
+        darkness = get_lighting_coefficient((RenderModeLighting *)self, state, x, y, z+1, NULL);
+    }
+    
+    /* if it's dark enough... */
+    if (darkness > 0.8) {
+        PyObject *block_py = PyInt_FromLong(state->block);
+        
+        /* make sure it's solid */
+        if (PySequence_Contains(self->solid_blocks, block_py)) {
+            int spawnable = 1;
+            
+            /* not spawnable if its in the nospawn list */
+            if (PySequence_Contains(self->nospawn_blocks, block_py))
+                spawnable = 0;
+            
+            /* check the block above for solid or fluid */
+            if (spawnable && z != 127) {
+                PyObject *top_block_py = PyInt_FromLong(getArrayByte3D(state->blocks, x, y, z+1));
+                if (PySequence_Contains(self->solid_blocks, top_block_py) ||
+                    PySequence_Contains(self->fluid_blocks, top_block_py)) {
+                    
+                    spawnable = 0;
+                }
+                
+                Py_DECREF(top_block_py);
+            }
+            
+            /* if we passed all the checks, replace black_color with red_color */
+            if (spawnable) {
+                old_black_color = lighting->black_color;
+                lighting->black_color = self->red_color;
+            }
+        }
+        
+        Py_DECREF(block_py);
+    }
+    
+    /* draw normally */
+    rendermode_night.draw(data, state, src, mask);
+    
+    /* reset black_color, if needed */
+    if (old_black_color != NULL) {
+        lighting->black_color = old_black_color;
+    }
+}
+
+RenderModeInterface rendermode_spawn = {
+    "spawn", "draws red where monsters can spawn at night",
+    sizeof(RenderModeSpawn),
+    rendermode_spawn_start,
+    rendermode_spawn_finish,
+    rendermode_spawn_occluded,
+    rendermode_spawn_draw,
+};

overviewer/src/rendermodes.c

@@ -0,0 +1,101 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "overviewer.h"
+#include <string.h>
+
+/* list of all render modes, ending in NULL
+   all of these will be available to the user, so DON'T include modes
+   that are only useful as a base for other modes. */
+static RenderModeInterface *render_modes[] = {
+    &rendermode_normal,
+    &rendermode_lighting,
+    &rendermode_night,
+    &rendermode_spawn,
+    NULL
+};
+
+/* decides which render mode to use */
+RenderModeInterface *get_render_mode(RenderState *state) {
+    unsigned int i;
+    /* default: NULL --> an error */
+    RenderModeInterface *iface = NULL;
+    PyObject *rendermode_py = PyObject_GetAttrString(state->self, "rendermode");
+    const char *rendermode = PyString_AsString(rendermode_py);
+    
+    for (i = 0; render_modes[i] != NULL; i++) {
+        if (strcmp(render_modes[i]->name, rendermode) == 0) {
+            iface = render_modes[i];
+            break;
+        }
+    }
+    
+    Py_DECREF(rendermode_py);
+    return iface;
+}
+
+/* bindings for python -- get all the rendermode names */
+PyObject *get_render_modes(PyObject *self, PyObject *args) {
+    PyObject *modes;
+    unsigned int i;
+    if (!PyArg_ParseTuple(args, ""))
+        return NULL;
+    
+    modes = PyList_New(0);
+    if (modes == NULL)
+        return NULL;
+    
+    for (i = 0; render_modes[i] != NULL; i++) {
+        PyObject *name = PyString_FromString(render_modes[i]->name);
+        PyList_Append(modes, name);
+        Py_DECREF(name);
+    }
+    
+    return modes;
+}
+
+/* more bindings -- return info for a given rendermode name */
+PyObject *get_render_mode_info(PyObject *self, PyObject *args) {
+    const char* rendermode;
+    PyObject *info;
+    unsigned int i;
+    if (!PyArg_ParseTuple(args, "s", &rendermode))
+        return NULL;
+    
+    info = PyDict_New();
+    if (info == NULL)
+        return NULL;
+    
+    for (i = 0; render_modes[i] != NULL; i++) {
+        if (strcmp(render_modes[i]->name, rendermode) == 0) {
+            PyObject *tmp;
+            
+            tmp = PyString_FromString(render_modes[i]->name);
+            PyDict_SetItemString(info, "name", tmp);
+            Py_DECREF(tmp);
+            
+            tmp = PyString_FromString(render_modes[i]->description);
+            PyDict_SetItemString(info, "description", tmp);
+            Py_DECREF(tmp);
+            
+            return info;
+        }
+    }
+    
+    Py_DECREF(info);
+    Py_RETURN_NONE;
+}

overviewer/src/rendermodes.h

@@ -0,0 +1,122 @@
+/* 
+ * This file is part of the Minecraft Overviewer.
+ *
+ * Minecraft Overviewer is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Minecraft Overviewer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * To make a new render mode (the C part, at least):
+ *
+ *     * add a data struct and extern'd interface declaration below
+ *
+ *     * fill in this interface struct in rendermode-(yourmode).c
+ *         (see rendermodes-normal.c for an example: the "normal" mode)
+ *
+ *     * if you want to derive from (say) the "normal" mode, put
+ *       a RenderModeNormal entry at the top of your data struct, and
+ *       be sure to call your parent's functions in your own!
+ *         (see rendermode-night.c for a simple example derived from
+ *          the "lighting" mode)
+ *
+ *     * add your mode to the list in rendermodes.c
+ */
+
+#ifndef __RENDERMODES_H_INCLUDED__
+#define __RENDERMODES_H_INCLUDED__
+
+#include <Python.h>
+
+/* rendermode interface */
+typedef struct {
+    /* the name of this mode */
+    const char* name;
+    /* the short description of this render mode */
+    const char* description;
+    
+    /* the size of the local storage for this rendermode */
+    unsigned int data_size;
+    
+    /* may return non-zero on error */
+    int (*start)(void *, RenderState *);
+    void (*finish)(void *, RenderState *);
+    /* returns non-zero to skip rendering this block */
+    int (*occluded)(void *, RenderState *);
+    /* last two arguments are img and mask, from texture lookup */
+    void (*draw)(void *, RenderState *, PyObject *, PyObject *);
+} RenderModeInterface;
+
+/* figures out the render mode to use from the given ChunkRenderer */
+RenderModeInterface *get_render_mode(RenderState *state);
+/* python bindings */
+PyObject *get_render_modes(PyObject *self, PyObject *args);
+PyObject *get_render_mode_info(PyObject *self, PyObject *args);
+
+/* individual rendermode interface declarations follow */
+
+/* NORMAL */
+typedef struct {
+    /* coordinates of the chunk, inside its region file */
+    int chunk_x, chunk_y;
+    /* biome data for the region */
+    PyObject *biome_data;
+    /* grasscolor and foliagecolor lookup tables */
+    PyObject *grasscolor, *foliagecolor;
+    /* biome-compatible grass/leaf textures */
+    PyObject *grass_texture, *leaf_texture;
+    /* top facemask for grass biome tinting */
+    PyObject *facemask_top;
+} RenderModeNormal;
+extern RenderModeInterface rendermode_normal;
+
+/* LIGHTING */
+typedef struct {
+    /* inherits from normal render mode */
+    RenderModeNormal parent;
+    
+    PyObject *black_color, *facemasks_py;
+    PyObject *facemasks[3];
+    
+    /* extra data, loaded off the chunk class */
+    PyObject *skylight, *blocklight;
+    PyObject *left_skylight, *left_blocklight;
+    PyObject *right_skylight, *right_blocklight;
+    
+    /* can be overridden in derived rendermodes to control lighting
+       arguments are skylight, blocklight */
+    float (*calculate_darkness)(unsigned char, unsigned char);
+} RenderModeLighting;
+extern RenderModeInterface rendermode_lighting;
+inline float get_lighting_coefficient(RenderModeLighting *self, RenderState *state,
+                                      int x, int y, int z, int *authoratative);
+
+/* NIGHT */
+typedef struct {
+    /* inherits from lighting */
+    RenderModeLighting parent;
+} RenderModeNight;
+extern RenderModeInterface rendermode_night;
+
+/* SPAWN */
+typedef struct {
+    /* inherits from night */
+    RenderModeNight parent;
+    
+    /* used to figure out which blocks are spawnable */
+    PyObject *solid_blocks, *nospawn_blocks, *fluid_blocks;
+    /* replacement for black_color */
+    PyObject *red_color;
+} RenderModeSpawn;
+extern RenderModeInterface rendermode_spawn;
+
+#endif /* __RENDERMODES_H_INCLUDED__ */

overviewer/util.py

@@ -0,0 +1,51 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Misc utility routines used by multiple files that don't belong anywhere else
+"""
+
+import imp
+import os
+import os.path
+import sys
+
+def get_program_path():
+    if hasattr(sys, "frozen") or imp.is_frozen("__main__"):
+        return os.path.dirname(sys.executable)
+    else:
+        try:
+            # normally, we're in ./overviewer/util.py
+            return os.path.dirname(os.path.dirname(__file__))
+        except NameError:
+            return os.path.dirname(sys.argv[0])
+
+
+
+def findGitVersion():
+    if os.path.exists(".git"):
+        with open(os.path.join(".git","HEAD")) as f:
+            data = f.read().strip()
+        if data.startswith("ref: "):
+            with open(os.path.join(".git", data[5:])) as g:
+                return g.read().strip()
+        else:
+            return data
+    else:
+        try:
+            import overviewer_version
+            return overviewer_version.VERSION
+        except:
+            return "unknown"

overviewer/world.py

@@ -0,0 +1,340 @@
+#    This file is part of the Minecraft Overviewer.
+#
+#    Minecraft Overviewer is free software: you can redistribute it and/or
+#    modify it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or (at
+#    your option) any later version.
+#
+#    Minecraft Overviewer is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+#    Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+
+import functools
+import os
+import os.path
+import multiprocessing
+import Queue
+import sys
+import logging
+import cPickle
+import collections
+import itertools
+
+import numpy
+
+import chunk
+import nbt
+import textures
+import time
+
+"""
+This module has routines for extracting information about available worlds
+
+"""
+
+base36decode = functools.partial(int, base=36)
+cached = collections.defaultdict(dict)
+
+def base36encode(number, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'):
+    '''
+    Convert an integer to a base36 string.
+    '''
+    if not isinstance(number, (int, long)):
+        raise TypeError('number must be an integer')
+    
+    newn = abs(number)
+ 
+    # Special case for zero
+    if number == 0:
+        return '0'
+ 
+    base36 = ''
+    while newn != 0:
+        newn, i = divmod(newn, len(alphabet))
+        base36 = alphabet[i] + base36
+
+    if number < 0:
+        return "-" + base36
+    return base36
+
+class World(object):
+    """Does world-level preprocessing to prepare for QuadtreeGen
+    worlddir is the path to the minecraft world
+    """
+
+    mincol = maxcol = minrow = maxrow = 0
+    
+    def __init__(self, worlddir, useBiomeData=False,regionlist=None):
+        self.worlddir = worlddir
+        self.useBiomeData = useBiomeData
+                
+        #find region files, or load the region list
+        #this also caches all the region file header info
+        logging.info("Scanning regions")
+        regionfiles = {}
+        self.regions = {}
+        for x, y, regionfile in self._iterate_regionfiles():  
+            mcr = self.reload_region(regionfile) 
+            mcr.get_chunk_info()
+            regionfiles[(x,y)]	= (x,y,regionfile,mcr)
+        self.regionfiles = regionfiles	
+        # set the number of region file handles we will permit open at any time before we start closing them
+#        self.regionlimit = 1000
+        # the max number of chunks we will keep before removing them (includes emptry chunks)
+        self.chunklimit = 1024 
+        self.chunkcount = 0
+        self.empty_chunk = [None,None]
+        logging.debug("Done scanning regions")
+        
+        # figure out chunk format is in use
+        # if not mcregion, error out early
+        data = nbt.load(os.path.join(self.worlddir, "level.dat"))[1]['Data']
+        #print data
+        if not ('version' in data and data['version'] == 19132):
+            logging.error("Sorry, This version of Minecraft-Overviewer only works with the new McRegion chunk format")
+            sys.exit(1)
+
+        if self.useBiomeData:
+            textures.prepareBiomeData(worlddir)
+
+        #  stores Points Of Interest to be mapped with markers
+        #  a list of dictionaries, see below for an example
+        self.POI = []
+
+        # if it exists, open overviewer.dat, and read in the data structure
+        # info self.persistentData.  This dictionary can hold any information
+        # that may be needed between runs.
+        # Currently only holds into about POIs (more more details, see quadtree)
+        # TODO maybe store this with the tiles, not with the world?
+        self.pickleFile = os.path.join(self.worlddir, "overviewer.dat")
+        if os.path.exists(self.pickleFile):
+            with open(self.pickleFile,"rb") as p:
+                self.persistentData = cPickle.load(p)
+        else:
+            # some defaults
+            self.persistentData = dict(POI=[])
+        
+ 
+    def get_region_path(self, chunkX, chunkY):
+        """Returns the path to the region that contains chunk (chunkX, chunkY)
+        """
+        _, _, regionfile,_ = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None,None));
+        return regionfile
+            
+    def load_from_region(self,filename, x, y):
+        #we need to manage the chunk cache
+        regioninfo = self.regions[filename]
+        if regioninfo is None:
+            return None
+        chunks = regioninfo[2]
+        chunk_data = chunks.get((x,y))
+        if chunk_data is None:        
+            #prune the cache if required
+            if self.chunkcount > self.chunklimit: #todo: make the emptying the chunk cache slightly less crazy
+                [self.reload_region(regionfile) for regionfile in self.regions if regionfile <> filename]
+                self.chunkcount = 0
+            self.chunkcount += 1  
+
+            nbt = self.load_region(filename).load_chunk(x, y)
+            if nbt is None:
+                chunks[(x,y)] = self.empty_chunk
+                return None ## return none.  I think this is who we should indicate missing chunks
+                #raise IOError("No such chunk in region: (%i, %i)" % (x, y))                 
+
+            #we cache the transformed data, not it's raw form
+            data = nbt.read_all()    
+            level = data[1]['Level']
+            chunk_data = level
+            #chunk_data = {}
+            #chunk_data['skylight'] = chunk.get_skylight_array(level)
+            #chunk_data['blocklight'] = chunk.get_blocklight_array(level)
+            #chunk_data['blockarray'] = chunk.get_blockdata_array(level)
+            #chunk_data['TileEntities'] = chunk.get_tileentity_data(level)
+            
+            chunks[(x,y)] = [level,time.time()]
+        else:
+            chunk_data = chunk_data[0]
+        return chunk_data      
+      
+    #used to reload a changed region
+    def reload_region(self,filename):
+        if self.regions.get(filename) is not None:
+            self.regions[filename][0].closefile()
+        chunkcache = {}    
+        mcr = nbt.MCRFileReader(filename)
+        self.regions[filename] = (mcr,os.path.getmtime(filename),chunkcache)
+        return mcr
+        
+    def load_region(self,filename):    
+        return self.regions[filename][0]
+        
+    def get_region_mtime(self,filename):                  
+        return (self.regions[filename][0],self.regions[filename][1])        
+        
+    def convert_coords(self, chunkx, chunky):
+        """Takes a coordinate (chunkx, chunky) where chunkx and chunky are
+        in the chunk coordinate system, and figures out the row and column
+        in the image each one should be. Returns (col, row)."""
+        
+        # columns are determined by the sum of the chunk coords, rows are the
+        # difference (TODO: be able to change direction of north)
+        # change this function, and you MUST change unconvert_coords
+        return (chunkx + chunky, chunky - chunkx)
+    
+    def unconvert_coords(self, col, row):
+        """Undoes what convert_coords does. Returns (chunkx, chunky)."""
+        
+        # col + row = chunky + chunky => (col + row)/2 = chunky
+        # col - row = chunkx + chunkx => (col - row)/2 = chunkx
+        return ((col - row) / 2, (col + row) / 2)
+    
+    def findTrueSpawn(self):
+        """Adds the true spawn location to self.POI.  The spawn Y coordinate
+        is almost always the default of 64.  Find the first air block above
+        that point for the true spawn location"""
+
+        ## read spawn info from level.dat
+        data = nbt.load(os.path.join(self.worlddir, "level.dat"))[1]
+        spawnX = data['Data']['SpawnX']
+        spawnY = data['Data']['SpawnY']
+        spawnZ = data['Data']['SpawnZ']
+   
+        ## The chunk that holds the spawn location 
+        chunkX = spawnX/16
+        chunkY = spawnZ/16
+
+        ## The filename of this chunk
+        chunkFile = self.get_region_path(chunkX, chunkY)
+
+        data=nbt.load_from_region(chunkFile, chunkX, chunkY)[1]
+        level = data['Level']
+        blockArray = numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
+
+        ## The block for spawn *within* the chunk
+        inChunkX = spawnX - (chunkX*16)
+        inChunkZ = spawnZ - (chunkY*16)
+
+        ## find the first air block
+        while (blockArray[inChunkX, inChunkZ, spawnY] != 0):
+            spawnY += 1
+            if spawnY == 128:
+                break
+
+        self.POI.append( dict(x=spawnX, y=spawnY, z=spawnZ, 
+                msg="Spawn", type="spawn", chunk=(inChunkX,inChunkZ)))
+        self.spawn = (spawnX, spawnY, spawnZ)
+
+    def go(self, procs):
+        """Scan the world directory, to fill in
+        self.{min,max}{col,row} for use later in quadtree.py. This
+        also does other world-level processing."""
+        
+        logging.info("Scanning chunks")
+        # find the dimensions of the map, in region files
+        minx = maxx = miny = maxy = 0
+        found_regions = False
+        for x, y in self.regionfiles:
+            found_regions = True
+            minx = min(minx, x)
+            maxx = max(maxx, x)
+            miny = min(miny, y)
+            maxy = max(maxy, y)
+        if not found_regions:
+            logging.error("Error: No chunks found!")
+            sys.exit(1)
+        logging.debug("Done scanning chunks")
+        
+        # turn our region coordinates into chunk coordinates
+        minx = minx * 32
+        miny = miny * 32
+        maxx = maxx * 32 + 32
+        maxy = maxy * 32 + 32
+        
+        # Translate chunks to our diagonal coordinate system
+        mincol = maxcol = minrow = maxrow = 0
+        for chunkx, chunky in [(minx, miny), (minx, maxy), (maxx, miny), (maxx, maxy)]:
+            col, row = self.convert_coords(chunkx, chunky)
+            mincol = min(mincol, col)
+            maxcol = max(maxcol, col)
+            minrow = min(minrow, row)
+            maxrow = max(maxrow, row)
+        
+        #logging.debug("map size: (%i, %i) to (%i, %i)" % (mincol, minrow, maxcol, maxrow))
+
+        self.mincol = mincol
+        self.maxcol = maxcol
+        self.minrow = minrow
+        self.maxrow = maxrow
+
+        self.findTrueSpawn()
+
+    def _iterate_regionfiles(self,regionlist=None):
+        """Returns an iterator of all of the region files, along with their 
+        coordinates
+
+        Returns (regionx, regiony, filename)"""
+        join = os.path.join
+        if regionlist is not None:
+            for path in regionlist:
+                if path.endswith("\n"):
+                    path = path[:-1]            
+                f = os.path.basename(path)
+                if f.startswith("r.") and f.endswith(".mcr"):
+                    p = f.split(".")
+                    yield (int(p[1]), int(p[2]), join(self.worlddir, 'region', f))        
+        else:                    
+            for dirpath, dirnames, filenames in os.walk(os.path.join(self.worlddir, 'region')):
+                if not dirnames and filenames and "DIM-1" not in dirpath:
+                    for f in filenames:
+                        if f.startswith("r.") and f.endswith(".mcr"):
+                            p = f.split(".")
+                            yield (int(p[1]), int(p[2]), join(dirpath, f))
+
+def get_save_dir():
+    """Returns the path to the local saves directory
+      * On Windows, at %APPDATA%/.minecraft/saves/
+      * On Darwin, at $HOME/Library/Application Support/minecraft/saves/
+      * at $HOME/.minecraft/saves/
+
+    """
+    
+    savepaths = []
+    if "APPDATA" in os.environ:
+        savepaths += [os.path.join(os.environ['APPDATA'], ".minecraft", "saves")]
+    if "HOME" in os.environ:
+        savepaths += [os.path.join(os.environ['HOME'], "Library",
+                "Application Support", "minecraft", "saves")]
+        savepaths += [os.path.join(os.environ['HOME'], ".minecraft", "saves")]
+
+    for path in savepaths:
+        if os.path.exists(path):
+            return path
+
+def get_worlds():
+    "Returns {world # or name : level.dat information}"
+    ret = {}
+    save_dir = get_save_dir()
+
+    # No dirs found - most likely not running from inside minecraft-dir
+    if save_dir is None:
+        return None
+
+    for dir in os.listdir(save_dir):
+        world_dat = os.path.join(save_dir, dir, "level.dat")
+        if not os.path.exists(world_dat): continue
+        info = nbt.load(world_dat)[1]
+        info['Data']['path'] = os.path.join(save_dir, dir)
+        if dir.startswith("World") and len(dir) == 6:
+            try:
+                world_n = int(dir[-1])
+                ret[world_n] = info['Data']
+            except ValueError:
+                pass
+        if 'LevelName' in info['Data'].keys():
+            ret[info['Data']['LevelName']] = info['Data']
+
+    return ret