Merge branch 'master' of http://github.com/brownan/Minecraft-Overviewer

README.rst

@@ -21,6 +21,9 @@ Features
 * Outputs a Google Map powered interface that is memory efficient, both in
   generating and viewing.
 
+* Renders efficiently in parallel, using as many simultaneous processes as you
+  want!
+
 * Utilizes 2 levels of caching to speed up subsequent renderings of your world.
 
 * Throw the output directory up on a web server to share your Minecraft world
@@ -34,8 +37,8 @@ This program requires:
 * PIL (Python Imaging Library) <http://www.pythonware.com/products/pil/>
 * Numpy <http://scipy.org/Download>
 
-I developed and tested this on Linux. It has been reported to work on Windows
+I develop and test this on Linux, but need help testing it on Windows and Mac.
-and Mac, but if something doesn't, let me know.
+If something doesn't work, let me know.
 
 Using the Google Map Tile Generator
 ===================================
@@ -77,9 +80,7 @@ greatly speeds up the rendering.
 Using more Cores
 ----------------
 Adding the "-p" option will utilize more cores to generate the chunk files.
-This can speed up rendering quite a bit. However, the tile generation routine
+This can speed up rendering quite a bit.
-is currently serial and not written to take advantage of multiple cores. This
-option will only affect the chunk generation (which is around half the process)
 
 Example::
 
@@ -96,8 +97,33 @@ render for my world from 85M to 67M.
 
     find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \;
 
-Windows users, you're on your own, but there's probably a way to do this. (If
+If you're on Windows, I've gotten word that this command line snippet works
-someone figures it out, let me know I'll update this README)
+provided pngout is installed and on your path. Note that the % symbols will
+need to be doubled up if this is in a batch file.
+
+::
+
+    FOR /R c:\path\to\tiles\folder %v IN (*.png) DO pngout %v /y
+
+Viewing the Results
+-------------------
+The output is two things: an index.html file, and a directory hierarchy full of
+images. To view your world, simply open index.html in a web browser. Internet
+access is required to load the Google Maps API files, but you otherwise don't
+need anything else.
+
+You can throw these files up to a web server to let others view your map. You
+do not need a Google Maps API key (as was the case with older versions of the
+API), so just copying the directory to your web server should suffice.
+
+Tip: Since Minecraft worlds rarely produce perfectly square worlds, there will
+be blank and non-existent tiles around the borders of your world. The Google
+Maps API has no way of knowing this until it requests them and the web server
+returns a 404 Not Found. If this doesn't bother you, then fine, stop reading.
+Otherwise: you can avoid a lot of 404s to your logs by configuring your web
+server to redirect all 404 requests in that directory to a single 1px
+"blank.png". This may or may not save on bandwidth, but it will probably save
+on log noise.
 
 Using the Large Image Renderer
 ==============================

chunk.py

@@ -6,7 +6,6 @@ import hashlib
 
 import nbt
 import textures
-from textures import texturemap as txtarray
 
 # General note about pasting transparent image objects onto an image with an
 # alpha channel:
@@ -52,6 +51,14 @@ def render_and_save(chunkfile, cave=False):
         import traceback
         traceback.print_exc()
         raise
+    except KeyboardInterrupt:
+        print
+        print "You pressed Ctrl-C. Unfortunately it got caught by a subprocess"
+        print "The program will terminate... eventually, but the main process"
+        print "may take a while to realize something went wrong."
+        print "To exit immediately, you'll need to kill this process some other"
+        print "way"
+        raise Exception()
 
 class ChunkRenderer(object):
     def __init__(self, chunkfile):
@@ -236,7 +243,7 @@ class ChunkRenderer(object):
                         if blockid not in transparent_blocks:
                             draw = ImageDraw.Draw(img)
                             if x != 15 and blocks[x+1,y,z] == 0:
-                                draw.line(((imgx+12,imgy), (imgx+24,imgy+6)), fill=(0,0,0), width=1)
+                                draw.line(((imgx+12,imgy), (imgx+22,imgy+5)), fill=(0,0,0), width=1)
                             if y != 0 and blocks[x,y-1,z] == 0:
                                 draw.line(((imgx,imgy+6), (imgx+12,imgy)), fill=(0,0,0), width=1)
 

gmap.py

@@ -35,8 +35,11 @@ def main():
     # Translate chunks from diagonal coordinate system
     mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks)
 
-    print "processing chunks in background"
+    print "Rendering chunks"
     results = world.render_chunks_async(chunks, False, options.procs)
+    for i, (col, row, filename) in enumerate(chunks):
+        results[col, row].wait()
+        print "{0}/{1} chunks rendered".format(i, len(chunks))
     
     print "Writing out html file"
     if not os.path.exists(destdir):
@@ -49,7 +52,7 @@ def main():
     tiledir = os.path.join(destdir, "tiles")
     if not os.path.exists(tiledir):
         os.mkdir(tiledir)
-    world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir)
+    world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir, options.procs)
 
     print "DONE"
 

textures.py

@@ -8,36 +8,70 @@ import math
 import numpy
 from PIL import Image, ImageEnhance
 
-def _get_terrain_image():
+def _find_file(filename, mode="rb"):
-    # Check the current directory for terrain.png first:
+    """Searches for the given file and returns an open handle to it.
-    if os.path.isfile("terrain.png"):
+    This searches the following locations in this order:
-        return Image.open("terrain.png")
     
-    if "darwin" in sys.platform:
+    * The program dir (same dir as this file)
-        # On Macs, terrain.png could lie at
+    * On Darwin, in /Applications/Minecraft
-        # "/Applications/minecraft/terrain.png" for custom terrain. Try this
+    * Inside minecraft.jar, which is looked for at these locations
-        # first.
-        png = "/Applications/Minecraft/terrain.png"
-        if os.access(png, os.F_OK):
-            return Image.open(png)
 
-        # Paths on a Mac are a bit different
+      * On Windows, at %APPDATA%/.minecraft/bin/minecraft.jar
-        minecraftdir = os.path.join(os.environ['HOME'], "Library",
+      * On Darwin, at $HOME/Library/Application Support/minecraft/bin/minecraft.jar
-                "Application Support", "minecraft")
+      * at $HOME/.minecraft/bin/minecraft.jar
-        minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, "bin", "minecraft.jar"))
-        textures = minecraftjar.open("terrain.png")
 
-    else:
+    * The current working directory
-        if "win" in sys.platform:
+    * The program dir / textures
-            minecraftdir = os.environ['APPDATA']
+
-        else:
+    """
-            minecraftdir = os.environ['HOME']
+    programdir = os.path.dirname(__file__)
-        minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, ".minecraft",
+    path = os.path.join(programdir, filename)
+    if os.path.exists(path):
+        return open(path, mode)
+
+    if sys.platform == "darwin":
+        path = os.path.join("/Applications/Minecraft", filename)
+        if os.path.exists(path):
+            return open(path, mode)
+
+    # Find minecraft.jar.
+    jarpaths = []
+    if "APPDATA" in os.environ:
+        jarpaths.append( os.path.join(os.environ['APPDATA'], ".minecraft",
             "bin", "minecraft.jar"))
-        textures = minecraftjar.open("terrain.png")
+    if "HOME" in os.environ:
-    buffer = StringIO(textures.read())
+        jarpaths.append(os.path.join(os.environ['HOME'], "Library",
+                "Application Support", "minecraft"))
+        jarpaths.append(os.path.join(os.environ['HOME'], ".minecraft", "bin",
+                "minecraft.jar"))
+
+    for jarpath in jarpaths:
+        if os.path.exists(jarpath):
+            jar = zipfile.ZipFile(jarpath)
+            try:
+                return jar.open(filename)
+            except KeyError:
+                pass
+
+    path = filename
+    if os.path.exists(path):
+        return open(path, mode)
+
+    path = os.path.join(programdir, "textures", filename)
+    if os.path.exists(path):
+        return open(path, mode)
+
+    raise IOError("Could not find the file {0}".format(filename))
+
+def _load_image(filename):
+    """Returns an image object"""
+    fileobj = _find_file(filename)
+    buffer = StringIO(fileobj.read())
     return Image.open(buffer)
 
+def _get_terrain_image():
+    return _load_image("terrain.png")
+
 def _split_terrain(terrain):
     """Builds and returns a length 256 array of each 16x16 chunk of texture"""
     textures = []
@@ -100,73 +134,26 @@ def _transform_image_side(img):
     return newimg
 
 
-def _build_texturemap():
+def _build_block(top, side):
-    """"""
+    """From a top texture and a side texture, build a block image.
-    t = terrain_images
+    top and side should be 16x16 image objects. Returns a 24x24 image
-
-    # Notes are for things I've left out or will probably have to make special
-    # exception for
-    top = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33,
-        34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, # Cloths are left out
-        -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, # Gold/iron blocks? Doublestep? TNT from above?
-        36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, # Torch from above? leaving out fire. Redstone wire? Crops left out. sign post
-        -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, # door,ladder left out. Minecart rail orientation
-        66,69,72,-1,74 # clay?
-        ]
-    side = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33,
-        34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-        -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35,
-        36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1,
-        -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67,
-        66,69,72,-1,74
-        ]
-    side[2] = 2
-
-    return (
-           [(t[x] if x != -1 else None) for x in top],
-           [(_transform_image(t[x]) if x != -1 else None) for x in top],
-           [(_transform_image_side(t[x]) if x != -1 else None) for x in side],
-           )
-# texturemap maps block ids to a 16x16 image that goes on the top face
-# perspective_texturemap does the same, except the texture is rotated and shrunk
-# shear_texturemap maps block ids to the image that goes on the side of the
-# block, sheared appropriately
-texturemap, perspective_texturemap, shear_texturemap = _build_texturemap()
-
-def _render_sprite(img):
-    """Takes a 16x16 sprite image, and returns a 22x22 image to go in the
-    blockmap
-    This is for rendering things that are sticking out of the ground, like
-    flowers and such
-    torches are drawn the same way, but torches that attach to walls are
-    handled differently
-    """
-    pass
-
-def _render_ground_image(img):
-    """Takes a 16x16 sprite image and skews it to look like it's on the ground.
-    This is for things like mine track and such
 
     """
-    pass
-
-def _build_blockimages():
-    """Returns a mapping from blockid to an image of that block in perspective
-    The values of the mapping are actually (image in RGB mode, alpha channel)"""
-    # This maps block id to the texture that goes on the side of the block
-    allimages = []
-    for top, side in zip(perspective_texturemap, shear_texturemap):
-        if not top or not side:
-            allimages.append(None)
-            continue
     img = Image.new("RGBA", (24,24))
 
+    top = _transform_image(top)
+
+    if not side:
+        img.paste(top, (0,0), top)
+        return img
+
+    side = _transform_image_side(side)
+
     otherside = side.transpose(Image.FLIP_LEFT_RIGHT)
     
     # Darken the sides slightly. These methods also affect the alpha layer,
     # so save them first (we don't want to "darken" the alpha layer making
     # the block transparent)
-        if 1:
     sidealpha = side.split()[3]
     side = ImageEnhance.Brightness(side).enhance(0.9)
     side.putalpha(sidealpha)
@@ -174,11 +161,8 @@ def _build_blockimages():
     otherside = ImageEnhance.Brightness(otherside).enhance(0.8)
     otherside.putalpha(othersidealpha)
 
-        # Copy on the left side
     img.paste(side, (0,6), side)
-        # Copy on the other side
     img.paste(otherside, (12,6), otherside)
-        # Copy on the top piece (last so it's on top)
     img.paste(top, (0,0), top)
 
     # Manually touch up 6 pixels that leave a gap because of how the
@@ -190,12 +174,73 @@ def _build_blockimages():
         # Copy a pixel to x,y from x+1,y
         img.putpixel((x,y), img.getpixel((x+1,y)))
 
-        allimages.append((img.convert("RGB"), img.split()[3]))
+    return img
-    return allimages
+
+
+def _build_blockimages():
+    """Returns a mapping from blockid to an image of that block in perspective
+    The values of the mapping are actually (image in RGB mode, alpha channel).
+    This is not appropriate for all block types, only block types that are
+    proper cubes"""
+
+    # Top textures of all block types. The number here is the index in the
+    # texture array (terrain_images), which comes from terrain.png's cells, left to right top to
+    # bottom.
+    topids = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33,
+        34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, # Cloths are left out
+        -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, # Gold/iron blocks? Doublestep? TNT from above?
+        36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, # Torch from above? leaving out fire. Redstone wire? Crops left out. sign post
+        -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, # door,ladder left out. Minecart rail orientation
+        66,69,72,-1,74 # clay?
+        ]
+
+    # And side textures of all block types
+    sideids = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33,
+        34,20,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+        -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35,
+        36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1,
+        -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67,
+        66,69,72,-1,74
+        ]
+
+    # This maps block id to the texture that goes on the side of the block
+    allimages = []
+    for toptextureid, sidetextureid in zip(topids, sideids):
+        if toptextureid == -1 or sidetextureid == -1:
+            allimages.append(None)
+            continue
+
+        toptexture = terrain_images[toptextureid]
+        sidetexture = terrain_images[sidetextureid]
+
+        img = _build_block(toptexture, sidetexture)
+
+        allimages.append((img.convert("RGB"), img.split()[3]))
 
-# Maps block images to the appropriate texture on each side. This map is not
-# appropriate for all block types
-blockmap = _build_blockimages()
     # Future block types:
-while len(blockmap) < 256:
+    while len(allimages) < 256:
-    blockmap.append(None)
+        allimages.append(None)
+    return allimages
+blockmap = _build_blockimages()
+
+def load_water():
+    """Evidentially, the water and lava textures are not loaded from any files
+    in the jar (that I can tell). They must be generated on the fly. While
+    terrain.png does have some water and lava cells, not all texture packs
+    include them. So I load them here from a couple pngs included.
+
+    This mutates the blockmap global list with the new water and lava blocks.
+    Block 9, standing water, is given a block with only the top face showing.
+    Block 8, flowing water, is given a full 3 sided cube."""
+
+    watertexture = _load_image("water.png")
+    w1 = _build_block(watertexture, None)
+    blockmap[9] = w1.convert("RGB"), w1
+    w2 = _build_block(watertexture, watertexture)
+    blockmap[8] = w2.convert("RGB"), w2
+
+    lavatexture = _load_image("lava.png")
+    lavablock = _build_block(lavatexture, lavatexture)
+    blockmap[10] = lavablock.convert("RGB"), lavablock
+    blockmap[11] = blockmap[10]
+load_water()

world.py

@@ -74,6 +74,8 @@ def render_chunks_async(chunks, caves, processes):
                 kwds=dict(cave=caves))
         resultsmap[(chunkx, chunky)] = result
 
+    pool.close()
+
     # Stick the pool object in the dict under the key "pool" so it isn't
     # garbage collected (which kills the subprocesses)
     resultsmap['pool'] = pool
@@ -326,7 +328,7 @@ def get_quadtree_depth(colstart, colend, rowstart, rowend):
     
     return p
 
-def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix):
+def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs):
     """Base call for quadtree_recurse. This sets up the recursion and generates
     a quadtree given a chunkmap and the ranges.
 
@@ -345,9 +347,12 @@ def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix):
     #print "     power is", p
     #print "     new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend)
 
-    quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base")
+    # procs is -1 here since the main process always runs as well, only spawn
+    # procs-1 /new/ processes
+    sem = multiprocessing.BoundedSemaphore(procs-1)
+    quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem)
 
-def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant):
+def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem):
     """Recursive method that generates a quadtree.
     A single call generates, saves, and returns an image with the range
     specified by colstart,colend,rowstart, and rowend.
@@ -382,6 +387,13 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr
 
     Each tile outputted is always 384 by 384 pixels.
 
+    The last parameter, sem, should be a multiprocessing.Semaphore or
+    BoundedSemaphore object. Before each recursive call, the semaphore is
+    acquired without blocking. If the acquire is successful, the recursive call
+    will spawn a new process. If it is not successful, the recursive call is
+    run in the same thread. The semaphore is passed to each recursive call, so
+    any call could spawn new processes if another one exits at some point.
+
     The return from this function is (path, hash) where path is the path to the
     file saved, and hash is a byte string that depends on the tile's contents.
     If the tile is blank, path will be None, but hash will still be valid.
@@ -476,18 +488,44 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr
         hasher = hashlib.md5()
 
         # Recurse to generate each quadrant of images
-        quad0file, hash0 = quadtree_recurse(chunkmap, 
+        # Quadrent 1:
-                colstart, colmid, rowstart, rowmid,
+        if sem.acquire(False):
-                newprefix, "0")
+            Procobj = ReturnableProcess
-        quad1file, hash1 = quadtree_recurse(chunkmap, 
+        else:
-                colmid, colend, rowstart, rowmid,
+            Procobj = FakeProcess
-                newprefix, "1")
+
-        quad2file, hash2 = quadtree_recurse(chunkmap, 
+        quad0result = Procobj(sem, target=quadtree_recurse,
-                colstart, colmid, rowmid, rowend,
+                args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem)
-                newprefix, "2")
+                )
+        quad0result.start()
+
+        if sem.acquire(False):
+            Procobj = ReturnableProcess
+        else:
+            Procobj = FakeProcess
+        quad1result = Procobj(sem, target=quadtree_recurse,
+                args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem)
+                )
+        quad1result.start()
+
+        if sem.acquire(False):
+            Procobj = ReturnableProcess
+        else:
+            Procobj = FakeProcess
+        quad2result = Procobj(sem, target=quadtree_recurse,
+                args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem)
+                )
+        quad2result.start()
+
+        # 3rd quadrent always runs in this process, no need to spawn a new one
+        # since we're just going to turn around and wait for it.
         quad3file, hash3 = quadtree_recurse(chunkmap, 
                 colmid, colend, rowmid, rowend,
-                newprefix, "3")
+                newprefix, "3", sem)
+
+        quad0file, hash0 = quad0result.get()
+        quad1file, hash1 = quad1result.get()
+        quad2file, hash2 = quad2result.get()
 
         #if dbg:
         #    print quad0file
@@ -567,3 +605,39 @@ def remove_tile(prefix, quadrent):
         os.unlink(img)
     if os.path.exists(hash):
         os.unlink(hash)
+
+class ReturnableProcess(multiprocessing.Process):
+    """Like the standard multiprocessing.Process class, but the return value of
+    the target method is available by calling get().
+    
+    The given semaphore is released when the target finishes running"""
+    def __init__(self, semaphore, *args, **kwargs):
+        self.__sem = semaphore
+        multiprocessing.Process.__init__(self, *args, **kwargs)
+
+    def run(self):
+        results = self._target(*self._args, **self._kwargs)
+        self._respipe_in.send(results)
+        self.__sem.release()
+
+    def get(self):
+        self.join()
+        return self._respipe_out.recv()
+
+    def start(self):
+        self._respipe_out, self._respipe_in = multiprocessing.Pipe()
+        multiprocessing.Process.start(self)
+
+class FakeProcess(object):
+    """Identical interface to the above class, but runs in the same thread.
+    Used to make the code simpler in quadtree_recurse
+
+    """
+    def __init__(self, semaphore, target, args=None, kwargs=None):
+        self._target = target
+        self._args = args if args else ()
+        self._kwargs = kwargs if kwargs else {}
+    def start(self):
+        self.ret = self._target(*self._args, **self._kwargs)
+    def get(self):
+        return self.ret