diff --git a/README.rst b/README.rst index 64dabf8..658c6e9 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,9 @@ Features * Outputs a Google Map powered interface that is memory efficient, both in generating and viewing. +* Renders efficiently in parallel, using as many simultaneous processes as you + want! + * Utilizes 2 levels of caching to speed up subsequent renderings of your world. * Throw the output directory up on a web server to share your Minecraft world @@ -34,8 +37,8 @@ This program requires: * PIL (Python Imaging Library) * Numpy -I developed and tested this on Linux. It has been reported to work on Windows -and Mac, but if something doesn't, let me know. +I develop and test this on Linux, but need help testing it on Windows and Mac. +If something doesn't work, let me know. Using the Google Map Tile Generator =================================== @@ -77,9 +80,7 @@ greatly speeds up the rendering. Using more Cores ---------------- Adding the "-p" option will utilize more cores to generate the chunk files. -This can speed up rendering quite a bit. However, the tile generation routine -is currently serial and not written to take advantage of multiple cores. This -option will only affect the chunk generation (which is around half the process) +This can speed up rendering quite a bit. Example:: @@ -96,8 +97,33 @@ render for my world from 85M to 67M. find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \; -Windows users, you're on your own, but there's probably a way to do this. (If -someone figures it out, let me know I'll update this README) +If you're on Windows, I've gotten word that this command line snippet works +provided pngout is installed and on your path. Note that the % symbols will +need to be doubled up if this is in a batch file. + +:: + + FOR /R c:\path\to\tiles\folder %v IN (*.png) DO pngout %v /y + +Viewing the Results +------------------- +The output is two things: an index.html file, and a directory hierarchy full of +images. To view your world, simply open index.html in a web browser. Internet +access is required to load the Google Maps API files, but you otherwise don't +need anything else. + +You can throw these files up to a web server to let others view your map. You +do not need a Google Maps API key (as was the case with older versions of the +API), so just copying the directory to your web server should suffice. + +Tip: Since Minecraft worlds rarely produce perfectly square worlds, there will +be blank and non-existent tiles around the borders of your world. The Google +Maps API has no way of knowing this until it requests them and the web server +returns a 404 Not Found. If this doesn't bother you, then fine, stop reading. +Otherwise: you can avoid a lot of 404s to your logs by configuring your web +server to redirect all 404 requests in that directory to a single 1px +"blank.png". This may or may not save on bandwidth, but it will probably save +on log noise. Using the Large Image Renderer ============================== diff --git a/chunk.py b/chunk.py index 3e9db04..d32bc2c 100644 --- a/chunk.py +++ b/chunk.py @@ -6,7 +6,6 @@ import hashlib import nbt import textures -from textures import texturemap as txtarray # General note about pasting transparent image objects onto an image with an # alpha channel: @@ -52,6 +51,14 @@ def render_and_save(chunkfile, cave=False): import traceback traceback.print_exc() raise + except KeyboardInterrupt: + print + print "You pressed Ctrl-C. Unfortunately it got caught by a subprocess" + print "The program will terminate... eventually, but the main process" + print "may take a while to realize something went wrong." + print "To exit immediately, you'll need to kill this process some other" + print "way" + raise Exception() class ChunkRenderer(object): def __init__(self, chunkfile): @@ -236,7 +243,7 @@ class ChunkRenderer(object): if blockid not in transparent_blocks: draw = ImageDraw.Draw(img) if x != 15 and blocks[x+1,y,z] == 0: - draw.line(((imgx+12,imgy), (imgx+24,imgy+6)), fill=(0,0,0), width=1) + draw.line(((imgx+12,imgy), (imgx+22,imgy+5)), fill=(0,0,0), width=1) if y != 0 and blocks[x,y-1,z] == 0: draw.line(((imgx,imgy+6), (imgx+12,imgy)), fill=(0,0,0), width=1) diff --git a/gmap.py b/gmap.py index 3d04e4d..6eba965 100755 --- a/gmap.py +++ b/gmap.py @@ -35,8 +35,11 @@ def main(): # Translate chunks from diagonal coordinate system mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks) - print "processing chunks in background" + print "Rendering chunks" results = world.render_chunks_async(chunks, False, options.procs) + for i, (col, row, filename) in enumerate(chunks): + results[col, row].wait() + print "{0}/{1} chunks rendered".format(i, len(chunks)) print "Writing out html file" if not os.path.exists(destdir): @@ -49,7 +52,7 @@ def main(): tiledir = os.path.join(destdir, "tiles") if not os.path.exists(tiledir): os.mkdir(tiledir) - world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir) + world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir, options.procs) print "DONE" diff --git a/textures.py b/textures.py index b857781..499b1a4 100644 --- a/textures.py +++ b/textures.py @@ -8,36 +8,70 @@ import math import numpy from PIL import Image, ImageEnhance -def _get_terrain_image(): - # Check the current directory for terrain.png first: - if os.path.isfile("terrain.png"): - return Image.open("terrain.png") +def _find_file(filename, mode="rb"): + """Searches for the given file and returns an open handle to it. + This searches the following locations in this order: + + * The program dir (same dir as this file) + * On Darwin, in /Applications/Minecraft + * Inside minecraft.jar, which is looked for at these locations - if "darwin" in sys.platform: - # On Macs, terrain.png could lie at - # "/Applications/minecraft/terrain.png" for custom terrain. Try this - # first. - png = "/Applications/Minecraft/terrain.png" - if os.access(png, os.F_OK): - return Image.open(png) + * On Windows, at %APPDATA%/.minecraft/bin/minecraft.jar + * On Darwin, at $HOME/Library/Application Support/minecraft/bin/minecraft.jar + * at $HOME/.minecraft/bin/minecraft.jar - # Paths on a Mac are a bit different - minecraftdir = os.path.join(os.environ['HOME'], "Library", - "Application Support", "minecraft") - minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, "bin", "minecraft.jar")) - textures = minecraftjar.open("terrain.png") + * The current working directory + * The program dir / textures - else: - if "win" in sys.platform: - minecraftdir = os.environ['APPDATA'] - else: - minecraftdir = os.environ['HOME'] - minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, ".minecraft", + """ + programdir = os.path.dirname(__file__) + path = os.path.join(programdir, filename) + if os.path.exists(path): + return open(path, mode) + + if sys.platform == "darwin": + path = os.path.join("/Applications/Minecraft", filename) + if os.path.exists(path): + return open(path, mode) + + # Find minecraft.jar. + jarpaths = [] + if "APPDATA" in os.environ: + jarpaths.append( os.path.join(os.environ['APPDATA'], ".minecraft", "bin", "minecraft.jar")) - textures = minecraftjar.open("terrain.png") - buffer = StringIO(textures.read()) + if "HOME" in os.environ: + jarpaths.append(os.path.join(os.environ['HOME'], "Library", + "Application Support", "minecraft")) + jarpaths.append(os.path.join(os.environ['HOME'], ".minecraft", "bin", + "minecraft.jar")) + + for jarpath in jarpaths: + if os.path.exists(jarpath): + jar = zipfile.ZipFile(jarpath) + try: + return jar.open(filename) + except KeyError: + pass + + path = filename + if os.path.exists(path): + return open(path, mode) + + path = os.path.join(programdir, "textures", filename) + if os.path.exists(path): + return open(path, mode) + + raise IOError("Could not find the file {0}".format(filename)) + +def _load_image(filename): + """Returns an image object""" + fileobj = _find_file(filename) + buffer = StringIO(fileobj.read()) return Image.open(buffer) +def _get_terrain_image(): + return _load_image("terrain.png") + def _split_terrain(terrain): """Builds and returns a length 256 array of each 16x16 chunk of texture""" textures = [] @@ -100,102 +134,113 @@ def _transform_image_side(img): return newimg -def _build_texturemap(): - """""" - t = terrain_images +def _build_block(top, side): + """From a top texture and a side texture, build a block image. + top and side should be 16x16 image objects. Returns a 24x24 image - # Notes are for things I've left out or will probably have to make special - # exception for - top = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33, + """ + img = Image.new("RGBA", (24,24)) + + top = _transform_image(top) + + if not side: + img.paste(top, (0,0), top) + return img + + side = _transform_image_side(side) + + otherside = side.transpose(Image.FLIP_LEFT_RIGHT) + + # Darken the sides slightly. These methods also affect the alpha layer, + # so save them first (we don't want to "darken" the alpha layer making + # the block transparent) + sidealpha = side.split()[3] + side = ImageEnhance.Brightness(side).enhance(0.9) + side.putalpha(sidealpha) + othersidealpha = otherside.split()[3] + otherside = ImageEnhance.Brightness(otherside).enhance(0.8) + otherside.putalpha(othersidealpha) + + img.paste(side, (0,6), side) + img.paste(otherside, (12,6), otherside) + img.paste(top, (0,0), top) + + # Manually touch up 6 pixels that leave a gap because of how the + # shearing works out. This makes the blocks perfectly tessellate-able + for x,y in [(13,23), (17,21), (21,19)]: + # Copy a pixel to x,y from x-1,y + img.putpixel((x,y), img.getpixel((x-1,y))) + for x,y in [(3,4), (7,2), (11,0)]: + # Copy a pixel to x,y from x+1,y + img.putpixel((x,y), img.getpixel((x+1,y))) + + return img + + +def _build_blockimages(): + """Returns a mapping from blockid to an image of that block in perspective + The values of the mapping are actually (image in RGB mode, alpha channel). + This is not appropriate for all block types, only block types that are + proper cubes""" + + # Top textures of all block types. The number here is the index in the + # texture array (terrain_images), which comes from terrain.png's cells, left to right top to + # bottom. + topids = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33, 34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, # Cloths are left out -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, # Gold/iron blocks? Doublestep? TNT from above? 36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, # Torch from above? leaving out fire. Redstone wire? Crops left out. sign post -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, # door,ladder left out. Minecart rail orientation 66,69,72,-1,74 # clay? ] - side = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33, - 34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + + # And side textures of all block types + sideids = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33, + 34,20,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, 36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, 66,69,72,-1,74 ] - side[2] = 2 - return ( - [(t[x] if x != -1 else None) for x in top], - [(_transform_image(t[x]) if x != -1 else None) for x in top], - [(_transform_image_side(t[x]) if x != -1 else None) for x in side], - ) -# texturemap maps block ids to a 16x16 image that goes on the top face -# perspective_texturemap does the same, except the texture is rotated and shrunk -# shear_texturemap maps block ids to the image that goes on the side of the -# block, sheared appropriately -texturemap, perspective_texturemap, shear_texturemap = _build_texturemap() - -def _render_sprite(img): - """Takes a 16x16 sprite image, and returns a 22x22 image to go in the - blockmap - This is for rendering things that are sticking out of the ground, like - flowers and such - torches are drawn the same way, but torches that attach to walls are - handled differently - """ - pass - -def _render_ground_image(img): - """Takes a 16x16 sprite image and skews it to look like it's on the ground. - This is for things like mine track and such - - """ - pass - -def _build_blockimages(): - """Returns a mapping from blockid to an image of that block in perspective - The values of the mapping are actually (image in RGB mode, alpha channel)""" # This maps block id to the texture that goes on the side of the block allimages = [] - for top, side in zip(perspective_texturemap, shear_texturemap): - if not top or not side: + for toptextureid, sidetextureid in zip(topids, sideids): + if toptextureid == -1 or sidetextureid == -1: allimages.append(None) continue - img = Image.new("RGBA", (24,24)) - - otherside = side.transpose(Image.FLIP_LEFT_RIGHT) - # Darken the sides slightly. These methods also affect the alpha layer, - # so save them first (we don't want to "darken" the alpha layer making - # the block transparent) - if 1: - sidealpha = side.split()[3] - side = ImageEnhance.Brightness(side).enhance(0.9) - side.putalpha(sidealpha) - othersidealpha = otherside.split()[3] - otherside = ImageEnhance.Brightness(otherside).enhance(0.8) - otherside.putalpha(othersidealpha) + toptexture = terrain_images[toptextureid] + sidetexture = terrain_images[sidetextureid] - # Copy on the left side - img.paste(side, (0,6), side) - # Copy on the other side - img.paste(otherside, (12,6), otherside) - # Copy on the top piece (last so it's on top) - img.paste(top, (0,0), top) - - # Manually touch up 6 pixels that leave a gap because of how the - # shearing works out. This makes the blocks perfectly tessellate-able - for x,y in [(13,23), (17,21), (21,19)]: - # Copy a pixel to x,y from x-1,y - img.putpixel((x,y), img.getpixel((x-1,y))) - for x,y in [(3,4), (7,2), (11,0)]: - # Copy a pixel to x,y from x+1,y - img.putpixel((x,y), img.getpixel((x+1,y))) + img = _build_block(toptexture, sidetexture) allimages.append((img.convert("RGB"), img.split()[3])) - return allimages -# Maps block images to the appropriate texture on each side. This map is not -# appropriate for all block types + # Future block types: + while len(allimages) < 256: + allimages.append(None) + return allimages blockmap = _build_blockimages() -# Future block types: -while len(blockmap) < 256: - blockmap.append(None) + +def load_water(): + """Evidentially, the water and lava textures are not loaded from any files + in the jar (that I can tell). They must be generated on the fly. While + terrain.png does have some water and lava cells, not all texture packs + include them. So I load them here from a couple pngs included. + + This mutates the blockmap global list with the new water and lava blocks. + Block 9, standing water, is given a block with only the top face showing. + Block 8, flowing water, is given a full 3 sided cube.""" + + watertexture = _load_image("water.png") + w1 = _build_block(watertexture, None) + blockmap[9] = w1.convert("RGB"), w1 + w2 = _build_block(watertexture, watertexture) + blockmap[8] = w2.convert("RGB"), w2 + + lavatexture = _load_image("lava.png") + lavablock = _build_block(lavatexture, lavatexture) + blockmap[10] = lavablock.convert("RGB"), lavablock + blockmap[11] = blockmap[10] +load_water() diff --git a/textures/lava.png b/textures/lava.png new file mode 100644 index 0000000..b70c6c1 Binary files /dev/null and b/textures/lava.png differ diff --git a/textures/water.png b/textures/water.png new file mode 100644 index 0000000..2c0e69f Binary files /dev/null and b/textures/water.png differ diff --git a/world.py b/world.py index 4d36efe..79aed63 100644 --- a/world.py +++ b/world.py @@ -74,6 +74,8 @@ def render_chunks_async(chunks, caves, processes): kwds=dict(cave=caves)) resultsmap[(chunkx, chunky)] = result + pool.close() + # Stick the pool object in the dict under the key "pool" so it isn't # garbage collected (which kills the subprocesses) resultsmap['pool'] = pool @@ -326,7 +328,7 @@ def get_quadtree_depth(colstart, colend, rowstart, rowend): return p -def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix): +def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs): """Base call for quadtree_recurse. This sets up the recursion and generates a quadtree given a chunkmap and the ranges. @@ -345,9 +347,12 @@ def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix): #print " power is", p #print " new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend) - quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base") + # procs is -1 here since the main process always runs as well, only spawn + # procs-1 /new/ processes + sem = multiprocessing.BoundedSemaphore(procs-1) + quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem) -def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant): +def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem): """Recursive method that generates a quadtree. A single call generates, saves, and returns an image with the range specified by colstart,colend,rowstart, and rowend. @@ -382,6 +387,13 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr Each tile outputted is always 384 by 384 pixels. + The last parameter, sem, should be a multiprocessing.Semaphore or + BoundedSemaphore object. Before each recursive call, the semaphore is + acquired without blocking. If the acquire is successful, the recursive call + will spawn a new process. If it is not successful, the recursive call is + run in the same thread. The semaphore is passed to each recursive call, so + any call could spawn new processes if another one exits at some point. + The return from this function is (path, hash) where path is the path to the file saved, and hash is a byte string that depends on the tile's contents. If the tile is blank, path will be None, but hash will still be valid. @@ -476,18 +488,44 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr hasher = hashlib.md5() # Recurse to generate each quadrant of images - quad0file, hash0 = quadtree_recurse(chunkmap, - colstart, colmid, rowstart, rowmid, - newprefix, "0") - quad1file, hash1 = quadtree_recurse(chunkmap, - colmid, colend, rowstart, rowmid, - newprefix, "1") - quad2file, hash2 = quadtree_recurse(chunkmap, - colstart, colmid, rowmid, rowend, - newprefix, "2") + # Quadrent 1: + if sem.acquire(False): + Procobj = ReturnableProcess + else: + Procobj = FakeProcess + + quad0result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem) + ) + quad0result.start() + + if sem.acquire(False): + Procobj = ReturnableProcess + else: + Procobj = FakeProcess + quad1result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem) + ) + quad1result.start() + + if sem.acquire(False): + Procobj = ReturnableProcess + else: + Procobj = FakeProcess + quad2result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem) + ) + quad2result.start() + + # 3rd quadrent always runs in this process, no need to spawn a new one + # since we're just going to turn around and wait for it. quad3file, hash3 = quadtree_recurse(chunkmap, colmid, colend, rowmid, rowend, - newprefix, "3") + newprefix, "3", sem) + + quad0file, hash0 = quad0result.get() + quad1file, hash1 = quad1result.get() + quad2file, hash2 = quad2result.get() #if dbg: # print quad0file @@ -567,3 +605,39 @@ def remove_tile(prefix, quadrent): os.unlink(img) if os.path.exists(hash): os.unlink(hash) + +class ReturnableProcess(multiprocessing.Process): + """Like the standard multiprocessing.Process class, but the return value of + the target method is available by calling get(). + + The given semaphore is released when the target finishes running""" + def __init__(self, semaphore, *args, **kwargs): + self.__sem = semaphore + multiprocessing.Process.__init__(self, *args, **kwargs) + + def run(self): + results = self._target(*self._args, **self._kwargs) + self._respipe_in.send(results) + self.__sem.release() + + def get(self): + self.join() + return self._respipe_out.recv() + + def start(self): + self._respipe_out, self._respipe_in = multiprocessing.Pipe() + multiprocessing.Process.start(self) + +class FakeProcess(object): + """Identical interface to the above class, but runs in the same thread. + Used to make the code simpler in quadtree_recurse + + """ + def __init__(self, semaphore, target, args=None, kwargs=None): + self._target = target + self._args = args if args else () + self._kwargs = kwargs if kwargs else {} + def start(self): + self.ret = self._target(*self._args, **self._kwargs) + def get(self): + return self.ret