From 91440ec621eb22412d2062f93ec1807b156f7d9b Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 9 Sep 2010 22:41:28 -0400 Subject: [PATCH 1/5] subprocesses catch keyboardinterrupt and raise a proper exception. This causes the parent process to re-raise the exception when it eventually tries to get() the result from the child, instead of hanging forever on a ctrl-C. It doesn't happen immediately, which is not ideal, but at least things will eventually exit on ctrl-C now. --- chunk.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/chunk.py b/chunk.py index 3e9db04..ffdbbb9 100644 --- a/chunk.py +++ b/chunk.py @@ -52,6 +52,14 @@ def render_and_save(chunkfile, cave=False): import traceback traceback.print_exc() raise + except KeyboardInterrupt: + print + print "You pressed Ctrl-C. Unfortunately it got caught by a subprocess" + print "The program will terminate... eventually, but the main process" + print "may take a while to realize something went wrong." + print "To exit immediately, you'll need to kill this process some other" + print "way" + raise Exception() class ChunkRenderer(object): def __init__(self, chunkfile): From 1e296e858a8a1c2eb075f8ca6f3c61595683fb2d Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 10 Sep 2010 00:04:02 -0400 Subject: [PATCH 2/5] readme updates, texture code updates. Re-factored the way the textures and blocks are being built. It should be easier to understand and add new exceptions (sorta). Also fixed the water and lava with other texture packs by putting a static water.png and lava.png in with the code. --- README.rst | 29 +++++- chunk.py | 3 +- textures.py | 245 +++++++++++++++++++++++++++------------------ textures/lava.png | Bin 0 -> 401 bytes textures/water.png | Bin 0 -> 374 bytes world.py | 2 + 6 files changed, 175 insertions(+), 104 deletions(-) create mode 100644 textures/lava.png create mode 100644 textures/water.png diff --git a/README.rst b/README.rst index 64dabf8..a652933 100644 --- a/README.rst +++ b/README.rst @@ -96,8 +96,33 @@ render for my world from 85M to 67M. find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \; -Windows users, you're on your own, but there's probably a way to do this. (If -someone figures it out, let me know I'll update this README) +If you're on Windows, I've gotten word that this command line snippet works +provided pngout is installed and on your path. Note that the % symbols will +need to be doubled up if this is in a batch file. + +:: + + FOR /R c:\path\to\tiles\folder %v IN (*.png) DO pngout %v /y + +Viewing the Results +------------------- +The output is two things: an index.html file, and a directory hierarchy full of +images. To view your world, simply open index.html in a web browser. Internet +access is required to load the Google Maps API files, but you otherwise don't +need anything else. + +You can throw these files up to a web server to let others view your map. You +do not need a Google Maps API key (as was the case with older versions of the +API), so just copying the directory to your web server should suffice. + +Tip: Since Minecraft worlds rarely produce perfectly square worlds, there will +be blank and non-existent tiles around the borders of your world. The Google +Maps API has no way of knowing this until it requests them and the web server +returns a 404 Not Found. If this doesn't bother you, then fine, stop reading. +Otherwise: you can avoid a lot of 404s to your logs by configuring your web +server to redirect all 404 requests in that directory to a single 1px +"blank.png". This may or may not save on bandwidth, but it will probably save +on log noise. Using the Large Image Renderer ============================== diff --git a/chunk.py b/chunk.py index ffdbbb9..d32bc2c 100644 --- a/chunk.py +++ b/chunk.py @@ -6,7 +6,6 @@ import hashlib import nbt import textures -from textures import texturemap as txtarray # General note about pasting transparent image objects onto an image with an # alpha channel: @@ -244,7 +243,7 @@ class ChunkRenderer(object): if blockid not in transparent_blocks: draw = ImageDraw.Draw(img) if x != 15 and blocks[x+1,y,z] == 0: - draw.line(((imgx+12,imgy), (imgx+24,imgy+6)), fill=(0,0,0), width=1) + draw.line(((imgx+12,imgy), (imgx+22,imgy+5)), fill=(0,0,0), width=1) if y != 0 and blocks[x,y-1,z] == 0: draw.line(((imgx,imgy+6), (imgx+12,imgy)), fill=(0,0,0), width=1) diff --git a/textures.py b/textures.py index b857781..499b1a4 100644 --- a/textures.py +++ b/textures.py @@ -8,36 +8,70 @@ import math import numpy from PIL import Image, ImageEnhance -def _get_terrain_image(): - # Check the current directory for terrain.png first: - if os.path.isfile("terrain.png"): - return Image.open("terrain.png") +def _find_file(filename, mode="rb"): + """Searches for the given file and returns an open handle to it. + This searches the following locations in this order: + + * The program dir (same dir as this file) + * On Darwin, in /Applications/Minecraft + * Inside minecraft.jar, which is looked for at these locations - if "darwin" in sys.platform: - # On Macs, terrain.png could lie at - # "/Applications/minecraft/terrain.png" for custom terrain. Try this - # first. - png = "/Applications/Minecraft/terrain.png" - if os.access(png, os.F_OK): - return Image.open(png) + * On Windows, at %APPDATA%/.minecraft/bin/minecraft.jar + * On Darwin, at $HOME/Library/Application Support/minecraft/bin/minecraft.jar + * at $HOME/.minecraft/bin/minecraft.jar - # Paths on a Mac are a bit different - minecraftdir = os.path.join(os.environ['HOME'], "Library", - "Application Support", "minecraft") - minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, "bin", "minecraft.jar")) - textures = minecraftjar.open("terrain.png") + * The current working directory + * The program dir / textures - else: - if "win" in sys.platform: - minecraftdir = os.environ['APPDATA'] - else: - minecraftdir = os.environ['HOME'] - minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, ".minecraft", + """ + programdir = os.path.dirname(__file__) + path = os.path.join(programdir, filename) + if os.path.exists(path): + return open(path, mode) + + if sys.platform == "darwin": + path = os.path.join("/Applications/Minecraft", filename) + if os.path.exists(path): + return open(path, mode) + + # Find minecraft.jar. + jarpaths = [] + if "APPDATA" in os.environ: + jarpaths.append( os.path.join(os.environ['APPDATA'], ".minecraft", "bin", "minecraft.jar")) - textures = minecraftjar.open("terrain.png") - buffer = StringIO(textures.read()) + if "HOME" in os.environ: + jarpaths.append(os.path.join(os.environ['HOME'], "Library", + "Application Support", "minecraft")) + jarpaths.append(os.path.join(os.environ['HOME'], ".minecraft", "bin", + "minecraft.jar")) + + for jarpath in jarpaths: + if os.path.exists(jarpath): + jar = zipfile.ZipFile(jarpath) + try: + return jar.open(filename) + except KeyError: + pass + + path = filename + if os.path.exists(path): + return open(path, mode) + + path = os.path.join(programdir, "textures", filename) + if os.path.exists(path): + return open(path, mode) + + raise IOError("Could not find the file {0}".format(filename)) + +def _load_image(filename): + """Returns an image object""" + fileobj = _find_file(filename) + buffer = StringIO(fileobj.read()) return Image.open(buffer) +def _get_terrain_image(): + return _load_image("terrain.png") + def _split_terrain(terrain): """Builds and returns a length 256 array of each 16x16 chunk of texture""" textures = [] @@ -100,102 +134,113 @@ def _transform_image_side(img): return newimg -def _build_texturemap(): - """""" - t = terrain_images +def _build_block(top, side): + """From a top texture and a side texture, build a block image. + top and side should be 16x16 image objects. Returns a 24x24 image - # Notes are for things I've left out or will probably have to make special - # exception for - top = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33, + """ + img = Image.new("RGBA", (24,24)) + + top = _transform_image(top) + + if not side: + img.paste(top, (0,0), top) + return img + + side = _transform_image_side(side) + + otherside = side.transpose(Image.FLIP_LEFT_RIGHT) + + # Darken the sides slightly. These methods also affect the alpha layer, + # so save them first (we don't want to "darken" the alpha layer making + # the block transparent) + sidealpha = side.split()[3] + side = ImageEnhance.Brightness(side).enhance(0.9) + side.putalpha(sidealpha) + othersidealpha = otherside.split()[3] + otherside = ImageEnhance.Brightness(otherside).enhance(0.8) + otherside.putalpha(othersidealpha) + + img.paste(side, (0,6), side) + img.paste(otherside, (12,6), otherside) + img.paste(top, (0,0), top) + + # Manually touch up 6 pixels that leave a gap because of how the + # shearing works out. This makes the blocks perfectly tessellate-able + for x,y in [(13,23), (17,21), (21,19)]: + # Copy a pixel to x,y from x-1,y + img.putpixel((x,y), img.getpixel((x-1,y))) + for x,y in [(3,4), (7,2), (11,0)]: + # Copy a pixel to x,y from x+1,y + img.putpixel((x,y), img.getpixel((x+1,y))) + + return img + + +def _build_blockimages(): + """Returns a mapping from blockid to an image of that block in perspective + The values of the mapping are actually (image in RGB mode, alpha channel). + This is not appropriate for all block types, only block types that are + proper cubes""" + + # Top textures of all block types. The number here is the index in the + # texture array (terrain_images), which comes from terrain.png's cells, left to right top to + # bottom. + topids = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33, 34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, # Cloths are left out -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, # Gold/iron blocks? Doublestep? TNT from above? 36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, # Torch from above? leaving out fire. Redstone wire? Crops left out. sign post -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, # door,ladder left out. Minecart rail orientation 66,69,72,-1,74 # clay? ] - side = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33, - 34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + + # And side textures of all block types + sideids = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33, + 34,20,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, 36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, -1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, 66,69,72,-1,74 ] - side[2] = 2 - return ( - [(t[x] if x != -1 else None) for x in top], - [(_transform_image(t[x]) if x != -1 else None) for x in top], - [(_transform_image_side(t[x]) if x != -1 else None) for x in side], - ) -# texturemap maps block ids to a 16x16 image that goes on the top face -# perspective_texturemap does the same, except the texture is rotated and shrunk -# shear_texturemap maps block ids to the image that goes on the side of the -# block, sheared appropriately -texturemap, perspective_texturemap, shear_texturemap = _build_texturemap() - -def _render_sprite(img): - """Takes a 16x16 sprite image, and returns a 22x22 image to go in the - blockmap - This is for rendering things that are sticking out of the ground, like - flowers and such - torches are drawn the same way, but torches that attach to walls are - handled differently - """ - pass - -def _render_ground_image(img): - """Takes a 16x16 sprite image and skews it to look like it's on the ground. - This is for things like mine track and such - - """ - pass - -def _build_blockimages(): - """Returns a mapping from blockid to an image of that block in perspective - The values of the mapping are actually (image in RGB mode, alpha channel)""" # This maps block id to the texture that goes on the side of the block allimages = [] - for top, side in zip(perspective_texturemap, shear_texturemap): - if not top or not side: + for toptextureid, sidetextureid in zip(topids, sideids): + if toptextureid == -1 or sidetextureid == -1: allimages.append(None) continue - img = Image.new("RGBA", (24,24)) - - otherside = side.transpose(Image.FLIP_LEFT_RIGHT) - # Darken the sides slightly. These methods also affect the alpha layer, - # so save them first (we don't want to "darken" the alpha layer making - # the block transparent) - if 1: - sidealpha = side.split()[3] - side = ImageEnhance.Brightness(side).enhance(0.9) - side.putalpha(sidealpha) - othersidealpha = otherside.split()[3] - otherside = ImageEnhance.Brightness(otherside).enhance(0.8) - otherside.putalpha(othersidealpha) + toptexture = terrain_images[toptextureid] + sidetexture = terrain_images[sidetextureid] - # Copy on the left side - img.paste(side, (0,6), side) - # Copy on the other side - img.paste(otherside, (12,6), otherside) - # Copy on the top piece (last so it's on top) - img.paste(top, (0,0), top) - - # Manually touch up 6 pixels that leave a gap because of how the - # shearing works out. This makes the blocks perfectly tessellate-able - for x,y in [(13,23), (17,21), (21,19)]: - # Copy a pixel to x,y from x-1,y - img.putpixel((x,y), img.getpixel((x-1,y))) - for x,y in [(3,4), (7,2), (11,0)]: - # Copy a pixel to x,y from x+1,y - img.putpixel((x,y), img.getpixel((x+1,y))) + img = _build_block(toptexture, sidetexture) allimages.append((img.convert("RGB"), img.split()[3])) - return allimages -# Maps block images to the appropriate texture on each side. This map is not -# appropriate for all block types + # Future block types: + while len(allimages) < 256: + allimages.append(None) + return allimages blockmap = _build_blockimages() -# Future block types: -while len(blockmap) < 256: - blockmap.append(None) + +def load_water(): + """Evidentially, the water and lava textures are not loaded from any files + in the jar (that I can tell). They must be generated on the fly. While + terrain.png does have some water and lava cells, not all texture packs + include them. So I load them here from a couple pngs included. + + This mutates the blockmap global list with the new water and lava blocks. + Block 9, standing water, is given a block with only the top face showing. + Block 8, flowing water, is given a full 3 sided cube.""" + + watertexture = _load_image("water.png") + w1 = _build_block(watertexture, None) + blockmap[9] = w1.convert("RGB"), w1 + w2 = _build_block(watertexture, watertexture) + blockmap[8] = w2.convert("RGB"), w2 + + lavatexture = _load_image("lava.png") + lavablock = _build_block(lavatexture, lavatexture) + blockmap[10] = lavablock.convert("RGB"), lavablock + blockmap[11] = blockmap[10] +load_water() diff --git a/textures/lava.png b/textures/lava.png new file mode 100644 index 0000000000000000000000000000000000000000..b70c6c152088e2799e5fa5bbe81205c67f7c29ef GIT binary patch literal 401 zcmeAS@N?(olHy`uVBq!ia0y~yU=RRd4mJh`2Kmqb6B!s77>k44ofy`glX=O&z`&N| z?e4&kwMlZ#P+Ya3@<3j+g# zWQl7;iF1B#Zfaf$gL6@8Vo7R>LV0FMhJw4NZ$Nk>pEv^p!y8W*#}JFtrIXL{9#-IK z-S2j@-F6p4yU+23>|!s(WH#{&*|HqBH<_lR$n&lG{I}HyJz7uib7&Rbs?uI_^uY0# zGcW%Ba^^+My$hfJsvVK;;ce(wv&nthvweHN`h%;979ElD#XkSjLp36Hh^?wHEO*+f zA)7bvXnmQK$HXK%Z>I3Mp-beBZedkz^bQhnJRF`an49?KhTZbP0l+XkKPPU!{ literal 0 HcmV?d00001 diff --git a/textures/water.png b/textures/water.png new file mode 100644 index 0000000000000000000000000000000000000000..2c0e69f3471246184a3cb2b3e8f7d98919a4086d GIT binary patch literal 374 zcmeAS@N?(olHy`uVBq!ia0y~yU=RRd4mJh`2Kmqb6B!s77>k44ofy`glX=O&z`&N| z?e4&kwMlZ%nx^sMPpbp{3o z$r9Iy66gHf+|;}h2Ir#G#FEq$h4Rdj37L3lj- zGu&C4&mGFH@m_6b z#pP$O+g90>DyDw8Gd=F$!n9ANuJi6WxyeMHzvr=w)qjro6>Z*(fA@7)oL^}Cjdfns Xobs%bt;-k~7#KWV{an^LB{Ts5N|28W literal 0 HcmV?d00001 diff --git a/world.py b/world.py index 4d36efe..c7d084c 100644 --- a/world.py +++ b/world.py @@ -74,6 +74,8 @@ def render_chunks_async(chunks, caves, processes): kwds=dict(cave=caves)) resultsmap[(chunkx, chunky)] = result + pool.close() + # Stick the pool object in the dict under the key "pool" so it isn't # garbage collected (which kills the subprocesses) resultsmap['pool'] = pool From f9783d7a20e0de7fff60d08e195af03f49c665a8 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sat, 11 Sep 2010 00:12:38 -0400 Subject: [PATCH 3/5] Tile rendering is now mostly parallel up to 4 procs. The initial recursive call for tile generation will spawn up to 3 extra processes to work on each quadrant. It's not perfect yet since some quadrants may have more or less work to do, and only 4 total workers are supported. Also, it waits for all chunks are finished before it dives into the tiles, to prevent it from using more resources than requested. --- gmap.py | 7 +++-- world.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/gmap.py b/gmap.py index 5578f30..1101e1c 100755 --- a/gmap.py +++ b/gmap.py @@ -35,8 +35,11 @@ def main(): # Translate chunks from diagonal coordinate system mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks) - print "processing chunks in background" + print "Rendering chunks" results = world.render_chunks_async(chunks, False, options.procs) + for i, (col, row, filename) in enumerate(chunks): + results[col, row].wait() + print "{0}/{1} chunks rendered".format(i, len(chunks)) print "Writing out html file" if not os.path.exists(destdir): @@ -49,7 +52,7 @@ def main(): tiledir = os.path.join(destdir, "tiles") if not os.path.exists(tiledir): os.mkdir(tiledir) - world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir) + world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir, options.procs) print "DONE" diff --git a/world.py b/world.py index c7d084c..5a077a2 100644 --- a/world.py +++ b/world.py @@ -328,7 +328,7 @@ def get_quadtree_depth(colstart, colend, rowstart, rowend): return p -def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix): +def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs): """Base call for quadtree_recurse. This sets up the recursion and generates a quadtree given a chunkmap and the ranges. @@ -347,9 +347,11 @@ def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix): #print " power is", p #print " new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend) - quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base") + # procs is -1 here since the main process always runs as well, only spawn + # procs-1 /new/ processes + quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", procs-1) -def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant): +def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, procs): """Recursive method that generates a quadtree. A single call generates, saves, and returns an image with the range specified by colstart,colend,rowstart, and rowend. @@ -478,18 +480,47 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr hasher = hashlib.md5() # Recurse to generate each quadrant of images - quad0file, hash0 = quadtree_recurse(chunkmap, - colstart, colmid, rowstart, rowmid, - newprefix, "0") - quad1file, hash1 = quadtree_recurse(chunkmap, - colmid, colend, rowstart, rowmid, - newprefix, "1") - quad2file, hash2 = quadtree_recurse(chunkmap, - colstart, colmid, rowmid, rowend, - newprefix, "2") + # Quadrent 1: + if procs > 0: + Procobj = ReturnableProcess + procs -= 1 + else: + Procobj = FakeProcess + + quad0result = Procobj(target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", 0) + ) + quad0result.start() + + if procs > 0: + Procobj = ReturnableProcess + procs -= 1 + else: + Procobj = FakeProcess + quad1result = Procobj(target=quadtree_recurse, + args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", 0) + ) + quad1result.start() + + if procs > 0: + Procobj = ReturnableProcess + procs -= 1 + else: + Procobj = FakeProcess + quad2result = Procobj(target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", 0) + ) + quad2result.start() + + # 3rd quadrent always runs in this process, no need to spawn a new one + # since we're just going to turn around and wait for it. quad3file, hash3 = quadtree_recurse(chunkmap, colmid, colend, rowmid, rowend, - newprefix, "3") + newprefix, "3", 0) + + quad0file, hash0 = quad0result.get() + quad1file, hash1 = quad1result.get() + quad2file, hash2 = quad2result.get() #if dbg: # print quad0file @@ -569,3 +600,32 @@ def remove_tile(prefix, quadrent): os.unlink(img) if os.path.exists(hash): os.unlink(hash) + +class ReturnableProcess(multiprocessing.Process): + """Like the standard multiprocessing.Process class, but the return value of + the target method is available by calling get()""" + def run(self): + results = self._target(*self._args, **self._kwargs) + self._respipe_in.send(results) + + def get(self): + self.join() + return self._respipe_out.recv() + + def start(self): + self._respipe_out, self._respipe_in = multiprocessing.Pipe() + multiprocessing.Process.start(self) + +class FakeProcess(object): + """Identical interface to the above class, but runs in the same thread. + Used to make the code simpler in quadtree_recurse + + """ + def __init__(self, target, args=None, kwargs=None): + self._target = target + self._args = args if args else () + self._kwargs = kwargs if kwargs else {} + def start(self): + return + def get(self): + return self._target(*self._args, **self._kwargs) From be146385e4d8f64b18c5989fe49371b5e7716e0e Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sat, 11 Sep 2010 00:36:43 -0400 Subject: [PATCH 4/5] readme updates --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index a652933..658c6e9 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,9 @@ Features * Outputs a Google Map powered interface that is memory efficient, both in generating and viewing. +* Renders efficiently in parallel, using as many simultaneous processes as you + want! + * Utilizes 2 levels of caching to speed up subsequent renderings of your world. * Throw the output directory up on a web server to share your Minecraft world @@ -34,8 +37,8 @@ This program requires: * PIL (Python Imaging Library) * Numpy -I developed and tested this on Linux. It has been reported to work on Windows -and Mac, but if something doesn't, let me know. +I develop and test this on Linux, but need help testing it on Windows and Mac. +If something doesn't work, let me know. Using the Google Map Tile Generator =================================== @@ -77,9 +80,7 @@ greatly speeds up the rendering. Using more Cores ---------------- Adding the "-p" option will utilize more cores to generate the chunk files. -This can speed up rendering quite a bit. However, the tile generation routine -is currently serial and not written to take advantage of multiple cores. This -option will only affect the chunk generation (which is around half the process) +This can speed up rendering quite a bit. Example:: From 9309fd6c9687ae86f18f242b5d0e14f4b03b0f46 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sat, 11 Sep 2010 13:21:13 -0400 Subject: [PATCH 5/5] Uses a shared semaphore to spawn new processes when needed. This more effectively utilizes as many cores as you tell it. It should now spawn a new process whenever an old branch of the recursive tree finishes, to always use as many processes as you specify. --- world.py | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/world.py b/world.py index 5a077a2..79aed63 100644 --- a/world.py +++ b/world.py @@ -349,9 +349,10 @@ def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, proc # procs is -1 here since the main process always runs as well, only spawn # procs-1 /new/ processes - quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", procs-1) + sem = multiprocessing.BoundedSemaphore(procs-1) + quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem) -def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, procs): +def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem): """Recursive method that generates a quadtree. A single call generates, saves, and returns an image with the range specified by colstart,colend,rowstart, and rowend. @@ -386,6 +387,13 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr Each tile outputted is always 384 by 384 pixels. + The last parameter, sem, should be a multiprocessing.Semaphore or + BoundedSemaphore object. Before each recursive call, the semaphore is + acquired without blocking. If the acquire is successful, the recursive call + will spawn a new process. If it is not successful, the recursive call is + run in the same thread. The semaphore is passed to each recursive call, so + any call could spawn new processes if another one exits at some point. + The return from this function is (path, hash) where path is the path to the file saved, and hash is a byte string that depends on the tile's contents. If the tile is blank, path will be None, but hash will still be valid. @@ -481,34 +489,31 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr # Recurse to generate each quadrant of images # Quadrent 1: - if procs > 0: + if sem.acquire(False): Procobj = ReturnableProcess - procs -= 1 else: Procobj = FakeProcess - quad0result = Procobj(target=quadtree_recurse, - args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", 0) + quad0result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem) ) quad0result.start() - if procs > 0: + if sem.acquire(False): Procobj = ReturnableProcess - procs -= 1 else: Procobj = FakeProcess - quad1result = Procobj(target=quadtree_recurse, - args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", 0) + quad1result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem) ) quad1result.start() - if procs > 0: + if sem.acquire(False): Procobj = ReturnableProcess - procs -= 1 else: Procobj = FakeProcess - quad2result = Procobj(target=quadtree_recurse, - args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", 0) + quad2result = Procobj(sem, target=quadtree_recurse, + args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem) ) quad2result.start() @@ -516,7 +521,7 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr # since we're just going to turn around and wait for it. quad3file, hash3 = quadtree_recurse(chunkmap, colmid, colend, rowmid, rowend, - newprefix, "3", 0) + newprefix, "3", sem) quad0file, hash0 = quad0result.get() quad1file, hash1 = quad1result.get() @@ -603,10 +608,17 @@ def remove_tile(prefix, quadrent): class ReturnableProcess(multiprocessing.Process): """Like the standard multiprocessing.Process class, but the return value of - the target method is available by calling get()""" + the target method is available by calling get(). + + The given semaphore is released when the target finishes running""" + def __init__(self, semaphore, *args, **kwargs): + self.__sem = semaphore + multiprocessing.Process.__init__(self, *args, **kwargs) + def run(self): results = self._target(*self._args, **self._kwargs) self._respipe_in.send(results) + self.__sem.release() def get(self): self.join() @@ -621,11 +633,11 @@ class FakeProcess(object): Used to make the code simpler in quadtree_recurse """ - def __init__(self, target, args=None, kwargs=None): + def __init__(self, semaphore, target, args=None, kwargs=None): self._target = target self._args = args if args else () self._kwargs = kwargs if kwargs else {} def start(self): - return + self.ret = self._target(*self._args, **self._kwargs) def get(self): - return self._target(*self._args, **self._kwargs) + return self.ret