From 2d4f0cc08230804932b3e0a4dfc97c434504c0d4 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Tue, 14 Sep 2010 23:51:05 -0400 Subject: [PATCH] I believe I now have a usable program again --- gmap.py | 62 ++------- quadtree.py | 362 +++----------------------------------------------- template.html | 2 +- world.py | 1 + 4 files changed, 31 insertions(+), 396 deletions(-) diff --git a/gmap.py b/gmap.py index 26ad620..7165df6 100755 --- a/gmap.py +++ b/gmap.py @@ -5,17 +5,22 @@ import sys import os.path from optparse import OptionParser import re +import multiprocessing import world +import quadtree helptext = """ %prog [-p PROCS] """ def main(): + try: + cpus = multiprocessing.cpu_count() + except NotImplementedError: + cpus = 1 parser = OptionParser(usage=helptext) - parser.add_option("-p", "--processes", dest="procs", help="How many chunks to render in parallel. A good number for this is 1 more than the number of cores in your computer. Default 2", default=2, action="store", type="int") - parser.add_option("-c", "--cachelife", dest="cachelife", help="How many minutes a tile will be considered valid by the web browser before it fetches a new copy. Used if you have a crontab or similar running this every once in a while. Default is no expiration.", default=0, action="store", type="int") + parser.add_option("-p", "--processes", dest="procs", help="How many chunks to render in parallel. A good number for this is the number of cores in your computer. Default %s" % cpus, default=cpus, action="store", type="int") options, args = parser.parse_args() @@ -29,54 +34,13 @@ def main(): parser.error("Where do you want to save the tiles?") destdir = args[1] - print "Scanning chunks" - all_chunks = world.find_chunkfiles(worlddir) + # First generate the world's chunk images + w = world.WorldRenderer(worlddir) + w.go(options.procs) - # Translate chunks from diagonal coordinate system - mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks) - - print "Rendering chunks" - results = world.render_chunks_async(chunks, False, options.procs) - if options.procs > 1: - for i, (col, row, filename) in enumerate(chunks): - if i > 0: - if 1000 % i == 0 or i % 1000 == 0: - print "{0}/{1} chunks rendered".format(i, len(chunks)) - - results['pool'].join() - print "Done" - # Compat-change for windows (which can't pass result objects to - # subprocesses) - chunkmap = {} - for col, row, filename in chunks: - chunkmap[col, row] = results[col, row].get() - del results - - print "Writing out html file" - if not os.path.exists(destdir): - os.mkdir(destdir) - zoom = world.get_quadtree_depth(mincol, maxcol, minrow, maxrow) - write_html(destdir, zoom+1, options.cachelife) - print "Your map will have {0} zoom levels".format(zoom+1) - - print "Generating quad tree. This may take a while and has no progress bar right now, so sit tight." - tiledir = os.path.join(destdir, "tiles") - if not os.path.exists(tiledir): - os.mkdir(tiledir) - world.generate_quadtree(chunkmap, mincol, maxcol, minrow, maxrow, tiledir, options.procs) - - print "DONE" - -def write_html(path, zoomlevel, cachelife): - templatepath = os.path.join(os.path.split(__file__)[0], "template.html") - html = open(templatepath, 'r').read() - html = html.replace( - "{maxzoom}", str(zoomlevel)) - html = html.replace( - "{cachelife}", str(cachelife)) - - with open(os.path.join(path, "index.html"), 'w') as output: - output.write(html) + # Now generate the tiles + q = quadtree.QuadtreeGen(w, destdir) + q.go(options.procs) if __name__ == "__main__": main() diff --git a/quadtree.py b/quadtree.py index 3346627..82f61e6 100644 --- a/quadtree.py +++ b/quadtree.py @@ -61,6 +61,16 @@ class QuadtreeGen(object): self.world = worldobj self.destdir = destdir + def write_html(self, zoomlevel): + """Writes out index.html""" + templatepath = os.path.join(os.path.split(__file__)[0], "template.html") + html = open(templatepath, 'r').read() + html = html.replace( + "{maxzoom}", str(zoomlevel)) + + with open(os.path.join(self.destdir, "index.html"), 'w') as output: + output.write(html) + def go(self, procs): """Renders all tiles""" @@ -69,6 +79,7 @@ class QuadtreeGen(object): # Render the highest level of tiles from the chunks print "Computing the tile ranges and starting tile processers for inner-most tiles..." + print "This takes the longest. The other levels will go quicker" results = [] for path in iterate_base4(self.p+1): # Get the range for this tile @@ -97,6 +108,8 @@ class QuadtreeGen(object): ) ) + self.write_html(self.p+1) + # Wait for all results to finish print "Rendering inner most zoom level tiles now!" for i, result in enumerate(results): @@ -117,7 +130,6 @@ class QuadtreeGen(object): dest = os.path.join(self.destdir, *(str(x) for x in path[:-1])) name = str(path[-1]) - print "Applying", path, dest, name results.append( pool.apply_async(func=render_innertile, args= (dest, name) @@ -136,6 +148,9 @@ class QuadtreeGen(object): render_innertile(self.destdir, "base") print "Done!" + pool.close() + pool.join() + def _get_range_by_path(self, path): """Returns the x, y chunk coordinates of this tile""" x, y = self.mincol, self.minrow @@ -212,8 +227,6 @@ def render_innertile(dest, name): os.unlink(imgpath) if os.path.exists(hashpath): os.unlink(hashpath) - print "Not generating due to non-existance of subtiles" - print "\t", dest, name return # Now check the hashes @@ -234,8 +247,6 @@ def render_innertile(dest, name): if newhash == oldhash: # Nothing to do - print "Not generating due to hash match" - print "\t", dest, name return # Create the actual image now @@ -255,7 +266,6 @@ def render_innertile(dest, name): img.paste(quad3, (192, 192)) # Save it - print "Saving", imgpath img.save(imgpath) with open(hashpath, "wb") as hashout: hashout.write(newhash) @@ -372,343 +382,3 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): with open(hashpath, "wb") as hashout: hashout.write(digest) -def get_quadtree_depth(colstart, colend, rowstart, rowend): - """Determines the zoom depth of a requested quadtree. - - Return value is an integer >= 0. Higher integers mean higher resolution - maps. This is one less than the maximum zoom (level 0 is a single tile, - level 1 is 2 tiles wide by 2 tiles high, etc.) - - """ - # This determines how many zoom levels we need to encompass the entire map. - # We need to make sure that each recursive call splits both dimensions - # evenly into a power of 2 tiles wide and high, so this function determines - # how many splits to make, and generate_quadtree() uses this to adjust the - # row and column limits so that everything splits just right. - # - # This comment makes more sense if you consider it inlined in its call from - # generate_quadtree() - # Since a single tile has 3 columns of chunks and 5 rows of chunks, this - # split needs to be sized into the void so that it is some number of rows - # in the form 2*2^p. And columns must be in the form 4*2^p - # They need to be the same power - # In other words, I need to find the smallest power p such that - # colmid + 2*2^p >= colend and rowmid + 4*2^p >= rowend - # I hope that makes some sense. I don't know how to explain this very well, - # it was some trial and error. - colmid = (colstart + colend) // 2 - rowmid = (rowstart + rowend) // 2 - for p in xrange(15): # That should be a high enough upper limit - if colmid + 2*2**p >= colend and rowmid + 4*2**p >= rowend: - break - else: - raise Exception("Your map is waaaay to big") - - return p - -def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs): - """Base call for quadtree_recurse. This sets up the recursion and generates - a quadtree given a chunkmap and the ranges. - - """ - p = get_quadtree_depth(colstart, colend, rowstart, rowend); - colmid = (colstart + colend) // 2 - rowmid = (rowstart + rowend) // 2 - - # Modify the lower and upper bounds to be sized correctly. See comments in - # get_quadtree_depth() - colstart = colmid - 2*2**p - colend = colmid + 2*2**p - rowstart = rowmid - 4*2**p - rowend = rowmid + 4*2**p - - #print " power is", p - #print " new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend) - - # procs is -1 here since the main process always runs as well, only spawn - # procs-1 /new/ processes - sem = multiprocessing.BoundedSemaphore(procs-1) - quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem) - -def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem): - """Recursive method that generates a quadtree. - A single call generates, saves, and returns an image with the range - specified by colstart,colend,rowstart, and rowend. - - The image is saved as os.path.join(prefix, quadrant+".png") - - If the requested range is larger than a certain threshold, this method will - instead make 4 calls to itself to render the 4 quadrants of the image. The - four pieces are then resized and pasted into one image that is saved and - returned. - - If the requested range is not too large, it is generated with - render_worldtile() - - The path "prefix" should be a directory where this call should save its - image. - - quadrant is used in recursion. If it is "base", the image is saved in the - directory named by prefix, and recursive calls will have quadrant set to - "0" "1" "2" or "3" and prefix will remain unchanged. - - If quadrant is anything else, the tile will be saved just the same, but for - recursive calls a directory named quadrant will be created (if it doesn't - exist) and prefix will be set to os.path.join(prefix, quadrant) - - So the first call will have prefix "tiles" (e.g.) and quadrant "base" and - will save its image as "tiles/base.png" - The second call will have prefix "tiles" and quadrant "0" and will save its - image as "tiles/0.png". It will create the directory "tiles/0/" - The third call will have prefix "tiles/0", quadrant "0" and will save its image as - "tile/0/0.png" - - Each tile outputted is always 384 by 384 pixels. - - The last parameter, sem, should be a multiprocessing.Semaphore or - BoundedSemaphore object. Before each recursive call, the semaphore is - acquired without blocking. If the acquire is successful, the recursive call - will spawn a new process. If it is not successful, the recursive call is - run in the same thread. The semaphore is passed to each recursive call, so - any call could spawn new processes if another one exits at some point. - - The return from this function is (path, hash) where path is the path to the - file saved, and hash is a byte string that depends on the tile's contents. - If the tile is blank, path will be None, but hash will still be valid. - - """ - cols = colend - colstart - rows = rowend - rowstart - - # Get the tile's existing hash. Maybe it hasn't changed. Whether this - # function invocation is destined to recurse, or whether we end up calling - # render_worldtile(), the hash will help us short circuit a lot of pixel - # copying. - hashpath = os.path.join(prefix, quadrant+".hash") - if os.path.exists(hashpath): - oldhash = open(hashpath, "rb").read() - else: - # This method (should) never actually return None for a hash, this is - # used so it will always compare unequal. - oldhash = None - - if cols == 2 and rows == 4: - # base case: just render the image - img, newhash = render_worldtile(chunkmap, colstart, colend, rowstart, rowend, oldhash) - # There are a few cases to handle here: - # 1) img is None: the image doesn't exist (would have been blank, no - # chunks exist for that range. - # 2) img is True: the image hasn't changed according to the hashes. The - # image object is not returned by render_worldtile, but we do need to - # return the path to it. - # 3) img is a PIL.Image.Image object, a new tile was computed, we need - # to save it and its hash (newhash) to disk. - - if not img: - # The image returned is blank, there should not be an image here. - # If one does exist, from a previous world or something, it is not - # deleted, but None is returned to indicate to our caller this tile - # is blank. - remove_tile(prefix, quadrant) - return None, newhash - if img is True: - # No image was returned because the hashes matched. Return the path - # to the image that already exists and is up to date according to - # the hash - path = os.path.join(prefix, quadrant+".png") - if not os.path.exists(path): - # Oops, the image doesn't actually exist. User must have - # deleted it, or must be some bug? - raise Exception("Error, this image should have existed according to the hashes, but didn't") - return path, newhash - - # If img was not None or True, it is an image object. The image exists - # and the hashes did not match, so it must have changed. Fall through - # to the last part of this function which saves the image and its hash. - assert isinstance(img, Image.Image) - elif cols < 2 or rows < 4: - raise Exception("Something went wrong, this tile is too small. (Please send " - "me the traceback so I can fix this)") - else: - # Recursively generate each quadrant for this tile - - # Find the midpoint - colmid = (colstart + colend) // 2 - rowmid = (rowstart + rowend) // 2 - - # Assert that the split in the center still leaves everything sized - # exactly right by checking divisibility by the final row and - # column sizes. This isn't sufficient, but is necessary for - # success. (A better check would make sure the dimensions fit the - # above equations for the same power of 2) - assert (colmid - colstart) % 2 == 0 - assert (colend - colmid) % 2 == 0 - assert (rowmid - rowstart) % 4 == 0 - assert (rowend - rowmid) % 4 == 0 - - if quadrant == "base": - newprefix = prefix - else: - # Make the directory for the recursive subcalls - newprefix = os.path.join(prefix, quadrant) - if not os.path.exists(newprefix): - os.mkdir(newprefix) - - # Keep a hash of the concatenation of each returned hash. If it matches - # oldhash from above, skip rendering this tile - hasher = hashlib.md5() - - # Recurse to generate each quadrant of images - if sem.acquire(False): - Procobj = ReturnableProcess - else: - Procobj = FakeProcess - - quad0result = Procobj(sem, target=quadtree_recurse, - args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem) - ) - - if sem.acquire(False): - Procobj = ReturnableProcess - else: - Procobj = FakeProcess - quad1result = Procobj(sem, target=quadtree_recurse, - args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem) - ) - - if sem.acquire(False): - Procobj = ReturnableProcess - else: - Procobj = FakeProcess - quad2result = Procobj(sem, target=quadtree_recurse, - args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem) - ) - - # Start the processes. If one is a fakeprocess, it will do the - # processing right here instead. - quad0result.start() - quad1result.start() - quad2result.start() - - # 3rd quadrent always runs in this process, no need to spawn a new one - # since we're just going to turn around and wait for it. - quad3file, hash3 = quadtree_recurse(chunkmap, - colmid, colend, rowmid, rowend, - newprefix, "3", sem) - - quad0file, hash0 = quad0result.get() - quad1file, hash1 = quad1result.get() - quad2file, hash2 = quad2result.get() - - # Check the hashes. This is checked even if the tile files returned - # None, since that could happen if either the tile was blank or it - # hasn't changed. So the hashes returned should tell us whether we need - # to update this tile or not. - hasher.update(hash0) - hasher.update(hash1) - hasher.update(hash2) - hasher.update(hash3) - newhash = hasher.digest() - if newhash == oldhash: - # Nothing left to do, this tile already exists and hasn't changed. - #if dbg: print "hashes match, nothing to do" - return os.path.join(prefix, quadrant+".png"), oldhash - - # Check here if this tile is actually blank. If all 4 returned quadrant - # filenames are None, this tile should not be rendered. However, we - # still need to return a valid hash for it, so that's why this check is - # below the hash check. - if not (bool(quad0file) or bool(quad1file) or bool(quad2file) or - bool(quad3file)): - remove_tile(prefix, quadrant) - return None, newhash - - img = Image.new("RGBA", (384, 384)) - - if quad0file: - quad0 = Image.open(quad0file).resize((192,192), Image.ANTIALIAS) - img.paste(quad0, (0,0)) - if quad1file: - quad1 = Image.open(quad1file).resize((192,192), Image.ANTIALIAS) - img.paste(quad1, (192,0)) - if quad2file: - quad2 = Image.open(quad2file).resize((192,192), Image.ANTIALIAS) - img.paste(quad2, (0, 192)) - if quad3file: - quad3 = Image.open(quad3file).resize((192,192), Image.ANTIALIAS) - img.paste(quad3, (192, 192)) - - # At this point, if the tile hasn't change or is blank, the function should - # have returned by now. - assert bool(img) - - # Save the image - path = os.path.join(prefix, quadrant+".png") - img.save(path) - - print "Saving image", path - - # Save the hash - with open(os.path.join(prefix, quadrant+".hash"), 'wb') as hashout: - hashout.write(newhash) - - # Return the location and hash of this tile - return path, newhash - -def remove_tile(prefix, quadrent): - """Called when a tile doesn't exist, this deletes an existing tile if it - does - """ - path = os.path.join(prefix, quadrent) - img = path + ".png" - hash = path + ".hash" - - if os.path.exists(img): - print "removing", img - os.unlink(img) - if os.path.exists(hash): - os.unlink(hash) - -class ReturnableProcess(multiprocessing.Process): - """Like the standard multiprocessing.Process class, but the return value of - the target method is available by calling get(). - - The given semaphore is released when the target finishes running""" - def __init__(self, semaphore, *args, **kwargs): - self.__sem = semaphore - multiprocessing.Process.__init__(self, *args, **kwargs) - - def run(self): - try: - results = self._target(*self._args, **self._kwargs) - except BaseException, e: - self._respipe_in.send(e) - else: - self._respipe_in.send(results) - finally: - self.__sem.release() - - def get(self): - self.join() - ret = self._respipe_out.recv() - if isinstance(ret, BaseException): - raise ret - return ret - - def start(self): - self._respipe_out, self._respipe_in = multiprocessing.Pipe() - multiprocessing.Process.start(self) - -class FakeProcess(object): - """Identical interface to the above class, but runs in the same thread. - Used to make the code simpler in quadtree_recurse - - """ - def __init__(self, semaphore, target, args=None, kwargs=None): - self._target = target - self._args = args if args else () - self._kwargs = kwargs if kwargs else {} - def start(self): - self.ret = self._target(*self._args, **self._kwargs) - def get(self): - return self.ret diff --git a/template.html b/template.html index 39f4241..c0e5bc9 100644 --- a/template.html +++ b/template.html @@ -17,7 +17,7 @@ tileSize: 384, defaultZoom: 1, maxZoom: {maxzoom}, - cacheMinutes: {cachelife}, + cacheMinutes: 0, // Change this to have browsers automatically requiest new images every x minutes debug: false }; diff --git a/world.py b/world.py index e84d2b8..41a488a 100644 --- a/world.py +++ b/world.py @@ -117,6 +117,7 @@ class WorldRenderer(object): if 1000 % i == 0 or i % 1000 == 0: print "{0}/{1} chunks rendered".format(i, len(chunks)) + pool.join() print "Done!" return results