From 2d7bd248ed2b8476eb99e65061fa70d6651ff728 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 6 Sep 2010 12:58:30 -0400 Subject: [PATCH 1/5] fixed strangeness in tile caching code. I believe the tile caching and hashing mechanisms are working now. --- world.py | 92 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/world.py b/world.py index 4863544..d07e9de 100644 --- a/world.py +++ b/world.py @@ -216,11 +216,13 @@ def render_worldtile(chunkmap, colstart, colend, rowstart, rowend, oldhash): object) as returned from render_chunks_async() Return value is (image object, hash) where hash is some string that depends - on the image contents. If no tiles were found, the image object is None. + on the image contents. + + If no tiles were found, (None, hash) is returned. oldhash is a hash value of an existing tile. The hash of this tile is computed before it is rendered, and if they match, rendering is skipped and - (None, oldhash) is returned. + (True, oldhash) is returned. """ # width of one chunk is 384. Each column is half a chunk wide. The total # width is (384 + 192*(numcols-1)) since the first column contributes full @@ -265,11 +267,13 @@ def render_worldtile(chunkmap, colstart, colend, rowstart, rowend, oldhash): os.path.basename(chunkfile).split(".")[4] ) - if not tilelist: - return None, imghash.digest() digest = imghash.digest() + if not tilelist: + # No chunks were found in this tile + return None, digest if digest == oldhash: - return None, oldhash + # All the chunks for this tile have not changed according to the hash + return True, digest tileimg = Image.new("RGBA", (width, height)) @@ -364,15 +368,18 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr Each tile outputted is always 384 by 384 pixels. - The return from this function (path, hash) where path is the path to the + The return from this function is (path, hash) where path is the path to the file saved, and hash is a byte string that depends on the tile's contents. - If the tile is blank, path will be None. + If the tile is blank, path will be None, but hash will still be valid. """ - if 0 and prefix == "/tmp/testrender/2/1/0/1/3" and quadrant == "1": - print "Called with {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend) - print " prefix:", prefix - print " quadrant:", quadrant + #if 1 and prefix == "/tmp/testrender/2/1/0/1" and quadrant == "1": + # print "Called with {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend) + # print " prefix:", prefix + # print " quadrant:", quadrant + # dbg = True + #else: + # dbg = False cols = colend - colstart rows = rowend - rowstart @@ -384,14 +391,43 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr if os.path.exists(hashpath): oldhash = open(hashpath, "rb").read() else: + # This method (should) never actually return None for a hash, this is + # used so it will always compare unequal. oldhash = None if cols == 2 and rows == 4: # base case: just render the image img, newhash = render_worldtile(chunkmap, colstart, colend, rowstart, rowend, oldhash) + # There are a few cases to handle here: + # 1) img is None: the image doesn't exist (would have been blank, no + # chunks exist for that range. + # 2) img is True: the image hasn't changed according to the hashes. The + # image object is not returned by render_worldtile, but we do need to + # return the path to it. + # 3) img is a PIL.Image.Image object, a new tile was computed, we need + # to save it and its hash (newhash) to disk. + if not img: - # Image doesn't exist, exit now + # The image returned is blank, there should not be an image here. + # If one does exist, from a previous world or something, it is not + # deleted, but None is returned to indicate to our caller this tile + # is blank. return None, newhash + if img is True: + # No image was returned because the hashes matched. Return the path + # to the image that already exists and is up to date according to + # the hash + path = os.path.join(prefix, quadrant+".png") + if not os.path.exists(path): + # Oops, the image doesn't actually exist. User must have + # deleted it, or must be some bug? + raise Exception("Error, this image should have existed according to the hashes, but didn't") + return path, newhash + + # If img was not None or True, it is an image object. The image exists + # and the hashes did not match, so it must have changed. Fall through + # to the last part of this function which saves the image and its hash. + assert isinstance(img, Image.Image) elif cols < 2 or rows < 4: raise Exception("Something went wrong, this tile is too small. (Please send " "me the traceback so I can fix this)") @@ -438,15 +474,20 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr colmid, colend, rowmid, rowend, newprefix, "3") - # Is this tile blank? If so, it doesn't matter what the old hash was, - # we can exit right now. - # Note for the confused: python's True value is a subclass of int and - # has value 1, so I can do this: - if (bool(quad0file) + bool(quad1file) + bool(quad2file) + - bool(quad3file)) == 0: - return None, hasher.digest() + #if dbg: + # print quad0file + # print repr(hash0) + # print quad1file + # print repr(hash1) + # print quad2file + # print repr(hash2) + # print quad3file + # print repr(hash3) - # Check the hashes. + # Check the hashes. This is checked even if the tile files returned + # None, since that could happen if either the tile was blank or it + # hasn't changed. So the hashes returned should tell us whether we need + # to update this tile or not. hasher.update(hash0) hasher.update(hash1) hasher.update(hash2) @@ -454,8 +495,19 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr newhash = hasher.digest() if newhash == oldhash: # Nothing left to do, this tile already exists and hasn't changed. + #if dbg: print "hashes match, nothing to do" return os.path.join(prefix, quadrant+".png"), oldhash + # Check here if this tile is actually blank. If all 4 returned quadrant + # filenames are None, this tile should not be rendered. However, we + # still need to return a valid hash for it, so that's why this check is + # below the hash check. + # For the confused: Python boolean values are a subclass of integers, + # and True has value 1, so I can do this: + if (bool(quad0file) + bool(quad1file) + bool(quad2file) + + bool(quad3file)) == 0: + return None, newhash + img = Image.new("RGBA", (384, 384)) if quad0file: From 040e2693313869fdbf10be8a5357d18042d1064e Mon Sep 17 00:00:00 2001 From: Gregory Short Date: Tue, 7 Sep 2010 06:44:08 +0800 Subject: [PATCH 2/5] Updated template.html to use google maps api v3. This also resolves the bug where double-clicking on the right side of the map to zoom in would zoom somewhere else entirely. --- template.html | 207 +++++++++++++++++--------------------------------- 1 file changed, 70 insertions(+), 137 deletions(-) diff --git a/template.html b/template.html index f960e14..fbb5b27 100644 --- a/template.html +++ b/template.html @@ -1,140 +1,73 @@ - - - - - + - - Minecraft Map Viewer - - - -
-
-
- - + }; + map = new google.maps.Map(document.getElementById("mcmap"), mapOptions); + + // Now attach the coordinate map type to the map's registry + map.mapTypes.set('mcmap', MCMapType); + + // We can now set the map to use the 'coordinate' map type + map.setMapTypeId('mcmap'); + } + + + +
+ + \ No newline at end of file From 92724e2688f17728a4d6efbf867ae9c0789b8d94 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 6 Sep 2010 22:05:32 -0400 Subject: [PATCH 3/5] updated readme --- README.rst | 80 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index df6984f..93e8132 100644 --- a/README.rst +++ b/README.rst @@ -9,13 +9,11 @@ Generates large resolution images of a Minecraft map. In short, this program reads in Minecraft world files and renders very large resolution images. It performs a similar function to the existing Minecraft -Cartographer program. - -I wrote this with an additional goal in mind: to generate large images that I -could zoom in and see details. +Cartographer program but with a slightly different goal in mind: to generate +large resolution images such that one can zoom in and see details. **New**: gmap.py generates tiles for a Google Map interface, so that people -with large worlds can still benefit! +with large worlds and/or limited computer memory can still view their worlds! Requirements ============ @@ -36,37 +34,74 @@ Disclaimers ----------- Before you dive into using this, let it be known that there are a few minor problems. First, it's slow. If your map is really large, this could take at -least half an hour, and for really large maps, several hours. Second, there's -no progress bar. You can watch the tiles get generated, but the program gives -no feedback at this time on how far it is. +least half an hour, and for really large maps, several hours (Subsequent runs +will be quicker since it only re-renders tiles that have changed). Second, +there's no progress bar. You can watch the tiles get generated, but the program +gives no feedback at this time on how far it is. There are probably some other minor glitches along the way, hopefully they will be fixed soon. See the `Bugs`_ section below. Running ------- -To generate a set of Google Map tiles, use the gmap.py script like this: +To generate a set of Google Map tiles, use the gmap.py script like this:: python gmap.py -The output directory must already exist. This will generate a set of image -tiles for your world. When it's done, it will put an index.html file in the -same directory that you can use to view it. +The output directory will be created if it doesn't exist. This will generate a +set of image tiles for your world in the directory you choose. When it's done, +it will put an index.html file in the same directory that you can use to view +it. Note that this program renders each chunk of your world as an intermediate step and stores the images in your world directory as a cache. You usually don't need to worry about this, but if you want to delete them, see the section below about `Deleting the Cache`_. +Also note that this program outputs hash files alongside the tile images in the +output directory. These files are used to quickly determine if a tile needs to +be re-generated on subsequent runs of the program on the same world. This +greatly speeds up the rendering. + +Using more Cores +---------------- +Adding the "-p" option will utilize more cores to generate the chunk files. +This can speed up rendering quite a bit. However, the tile generation routine +is currently serial and not written to take advantage of multiple cores. This +option will only affect the chunk generation (which is around half the process) + +Example:: + + python gmap.py -p 5 + +Crushing the Output Tiles +------------------------- +Image files taking too much disk space? Try using pngcrush. On Linux and +probably Mac, if you have pngcrush installed, this command will go and crush +all your images in the given destination. This took the total disk usage of my +world from 85M to 67M. + +:: + + find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \; + +Windows users, you're on your own, but there's probably a way to do this. (If +someone figures it out, let me know I'll update this README) + Using the Large Image Renderer ============================== The Large Image Renderer creates one large image of your world. This was -originally the only option, but would crash and use too much memory for very -large worlds. You may still find a use for it though. +originally the only option, but uses a large amount of memory and generates +unwieldy large images. It is still included in this package in case someone +finds it useful, but the preferred method is the Google Map tile generator. -Right now there's only a console interface. Here's how to use it: +Be warned: For even moderately large worlds this may eat up all your memory, +take a long time, or even just outright crash. It allocates an image large +enough to accommodate your entire world and then draws each block on it. It +would not be surprising to need gigabytes of memory for extremely large +worlds. -To render a world, run the renderer.py script like this: +To render a world, run the renderer.py script like this:: python renderer.py @@ -78,7 +113,7 @@ Cave mode renders all blocks that have no sunlight hitting them. Additionally, blocks are given a colored tint according to how deep they are. Red are closest to bedrock, green is close to sea level, and blue is close to the sky. -Cave mode is like normal mode, but give it the "-c" flag. Like this: +Cave mode is like normal mode, but give it the "-c" flag. Like this:: python renderer.py -c @@ -88,12 +123,14 @@ The Overviewer keeps a cache of each world chunk it renders stored within your world directory. When you generate a new image of the same world, it will only re-render chunks that have changed, speeding things up a lot. -If you want to delete these images, run the renderer.py script with the -d flag: +If you want to delete these images, run the renderer.py script with the -d flag:: python renderer.py -d To delete the cave mode images, run it with -d and -c +:: + python renderer.py -d -c You may want to do this for example to save space. Or perhaps you've changed @@ -105,7 +142,7 @@ The Overviewer will render each chunk separately in parallel. You can tell it how many processes to start with the -p option. This is set to a default of 2, which will use 2 processes to render chunks, and 1 to render the final image. -To bump that up to 3 processes, use a command in this form: +To bump that up to 3 processes, use a command in this form:: python renderer.py -p 3 @@ -132,7 +169,8 @@ An incomplete list of things I want to fix soon is: * Add lighting -* Speed up the tile rendering. I can parallelize that process, and add more - caches to the tiles so subsequent renderings go faster. +* Speed up the tile rendering. I can parallelize that process. * I want to add some indication of progress to the tile generation. + +* Some kind of graphical interface. From dfc336e46aca197acc4b9f1040492b914c61a0ac Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 6 Sep 2010 22:17:10 -0400 Subject: [PATCH 4/5] should still make the tiledir even if destdir exists --- gmap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gmap.py b/gmap.py index 77e9be3..024f6f0 100755 --- a/gmap.py +++ b/gmap.py @@ -41,9 +41,9 @@ def main(): if not os.path.exists(destdir): os.mkdir(destdir) - tiledir = os.path.join(destdir, "tiles"); - if not os.path.exists(tiledir): - os.mkdir(tiledir) + tiledir = os.path.join(destdir, "tiles"); + if not os.path.exists(tiledir): + os.mkdir(tiledir) zoom = world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir) From 41f5eab06748d65bdea311da0b1a5afe28d3bfcc Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 6 Sep 2010 22:53:16 -0400 Subject: [PATCH 5/5] fixed template.html to use the right path --- template.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.html b/template.html index c04bfa6..079a19a 100644 --- a/template.html +++ b/template.html @@ -33,7 +33,7 @@ url += '/' + (x + 2 * y); } } - url = config.path + url + '.' + config.fileExt; + url = url + '.' + config.fileExt; return(url); }, tileSize: new google.maps.Size(config.tileSize, config.tileSize),