diff --git a/README.rst b/README.rst index 007de92..a44cfed 100644 --- a/README.rst +++ b/README.rst @@ -100,7 +100,7 @@ it. The Overviewer will put a cached image for every chunk *directly in your world directory by default*. If you do not like this behavior, you can specify -another location with the --chunkdir option. See below for details. +another location with the --cachedir option. See below for details. Options ------- @@ -121,6 +121,12 @@ Options python gmap.py --cachedir= +--imgformat=FORMAT + Set the output image format used for the tiles. The default is 'png', + but 'jpg' is also supported. Note that regardless of what you choose, + Overviewer will still use PNG for cached images to avoid recompression + artifacts. + -p PROCS, --processes=PROCS Adding the "-p" option will utilize more cores during processing. This can speed up rendering quite a bit. The default is set to the same @@ -209,6 +215,10 @@ render for my world from 85M to 67M. find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \; +Or if you prefer a more parallel solution, try something like this:: + + find /path/to/destination -print0 | xargs -0 -n 1 -P sh -c 'pngcrush $0 temp.$$ && mv temp.$$ $0' + If you're on Windows, I've gotten word that this command line snippet works provided pngout is installed and on your path. Note that the % symbols will need to be doubled up if this is in a batch file. diff --git a/chunk.py b/chunk.py index c548afa..aaee2e4 100644 --- a/chunk.py +++ b/chunk.py @@ -15,7 +15,6 @@ import numpy from PIL import Image, ImageDraw -from itertools import izip, count import os.path import hashlib @@ -60,9 +59,14 @@ def get_skylight_array(level): """ return numpy.frombuffer(level['SkyLight'], dtype=numpy.uint8).reshape((16,16,64)) +def get_blockdata_array(level): + """Returns the ancillary data from the 'Data' byte array. Data is packed + in a similar manner to skylight data""" + return numpy.frombuffer(level['Data'], dtype=numpy.uint8).reshape((16,16,64)) + # This set holds blocks ids that can be seen through, for occlusion calculations -transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 50, 51, 52, 53, - 59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 79, 83, 85]) +transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 44, 50, 51, 52, 53, + 59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 79, 81, 83, 85]) def render_and_save(chunkfile, cachedir, cave=False): """Used as the entry point for the multiprocessing workers (since processes @@ -238,6 +242,13 @@ class ChunkRenderer(object): blocks = blocks.copy() blocks[skylight_expanded != 0] = 21 + blockData = get_blockdata_array(self.level) + blockData_expanded = numpy.empty((16,16,128), dtype=numpy.uint8) + # Even elements get the lower 4 bits + blockData_expanded[:,:,::2] = blockData & 0x0F + # Odd elements get the upper 4 bits + blockData_expanded[:,:,1::2] = blockData >> 4 + # Each block is 24x24 # The next block on the X axis adds 12px to x and subtracts 6px from y in the image @@ -256,7 +267,17 @@ class ChunkRenderer(object): for z in xrange(128): try: blockid = blocks[x,y,z] - t = textures.blockmap[blockid] + + # the following blocks don't have textures that can be pre-computed from the blockid + # alone. additional data is required. + # TODO torches, redstone torches, crops, ladders, stairs, + # levers, doors, buttons, and signs all need to be handled here (and in textures.py) + if blockid in (66,): ## minecart track + ancilData = blockData_expanded[x,y,z] + t = textures.generate_special_texture(blockid, ancilData) + + else: + t = textures.blockmap[blockid] if not t: continue diff --git a/gmap.py b/gmap.py index 97eeb97..aad5959 100755 --- a/gmap.py +++ b/gmap.py @@ -26,14 +26,16 @@ from optparse import OptionParser import re import multiprocessing import time +import logging + +logging.basicConfig(level=logging.INFO,format="%(asctime)s [%(levelname)s] %(message)s") import world import quadtree helptext = """ %prog [OPTIONS] -%prog -d [tiles dest dir] -""" +%prog -d [tiles dest dir]""" def main(): try: @@ -46,6 +48,9 @@ def main(): parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true") parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") + parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") + parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") + parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") options, args = parser.parse_args() @@ -85,12 +90,28 @@ def main(): else: chunklist = None + if options.imgformat: + if options.imgformat not in ('jpg','png'): + parser.error("Unknown imgformat!") + else: + imgformat = options.imgformat + else: + imgformat = 'png' + + logging.getLogger().setLevel( + logging.getLogger().level + 10*options.quiet) + logging.getLogger().setLevel( + logging.getLogger().level - 10*options.verbose) + + logging.info("Welcome to Minecraft Overviewer!") + logging.debug("Current log level: {0}".format(logging.getLogger().level)) + # First generate the world's chunk images w = world.WorldRenderer(worlddir, cachedir, chunklist=chunklist) w.go(options.procs) # Now generate the tiles - q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom) + q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat) q.go(options.procs) def delete_all(worlddir, tiledir): @@ -102,7 +123,7 @@ def delete_all(worlddir, tiledir): for f in filenames: if matcher.match(f): filepath = os.path.join(dirpath, f) - print "Deleting {0}".format(filepath) + logging.info("Deleting {0}".format(filepath)) os.unlink(filepath) # Now delete all /hash/ files in the tile dir. @@ -111,7 +132,7 @@ def delete_all(worlddir, tiledir): for f in filenames: if f.endswith(".hash"): filepath = os.path.join(dirpath, f) - print "Deleting {0}".format(filepath) + logging.info("Deleting {0}".format(filepath)) os.unlink(filepath) def list_worlds(): @@ -132,4 +153,5 @@ def list_worlds(): if __name__ == "__main__": + multiprocessing.freeze_support() main() diff --git a/quadtree.py b/quadtree.py index a075e77..553c67f 100644 --- a/quadtree.py +++ b/quadtree.py @@ -23,9 +23,12 @@ import re import shutil import collections import json +import logging from PIL import Image +import util + """ This module has routines related to generating a quadtree of tiles @@ -43,7 +46,7 @@ def catch_keyboardinterrupt(func): try: return func(*args, **kwargs) except KeyboardInterrupt: - print "Ctrl-C caught!" + logging.error("Ctrl-C caught!") raise Exception("Exiting") except: import traceback @@ -52,7 +55,7 @@ def catch_keyboardinterrupt(func): return newfunc class QuadtreeGen(object): - def __init__(self, worldobj, destdir, depth=None): + def __init__(self, worldobj, destdir, depth=None, imgformat=None): """Generates a quadtree from the world given into the given dest directory @@ -62,6 +65,9 @@ class QuadtreeGen(object): minimum depth that contains all chunks is calculated and used. """ + assert(imgformat) + self.imgformat = imgformat + if depth is None: # Determine quadtree depth (midpoint is always 0,0) for p in xrange(15): @@ -106,16 +112,18 @@ class QuadtreeGen(object): else: if not complete % 1000 == 0: return - print "{0}/{1} tiles complete on level {2}/{3}".format( - complete, total, level, self.p) + logging.info("{0}/{1} tiles complete on level {2}/{3}".format( + complete, total, level, self.p)) - def write_html(self, zoomlevel): + def write_html(self, zoomlevel, imgformat): """Writes out index.html""" - templatepath = os.path.join(os.path.split(__file__)[0], "template.html") + templatepath = os.path.join(util.get_program_path(), "template.html") html = open(templatepath, 'r').read() html = html.replace( "{maxzoom}", str(zoomlevel)) + html = html.replace( + "{imgformat}", str(imgformat)) with open(os.path.join(self.destdir, "index.html"), 'w') as output: output.write(html) @@ -125,6 +133,12 @@ class QuadtreeGen(object): with open(os.path.join(self.destdir, "markers.js"), 'w') as output: output.write("var markerData=%s" % json.dumps(self.world.POI)) + # Write a blank image + blank = Image.new("RGBA", (1,1)) + tileDir = os.path.join(self.destdir, "tiles") + if not os.path.exists(tileDir): os.mkdir(tileDir) + blank.save(os.path.join(tileDir, "blank."+self.imgformat)) + def _get_cur_depth(self): """How deep is the quadtree currently in the destdir? This glances in index.html to see what maxZoom is set to. @@ -159,8 +173,8 @@ class QuadtreeGen(object): newdir = "new" + str(dirnum) newdirpath = getpath(newdir) - files = [str(dirnum)+".png", str(dirnum)+".hash", str(dirnum)] - newfiles = [str(newnum)+".png", str(newnum)+".hash", str(newnum)] + files = [str(dirnum)+"."+self.imgformat, str(dirnum)+".hash", str(dirnum)] + newfiles = [str(newnum)+"."+self.imgformat, str(newnum)+".hash", str(newnum)] os.mkdir(newdirpath) for f, newf in zip(files, newfiles): @@ -221,7 +235,7 @@ class QuadtreeGen(object): # (even if tilechunks is empty, render_worldtile will delete # existing images if appropriate) yield pool.apply_async(func=render_worldtile, args= (tilechunks, - colstart, colend, rowstart, rowend, dest)) + colstart, colend, rowstart, rowend, dest, self.imgformat)) def _apply_render_inntertile(self, pool, zoom): """Same as _apply_render_worltiles but for the inntertile routine. @@ -233,7 +247,7 @@ class QuadtreeGen(object): dest = os.path.join(self.destdir, "tiles", *(str(x) for x in path[:-1])) name = str(path[-1]) - yield pool.apply_async(func=render_innertile, args= (dest, name)) + yield pool.apply_async(func=render_innertile, args= (dest, name, self.imgformat)) def go(self, procs): """Renders all tiles""" @@ -245,12 +259,12 @@ class QuadtreeGen(object): curdepth = self._get_cur_depth() if curdepth != -1: if self.p > curdepth: - print "Your map seemes to have expanded beyond its previous bounds." - print "Doing some tile re-arrangements... just a sec..." + logging.warning("Your map seemes to have expanded beyond its previous bounds.") + logging.warning( "Doing some tile re-arrangements... just a sec...") for _ in xrange(self.p-curdepth): self._increase_depth() elif self.p < curdepth: - print "Your map seems to have shrunk. Re-arranging tiles, just a sec..." + logging.warning("Your map seems to have shrunk. Re-arranging tiles, just a sec...") for _ in xrange(curdepth - self.p): self._decrease_depth() @@ -260,17 +274,17 @@ class QuadtreeGen(object): else: pool = multiprocessing.Pool(processes=procs) - self.write_html(self.p) + self.write_html(self.p, self.imgformat) # Render the highest level of tiles from the chunks results = collections.deque() complete = 0 total = 4**self.p - print "Rendering highest zoom level of tiles now." - print "There are {0} tiles to render".format(total) - print "There are {0} total levels to render".format(self.p) - print "Don't worry, each level has only 25% as many tiles as the last." - print "The others will go faster" + logging.info("Rendering highest zoom level of tiles now.") + logging.info("There are {0} tiles to render".format(total)) + logging.info("There are {0} total levels to render".format(self.p)) + logging.info("Don't worry, each level has only 25% as many tiles as the last.") + logging.info("The others will go faster") for result in self._apply_render_worldtiles(pool): results.append(result) if len(results) > 10000: @@ -295,7 +309,7 @@ class QuadtreeGen(object): assert len(results) == 0 complete = 0 total = 4**zoom - print "Starting level", level + logging.info("Starting level {0}".format(level)) for result in self._apply_render_inntertile(pool, zoom): results.append(result) if len(results) > 10000: @@ -311,13 +325,13 @@ class QuadtreeGen(object): self.print_statusline(complete, total, level, True) - print "Done" + logging.info("Done") pool.close() pool.join() # Do the final one right here: - render_innertile(os.path.join(self.destdir, "tiles"), "base") + render_innertile(os.path.join(self.destdir, "tiles"), "base", self.imgformat) def _get_range_by_path(self, path): """Returns the x, y chunk coordinates of this tile""" @@ -348,28 +362,28 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile(dest, name): +def render_innertile(dest, name, imgformat): """ - Renders a tile at os.path.join(dest, name)+".png" by taking tiles from + Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from os.path.join(dest, name, "{0,1,2,3}.png") """ - imgpath = os.path.join(dest, name) + ".png" + imgpath = os.path.join(dest, name) + "." + imgformat hashpath = os.path.join(dest, name) + ".hash" if name == "base": - q0path = os.path.join(dest, "0.png") - q1path = os.path.join(dest, "1.png") - q2path = os.path.join(dest, "2.png") - q3path = os.path.join(dest, "3.png") + q0path = os.path.join(dest, "0." + imgformat) + q1path = os.path.join(dest, "1." + imgformat) + q2path = os.path.join(dest, "2." + imgformat) + q3path = os.path.join(dest, "3." + imgformat) q0hash = os.path.join(dest, "0.hash") q1hash = os.path.join(dest, "1.hash") q2hash = os.path.join(dest, "2.hash") q3hash = os.path.join(dest, "3.hash") else: - q0path = os.path.join(dest, name, "0.png") - q1path = os.path.join(dest, name, "1.png") - q2path = os.path.join(dest, name, "2.png") - q3path = os.path.join(dest, name, "3.png") + q0path = os.path.join(dest, name, "0." + imgformat) + q1path = os.path.join(dest, name, "1." + imgformat) + q2path = os.path.join(dest, name, "2." + imgformat) + q3path = os.path.join(dest, name, "3." + imgformat) q0hash = os.path.join(dest, name, "0.hash") q1hash = os.path.join(dest, name, "1.hash") q2hash = os.path.join(dest, name, "2.hash") @@ -434,13 +448,16 @@ def render_innertile(dest, name): img.paste(quad3, (192, 192)) # Save it - img.save(imgpath) + if imgformat == 'jpg': + img.save(imgpath, quality=95, subsampling=0) + else: # png + img.save(imgpath) with open(hashpath, "wb") as hashout: hashout.write(newhash) @catch_keyboardinterrupt -def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): +def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat): """Renders just the specified chunks into a tile and save it. Unlike usual python conventions, rowend and colend are inclusive. Additionally, the chunks around the edges are half-way cut off (so that neighboring tiles @@ -449,7 +466,7 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): chunks is a list of (col, row, filename) of chunk images that are relevant to this call - The image is saved to path+".png" and a hash is saved to path+".hash" + The image is saved to path+".ext" and a hash is saved to path+".hash" If there are no chunks, this tile is not saved (if it already exists, it is deleted) @@ -490,7 +507,7 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): # Before we render any tiles, check the hash of each image in this tile to # see if it's changed. hashpath = path + ".hash" - imgpath = path + ".png" + imgpath = path + "." + imgformat if not chunks: # No chunks were found in this tile @@ -545,8 +562,7 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): # corrupting it), then this could error. # Since we have no easy way of determining how this chunk was # generated, we need to just ignore it. - print "Error opening file", chunkfile - print "(Error was {0})".format(e) + logging.warning("Could not open chunk '{0}' ({1})".format(chunkfile,e)) try: # Remove the file so that the next run will re-generate it. os.unlink(chunkfile) @@ -555,12 +571,12 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path): # Ignore if file doesn't exist, another task could have already # removed it. if e.errno != errno.ENOENT: - print "Could not remove the corrupt chunk!" + logging.warning("Could not remove chunk '{0}'!".format(chunkfile)) raise else: - print "Removed the corrupt file" + logging.warning("Removed the corrupt file") - print "You will need to re-run the Overviewer to fix this chunk" + logging.warning("You will need to re-run the Overviewer to fix this chunk") continue xpos = -192 + (col-colstart)*192 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..49d1d22 --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +from distutils.core import setup +import py2exe + +setup(console=['gmap.py'], + data_files=[('textures', ['textures/lava.png', 'textures/water.png']), + ('', ['template.html'])], + zipfile = None, + options = {'py2exe': { + 'bundle_files': 1, + }}, + + + ) diff --git a/template.html b/template.html index 63ebdb6..4387e8a 100644 --- a/template.html +++ b/template.html @@ -14,11 +14,11 @@