From e989e97c5eff7613f2799801f1c4f9f5ec2dbd7e Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Fri, 31 Dec 2010 00:53:57 -0500 Subject: [PATCH 01/24] Added a new config file parser. The new config file parser has an interface that's nearly identical to the OptionParser of optparse. Below is a sample settings.py config file: $ cat settings.py import multiprocessing if 'rendermode' not in locals(): rendermode="lighting" cachedir = "cache.%s.cachedir" % rendermode procs = multiprocessing.cpu_count() - 1 --- configParser.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++++ gmap.py | 19 ++++--- world.py | 8 +-- 3 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 configParser.py diff --git a/configParser.py b/configParser.py new file mode 100644 index 0000000..f7bdb65 --- /dev/null +++ b/configParser.py @@ -0,0 +1,134 @@ +from optparse import OptionParser +import sys + +class OptionsResults(object): + pass + +class ConfigOptionParser(object): + def __init__(self, **kwargs): + self.cmdParser = OptionParser(usage=kwargs.get("usage","")) + self.configFile = kwargs.get("config","settings.py") + self.configVars = [] + + # these are arguments not understood by OptionParser, so they must be removed + # in add_option before being passed to the OptionParser + # note that default is a valid OptionParser argument, but we remove it + # because we want to do our default value handling + self.customArgs = ["required", "commandLineOnly", "default"] + + self.requiredArgs = [] + + def display_config(self): + for x in self.configVars: + n = x['dest'] + print "%s: %r" % (n, self.configResults.__dict__[n]) + + def add_option(self, *args, **kwargs): + self.configVars.append(kwargs.copy()) + + if not kwargs.get("configFileOnly", False): + for arg in self.customArgs: + if arg in kwargs.keys(): del kwargs[arg] + + self.cmdParser.add_option(*args, **kwargs) + + def print_help(self): + self.cmdParser.print_help() + + def parse_args(self): + + # first, load the results from the command line: + options, args = self.cmdParser.parse_args() + + + # second, use these values to seed the locals dict + l = dict() + g = dict() + for a in self.configVars: + n = a['dest'] + if a.get('configFileOnly', False): continue + if a.get('commandLineOnly', False): continue + if getattr(options, n) != None: + l[n] = getattr(options, n) + g['args'] = args + try: + execfile(self.configFile, g, l) + except NameError, ex: + pass + except SyntaxError, ex: + print "Error parsing %s. Please check the trackback below:" % self.configFile + import traceback + traceback.print_exc() + tb = sys.exc_info()[2] + #print tb.tb_frame.f_code.co_filename + sys.exit(1) + + #print l.keys() + + configResults = OptionsResults() + # first, load the results from the config file: + for a in self.configVars: + n = a['dest'] + if a.get('commandLineOnly', False): + if n in l.keys(): + print "Error: %s can only be specified on the command line. It is not valid in the config file" % n + sys.exit(1) + + configResults.__dict__[n] = l.get(n) + + + + # third, merge options into configReslts (with options overwriting anything in configResults) + for a in self.configVars: + n = a['dest'] + if a.get('configFileOnly', False): continue + if getattr(options, n) != None: + configResults.__dict__[n] = getattr(options, n) + + # forth, set defaults for any empty values + for a in self.configVars: + n = a['dest'] + if (n not in configResults.__dict__.keys() or configResults.__dict__[n] == None) and 'default' in a.keys(): + configResults.__dict__[n] = a['default'] + + # fifth, check required args: + for a in self.configVars: + n = a['dest'] + if configResults.__dict__[n] == None and a.get('required',False): + raise Exception("%s is required" % n) + + # sixth, check types + for a in self.configVars: + n = a['dest'] + if 'type' in a.keys() and configResults.__dict__[n] != None: + try: + # switch on type. there are only 6 types that can be used with optparse + if a['type'] == "int": + configResults.__dict__[n] = int(configResults.__dict__[n]) + elif a['type'] == "string": + configResults.__dict__[n] = str(configResults.__dict__[n]) + elif a['type'] == "long": + configResults.__dict__[n] = long(configResults.__dict__[n]) + elif a['type'] == "choice": + if configResults.__dict__[n] not in a['choices']: + print "The value '%s' is not valid for config parameter '%s'" % (configResults.__dict__[n], n) + sys.exit(1) + elif a['type'] == "float": + configResults.__dict__[n] = long(configResults.__dict__[n]) + elif a['type'] == "complex": + configResults.__dict__[n] = complex(configResults.__dict__[n]) + else: + print "Unknown type!" + sys.exit(1) + except ValueError, ex: + print "There was a problem converting the value '%s' to type %s for config parameter '%s'" % (configResults.__dict__[n], a['type'], n) + import traceback + #traceback.print_exc() + sys.exit(1) + + + + self.configResults = configResults + + return configResults, args + diff --git a/gmap.py b/gmap.py index 8a41c40..728ef77 100755 --- a/gmap.py +++ b/gmap.py @@ -22,7 +22,7 @@ if not (sys.version_info[0] == 2 and sys.version_info[1] >= 6): import os import os.path -from optparse import OptionParser +from configParser import ConfigOptionParser import re import multiprocessing import time @@ -43,20 +43,20 @@ def main(): cpus = multiprocessing.cpu_count() except NotImplementedError: cpus = 1 - parser = OptionParser(usage=helptext) + parser = ConfigOptionParser(usage=helptext, config="settings.py") parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int") - parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true") + parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") - parser.add_option("--lighting", dest="lighting", help="Renders shadows using light data from each chunk.", action="store_true") - parser.add_option("--night", dest="night", help="Renders shadows using light data from each chunk, as if it were night. Implies --lighting.", action="store_true") - parser.add_option("--spawn", dest="spawn", help="Renders shadows using light data from each chunk, as if it were night, while also highlighting areas that are dark enough to spawn mobs. Implies --lighting and --night.", action="store_true") + parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%") parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") + parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map", commandLineOnly=True) + #parser.add_option("--write-config", dest="write_config", action="store_true", help="Writes out a sample config file", commandLineOnly=True) options, args = parser.parse_args() @@ -87,6 +87,11 @@ def main(): parser.error("Where do you want to save the tiles?") destdir = args[1] + if options.display_config: + # just display the config file and exit + parser.display_config() + sys.exit(0) + if options.delete: return delete_all(cachedir, destdir) @@ -124,7 +129,7 @@ def main(): logging.info("Notice: Not using biome data for tinting") # First generate the world's chunk images - w = world.WorldRenderer(worlddir, cachedir, chunklist=chunklist, lighting=options.lighting, night=options.night, spawn=options.spawn, useBiomeData=useBiomeData) + w = world.WorldRenderer(worlddir, cachedir, chunklist=chunklist, rendermode=options.rendermode, useBiomeData=useBiomeData) w.go(options.procs) diff --git a/world.py b/world.py index 5064baf..d1ea20e 100644 --- a/world.py +++ b/world.py @@ -103,12 +103,12 @@ class WorldRenderer(object): files to update. If it includes a trailing newline, it is stripped, so you can pass in file handles just fine. """ - def __init__(self, worlddir, cachedir, chunklist=None, lighting=False, night=False, spawn=False, useBiomeData=False): + def __init__(self, worlddir, cachedir, chunklist=None, rendermode="normal", useBiomeData=False): self.worlddir = worlddir self.caves = False - self.lighting = lighting or night or spawn - self.night = night or spawn - self.spawn = spawn + self.lighting = rendermode in ("lighting","night","spawn") + self.night = rendermode in ("night","spawn") + self.spawn = rendermode in ("spawn",) self.cachedir = cachedir self.useBiomeData = useBiomeData From feeb3283e0b27d23b27ff922c1427cc69e6e36bc Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 8 Jan 2011 01:21:41 -0500 Subject: [PATCH 02/24] New benchmarking script Useful for examining how a code change affects performance --- chunk.py | 3 ++- contrib/benchmark.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 contrib/benchmark.py diff --git a/chunk.py b/chunk.py index 13a387b..7a5fc15 100644 --- a/chunk.py +++ b/chunk.py @@ -924,7 +924,8 @@ class ChunkRenderer(object): # check to see if there are any signs in the persistentData list that are from this chunk. # if so, remove them from the persistentData list (since they're have been added to the world.POI # list above. - self.queue.put(['removePOI', (self.chunkX, self.chunkY)]) + if self.queue: + self.queue.put(['removePOI', (self.chunkX, self.chunkY)]) return img diff --git a/contrib/benchmark.py b/contrib/benchmark.py new file mode 100644 index 0000000..145654e --- /dev/null +++ b/contrib/benchmark.py @@ -0,0 +1,51 @@ +import chunk +import world +import tempfile +import glob +import time +import cProfile +import os +import sys +import shutil + +# Simple Benchmarking script. Usage and example: + +# $ python contrib/benchmark.py World4/ +# Rendering 50 chunks... +# Took 20.290062 seconds or 0.405801 seconds per chunk, or 2.464261 chunks per second + + +# create a new, empty, cache dir +cachedir = tempfile.mkdtemp(prefix="benchmark_cache", dir=".") +if os.path.exists("benchmark.prof"): os.unlink("benchmark.prof") + +w = world.WorldRenderer("World4", cachedir) + +numchunks = 50 +chunklist = w._find_chunkfiles()[:numchunks] + +print "Rendering %d chunks..." % (numchunks) +def go(): + for f in chunklist: + chunk.render_and_save(f[2], w.cachedir, w, (None,None), None) +start = time.time() +if "-profile" in sys.argv: + cProfile.run("go()", 'benchmark.prof') +else: + go() +stop = time.time() + +delta = stop - start + +print "Took %f seconds or %f seconds per chunk, or %f chunks per second" % (delta, delta/numchunks, numchunks/delta) + +if "-profile" in sys.argv: + print "Profile is below:\n----\n" + + import pstats + p = pstats.Stats('benchmark.prof') + + p.strip_dirs().sort_stats("cumulative").print_stats(20) + + +shutil.rmtree(cachedir) From f46d0ce4addb71c2944a034c8f3ffad645b274c8 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 22 Jan 2011 14:23:20 -0500 Subject: [PATCH 03/24] options with default args are now seeded in the globals dict --- configParser.py | 24 ++++++++++++++++++------ gmap.py | 8 ++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/configParser.py b/configParser.py index f7bdb65..0f43a34 100644 --- a/configParser.py +++ b/configParser.py @@ -24,6 +24,10 @@ class ConfigOptionParser(object): print "%s: %r" % (n, self.configResults.__dict__[n]) def add_option(self, *args, **kwargs): + + if kwargs.get("configFileOnly", False) and kwargs.get("commandLineOnly", False): + raise Exception(args, "configFileOnly and commandLineOnly are mututally exclusive") + self.configVars.append(kwargs.copy()) if not kwargs.get("configFileOnly", False): @@ -40,7 +44,6 @@ class ConfigOptionParser(object): # first, load the results from the command line: options, args = self.cmdParser.parse_args() - # second, use these values to seed the locals dict l = dict() g = dict() @@ -48,25 +51,34 @@ class ConfigOptionParser(object): n = a['dest'] if a.get('configFileOnly', False): continue if a.get('commandLineOnly', False): continue - if getattr(options, n) != None: - l[n] = getattr(options, n) + v = getattr(options, n) + if v != None: + #print "seeding %s with %s" % (n, v) + l[n] = v + else: + # if this has a default, use that to seed the globals dict + if a.get("default", None): g[n] = a['default'] g['args'] = args + try: execfile(self.configFile, g, l) except NameError, ex: - pass + import traceback + traceback.print_exc() + print "\nError parsing %s. Please check the trackback above" % self.configFile + sys.exit(1) except SyntaxError, ex: - print "Error parsing %s. Please check the trackback below:" % self.configFile import traceback traceback.print_exc() tb = sys.exc_info()[2] #print tb.tb_frame.f_code.co_filename + print "\nError parsing %s. Please check the trackback above" % self.configFile sys.exit(1) #print l.keys() configResults = OptionsResults() - # first, load the results from the config file: + # third, load the results from the config file: for a in self.configVars: n = a['dest'] if a.get('commandLineOnly', False): diff --git a/gmap.py b/gmap.py index 5ab81d4..749a41b 100755 --- a/gmap.py +++ b/gmap.py @@ -46,17 +46,17 @@ def main(): cpus = 1 parser = ConfigOptionParser(usage=helptext, config="settings.py") parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") - parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int") + parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int", configFileOnly=True) parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") - parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") - parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%") + parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True ) + parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True) parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") - parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map", commandLineOnly=True) + parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map. Requires all required options to be specified", commandLineOnly=True) #parser.add_option("--write-config", dest="write_config", action="store_true", help="Writes out a sample config file", commandLineOnly=True) options, args = parser.parse_args() From 2be64f2aa7a041becea95c3d2014c8ce8b527025 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 16:01:33 +0800 Subject: [PATCH 04/24] render_inntertile & render_inntertile now called in batches in the worker process, speeds up update scan with a lot of tiles to skip. --- quadtree.py | 84 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/quadtree.py b/quadtree.py index fbee945..67eb7d3 100644 --- a/quadtree.py +++ b/quadtree.py @@ -292,10 +292,13 @@ class QuadtreeGen(object): shutil.rmtree(getpath("3")) os.rename(getpath("new3"), getpath("3")) - def _apply_render_worldtiles(self, pool): + def _apply_render_worldtiles(self, pool,batch_size): """Returns an iterator over result objects. Each time a new result is requested, a new task is added to the pool and a result returned. """ + + batch = [] + tiles = 0 for path in iterate_base4(self.p): # Get the range for this tile colstart, rowstart = self._get_range_by_path(path) @@ -310,24 +313,42 @@ class QuadtreeGen(object): tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, rowend) #logging.debug(" tilechunks: %r", tilechunks) - - # Put this in the pool + + # Put this in the batch to be submited to the pool # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - yield pool.apply_async(func=render_worldtile, args= (self, - tilechunks, colstart, colend, rowstart, rowend, dest)) + # existing images if appropriate) + batch.append((tilechunks, colstart, colend, rowstart, rowend, dest)) + tiles += 1 + if tiles >= batch_size: + tiles = 0 + yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + batch = [] - def _apply_render_inntertile(self, pool, zoom): + if tiles > 0: + yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + + + def _apply_render_inntertile(self, pool, zoom,batch_size): """Same as _apply_render_worltiles but for the inntertile routine. Returns an iterator that yields result objects from tasks that have been applied to the pool. """ + batch = [] + tiles = 0 for path in iterate_base4(zoom): # This image is rendered at: dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1])) name = str(path[-1]) - - yield pool.apply_async(func=render_innertile, args= (dest, name, self.imgformat, self.optimizeimg)) + + batch.append((dest, name, self.imgformat, self.optimizeimg)) + tiles += 1 + if tiles >= batch_size: + tiles = 0 + yield pool.apply_async(func=render_innertile_batch, args= (batch)) + batch = [] + + if tiles > 0: + yield pool.apply_async(func=render_innertile_batch, args= (batch)) def go(self, procs): """Renders all tiles""" @@ -359,20 +380,21 @@ class QuadtreeGen(object): logging.info("There are {0} total levels to render".format(self.p)) logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") - for result in self._apply_render_worldtiles(pool): + count = 0 + batch_size = 50 + for result in self._apply_render_worldtiles(pool,batch_size): results.append(result) - if len(results) > 10000: + if len(results) > (10000/batch_size): # Empty the queue before adding any more, so that memory # required has an upper bound - while len(results) > 500: - results.popleft().get() - complete += 1 + while len(results) > (500/batch_size): + complete += results.popleft().get() self.print_statusline(complete, total, 1) # Wait for the rest of the results while len(results) > 0: - results.popleft().get() - complete += 1 + + complete += results.popleft().get() self.print_statusline(complete, total, 1) self.print_statusline(complete, total, 1, True) @@ -384,17 +406,15 @@ class QuadtreeGen(object): complete = 0 total = 4**zoom logging.info("Starting level {0}".format(level)) - for result in self._apply_render_inntertile(pool, zoom): + for result in self._apply_render_inntertile(pool, zoom,batch_size): results.append(result) - if len(results) > 10000: - while len(results) > 500: - results.popleft().get() - complete += 1 + if len(results) > (10000/batch_size): + while len(results) > (500/batch_size): + complete += results.popleft().get() self.print_statusline(complete, total, level) # Empty the queue while len(results) > 0: - results.popleft().get() - complete += 1 + complete += results.popleft().get() self.print_statusline(complete, total, level) self.print_statusline(complete, total, level, True) @@ -448,6 +468,14 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt +def render_innertile_batch( batch): + count = 0 + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + render_worldtile(job[0],job[1],job[2],job[3]) + return count + def render_innertile(dest, name, imgformat, optimizeimg): """ Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from @@ -512,6 +540,14 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt +def render_worldtile_batch(quadtree, batch): + count = 0 + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5]) + return count + def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path): """Renders just the specified chunks into a tile and save it. Unlike usual python conventions, rowend and colend are inclusive. Additionally, the @@ -529,7 +565,7 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path) Standard tile size has colend-colstart=2 and rowend-rowstart=4 There is no return value - """ + """ # width of one chunk is 384. Each column is half a chunk wide. The total # width is (384 + 192*(numcols-1)) since the first column contributes full From e113a24ae0279637abfb1d92a60cd14b14a7e135 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 16:23:11 +0800 Subject: [PATCH 05/24] Fixed render_innertile_batch --- quadtree.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/quadtree.py b/quadtree.py index 67eb7d3..d577325 100644 --- a/quadtree.py +++ b/quadtree.py @@ -344,11 +344,11 @@ class QuadtreeGen(object): tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= (batch)) + yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) batch = [] if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= (batch)) + yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) def go(self, procs): """Renders all tiles""" @@ -468,12 +468,12 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile_batch( batch): +def render_innertile_batch(quadtree, batch): count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_worldtile(job[0],job[1],job[2],job[3]) + render_innertile(job[0],job[1],job[2],job[3]) return count def render_innertile(dest, name, imgformat, optimizeimg): From b9433173c92498684d8397717318055457b3a9a3 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 23:36:26 +0800 Subject: [PATCH 06/24] Fixed worker processes being passed the full quadtree object. Caused massive performance regressions when caching stuff in quadtree.world Offloaded self._get_chunks_in_range into worker process. --- quadtree.py | 61 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/quadtree.py b/quadtree.py index d577325..452b481 100644 --- a/quadtree.py +++ b/quadtree.py @@ -82,6 +82,8 @@ def catch_keyboardinterrupt(func): raise return newfunc +child_quadtree = None + class QuadtreeGen(object): def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False): """Generates a quadtree from the world given into the @@ -308,24 +310,17 @@ class QuadtreeGen(object): # This image is rendered at: dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path)) #logging.debug("this is rendered at %s", dest) - - # And uses these chunks - tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, - rowend) - #logging.debug(" tilechunks: %r", tilechunks) - # Put this in the batch to be submited to the pool - # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - batch.append((tilechunks, colstart, colend, rowstart, rowend, dest)) + # Put this in the batch to be submited to the pool + batch.append((colstart, colend, rowstart, rowend, dest)) tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) batch = [] if tiles > 0: - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= (batch,)) def _apply_render_inntertile(self, pool, zoom,batch_size): @@ -344,12 +339,15 @@ class QuadtreeGen(object): tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + yield pool.apply_async(func=render_innertile_batch, args= [batch]) batch = [] - if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + if tiles > 0: + yield pool.apply_async(func=render_innertile_batch, args= [batch]) + def pool_initializer(args): + logging.debug("Child process {0}".format(os.getpid())) + def go(self, procs): """Renders all tiles""" @@ -365,12 +363,18 @@ class QuadtreeGen(object): for _ in xrange(curdepth - self.p): self._decrease_depth() + logging.debug("Parent process {0}".format(os.getpid())) + #stash the quadtree object so child process's can + global child_quadtree + child_quadtree = self # Create a pool if procs == 1: pool = FakePool() else: - pool = multiprocessing.Pool(processes=procs) - + pool = multiprocessing.Pool(processes=procs,initializer=self.pool_initializer,initargs=()) + #warm up the pool so it reports all the worker id's + pool.map(bool,xrange(multiprocessing.cpu_count()),1) + # Render the highest level of tiles from the chunks results = collections.deque() complete = 0 @@ -381,7 +385,7 @@ class QuadtreeGen(object): logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") count = 0 - batch_size = 50 + batch_size = 10 for result in self._apply_render_worldtiles(pool,batch_size): results.append(result) if len(results) > (10000/batch_size): @@ -393,7 +397,6 @@ class QuadtreeGen(object): # Wait for the rest of the results while len(results) > 0: - complete += results.popleft().get() self.print_statusline(complete, total, 1) @@ -468,7 +471,7 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile_batch(quadtree, batch): +def render_innertile_batch(batch): count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: @@ -540,12 +543,28 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt -def render_worldtile_batch(quadtree, batch): +def render_worldtile_batch(batch): + global child_quadtree + return render_worldtile_batch_(child_quadtree, batch) + +def render_worldtile_batch_(quadtree, batch): count = 0 + _get_chunks_in_range = quadtree._get_chunks_in_range #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5]) + colstart = job[0] + colend = job[1] + rowstart = job[2] + rowend = job[3] + path = job[4] + # (even if tilechunks is empty, render_worldtile will delete + # existing images if appropriate) + # And uses these chunks + tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) + #logging.debug(" tilechunks: %r", tilechunks) + + render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path) return count def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path): From 6f340dceee3fda5cb1a9a52b9b849196211fad57 Mon Sep 17 00:00:00 2001 From: Xon Date: Sun, 20 Mar 2011 01:19:23 +0800 Subject: [PATCH 07/24] New conrtib script to validate a region file --- contrib/validateRegionFile.py | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 contrib/validateRegionFile.py diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py new file mode 100644 index 0000000..5bd7e14 --- /dev/null +++ b/contrib/validateRegionFile.py @@ -0,0 +1,76 @@ +#!/usr/bin/python + +usage = "python contrib/%prog [OPTIONS] " + +description = """ +This script will delete files from the old chunk-based cache, a lot +like the old `gmap.py -d World/` command. You should only use this if +you're updating from an older version of Overviewer, and you want to +clean up your world folder. +""" + +from optparse import OptionParser +import sys +import re +import os.path +import logging + +# sys.path wrangling, so we can access Overviewer code +overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0] +sys.path.insert(0, overviewer_dir) + +import nbt +import chunk + +def main(): + parser = OptionParser(usage=usage, description=description) +# parser.add_option("-d", "--dry-run", dest="dry", action="store_true", +# help="Don't actually delete anything. Best used with -v.") + + opt, args = parser.parse_args() + + if not len(args) == 1: + parser.print_help() + sys.exit(1) + + regionfile = args[0] + + if not os.path.exists(regionfile): + parser.print_help() + print "\nFile not found" + sys.exit(1) + chunk_pass = 0 + chunk_total = 0 + print( "Loading region: %s" % ( regionfile)) + try: + mcr = nbt.load_region(regionfile) + except IOError, e: + print("Error opening regionfile. It may be corrupt. %s"%( e)) + pass + if mcr is not None: + try: + chunks = mcr.get_chunk_info(False) + except IOError, e: + print("Error opening regionfile. It may be corrupt. %s"%( e)) + chunks = [] + pass + for x, y in chunks: + chunk_total += 1 + #try: + chunk_data = mcr.load_chunk(x, y) + if chunk_data is None: + print("Chunk %s:%s is unexpectedly empty"%(x, y)) + else: + try: + processed = chunk_data.read_all() + if processed == []: + print("Chunk %s:%s is an unexpectedly empty set"%(x, y)) + else: + chunk_pass += 1 + except Exception, e: + print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e)) + else: + print("Error opening regionfile.") + print("Done; Passed %s/%s"%(chunk_pass,chunk_total)) +if __name__ == "__main__": + main() From cfabf161485d57879de91b963938b75504af850d Mon Sep 17 00:00:00 2001 From: Xon Date: Sun, 20 Mar 2011 01:22:34 +0800 Subject: [PATCH 08/24] Clarify if opening the region file failed or if the headers are wonky --- contrib/validateRegionFile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py index 5bd7e14..3846784 100644 --- a/contrib/validateRegionFile.py +++ b/contrib/validateRegionFile.py @@ -51,7 +51,7 @@ def main(): try: chunks = mcr.get_chunk_info(False) except IOError, e: - print("Error opening regionfile. It may be corrupt. %s"%( e)) + print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e)) chunks = [] pass for x, y in chunks: From 76f85d0d2c68d805c1d2e4190f6913ab7a9bc0b1 Mon Sep 17 00:00:00 2001 From: Xon Date: Sun, 20 Mar 2011 01:57:47 +0800 Subject: [PATCH 09/24] Added ctrl-c handling, output is a single line (verbose reports what error occured), added optparsing, supports multipule files or a dir --- contrib/validateRegionFile.py | 109 +++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py index 3846784..1e5320a 100644 --- a/contrib/validateRegionFile.py +++ b/contrib/validateRegionFile.py @@ -1,12 +1,9 @@ #!/usr/bin/python -usage = "python contrib/%prog [OPTIONS] " +usage = "python contrib/%prog [OPTIONS] ()*" description = """ -This script will delete files from the old chunk-based cache, a lot -like the old `gmap.py -d World/` command. You should only use this if -you're updating from an older version of Overviewer, and you want to -clean up your world folder. +This script will valide a minecraft region file for errors """ from optparse import OptionParser @@ -21,56 +18,74 @@ sys.path.insert(0, overviewer_dir) import nbt import chunk +import quadtree def main(): parser = OptionParser(usage=usage, description=description) -# parser.add_option("-d", "--dry-run", dest="dry", action="store_true", -# help="Don't actually delete anything. Best used with -v.") + parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files") + parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed") opt, args = parser.parse_args() - if not len(args) == 1: + if opt.regiondir: + if os.path.exists(opt.regiondir): + for dirpath, dirnames, filenames in os.walk(opt.regiondir, 'region'): + if not dirnames and filenames and "DIM-1" not in dirpath: + for f in filenames: + if f.startswith("r.") and f.endswith(".mcr"): + p = f.split(".") + args.append(os.path.join(dirpath, f)) + + if len(args) < 1: + print "You must list at least one region file" parser.print_help() sys.exit(1) - - regionfile = args[0] - - if not os.path.exists(regionfile): - parser.print_help() - print "\nFile not found" - sys.exit(1) - chunk_pass = 0 - chunk_total = 0 - print( "Loading region: %s" % ( regionfile)) - try: - mcr = nbt.load_region(regionfile) - except IOError, e: - print("Error opening regionfile. It may be corrupt. %s"%( e)) - pass - if mcr is not None: + + for regionfile in args: + _,shortname = os.path.split(regionfile) + chunk_pass = 0 + chunk_total = 0 + if not os.path.exists(regionfile): + print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) + continue try: - chunks = mcr.get_chunk_info(False) + mcr = nbt.load_region(regionfile) except IOError, e: - print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e)) - chunks = [] - pass - for x, y in chunks: - chunk_total += 1 - #try: - chunk_data = mcr.load_chunk(x, y) - if chunk_data is None: - print("Chunk %s:%s is unexpectedly empty"%(x, y)) - else: - try: - processed = chunk_data.read_all() - if processed == []: - print("Chunk %s:%s is an unexpectedly empty set"%(x, y)) - else: - chunk_pass += 1 - except Exception, e: - print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e)) - else: - print("Error opening regionfile.") - print("Done; Passed %s/%s"%(chunk_pass,chunk_total)) + if options.verbose: + print("Error opening regionfile. It may be corrupt. %s"%( e)) + if mcr is not None: + try: + chunks = mcr.get_chunk_info(False) + except IOError, e: + if options.verbose: + print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e)) + chunks = [] + for x, y in chunks: + chunk_total += 1 + #try: + chunk_data = mcr.load_chunk(x, y) + if chunk_data is None: + if options.verbose: + print("Chunk %s:%s is unexpectedly empty"%(x, y)) + else: + try: + processed = chunk_data.read_all() + if processed == []: + if options.verbose: + print("Chunk %s:%s is an unexpectedly empty set"%(x, y)) + else: + chunk_pass += 1 + except Exception, e: + if options.verbose: + print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e)) + else: + if options.verbose: + print("Error opening regionfile.") + + print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + print "Caught Ctrl-C" + From fae67de9f0fb156b5a3d2408ab3431130aab9ccc Mon Sep 17 00:00:00 2001 From: Alex Headley Date: Sat, 19 Mar 2011 14:28:19 -0400 Subject: [PATCH 10/24] fixed some bugs in the validateRegion script --- contrib/validateRegionFile.py | 45 +++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py index 1e5320a..34ec1fe 100644 --- a/contrib/validateRegionFile.py +++ b/contrib/validateRegionFile.py @@ -12,15 +12,15 @@ import re import os.path import logging -# sys.path wrangling, so we can access Overviewer code -overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0] -sys.path.insert(0, overviewer_dir) -import nbt -import chunk -import quadtree def main(): + # sys.path wrangling, so we can access Overviewer code + overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0] + sys.path.insert(0, overviewer_dir) + import nbt + #import chunk + #import quadtree parser = OptionParser(usage=usage, description=description) parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files") parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed") @@ -46,41 +46,56 @@ def main(): chunk_pass = 0 chunk_total = 0 if not os.path.exists(regionfile): - print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) + print("File not found: %s"%( regionfile)) + #print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) continue try: mcr = nbt.load_region(regionfile) except IOError, e: - if options.verbose: + if opt.verbose: print("Error opening regionfile. It may be corrupt. %s"%( e)) + continue if mcr is not None: try: chunks = mcr.get_chunk_info(False) except IOError, e: - if options.verbose: + if opt.verbose: print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e)) chunks = [] + continue + except Exception, e: + if opt.verbose: + print("Error opening regionfile (%s): %s"%( regionfile,e)) + continue for x, y in chunks: chunk_total += 1 - #try: - chunk_data = mcr.load_chunk(x, y) + try: + chunk_data = mcr.load_chunk(x, y) + except Exception, e: + if opt.verbose: + print("Error reading chunk (%i,%i): %s"%(x,y,e)) + continue if chunk_data is None: - if options.verbose: + if opt.verbose: print("Chunk %s:%s is unexpectedly empty"%(x, y)) + continue else: try: processed = chunk_data.read_all() if processed == []: - if options.verbose: + if opt.verbose: print("Chunk %s:%s is an unexpectedly empty set"%(x, y)) + continue else: chunk_pass += 1 except Exception, e: - if options.verbose: + if opt.verbose: print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e)) + continue else: - if options.verbose: + if opt.verbose: print("Error opening regionfile.") + continue print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) if __name__ == "__main__": From 5ae361824bf6ce00360b30dd9277a92a33005945 Mon Sep 17 00:00:00 2001 From: Alex Headley Date: Sat, 19 Mar 2011 17:45:20 -0400 Subject: [PATCH 11/24] cleaned up contrib/validateRegionFile script --- contrib/validateRegionFile.py | 201 +++++++++++++++++----------------- 1 file changed, 100 insertions(+), 101 deletions(-) diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py index 34ec1fe..7dd2763 100644 --- a/contrib/validateRegionFile.py +++ b/contrib/validateRegionFile.py @@ -1,106 +1,105 @@ -#!/usr/bin/python +#!/usr/bin/env python -usage = "python contrib/%prog [OPTIONS] ()*" - -description = """ -This script will valide a minecraft region file for errors -""" - -from optparse import OptionParser -import sys -import re import os.path -import logging +import sys +overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0] +sys.path.insert(0, overviewer_dir) +import nbt - - -def main(): - # sys.path wrangling, so we can access Overviewer code - overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0] - sys.path.insert(0, overviewer_dir) - import nbt - #import chunk - #import quadtree - parser = OptionParser(usage=usage, description=description) - parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files") - parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed") - - opt, args = parser.parse_args() - - if opt.regiondir: - if os.path.exists(opt.regiondir): - for dirpath, dirnames, filenames in os.walk(opt.regiondir, 'region'): - if not dirnames and filenames and "DIM-1" not in dirpath: - for f in filenames: - if f.startswith("r.") and f.endswith(".mcr"): - p = f.split(".") - args.append(os.path.join(dirpath, f)) - - if len(args) < 1: - print "You must list at least one region file" - parser.print_help() - sys.exit(1) - - for regionfile in args: - _,shortname = os.path.split(regionfile) - chunk_pass = 0 - chunk_total = 0 - if not os.path.exists(regionfile): - print("File not found: %s"%( regionfile)) - #print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) - continue - try: - mcr = nbt.load_region(regionfile) - except IOError, e: - if opt.verbose: - print("Error opening regionfile. It may be corrupt. %s"%( e)) - continue - if mcr is not None: - try: - chunks = mcr.get_chunk_info(False) - except IOError, e: - if opt.verbose: - print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e)) - chunks = [] - continue - except Exception, e: - if opt.verbose: - print("Error opening regionfile (%s): %s"%( regionfile,e)) - continue - for x, y in chunks: - chunk_total += 1 - try: - chunk_data = mcr.load_chunk(x, y) - except Exception, e: - if opt.verbose: - print("Error reading chunk (%i,%i): %s"%(x,y,e)) - continue - if chunk_data is None: - if opt.verbose: - print("Chunk %s:%s is unexpectedly empty"%(x, y)) - continue - else: - try: - processed = chunk_data.read_all() - if processed == []: - if opt.verbose: - print("Chunk %s:%s is an unexpectedly empty set"%(x, y)) - continue - else: - chunk_pass += 1 - except Exception, e: - if opt.verbose: - print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e)) - continue - else: - if opt.verbose: - print("Error opening regionfile.") - continue - - print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total)) -if __name__ == "__main__": +def check_region(region_filename): + chunk_errors = [] + if not os.path.exists(region_filename): + raise Exception('Region file not found: %s' % region_filename) try: - main() - except KeyboardInterrupt: - print "Caught Ctrl-C" + region = nbt.load_region(region_filename) + except IOError, e: + raise Exception('Error loading region (%s): %s' % (region_filename, e)) + try: + chunks = region.get_chunk_info(False) + except IOError, e: + raise Exception('Error reading region header (%s): %s' % (region_filename, e)) + except Exception, e: + raise Exception('Error reading region (%s): %s' % (region_filename, e)) + for x,y in chunks: + try: + check_chunk(region, x, y) + except Exception, e: + chunk_errors.append(e) + return (chunk_errors, len(chunks)) + +def check_chunk(region, x, y): + try: + data = region.load_chunk(x ,y) + except Exception, e: + raise Exception('Error reading chunk (%i, %i): %s' % (x, y, e)) + if data is None: + raise Exception('Chunk (%i, %i) is unexpectedly empty' % (x, y)) + else: + try: + processed_data = data.read_all() + except Exception, e: + raise Exception('Error reading chunk (%i, %i) data: %s' % (x, y, e)) + if processed_data == []: + raise Exception('Chunk (%i, %i) is an unexpectedly empty set' % (x, y)) + +if __name__ == '__main__': + try: + from optparse import OptionParser + + parser = OptionParser(usage='python contrib/%prog [OPTIONS] ', + description='This script will valide a minecraft region file for errors.') + parser.add_option('-v', dest='verbose', action='store_true', help='Print additional information.') + opts, args = parser.parse_args() + region_files = [] + for path in args: + if os.path.isdir(path): + for dirpath, dirnames, filenames in os.walk(path, True): + for filename in filenames: + if filename.startswith('r.') and filename.endswith('.mcr'): + if filename not in region_files: + region_files.append(os.path.join(dirpath, filename)) + elif opts.verbose: + print('Ignoring non-region file: %s' % os.path.join(dirpath, filename)) + elif os.path.isfile(path): + dirpath,filename = os.path.split(path) + if filename.startswith('r.') and filename.endswith('.mcr'): + if path not in region_files: + region_files.append(path) + else: + print('Ignoring non-region file: %s' % path) + else: + if opts.verbose: + print('Ignoring arg: %s' % path) + if len(region_files) < 1: + print 'You must list at least one region file.' + parser.print_help() + sys.exit(1) + else: + overall_chunk_total = 0 + bad_chunk_total = 0 + bad_region_total = 0 + for region_file in region_files: + try: + (chunk_errors, region_chunks) = check_region(region_file) + bad_chunk_total += len(chunk_errors) + overall_chunk_total += region_chunks + except Exception, e: + bad_region_total += 1 + print('FAILED(%s): %s' % (region_file, e)) + else: + if len(chunk_errors) is not 0: + print('WARNING(%s) Chunks: %i/%' % (region_file, region_chunks - len(chunk_errors), region_chunks)) + if opts.verbose: + for error in chunk_errors: + print(error) + elif opts.verbose: + print ('PASSED(%s) Chunks: %i/%i' % (region_file, region_chunks - len(chunk_errors), region_chunks)) + if opts.verbose: + print 'REGIONS: %i/%i' % (len(region_files) - bad_region_total, len(region_files)) + print 'CHUNKS: %i/%i' % (overall_chunk_total - bad_chunk_total, overall_chunk_total) + except KeyboardInterrupt: + sys.exit(1) + except Exception, e: + print('ERROR: %s' % e) + From edf34f6d59b6e015669d599a29b4175fd501fb30 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 19 Mar 2011 18:23:23 -0400 Subject: [PATCH 12/24] changed level data loader to retry up to twice before declaring a chunk corrupt --- chunk.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/chunk.py b/chunk.py index 48d80bb..84b5774 100644 --- a/chunk.py +++ b/chunk.py @@ -46,15 +46,20 @@ image # alpha_over extension, BUT this extension may fall back to PIL's # paste(), which DOES need the workaround.) -def get_lvldata(filename, x, y): +def get_lvldata(filename, x, y, retries=2): """Takes a filename and chunkcoords and returns the Level struct, which contains all the level info""" try: d = nbt.load_from_region(filename, x, y) except Exception, e: - logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e) - raise ChunkCorrupt(str(e)) + if retries > 0: + # wait a little bit, and try again (up to `retries` times) + time.sleep(1) + return get_lvldata(filename, x, y, retries=retries-1) + else: + logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e) + raise ChunkCorrupt(str(e)) if not d: raise NoSuchChunk(x,y) return d[1]['Level'] From 494caba5980949ce856e75fea22346489c3cd0fb Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 19 Mar 2011 18:53:29 -0400 Subject: [PATCH 13/24] added a nice error if c_overviewer is missing, simplified composite.py now that c_overviewer is required, we don't need to have a PIL paste() fallback. The next step is to remove composite.py entirely! --- composite.py | 17 ++--------------- gmap.py | 12 ++++++++++-- src/composite.c | 1 + 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/composite.py b/composite.py index 6cbe44e..be5387f 100644 --- a/composite.py +++ b/composite.py @@ -23,12 +23,7 @@ Overviewer. It defaults to the PIL paste function when the custom alpha-over extension cannot be found. """ -extension_alpha_over = None -try: - from c_overviewer import alpha_over as _extension_alpha_over - extension_alpha_over = _extension_alpha_over -except ImportError: - pass +from c_overviewer import alpha_over as extension_alpha_over def alpha_over(dest, src, pos_or_rect=(0, 0), mask=None): """Composite src over dest, using mask as the alpha channel (if @@ -40,12 +35,4 @@ def alpha_over(dest, src, pos_or_rect=(0, 0), mask=None): mask = src global extension_alpha_over - if extension_alpha_over is not None: - # extension ALWAYS expects rects, so convert if needed - if len(pos_or_rect) == 2: - pos_or_rect = (pos_or_rect[0], pos_or_rect[1], src.size[0], src.size[1]) - extension_alpha_over(dest, src, pos_or_rect, mask) - else: - # fallback - dest.paste(src, pos_or_rect, mask) - + return extension_alpha_over(dest, src, pos_or_rect, mask) diff --git a/gmap.py b/gmap.py index 57edb0c..13a24c9 100755 --- a/gmap.py +++ b/gmap.py @@ -27,11 +27,19 @@ import re import multiprocessing import time import logging -import optimizeimages -import composite logging.basicConfig(level=logging.INFO,format="%(asctime)s [%(levelname)s] %(message)s") +# make sure the c_overviewer extension is available +try: + import c_overviewer +except ImportError: + print "You need to compile the c_overviewer module to run Minecraft Overviewer." + print "Run `python setup.py build`, or see the README for details." + sys.exit(1) + +import optimizeimages +import composite import world import quadtree diff --git a/src/composite.c b/src/composite.c index 4b58403..1cfdf7c 100644 --- a/src/composite.c +++ b/src/composite.c @@ -223,6 +223,7 @@ alpha_over_wrap(PyObject *self, PyObject *args) /* destination position read */ if (!PyArg_ParseTuple(pos, "iiii", &dx, &dy, &xsize, &ysize)) { /* try again, but this time try to read a point */ + PyErr_Clear(); xsize = 0; ysize = 0; if (!PyArg_ParseTuple(pos, "ii", &dx, &dy)) { From ec255bf29b1f5d0245abd15548aa41e1e75c17b3 Mon Sep 17 00:00:00 2001 From: Xon Date: Sun, 20 Mar 2011 08:24:43 +0800 Subject: [PATCH 14/24] Added region cache invalidation & reloading. Cached region mtimes. --- quadtree.py | 8 +------- world.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/quadtree.py b/quadtree.py index fbee945..0267113 100644 --- a/quadtree.py +++ b/quadtree.py @@ -597,16 +597,10 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path) # check chunk mtimes to see if they are newer try: - #tile_mtime = os.path.getmtime(imgpath) - regionMtimes = {} needs_rerender = False for col, row, chunkx, chunky, regionfile in chunks: # check region file mtime first. - # Note: we cache the value since it's actually very likely we will have multipule chunks in the same region, and syscalls are expensive - regionMtime = regionMtimes.get(regionfile,None) - if regionMtime is None: - regionMtime = os.path.getmtime(regionfile) - regionMtimes[regionfile] = regionMtime + regionMtime = world.get_region_mtime(regionfile) if regionMtime <= tile_mtime: continue diff --git a/world.py b/world.py index 7ed790c..f8e880b 100644 --- a/world.py +++ b/world.py @@ -78,7 +78,7 @@ class World(object): for x, y, regionfile in self._iterate_regionfiles(): mcr = nbt.MCRFileReader(regionfile) mcr.get_chunk_info() - regions[regionfile] = mcr + regions[regionfile] = (mcr,os.path.getmtime(regionfile)) regionfiles[(x,y)] = (x,y,regionfile) self.regionfiles = regionfiles self.regions = regions @@ -117,8 +117,6 @@ class World(object): _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None)); return regionfile - - def load_from_region(self,filename, x, y): nbt = self.load_region(filename).load_chunk(x, y) if nbt is None: @@ -127,11 +125,15 @@ class World(object): return nbt.read_all() - #filo region cache - def load_region(self,filename): - #return nbt.MCRFileReader(filename) - return self.regions[filename] + #used to reload a changed region + def reload_region(self,filename): + self.regions[filename] = (nbt.MCRFileReader(filename),os.path.getmtime(regionfile)) + def load_region(self,filename): + return self.regions[filename][0] + + def get_region_mtime(self,filename): + return self.regions[filename][1] def convert_coords(self, chunkx, chunky): """Takes a coordinate (chunkx, chunky) where chunkx and chunky are From 0046da559477a9e6be5e0603743f0f7b61265b26 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 16:01:33 +0800 Subject: [PATCH 15/24] render_inntertile & render_inntertile now called in batches in the worker process, speeds up update scan with a lot of tiles to skip. --- quadtree.py | 84 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/quadtree.py b/quadtree.py index fbee945..67eb7d3 100644 --- a/quadtree.py +++ b/quadtree.py @@ -292,10 +292,13 @@ class QuadtreeGen(object): shutil.rmtree(getpath("3")) os.rename(getpath("new3"), getpath("3")) - def _apply_render_worldtiles(self, pool): + def _apply_render_worldtiles(self, pool,batch_size): """Returns an iterator over result objects. Each time a new result is requested, a new task is added to the pool and a result returned. """ + + batch = [] + tiles = 0 for path in iterate_base4(self.p): # Get the range for this tile colstart, rowstart = self._get_range_by_path(path) @@ -310,24 +313,42 @@ class QuadtreeGen(object): tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, rowend) #logging.debug(" tilechunks: %r", tilechunks) - - # Put this in the pool + + # Put this in the batch to be submited to the pool # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - yield pool.apply_async(func=render_worldtile, args= (self, - tilechunks, colstart, colend, rowstart, rowend, dest)) + # existing images if appropriate) + batch.append((tilechunks, colstart, colend, rowstart, rowend, dest)) + tiles += 1 + if tiles >= batch_size: + tiles = 0 + yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + batch = [] - def _apply_render_inntertile(self, pool, zoom): + if tiles > 0: + yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + + + def _apply_render_inntertile(self, pool, zoom,batch_size): """Same as _apply_render_worltiles but for the inntertile routine. Returns an iterator that yields result objects from tasks that have been applied to the pool. """ + batch = [] + tiles = 0 for path in iterate_base4(zoom): # This image is rendered at: dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1])) name = str(path[-1]) - - yield pool.apply_async(func=render_innertile, args= (dest, name, self.imgformat, self.optimizeimg)) + + batch.append((dest, name, self.imgformat, self.optimizeimg)) + tiles += 1 + if tiles >= batch_size: + tiles = 0 + yield pool.apply_async(func=render_innertile_batch, args= (batch)) + batch = [] + + if tiles > 0: + yield pool.apply_async(func=render_innertile_batch, args= (batch)) def go(self, procs): """Renders all tiles""" @@ -359,20 +380,21 @@ class QuadtreeGen(object): logging.info("There are {0} total levels to render".format(self.p)) logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") - for result in self._apply_render_worldtiles(pool): + count = 0 + batch_size = 50 + for result in self._apply_render_worldtiles(pool,batch_size): results.append(result) - if len(results) > 10000: + if len(results) > (10000/batch_size): # Empty the queue before adding any more, so that memory # required has an upper bound - while len(results) > 500: - results.popleft().get() - complete += 1 + while len(results) > (500/batch_size): + complete += results.popleft().get() self.print_statusline(complete, total, 1) # Wait for the rest of the results while len(results) > 0: - results.popleft().get() - complete += 1 + + complete += results.popleft().get() self.print_statusline(complete, total, 1) self.print_statusline(complete, total, 1, True) @@ -384,17 +406,15 @@ class QuadtreeGen(object): complete = 0 total = 4**zoom logging.info("Starting level {0}".format(level)) - for result in self._apply_render_inntertile(pool, zoom): + for result in self._apply_render_inntertile(pool, zoom,batch_size): results.append(result) - if len(results) > 10000: - while len(results) > 500: - results.popleft().get() - complete += 1 + if len(results) > (10000/batch_size): + while len(results) > (500/batch_size): + complete += results.popleft().get() self.print_statusline(complete, total, level) # Empty the queue while len(results) > 0: - results.popleft().get() - complete += 1 + complete += results.popleft().get() self.print_statusline(complete, total, level) self.print_statusline(complete, total, level, True) @@ -448,6 +468,14 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt +def render_innertile_batch( batch): + count = 0 + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + render_worldtile(job[0],job[1],job[2],job[3]) + return count + def render_innertile(dest, name, imgformat, optimizeimg): """ Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from @@ -512,6 +540,14 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt +def render_worldtile_batch(quadtree, batch): + count = 0 + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5]) + return count + def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path): """Renders just the specified chunks into a tile and save it. Unlike usual python conventions, rowend and colend are inclusive. Additionally, the @@ -529,7 +565,7 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path) Standard tile size has colend-colstart=2 and rowend-rowstart=4 There is no return value - """ + """ # width of one chunk is 384. Each column is half a chunk wide. The total # width is (384 + 192*(numcols-1)) since the first column contributes full From 1d666c7be756d94126bcf7b0fa7d8230f40b6a50 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 16:23:11 +0800 Subject: [PATCH 16/24] Fixed render_innertile_batch --- quadtree.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/quadtree.py b/quadtree.py index 67eb7d3..d577325 100644 --- a/quadtree.py +++ b/quadtree.py @@ -344,11 +344,11 @@ class QuadtreeGen(object): tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= (batch)) + yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) batch = [] if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= (batch)) + yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) def go(self, procs): """Renders all tiles""" @@ -468,12 +468,12 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile_batch( batch): +def render_innertile_batch(quadtree, batch): count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_worldtile(job[0],job[1],job[2],job[3]) + render_innertile(job[0],job[1],job[2],job[3]) return count def render_innertile(dest, name, imgformat, optimizeimg): From cd7b9456a956accc9c8ff6429af78d88eaecf2b7 Mon Sep 17 00:00:00 2001 From: Xon Date: Sat, 19 Mar 2011 23:36:26 +0800 Subject: [PATCH 17/24] Fixed worker processes being passed the full quadtree object. Caused massive performance regressions when caching stuff in quadtree.world Offloaded self._get_chunks_in_range into worker process. --- quadtree.py | 61 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/quadtree.py b/quadtree.py index d577325..452b481 100644 --- a/quadtree.py +++ b/quadtree.py @@ -82,6 +82,8 @@ def catch_keyboardinterrupt(func): raise return newfunc +child_quadtree = None + class QuadtreeGen(object): def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False): """Generates a quadtree from the world given into the @@ -308,24 +310,17 @@ class QuadtreeGen(object): # This image is rendered at: dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path)) #logging.debug("this is rendered at %s", dest) - - # And uses these chunks - tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, - rowend) - #logging.debug(" tilechunks: %r", tilechunks) - # Put this in the batch to be submited to the pool - # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - batch.append((tilechunks, colstart, colend, rowstart, rowend, dest)) + # Put this in the batch to be submited to the pool + batch.append((colstart, colend, rowstart, rowend, dest)) tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) batch = [] if tiles > 0: - yield pool.apply_async(func=render_worldtile_batch, args= (self,batch)) + yield pool.apply_async(func=render_worldtile_batch, args= (batch,)) def _apply_render_inntertile(self, pool, zoom,batch_size): @@ -344,12 +339,15 @@ class QuadtreeGen(object): tiles += 1 if tiles >= batch_size: tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + yield pool.apply_async(func=render_innertile_batch, args= [batch]) batch = [] - if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= (self,batch)) + if tiles > 0: + yield pool.apply_async(func=render_innertile_batch, args= [batch]) + def pool_initializer(args): + logging.debug("Child process {0}".format(os.getpid())) + def go(self, procs): """Renders all tiles""" @@ -365,12 +363,18 @@ class QuadtreeGen(object): for _ in xrange(curdepth - self.p): self._decrease_depth() + logging.debug("Parent process {0}".format(os.getpid())) + #stash the quadtree object so child process's can + global child_quadtree + child_quadtree = self # Create a pool if procs == 1: pool = FakePool() else: - pool = multiprocessing.Pool(processes=procs) - + pool = multiprocessing.Pool(processes=procs,initializer=self.pool_initializer,initargs=()) + #warm up the pool so it reports all the worker id's + pool.map(bool,xrange(multiprocessing.cpu_count()),1) + # Render the highest level of tiles from the chunks results = collections.deque() complete = 0 @@ -381,7 +385,7 @@ class QuadtreeGen(object): logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") count = 0 - batch_size = 50 + batch_size = 10 for result in self._apply_render_worldtiles(pool,batch_size): results.append(result) if len(results) > (10000/batch_size): @@ -393,7 +397,6 @@ class QuadtreeGen(object): # Wait for the rest of the results while len(results) > 0: - complete += results.popleft().get() self.print_statusline(complete, total, 1) @@ -468,7 +471,7 @@ class QuadtreeGen(object): return chunklist @catch_keyboardinterrupt -def render_innertile_batch(quadtree, batch): +def render_innertile_batch(batch): count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: @@ -540,12 +543,28 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt -def render_worldtile_batch(quadtree, batch): +def render_worldtile_batch(batch): + global child_quadtree + return render_worldtile_batch_(child_quadtree, batch) + +def render_worldtile_batch_(quadtree, batch): count = 0 + _get_chunks_in_range = quadtree._get_chunks_in_range #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5]) + colstart = job[0] + colend = job[1] + rowstart = job[2] + rowend = job[3] + path = job[4] + # (even if tilechunks is empty, render_worldtile will delete + # existing images if appropriate) + # And uses these chunks + tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) + #logging.debug(" tilechunks: %r", tilechunks) + + render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path) return count def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path): From ac0055a6d71dae10bda681cb7cd9e13e6ff0ab03 Mon Sep 17 00:00:00 2001 From: Xon Date: Sun, 20 Mar 2011 07:30:15 +0800 Subject: [PATCH 18/24] Fix Windows compat with stashing the quadtree object into each worker process --- quadtree.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/quadtree.py b/quadtree.py index 452b481..f7dcbb0 100644 --- a/quadtree.py +++ b/quadtree.py @@ -83,7 +83,12 @@ def catch_keyboardinterrupt(func): return newfunc child_quadtree = None - +def pool_initializer(quadtree): + logging.debug("Child process {0}".format(os.getpid())) + #stash the quadtree object in a global variable after fork() for windows compat. + global child_quadtree + child_quadtree = quadtree + class QuadtreeGen(object): def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False): """Generates a quadtree from the world given into the @@ -344,9 +349,6 @@ class QuadtreeGen(object): if tiles > 0: yield pool.apply_async(func=render_innertile_batch, args= [batch]) - - def pool_initializer(args): - logging.debug("Child process {0}".format(os.getpid())) def go(self, procs): """Renders all tiles""" @@ -364,14 +366,11 @@ class QuadtreeGen(object): self._decrease_depth() logging.debug("Parent process {0}".format(os.getpid())) - #stash the quadtree object so child process's can - global child_quadtree - child_quadtree = self # Create a pool if procs == 1: pool = FakePool() else: - pool = multiprocessing.Pool(processes=procs,initializer=self.pool_initializer,initargs=()) + pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's pool.map(bool,xrange(multiprocessing.cpu_count()),1) From 38deb98d5a961d2e8aadcb43bad84ee1891a6612 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 19 Mar 2011 20:45:40 -0400 Subject: [PATCH 19/24] added command line option for web_assets postprocessing hook --- README.rst | 9 +++++++++ gmap.py | 21 +++++++++++++++++++-- quadtree.py | 8 +++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index f3f23b9..3164ef7 100644 --- a/README.rst +++ b/README.rst @@ -246,6 +246,15 @@ Options --night This option enables --lighting, and renders the world at night. +--web-assets-hook=HOOK + This option lets you specify a script to run after the web assets have been + copied into the output directory, but before any tile rendering takes + place. This is an ideal time to do any custom postprocessing for markers.js + or other web assets. + + The script should be executable, and it should accept one argument: + the path to the output directory. + Viewing the Results ------------------- Within the output directory you will find two things: an index.html file, and a diff --git a/gmap.py b/gmap.py index 65545ad..856eaee 100755 --- a/gmap.py +++ b/gmap.py @@ -24,6 +24,7 @@ import os import os.path from optparse import OptionParser import re +import subprocess import multiprocessing import time import logging @@ -55,6 +56,7 @@ def main(): parser.add_option("--spawn", dest="spawn", help="Renders shadows using light data from each chunk, as if it were night, while also highlighting areas that are dark enough to spawn mobs. Implies --lighting and --night.", action="store_true") parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%") + parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this script after the web assets have been copied, but before actual tile rendering begins. See the README for details.", action="store", metavar="SCRIPT", type="string") parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") @@ -129,7 +131,22 @@ def main(): optimizeimages.check_programs(optimizeimg) else: optimizeimg = None - + + if options.web_assets_hook: + if not os.path.exists(options.web_assets_hook): + parser.error("Provided hook script does not exist!") + def web_assets_hook(quadtree): + if options.web_assets_hook == None: + return + try: + subprocess.check_call((options.web_assets_hook, os.path.abspath(quadtree.destdir))) + except OSError, e: + logging.error("could not call web assets hook: %s" % (e,)) + sys.exit(1) + except subprocess.CalledProcessError: + logging.error("web assets hook returned error") + sys.exit(1) + logging.getLogger().setLevel( logging.getLogger().level + 10*options.quiet) logging.getLogger().setLevel( @@ -151,7 +168,7 @@ def main(): w.go(options.procs) # Now generate the tiles - q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg) + q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, web_assets_hook=web_assets_hook) q.write_html(options.skipjs) q.go(options.procs) diff --git a/quadtree.py b/quadtree.py index eefc7de..f376608 100644 --- a/quadtree.py +++ b/quadtree.py @@ -81,7 +81,7 @@ def catch_keyboardinterrupt(func): return newfunc class QuadtreeGen(object): - def __init__(self, worldobj, destdir, depth=None, imgformat=None, optimizeimg=None): + def __init__(self, worldobj, destdir, depth=None, imgformat=None, optimizeimg=None, web_assets_hook=None): """Generates a quadtree from the world given into the given dest directory @@ -94,6 +94,7 @@ class QuadtreeGen(object): assert(imgformat) self.imgformat = imgformat self.optimizeimg = optimizeimg + self.web_assets_hook = web_assets_hook # Make the destination dir if not os.path.exists(destdir): @@ -182,6 +183,8 @@ class QuadtreeGen(object): output.write(index) if skipjs: + if self.web_assets_hook: + self.web_assets_hook(self) return # since we will only discover PointsOfInterest in chunks that need to be @@ -210,6 +213,9 @@ class QuadtreeGen(object): output.write(' // ]},\n') output.write('];') + if self.web_assets_hook: + self.web_assets_hook(self) + def _get_cur_depth(self): """How deep is the quadtree currently in the destdir? This glances in config.js to see what maxZoom is set to. From 346ee004e87d73ba95a2af5cb58bd1c9f137aade Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 19 Mar 2011 21:17:29 -0400 Subject: [PATCH 20/24] fixed some things left over from configfile merge --- configParser.py | 4 +++- gmap.py | 1 - quadtree.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/configParser.py b/configParser.py index 0f43a34..7ed285d 100644 --- a/configParser.py +++ b/configParser.py @@ -1,5 +1,6 @@ from optparse import OptionParser import sys +import os.path class OptionsResults(object): pass @@ -61,7 +62,8 @@ class ConfigOptionParser(object): g['args'] = args try: - execfile(self.configFile, g, l) + if os.path.exists(self.configFile): + execfile(self.configFile, g, l) except NameError, ex: import traceback traceback.print_exc() diff --git a/gmap.py b/gmap.py index 50b6822..0dc7191 100755 --- a/gmap.py +++ b/gmap.py @@ -58,7 +58,6 @@ def main(): parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int", configFileOnly=True) parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) - parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True ) diff --git a/quadtree.py b/quadtree.py index ec97e50..73d982b 100644 --- a/quadtree.py +++ b/quadtree.py @@ -107,7 +107,7 @@ class QuadtreeGen(object): self.lighting = rendermode in ("lighting", "night", "spawn") self.night = rendermode in ("night", "spawn") - self.spawn = spawn in ("spawn",) + self.spawn = rendermode in ("spawn",) # Make the destination dir if not os.path.exists(destdir): From 29dc98ce2ab857664dfd80b4474b53a8d4251162 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 19 Mar 2011 23:49:56 -0400 Subject: [PATCH 21/24] Fix a -p 1 regression --- quadtree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quadtree.py b/quadtree.py index 73d982b..25855fc 100644 --- a/quadtree.py +++ b/quadtree.py @@ -375,6 +375,8 @@ class QuadtreeGen(object): # Create a pool if procs == 1: pool = FakePool() + global child_quadtree + child_quadtree = self else: pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's From 6f60439f14132bb5db4499a0d47f7561c60408ce Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 20 Mar 2011 00:37:34 -0400 Subject: [PATCH 22/24] accept callables in the configFile with type="function" --- configParser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configParser.py b/configParser.py index 7ed285d..8ca9eba 100644 --- a/configParser.py +++ b/configParser.py @@ -131,6 +131,9 @@ class ConfigOptionParser(object): configResults.__dict__[n] = long(configResults.__dict__[n]) elif a['type'] == "complex": configResults.__dict__[n] = complex(configResults.__dict__[n]) + elif a['type'] == "function": + if not callable(configResults.__dict__[n]): + raise ValueError("Not callable") else: print "Unknown type!" sys.exit(1) From 4e9985c61167aa7335b353527da7002b00c34317 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sun, 20 Mar 2011 00:46:56 -0400 Subject: [PATCH 23/24] changed web_assets_hook to use new config-file function support --- gmap.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/gmap.py b/gmap.py index 0dc7191..00acbde 100755 --- a/gmap.py +++ b/gmap.py @@ -62,7 +62,7 @@ def main(): parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True ) parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True) - parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this script after the web assets have been copied, but before actual tile rendering begins. See the README for details.", action="store", metavar="SCRIPT", type="string", configFileOnly=True) + parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this function after the web assets have been copied, but before actual tile rendering begins. It should accept a QuadtreeGen object as its only argument.", action="store", metavar="SCRIPT", type="function", configFileOnly=True) parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") @@ -140,21 +140,6 @@ def main(): else: optimizeimg = None - if options.web_assets_hook: - if not os.path.exists(options.web_assets_hook): - parser.error("Provided hook script does not exist!") - def web_assets_hook(quadtree): - if options.web_assets_hook == None: - return - try: - subprocess.check_call((options.web_assets_hook, os.path.abspath(quadtree.destdir))) - except OSError, e: - logging.error("could not call web assets hook: %s" % (e,)) - sys.exit(1) - except subprocess.CalledProcessError: - logging.error("web assets hook returned error") - sys.exit(1) - logging.getLogger().setLevel( logging.getLogger().level + 10*options.quiet) logging.getLogger().setLevel( @@ -176,7 +161,7 @@ def main(): # Now generate the tiles # TODO chunklist - q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode, web_assets_hook=web_assets_hook) + q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode, web_assets_hook=options.web_assets_hook) q.write_html(options.skipjs) q.go(options.procs) From 329c7557f644b64f6da7b3b0054dbd5d0d2c9304 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 20 Mar 2011 21:13:17 -0400 Subject: [PATCH 24/24] Move some code out of the render_loop function and into a 1-time init routine --- src/iterate.c | 56 +++++++++++++++++++++++++++++------------------- src/main.c | 6 ++++++ src/overviewer.h | 1 + 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/iterate.c b/src/iterate.c index 031864c..3a12d53 100644 --- a/src/iterate.c +++ b/src/iterate.c @@ -22,6 +22,40 @@ /* macro for getting blockID from a chunk of memory */ #define getBlock(blockThing, x,y,z) (*(unsigned char *)(PyArray_GETPTR3(blockThing, (x), (y), (z)))) +static PyObject *textures = NULL; +static PyObject *chunk_mod = NULL; +static PyObject *blockmap = NULL; +static PyObject *special_blocks = NULL; +static PyObject *specialblockmap = NULL; +static PyObject *transparent_blocks = NULL; + +int init_chunk_render(void) { + + /* if blockmap (or any of these) is not NULL, then that means that we've + * somehow called this function twice. error out so we can notice this + * */ + if (blockmap) return 1; + + textures = PyImport_ImportModule("textures"); + chunk_mod = PyImport_ImportModule("chunk"); + + blockmap = PyObject_GetAttrString(textures, "blockmap"); + special_blocks = PyObject_GetAttrString(textures, "special_blocks"); + specialblockmap = PyObject_GetAttrString(textures, "specialblockmap"); + transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks"); + + /* ensure none of these pointers are NULL */ + if ((!transparent_blocks) || (!blockmap) || (!special_blocks) || (!specialblockmap)){ + fprintf(stderr, "\ninit_chunk_render failed\n"); + return 1; + } + + Py_DECREF(textures); + Py_DECREF(chunk_mod); + return 0; + +} + static inline int isTransparent(PyObject* tup, unsigned char b) { PyObject *block = PyInt_FromLong(b); int ret = PySequence_Contains(tup, block); @@ -59,8 +93,6 @@ chunk_render(PyObject *self, PyObject *args) { PyObject *blocks_py; - PyObject *textures, *blockmap, *special_blocks, *specialblockmap, *chunk_mod, *transparent_blocks; - int imgx, imgy; int x, y, z; @@ -88,23 +120,6 @@ chunk_render(PyObject *self, PyObject *args) { PyObject *right_blocks = PyObject_GetAttrString(chunk, "right_blocks"); */ - textures = PyImport_ImportModule("textures"); - chunk_mod = PyImport_ImportModule("chunk"); - - /* TODO can these be global static? these don't change during program execution */ - blockmap = PyObject_GetAttrString(textures, "blockmap"); - special_blocks = PyObject_GetAttrString(textures, "special_blocks"); - specialblockmap = PyObject_GetAttrString(textures, "specialblockmap"); - transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks"); - if (transparent_blocks == NULL) { - PyErr_SetString(PyExc_ValueError, - "transparent_blocks is NULL"); - return NULL; - } - - - Py_DECREF(textures); - Py_DECREF(chunk_mod); for (x = 15; x > -1; x--) { for (y = 0; y < 16; y++) { @@ -184,9 +199,6 @@ chunk_render(PyObject *self, PyObject *args) { } Py_DECREF(blocks_py); - Py_DECREF(blockmap); - Py_DECREF(special_blocks); - Py_DECREF(specialblockmap); return Py_BuildValue("i",2); } diff --git a/src/main.c b/src/main.c index f56fb41..8bdd91d 100644 --- a/src/main.c +++ b/src/main.c @@ -33,4 +33,10 @@ initc_overviewer(void) (void)Py_InitModule("c_overviewer", COverviewerMethods); /* for numpy */ import_array(); + + /* initialize some required variables in iterage.c */ + if (init_chunk_render()) { + fprintf(stderr, "failed to init_chunk_render\n"); + exit(1); // TODO better way to indicate error? + } } diff --git a/src/overviewer.h b/src/overviewer.h index 5746b53..fa583fc 100644 --- a/src/overviewer.h +++ b/src/overviewer.h @@ -36,5 +36,6 @@ PyObject *alpha_over_wrap(PyObject *self, PyObject *args); /* in iterate.c */ PyObject *chunk_render(PyObject *self, PyObject *args); +int init_chunk_render(void); #endif /* __OVERVIEWER_H_INCLUDED__ */