From e989e97c5eff7613f2799801f1c4f9f5ec2dbd7e Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Fri, 31 Dec 2010 00:53:57 -0500 Subject: [PATCH 1/3] Added a new config file parser. The new config file parser has an interface that's nearly identical to the OptionParser of optparse. Below is a sample settings.py config file: $ cat settings.py import multiprocessing if 'rendermode' not in locals(): rendermode="lighting" cachedir = "cache.%s.cachedir" % rendermode procs = multiprocessing.cpu_count() - 1 --- configParser.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++++ gmap.py | 19 ++++--- world.py | 8 +-- 3 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 configParser.py diff --git a/configParser.py b/configParser.py new file mode 100644 index 0000000..f7bdb65 --- /dev/null +++ b/configParser.py @@ -0,0 +1,134 @@ +from optparse import OptionParser +import sys + +class OptionsResults(object): + pass + +class ConfigOptionParser(object): + def __init__(self, **kwargs): + self.cmdParser = OptionParser(usage=kwargs.get("usage","")) + self.configFile = kwargs.get("config","settings.py") + self.configVars = [] + + # these are arguments not understood by OptionParser, so they must be removed + # in add_option before being passed to the OptionParser + # note that default is a valid OptionParser argument, but we remove it + # because we want to do our default value handling + self.customArgs = ["required", "commandLineOnly", "default"] + + self.requiredArgs = [] + + def display_config(self): + for x in self.configVars: + n = x['dest'] + print "%s: %r" % (n, self.configResults.__dict__[n]) + + def add_option(self, *args, **kwargs): + self.configVars.append(kwargs.copy()) + + if not kwargs.get("configFileOnly", False): + for arg in self.customArgs: + if arg in kwargs.keys(): del kwargs[arg] + + self.cmdParser.add_option(*args, **kwargs) + + def print_help(self): + self.cmdParser.print_help() + + def parse_args(self): + + # first, load the results from the command line: + options, args = self.cmdParser.parse_args() + + + # second, use these values to seed the locals dict + l = dict() + g = dict() + for a in self.configVars: + n = a['dest'] + if a.get('configFileOnly', False): continue + if a.get('commandLineOnly', False): continue + if getattr(options, n) != None: + l[n] = getattr(options, n) + g['args'] = args + try: + execfile(self.configFile, g, l) + except NameError, ex: + pass + except SyntaxError, ex: + print "Error parsing %s. Please check the trackback below:" % self.configFile + import traceback + traceback.print_exc() + tb = sys.exc_info()[2] + #print tb.tb_frame.f_code.co_filename + sys.exit(1) + + #print l.keys() + + configResults = OptionsResults() + # first, load the results from the config file: + for a in self.configVars: + n = a['dest'] + if a.get('commandLineOnly', False): + if n in l.keys(): + print "Error: %s can only be specified on the command line. It is not valid in the config file" % n + sys.exit(1) + + configResults.__dict__[n] = l.get(n) + + + + # third, merge options into configReslts (with options overwriting anything in configResults) + for a in self.configVars: + n = a['dest'] + if a.get('configFileOnly', False): continue + if getattr(options, n) != None: + configResults.__dict__[n] = getattr(options, n) + + # forth, set defaults for any empty values + for a in self.configVars: + n = a['dest'] + if (n not in configResults.__dict__.keys() or configResults.__dict__[n] == None) and 'default' in a.keys(): + configResults.__dict__[n] = a['default'] + + # fifth, check required args: + for a in self.configVars: + n = a['dest'] + if configResults.__dict__[n] == None and a.get('required',False): + raise Exception("%s is required" % n) + + # sixth, check types + for a in self.configVars: + n = a['dest'] + if 'type' in a.keys() and configResults.__dict__[n] != None: + try: + # switch on type. there are only 6 types that can be used with optparse + if a['type'] == "int": + configResults.__dict__[n] = int(configResults.__dict__[n]) + elif a['type'] == "string": + configResults.__dict__[n] = str(configResults.__dict__[n]) + elif a['type'] == "long": + configResults.__dict__[n] = long(configResults.__dict__[n]) + elif a['type'] == "choice": + if configResults.__dict__[n] not in a['choices']: + print "The value '%s' is not valid for config parameter '%s'" % (configResults.__dict__[n], n) + sys.exit(1) + elif a['type'] == "float": + configResults.__dict__[n] = long(configResults.__dict__[n]) + elif a['type'] == "complex": + configResults.__dict__[n] = complex(configResults.__dict__[n]) + else: + print "Unknown type!" + sys.exit(1) + except ValueError, ex: + print "There was a problem converting the value '%s' to type %s for config parameter '%s'" % (configResults.__dict__[n], a['type'], n) + import traceback + #traceback.print_exc() + sys.exit(1) + + + + self.configResults = configResults + + return configResults, args + diff --git a/gmap.py b/gmap.py index 8a41c40..728ef77 100755 --- a/gmap.py +++ b/gmap.py @@ -22,7 +22,7 @@ if not (sys.version_info[0] == 2 and sys.version_info[1] >= 6): import os import os.path -from optparse import OptionParser +from configParser import ConfigOptionParser import re import multiprocessing import time @@ -43,20 +43,20 @@ def main(): cpus = multiprocessing.cpu_count() except NotImplementedError: cpus = 1 - parser = OptionParser(usage=helptext) + parser = ConfigOptionParser(usage=helptext, config="settings.py") parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int") - parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true") + parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") - parser.add_option("--lighting", dest="lighting", help="Renders shadows using light data from each chunk.", action="store_true") - parser.add_option("--night", dest="night", help="Renders shadows using light data from each chunk, as if it were night. Implies --lighting.", action="store_true") - parser.add_option("--spawn", dest="spawn", help="Renders shadows using light data from each chunk, as if it were night, while also highlighting areas that are dark enough to spawn mobs. Implies --lighting and --night.", action="store_true") + parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%") parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") + parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map", commandLineOnly=True) + #parser.add_option("--write-config", dest="write_config", action="store_true", help="Writes out a sample config file", commandLineOnly=True) options, args = parser.parse_args() @@ -87,6 +87,11 @@ def main(): parser.error("Where do you want to save the tiles?") destdir = args[1] + if options.display_config: + # just display the config file and exit + parser.display_config() + sys.exit(0) + if options.delete: return delete_all(cachedir, destdir) @@ -124,7 +129,7 @@ def main(): logging.info("Notice: Not using biome data for tinting") # First generate the world's chunk images - w = world.WorldRenderer(worlddir, cachedir, chunklist=chunklist, lighting=options.lighting, night=options.night, spawn=options.spawn, useBiomeData=useBiomeData) + w = world.WorldRenderer(worlddir, cachedir, chunklist=chunklist, rendermode=options.rendermode, useBiomeData=useBiomeData) w.go(options.procs) diff --git a/world.py b/world.py index 5064baf..d1ea20e 100644 --- a/world.py +++ b/world.py @@ -103,12 +103,12 @@ class WorldRenderer(object): files to update. If it includes a trailing newline, it is stripped, so you can pass in file handles just fine. """ - def __init__(self, worlddir, cachedir, chunklist=None, lighting=False, night=False, spawn=False, useBiomeData=False): + def __init__(self, worlddir, cachedir, chunklist=None, rendermode="normal", useBiomeData=False): self.worlddir = worlddir self.caves = False - self.lighting = lighting or night or spawn - self.night = night or spawn - self.spawn = spawn + self.lighting = rendermode in ("lighting","night","spawn") + self.night = rendermode in ("night","spawn") + self.spawn = rendermode in ("spawn",) self.cachedir = cachedir self.useBiomeData = useBiomeData From feeb3283e0b27d23b27ff922c1427cc69e6e36bc Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 8 Jan 2011 01:21:41 -0500 Subject: [PATCH 2/3] New benchmarking script Useful for examining how a code change affects performance --- chunk.py | 3 ++- contrib/benchmark.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 contrib/benchmark.py diff --git a/chunk.py b/chunk.py index 13a387b..7a5fc15 100644 --- a/chunk.py +++ b/chunk.py @@ -924,7 +924,8 @@ class ChunkRenderer(object): # check to see if there are any signs in the persistentData list that are from this chunk. # if so, remove them from the persistentData list (since they're have been added to the world.POI # list above. - self.queue.put(['removePOI', (self.chunkX, self.chunkY)]) + if self.queue: + self.queue.put(['removePOI', (self.chunkX, self.chunkY)]) return img diff --git a/contrib/benchmark.py b/contrib/benchmark.py new file mode 100644 index 0000000..145654e --- /dev/null +++ b/contrib/benchmark.py @@ -0,0 +1,51 @@ +import chunk +import world +import tempfile +import glob +import time +import cProfile +import os +import sys +import shutil + +# Simple Benchmarking script. Usage and example: + +# $ python contrib/benchmark.py World4/ +# Rendering 50 chunks... +# Took 20.290062 seconds or 0.405801 seconds per chunk, or 2.464261 chunks per second + + +# create a new, empty, cache dir +cachedir = tempfile.mkdtemp(prefix="benchmark_cache", dir=".") +if os.path.exists("benchmark.prof"): os.unlink("benchmark.prof") + +w = world.WorldRenderer("World4", cachedir) + +numchunks = 50 +chunklist = w._find_chunkfiles()[:numchunks] + +print "Rendering %d chunks..." % (numchunks) +def go(): + for f in chunklist: + chunk.render_and_save(f[2], w.cachedir, w, (None,None), None) +start = time.time() +if "-profile" in sys.argv: + cProfile.run("go()", 'benchmark.prof') +else: + go() +stop = time.time() + +delta = stop - start + +print "Took %f seconds or %f seconds per chunk, or %f chunks per second" % (delta, delta/numchunks, numchunks/delta) + +if "-profile" in sys.argv: + print "Profile is below:\n----\n" + + import pstats + p = pstats.Stats('benchmark.prof') + + p.strip_dirs().sort_stats("cumulative").print_stats(20) + + +shutil.rmtree(cachedir) From f46d0ce4addb71c2944a034c8f3ffad645b274c8 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 22 Jan 2011 14:23:20 -0500 Subject: [PATCH 3/3] options with default args are now seeded in the globals dict --- configParser.py | 24 ++++++++++++++++++------ gmap.py | 8 ++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/configParser.py b/configParser.py index f7bdb65..0f43a34 100644 --- a/configParser.py +++ b/configParser.py @@ -24,6 +24,10 @@ class ConfigOptionParser(object): print "%s: %r" % (n, self.configResults.__dict__[n]) def add_option(self, *args, **kwargs): + + if kwargs.get("configFileOnly", False) and kwargs.get("commandLineOnly", False): + raise Exception(args, "configFileOnly and commandLineOnly are mututally exclusive") + self.configVars.append(kwargs.copy()) if not kwargs.get("configFileOnly", False): @@ -40,7 +44,6 @@ class ConfigOptionParser(object): # first, load the results from the command line: options, args = self.cmdParser.parse_args() - # second, use these values to seed the locals dict l = dict() g = dict() @@ -48,25 +51,34 @@ class ConfigOptionParser(object): n = a['dest'] if a.get('configFileOnly', False): continue if a.get('commandLineOnly', False): continue - if getattr(options, n) != None: - l[n] = getattr(options, n) + v = getattr(options, n) + if v != None: + #print "seeding %s with %s" % (n, v) + l[n] = v + else: + # if this has a default, use that to seed the globals dict + if a.get("default", None): g[n] = a['default'] g['args'] = args + try: execfile(self.configFile, g, l) except NameError, ex: - pass + import traceback + traceback.print_exc() + print "\nError parsing %s. Please check the trackback above" % self.configFile + sys.exit(1) except SyntaxError, ex: - print "Error parsing %s. Please check the trackback below:" % self.configFile import traceback traceback.print_exc() tb = sys.exc_info()[2] #print tb.tb_frame.f_code.co_filename + print "\nError parsing %s. Please check the trackback above" % self.configFile sys.exit(1) #print l.keys() configResults = OptionsResults() - # first, load the results from the config file: + # third, load the results from the config file: for a in self.configVars: n = a['dest'] if a.get('commandLineOnly', False): diff --git a/gmap.py b/gmap.py index 5ab81d4..749a41b 100755 --- a/gmap.py +++ b/gmap.py @@ -46,17 +46,17 @@ def main(): cpus = 1 parser = ConfigOptionParser(usage=helptext, config="settings.py") parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") - parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int") + parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int", configFileOnly=True) parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) parser.add_option("--cachedir", dest="cachedir", help="Sets the directory where the Overviewer will save chunk images, which is an intermediate step before the tiles are generated. You must use the same directory each time to gain any benefit from the cache. If not set, this defaults to your world directory.") parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") parser.add_option("--rendermode", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal") - parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.") - parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%") + parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True ) + parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True) parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") - parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map", commandLineOnly=True) + parser.add_option("--display-config", dest="display_config", action="store_true", help="Display the configuration parameters, but don't render the map. Requires all required options to be specified", commandLineOnly=True) #parser.add_option("--write-config", dest="write_config", action="store_true", help="Writes out a sample config file", commandLineOnly=True) options, args = parser.parse_args()