From 93424c296d1b608ea8dd98613aa840aa8c72e8f1 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Fri, 13 May 2011 12:30:13 -0400 Subject: [PATCH 1/4] textures are now only generated when actual *rendering* is done --- rendernode.py | 14 ++++++++++++-- src/iterate.c | 42 ++++++++++++++++++++---------------------- src/main.c | 8 ++------ src/overviewer.h | 4 ++-- textures.py | 43 +++++++++++++++++++++++++++---------------- 5 files changed, 63 insertions(+), 48 deletions(-) diff --git a/rendernode.py b/rendernode.py index 9dce5cc..f99c087 100644 --- a/rendernode.py +++ b/rendernode.py @@ -26,6 +26,8 @@ import collections import json import logging import util +import textures +import c_overviewer import cPickle import stat import errno @@ -59,14 +61,22 @@ def pool_initializer(rendernode): logging.debug("Child process {0}".format(os.getpid())) #stash the quadtree objects in a global variable after fork() for windows compat. global child_rendernode - child_rendernode = rendernode + child_rendernode = rendernode + + # make sure textures are generated for this process + # and initialize c_overviewer + textures.generate() + c_overviewer.init_chunk_render() + + # load biome data in each process, if needed for quadtree in rendernode.quadtrees: if quadtree.world.useBiomeData: - import textures # make sure we've at least *tried* to load the color arrays in this process... textures.prepareBiomeData(quadtree.world.worlddir) if not textures.grasscolor or not textures.foliagecolor: raise Exception("Can't find grasscolor.png or foliagecolor.png") + # only load biome data once + break #http://docs.python.org/library/itertools.html def roundrobin(iterables): diff --git a/src/iterate.c b/src/iterate.c index 9f9be9a..b55e3df 100644 --- a/src/iterate.c +++ b/src/iterate.c @@ -24,43 +24,41 @@ static PyObject *special_blocks = NULL; static PyObject *specialblockmap = NULL; static PyObject *transparent_blocks = NULL; -int init_chunk_render(void) { +PyObject *init_chunk_render(PyObject *self, PyObject *args) { - /* if blockmap (or any of these) is not NULL, then that means that we've - * somehow called this function twice. error out so we can notice this - * */ - if (blockmap) return 1; + /* this function only needs to be called once, anything more is an + * error... */ + if (blockmap) { + PyErr_SetString(PyExc_RuntimeError, "init_chunk_render should only be called once per process."); + return NULL; + } textures = PyImport_ImportModule("textures"); /* ensure none of these pointers are NULL */ if ((!textures)) { - fprintf(stderr, "\ninit_chunk_render failed to load; textures\n"); - PyErr_Print(); - return 1; + return NULL; } chunk_mod = PyImport_ImportModule("chunk"); /* ensure none of these pointers are NULL */ if ((!chunk_mod)) { - fprintf(stderr, "\ninit_chunk_render failed to load; chunk\n"); - PyErr_Print(); - return 1; + return NULL; } blockmap = PyObject_GetAttrString(textures, "blockmap"); + if (!blockmap) + return NULL; special_blocks = PyObject_GetAttrString(textures, "special_blocks"); + if (!special_blocks) + return NULL; specialblockmap = PyObject_GetAttrString(textures, "specialblockmap"); + if (!specialblockmap) + return NULL; transparent_blocks = PyObject_GetAttrString(chunk_mod, "transparent_blocks"); + if (!transparent_blocks) + return NULL; - /* ensure none of these pointers are NULL */ - if ((!transparent_blocks) || (!blockmap) || (!special_blocks) || (!specialblockmap)) { - fprintf(stderr, "\ninit_chunk_render failed\n"); - PyErr_Print(); - return 1; - } - - return 0; - + Py_RETURN_NONE; } int @@ -310,7 +308,7 @@ chunk_render(PyObject *self, PyObject *args) { PyObject *t = NULL; if (!PyArg_ParseTuple(args, "OOiiO", &state.self, &state.img, &xoff, &yoff, &blockdata_expanded)) - return Py_BuildValue("i", "-1"); + return NULL; /* fill in important modules */ state.textures = textures; @@ -435,7 +433,7 @@ chunk_render(PyObject *self, PyObject *args) { blockid = NULL; } } - } + } /* free up the rendermode info */ rendermode->finish(rm_data, &state); diff --git a/src/main.c b/src/main.c index 4de7eec..19d3612 100644 --- a/src/main.c +++ b/src/main.c @@ -26,6 +26,8 @@ static PyMethodDef COverviewerMethods[] = { {"alpha_over", alpha_over_wrap, METH_VARARGS, "alpha over composite function"}, + {"init_chunk_render", init_chunk_render, METH_VARARGS, + "Initializes the stuffs renderer."}, {"render_loop", chunk_render, METH_VARARGS, "Renders stuffs"}, @@ -53,12 +55,6 @@ initc_overviewer(void) (void)Py_InitModule("c_overviewer", COverviewerMethods); /* for numpy */ import_array(); - - /* initialize some required variables in iterage.c */ - if (init_chunk_render()) { - fprintf(stderr, "failed to init_chunk_render\n"); - exit(1); // TODO better way to indicate error? - } init_endian(); } diff --git a/src/overviewer.h b/src/overviewer.h index 55243a5..5ee89dc 100644 --- a/src/overviewer.h +++ b/src/overviewer.h @@ -26,7 +26,7 @@ // increment this value if you've made a change to the c extesion // and want to force users to rebuild -#define OVERVIEWER_EXTENSION_VERSION 5 +#define OVERVIEWER_EXTENSION_VERSION 6 /* Python PIL, and numpy headers */ #include @@ -76,7 +76,7 @@ typedef struct { PyObject *left_blocks; PyObject *right_blocks; } RenderState; -int init_chunk_render(void); +PyObject *init_chunk_render(PyObject *self, PyObject *args); int is_transparent(unsigned char b); PyObject *chunk_render(PyObject *self, PyObject *args); diff --git a/textures.py b/textures.py index b740420..8a63d3c 100644 --- a/textures.py +++ b/textures.py @@ -112,9 +112,6 @@ def _split_terrain(terrain): return textures -# This maps terainids to 16x16 images -terrain_images = _split_terrain(_get_terrain_image()) - def transform_image(img, blockID=None): """Takes a PIL image and rotates it left 45 degrees and shrinks the y axis by a factor of 2. Returns the resulting image, which will be 24x12 pixels @@ -460,7 +457,6 @@ def _build_blockimages(): while len(allimages) < 256: allimages.append(None) return allimages -blockmap = _build_blockimages() def load_water(): """Evidentially, the water and lava textures are not loaded from any files @@ -482,8 +478,6 @@ def load_water(): lavablock = _build_block(lavatexture, lavatexture) blockmap[10] = lavablock.convert("RGB"), lavablock blockmap[11] = blockmap[10] -load_water() - def generate_special_texture(blockID, data): """Generates a special texture, such as a correctly facing minecraft track""" @@ -1541,11 +1535,6 @@ def tintTexture(im, c): i.putalpha(im.split()[3]); # copy the alpha band back in. assuming RGBA return i -# generate biome (still grayscale) leaf, grass textures -biome_grass_texture = _build_block(terrain_images[0], terrain_images[38], 2) -biome_leaf_texture = _build_block(terrain_images[52], terrain_images[52], 18) - - currentBiomeFile = None currentBiomeData = None grasscolor = None @@ -1665,9 +1654,31 @@ special_map[2] = range(11) + [0x10,] # grass, grass has not ancildata but is # and is harmless for normal maps special_map[18] = range(16) # leaves, birch, normal or pine leaves (not implemented) +# placeholders that are generated in generate() +terrain_images = None +blockmap = None +biome_grass_texture = None +biome_leaf_texture = None +specialblockmap = None -specialblockmap = {} - -for blockID in special_blocks: - for data in special_map[blockID]: - specialblockmap[(blockID, data)] = generate_special_texture(blockID, data) +def generate(): + # This maps terainids to 16x16 images + global terrain_images + terrain_images = _split_terrain(_get_terrain_image()) + + # generate the normal blocks + global blockmap + blockmap = _build_blockimages() + load_water() + + # generate biome (still grayscale) leaf, grass textures + global biome_grass_texture, biome_leaf_texture + biome_grass_texture = _build_block(terrain_images[0], terrain_images[38], 2) + biome_leaf_texture = _build_block(terrain_images[52], terrain_images[52], 18) + + # generate the special blocks + global specialblockmap, special_blocks + specialblockmap = {} + for blockID in special_blocks: + for data in special_map[blockID]: + specialblockmap[(blockID, data)] = generate_special_texture(blockID, data) From dacd45e5d32aab58258a0722495c6b86c9dc0534 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Fri, 13 May 2011 17:21:48 -0400 Subject: [PATCH 2/4] configurable settings.py, web_assets, textures paths --- configParser.py | 11 +++++- googlemap.py | 34 +++++++++++-------- overviewer.py | 4 ++- rendernode.py | 5 +-- textures.py | 29 +++++++++------- .../overviewerConfig.js | 0 6 files changed, 52 insertions(+), 31 deletions(-) rename overviewerConfig.js => web_assets/overviewerConfig.js (100%) diff --git a/configParser.py b/configParser.py index 5b275f0..c779fc2 100644 --- a/configParser.py +++ b/configParser.py @@ -22,6 +22,9 @@ class ConfigOptionParser(object): self.customArgs = ["required", "commandLineOnly", "default", "listify", "listdelim", "choices"] self.requiredArgs = [] + + # add the *very* special config-file path option + self.add_option("--config-file", dest="config_file", help="Specifies a configuration file to load, by name. This file's format is discussed in the README.", metavar="PATH", type="string", commandLineOnly=True) def display_config(self): logging.info("Using the following settings:") @@ -68,8 +71,14 @@ class ConfigOptionParser(object): # if this has a default, use that to seed the globals dict if a.get("default", None): g[n] = a['default'] g['args'] = args - + try: + if options.config_file: + self.configFile = options.config_file + elif os.path.exists(self.configFile): + # warn about automatic loading + logging.warning("Automatic settings.py loading is DEPRECATED, and may be removed in the future. Please use --config-file instead.") + if os.path.exists(self.configFile): execfile(self.configFile, g, l) except NameError, ex: diff --git a/googlemap.py b/googlemap.py index 331a4ea..daf0e0e 100644 --- a/googlemap.py +++ b/googlemap.py @@ -68,6 +68,7 @@ class MapGen(object): self.skipjs = configInfo.get('skipjs', None) self.web_assets_hook = configInfo.get('web_assets_hook', None) + self.web_assets_path = configInfo.get('web_assets_path', None) self.bg_color = configInfo.get('bg_color') if not len(quadtrees) > 0: @@ -81,14 +82,28 @@ class MapGen(object): raise ValueError("all the given quadtrees must have the same destdir and world") self.quadtrees = quadtrees - + def go(self, procs): """Writes out config.js, marker.js, and region.js Copies web assets into the destdir""" zoomlevel = self.p - configpath = os.path.join(util.get_program_path(), "overviewerConfig.js") - config = open(configpath, 'r').read() + bgcolor = (int(self.bg_color[1:3],16), int(self.bg_color[3:5],16), int(self.bg_color[5:7],16), 0) + blank = Image.new("RGBA", (1,1), bgcolor) + # Write a blank image + for quadtree in self.quadtrees: + tileDir = os.path.join(self.destdir, quadtree.tiledir) + if not os.path.exists(tileDir): os.mkdir(tileDir) + blank.save(os.path.join(tileDir, "blank."+quadtree.imgformat)) + + # copy web assets into destdir: + mirror_dir(os.path.join(util.get_program_path(), "web_assets"), self.destdir) + # do the same with the local copy, if we have it + if self.web_assets_path: + mirror_dir(self.web_assets_path, self.destdir) + + # replace the config js stuff + config = open(os.path.join(self.destdir, 'overviewerConfig.js'), 'r').read() config = config.replace( "{minzoom}", str(0)) config = config.replace( @@ -111,18 +126,7 @@ class MapGen(object): with open(os.path.join(self.destdir, "overviewerConfig.js"), 'w') as output: output.write(config) - bgcolor = (int(self.bg_color[1:3],16), int(self.bg_color[3:5],16), int(self.bg_color[5:7],16), 0) - blank = Image.new("RGBA", (1,1), bgcolor) - # Write a blank image - for quadtree in self.quadtrees: - tileDir = os.path.join(self.destdir, quadtree.tiledir) - if not os.path.exists(tileDir): os.mkdir(tileDir) - blank.save(os.path.join(tileDir, "blank."+quadtree.imgformat)) - - # copy web assets into destdir: - mirror_dir(os.path.join(util.get_program_path(), "web_assets"), self.destdir) - - # Add time in index.html + # Add time and version in index.html indexpath = os.path.join(self.destdir, "index.html") index = open(indexpath, 'r').read() diff --git a/overviewer.py b/overviewer.py index bd4ef7b..396cee8 100755 --- a/overviewer.py +++ b/overviewer.py @@ -101,6 +101,8 @@ def main(): parser.add_option("--bg_color", dest="bg_color", help="Configures the background color for the GoogleMap output. Specify in #RRGGBB format", configFileOnly=True, type="string", default="#1A1A1A") parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True) parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this function after the web assets have been copied, but before actual tile rendering begins. It should accept a QuadtreeGen object as its only argument.", action="store", metavar="SCRIPT", type="function", configFileOnly=True) + parser.add_option("--web-assets-path", dest="web_assets_path", help="Specifies a non-standard web_assets directory to use. Files here will overwrite the default web assets.", metavar="PATH", type="string", configFileOnly=True) + parser.add_option("--textures-path", dest="textures_path", help="Specifies a non-standard textures path, from which terrain.png and other textures are loaded.", metavar="PATH", type="string", configFileOnly=True) parser.add_option("-q", "--quiet", dest="quiet", action="count", default=0, help="Print less output. You can specify this option multiple times.") parser.add_option("-v", "--verbose", dest="verbose", action="count", default=0, help="Print more output. You can specify this option multiple times.") parser.add_option("--skip-js", dest="skipjs", action="store_true", help="Don't output marker.js or regions.js") @@ -242,7 +244,7 @@ def main(): qtree.go(options.procs) # create the distributed render - r = rendernode.RenderNode(q) + r = rendernode.RenderNode(q, options) # write out the map and web assets m = googlemap.MapGen(q, configInfo=options) diff --git a/rendernode.py b/rendernode.py index f99c087..6f6c76e 100644 --- a/rendernode.py +++ b/rendernode.py @@ -65,7 +65,7 @@ def pool_initializer(rendernode): # make sure textures are generated for this process # and initialize c_overviewer - textures.generate() + textures.generate(path=rendernode.options.get('textures_path', None)) c_overviewer.init_chunk_render() # load biome data in each process, if needed @@ -94,12 +94,13 @@ def roundrobin(iterables): class RenderNode(object): - def __init__(self, quadtrees): + def __init__(self, quadtrees, options): """Distributes the rendering of a list of quadtrees.""" if not len(quadtrees) > 0: raise ValueError("there must be at least one quadtree to work on") + self.options = options self.quadtrees = quadtrees #bind an index value to the quadtree so we can find it again #and figure out which worlds are where diff --git a/textures.py b/textures.py index 8a63d3c..611353a 100644 --- a/textures.py +++ b/textures.py @@ -26,11 +26,14 @@ from PIL import Image, ImageEnhance, ImageOps, ImageDraw import util import composite +_find_file_local_path = None def _find_file(filename, mode="rb"): """Searches for the given file and returns an open handle to it. This searches the following locations in this order: + * the textures_path given in the config file (if present) * The program dir (same dir as this file) + * The program dir / textures * On Darwin, in /Applications/Minecraft * Inside minecraft.jar, which is looked for at these locations @@ -38,14 +41,21 @@ def _find_file(filename, mode="rb"): * On Darwin, at $HOME/Library/Application Support/minecraft/bin/minecraft.jar * at $HOME/.minecraft/bin/minecraft.jar - * The current working directory - * The program dir / textures - """ + + if _find_file_local_path: + path = os.path.join(_find_file_local_path, filename) + if os.path.exists(path): + return open(path, mode) + programdir = util.get_program_path() path = os.path.join(programdir, filename) if os.path.exists(path): return open(path, mode) + + path = os.path.join(programdir, "textures", filename) + if os.path.exists(path): + return open(path, mode) if sys.platform == "darwin": path = os.path.join("/Applications/Minecraft", filename) @@ -73,14 +83,6 @@ def _find_file(filename, mode="rb"): except (KeyError, IOError): pass - path = filename - if os.path.exists(path): - return open(path, mode) - - path = os.path.join(programdir, "textures", filename) - if os.path.exists(path): - return open(path, mode) - raise IOError("Could not find the file {0}. Is Minecraft installed? If so, I couldn't find the minecraft.jar file.".format(filename)) def _load_image(filename): @@ -1661,7 +1663,10 @@ biome_grass_texture = None biome_leaf_texture = None specialblockmap = None -def generate(): +def generate(path=None): + global _find_file_local_path + _find_file_local_path = path + # This maps terainids to 16x16 images global terrain_images terrain_images = _split_terrain(_get_terrain_image()) diff --git a/overviewerConfig.js b/web_assets/overviewerConfig.js similarity index 100% rename from overviewerConfig.js rename to web_assets/overviewerConfig.js From 21de6222ca80a251e4fc92514e7278e7442b8e84 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Fri, 13 May 2011 18:22:00 -0400 Subject: [PATCH 3/4] renamed --config-file to --settings, updated README with new options --- README.rst | 23 +++++++++++++++++++++-- configParser.py | 4 ++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 1d0e9ab..11f9f10 100644 --- a/README.rst +++ b/README.rst @@ -191,11 +191,18 @@ Options --list-rendermodes List the available render modes, and a short description of each. +--settings=PATH + Use this option to load settings from a file. The format of this file is + given below. + Settings -------- -You can optionally store settings in a file named settings.py. It is a regular -python script, so you can use any python functions or modules you want. + +You can optionally store settings in a file named settings.py (or really, +anything you want). It is a regular python script, so you can use any python +functions or modules you want. To use a settings file, use the --settings +command line option. For a sample settings file, look at 'sample.settings.py'. Note that this file is not meant to be used directly, but instead it should be used as a @@ -241,6 +248,18 @@ web_assets_hook This function should accept one argument: a QuadtreeGen object. +web_assets_path + This option lets you provide alternative web assets to use when + rendering. The contents of this folder will be copied into the output folder + during render, and will overwrite any default files already copied by + Overviewer. See the web_assets folder included with Overviewer for the + default assets. + +textures_path + This is like web_assets_path, but instead it provides an alternative texture + source. Overviewer looks in here for terrain.png and other textures before + it looks anywhere else. + Viewing the Results ------------------- Within the output directory you will find two things: an index.html file, and a diff --git a/configParser.py b/configParser.py index c779fc2..01e6a14 100644 --- a/configParser.py +++ b/configParser.py @@ -24,7 +24,7 @@ class ConfigOptionParser(object): self.requiredArgs = [] # add the *very* special config-file path option - self.add_option("--config-file", dest="config_file", help="Specifies a configuration file to load, by name. This file's format is discussed in the README.", metavar="PATH", type="string", commandLineOnly=True) + self.add_option("--settings", dest="config_file", help="Specifies a settings file to load, by name. This file's format is discussed in the README.", metavar="PATH", type="string", commandLineOnly=True) def display_config(self): logging.info("Using the following settings:") @@ -77,7 +77,7 @@ class ConfigOptionParser(object): self.configFile = options.config_file elif os.path.exists(self.configFile): # warn about automatic loading - logging.warning("Automatic settings.py loading is DEPRECATED, and may be removed in the future. Please use --config-file instead.") + logging.warning("Automatic settings.py loading is DEPRECATED, and may be removed in the future. Please use --settings instead.") if os.path.exists(self.configFile): execfile(self.configFile, g, l) From 6dd3ac067c8d635ed91c5366633dcfeacbb62818 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Fri, 13 May 2011 21:30:26 -0400 Subject: [PATCH 4/4] fixed setup.py info to account for overviewerConfig.js move --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 415e1a4..a5baa97 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup_kwargs['cmdclass'] = {} if py2exe is not None: setup_kwargs['console'] = ['overviewer.py'] setup_kwargs['data_files'] = [('textures', ['textures/lava.png', 'textures/water.png', 'textures/fire.png', 'textures/portal.png']), - ('', ['overviewerConfig.js', 'COPYING.txt', 'README.rst']), + ('', ['COPYING.txt', 'README.rst']), ('web_assets', glob.glob('web_assets/*'))] setup_kwargs['zipfile'] = None if platform.system() == 'Windows' and '64bit' in platform.architecture():