From fe55d5343cd562a3fdadf59d79535cc37bc54836 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Tue, 6 Mar 2012 23:37:29 -0500 Subject: [PATCH] added a "changelist" option --- docs/config.rst | 17 +++++ overviewer.py | 25 +++++-- overviewer_core/settingsDefinition.py | 1 + overviewer_core/tileset.py | 97 +++++++++++++++++---------- 4 files changed, 101 insertions(+), 39 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index d2358a2..cac1d81 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -426,6 +426,23 @@ values. The valid configuration keys are listed below. 'forcerender': True, } +``changelist`` + This is a string. It names a file where it will write out, one per line, the + path to tiles that have been updated. You can specify the same file for + multiple (or all) renders and they will all be written to the same file. The + file is cleared when The Overviewer starts. + + This option is useful in conjunction with a simple upload script, to upload + the files that have changed. + + .. warning:: + + A solution like ``rsync -a --delete`` is much better because it also + watches for tiles that should be *deleted*, which is impossible to + convey with the changelist option. If your map ever shrinks or you've + removed some tiles, you may need to do some manual deletion on the + remote side. + .. _customrendermodes: Custom Rendermodes and Rendermode Primitives diff --git a/overviewer.py b/overviewer.py index f739bdd..d8800ba 100755 --- a/overviewer.py +++ b/overviewer.py @@ -252,9 +252,12 @@ dir but you forgot to put quotes around the directory, since it contains spaces. logging.exception("An error was encountered with your configuration. See the info below.") return 1 + ############################################################ # Final validation steps and creation of the destination directory + logging.info("Welcome to Minecraft Overviewer!") + logging.debug("Current log level: {0}".format(logging.getLogger().level)) # Override some render configdict options depending on one-time command line # modifiers @@ -315,13 +318,23 @@ dir but you forgot to put quotes around the directory, since it contains spaces. logging.exception("Could not create the output directory.") return 1 + # The changelist support. + changelists = {} + for render in config['renders'].itervalues(): + if 'changelist' in render: + path = render['changelist'] + if path not in changelists: + out = open(path, "w") + logging.debug("Opening changelist %s (%s)", out, out.fileno()) + changelists[path] = out + else: + out = changelists[path] + render['changelist'] = out.fileno() + ######################################################################## # Now we start the actual processing, now that all the configuration has # been gathered and validated - logging.info("Welcome to Minecraft Overviewer!") - logging.debug("Current log level: {0}".format(logging.getLogger().level)) - # create our asset manager... ASSMAN assetMrg = assetmanager.AssetManager(destdir) @@ -394,7 +407,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # only pass to the TileSet the options it really cares about render['name'] = render_name # perhaps a hack. This is stored here for the asset manager - tileSetOpts = util.dict_subset(render, ["name", "imgformat", "renderchecks", "rerenderprob", "bgcolor", "imgquality", "optimizeimg", "rendermode", "worldname_orig", "title", "dimension"]) + tileSetOpts = util.dict_subset(render, ["name", "imgformat", "renderchecks", "rerenderprob", "bgcolor", "imgquality", "optimizeimg", "rendermode", "worldname_orig", "title", "dimension", "changelist"]) tset = tileset.TileSet(rset, assetMrg, tex, tileSetOpts, tileset_dir) tilesets.append(tset) @@ -427,6 +440,10 @@ dir but you forgot to put quotes around the directory, since it contains spaces. assetMrg.finalize(tilesets) + for out in changelists.itervalues(): + logging.debug("Closing %s (%s)", out, out.fileno()) + out.close() + if config['processes'] == 1: logging.debug("Final cache stats:") for c in caches: diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index e023312..67fee81 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -74,6 +74,7 @@ renders = Setting(required=True, default=util.OrderedDict(), "renderchecks": Setting(required=False, validator=validateInt, default=None), "rerenderprob": Setting(required=True, validator=validateFloat, default=0), "crop": Setting(required=False, validator=validateCrop, default=None), + "changelist": Setting(required=False, validator=validateStr, default=None), # Remove this eventually (once people update their configs) "worldname": Setting(required=False, default=None, diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 5c409d3..b889286 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -97,57 +97,57 @@ Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow")) # # For reference, here's what the rendercheck modes are: # 0 -# Only render tiles that have chunks with a greater mtime than -# the last render timestamp, and their ancestors. +# Only render tiles that have chunks with a greater mtime than the last +# render timestamp, and their ancestors. # -# In other words, only renders parts of the map that have changed -# since last render, nothing more, nothing less. +# In other words, only renders parts of the map that have changed since +# last render, nothing more, nothing less. # -# This is the fastest option, but will not detect tiles that have -# e.g. been deleted from the directory tree, or pick up where a -# partial interrupted render left off. +# This is the fastest option, but will not detect tiles that have e.g. +# been deleted from the directory tree, or pick up where a partial +# interrupted render left off. # 1 -# For render-tiles, render all whose chunks have an mtime greater -# than the mtime of the tile on disk, and their upper-tile -# ancestors. +# For render-tiles, render all whose chunks have an mtime greater than +# the mtime of the tile on disk, and their composite-tile ancestors. # -# Also check all other upper-tiles and render any that have -# children with more rencent mtimes than itself. +# Also check all other composite-tiles and render any that have children +# with more rencent mtimes than itself. # -# This is slower due to stat calls to determine tile mtimes, but -# safe if the last render was interrupted. +# This is slower due to stat calls to determine tile mtimes, but safe if +# the last render was interrupted. # 2 -# Render all tiles unconditionally. This is a "forcerender" and -# is the slowest, but SHOULD be specified if this is the first -# render because the scan will forgo tile stat calls. It's also -# useful for changing texture packs or other options that effect -# the output. +# Render all tiles unconditionally. This is a "forcerender" and is the +# slowest, but SHOULD be specified if this is the first render because +# the scan will forgo tile stat calls. It's also useful for changing +# texture packs or other options that effect the output. # # For 0 our caller has explicitly requested not to check mtimes on disk to # speed things up. So the mode 0 chunk scan only looks at chunk mtimes and the -# last render mtime, and has marked only the render-tiles that need rendering. -# Mode 0 then iterates over all dirty render-tiles and upper-tiles that depend -# on them. It does not check mtimes of upper-tiles, so this is only a good -# option if the last render was not interrupted. +# last render mtime from the asset manager, and marks only the tiles that need +# rendering based on that. Mode 0 then iterates over all dirty render-tiles +# and composite-tiles that depend on them. It does not check mtimes of any +# tiles on disk, so this is only a good option if the last render was not +# interrupted. # For mode 2, this is a forcerender, the caller has requested we render # everything. The mode 2 chunk scan marks every tile as needing rendering, and # disregards mtimes completely. Mode 2 then iterates over all render-tiles and -# upper-tiles that depend on them, which is every tile that should exist. +# composite-tiles that depend on them, which is every tile. It therefore +# renders everything. # In both 0 and 2 the render iteration is the same: the dirtytile tree built is # authoritive on every tile that needs rendering. # In mode 1, things are most complicated. Mode 1 chunk scan is identical to a # forcerender, or mode 2 scan: every render tile that should exist is marked in -# the dirtytile tree. Then, a special recursive algorithm goes through and -# checks every tile that should exist and determines whether it needs -# rendering. This routine works in such a way so that every tile is stat()'d at -# most once, so it shouldn't be too bad. This logic happens in the -# iterate_work_items() method, and therefore in the master process, not the -# worker processes. +# the dirtytile tree. But instead of iterating over that tree directly, a +# special recursive algorithm goes through and checks every tile that should +# exist and determines whether it needs rendering. This routine works in such a +# way so that every tile is stat()'d at most once, so it shouldn't be too bad. +# This logic happens in the iterate_work_items() method, and therefore in the +# master process, not the worker processes. # In all three rendercheck modes, the results out of iterate_work_items() is # authoritive on what needs rendering. The do_work() method does not need to do @@ -253,6 +253,11 @@ class TileSet(object): that a tile which is not marked for render by any mtime checks will be rendered anyways. 0 disables this option. + changelist + Optional: A file descriptor which will be opened and used as the + changelist output: each tile written will get outputted to the + specified fd. + Other options that must be specified but aren't really documented (oops. consider it a TODO): * worldname_orig @@ -370,6 +375,29 @@ class TileSet(object): This method returns an iterator over (obj, [dependencies, ...]) """ + fd = self.options.get("changelist", None) + if fd: + logging.debug("Changelist activated for %s (fileno %s)", self, fd) + # This re-implements some of the logic from do_work() + def write_out(tilepath): + if len(tilepath) == self.treedepth: + rt = RenderTile.from_path(tilepath) + imgpath = rt.get_filepath(self.outputdir, self.imgextension) + elif len(tilepath) == 0: + imgpath = os.path.join(self.outputdir, "base."+self.imgextension) + else: + dest = os.path.join(self.outputdir, *(str(x) for x in tilepath[:-1])) + name = str(tilepath[-1]) + imgpath = os.path.join(dest, name) + "." + self.imgextension + # We use low-level file output because we don't want open file + # handles being passed to subprocesses. fd is just an integer. + # This method is only called from the master process anyways. + # We don't use os.fdopen() because this fd may be shared by + # many tileset objects, and as soon as this method exists the + # file object may be garbage collected, closing the file. + os.write(fd, imgpath + "\n") + + # See note at the top of this file about the rendercheck modes for an # explanation of what this method does in different situations. # @@ -384,6 +412,8 @@ class TileSet(object): # wait for the items that do exist and are in the queue. for i in range(4): dependencies.append( tilepath + (i,) ) + if fd: + write_out(tilepath) yield tilepath, dependencies else: @@ -395,8 +425,9 @@ class TileSet(object): dependencies = [] for i in range(4): dependencies.append( tilepath + (i,) ) + if fd: + write_out(tilepath) yield tilepath, dependencies - def do_work(self, tilepath): """Renders the given tile. @@ -405,10 +436,6 @@ class TileSet(object): integers representing the path of the tile to render. """ - # For rendercheck modes 0 and 2: unconditionally render the specified - # tile. - # For rendercheck mode 1, unconditionally render render-tiles, but - # check if the given upper-tile needs rendering if len(tilepath) == self.treedepth: # A render-tile self._render_rendertile(RenderTile.from_path(tilepath))