# This file is part of the Minecraft Overviewer. # # Minecraft Overviewer is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation, either version 3 of the License, or (at # your option) any later version. # # Minecraft Overviewer is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along # with the Overviewer. If not, see . import os import os.path from collections import namedtuple import logging import shutil import itertools from .util import iterate_base4, convert_coords """ tileset.py contains the TileSet class, and in general, routines that manage a set of output tiles corresponding to a requested rendermode for a world. In general, there will be one TileSet object per world per rendermode requested by the user. The TileSet class implements the Worker interface. This interface has the following methods: do_preprocessing() This method is called before iterate_work_items(). It should do any work that needs to be done prior to iterate_work_items(). It is not called for instances that will not have iterate_work_items() called. get_num_phases() This method returns an integer indicating how many phases of work this worker has to perform. Each phase of work is completed serially with the other phases... all work done by one phase is done before the next phase is started. iterate_work_items(phase) Takes a phase number (a non-negative integer). This method should return an iterator over work items and a list of dependencies i.e. (work_item, [d1, d2, ...]). The work items and dependencies can be any pickelable object; they are treated as opaque by the Dispatcher. The work item objects are passed back in to the do_work() method (perhaps in a different, identically configured instance). The dependency items are other work items that are compared for equality with work items that are already in the queue. The dispatcher guarantees that dependent items which are currently in the queue or in progress finish before the corresponding work item is started. Note that dependencies must have already been yielded as work items before they can be used as dependencies; the dispatcher requires this ordering or it cannot guarantee the dependencies are met. do_work(workobj) Does the work for a given work object. This method is not expected to return anything, so the results of its work should be reflected on the filesystem or by sending signals. """ # A named tuple class storing the row and column bounds for the to-be-rendered # world Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow")) __all__ = ["TileSet"] class TileSet(object): """The TileSet object manages the work required to produce a set of tiles on disk. It calculates the work that needs to be done and tells the dipatcher (through the Worker interface) this information. The Dispatcher then tells this object when and where to do the work of rendering the tiles. """ def __init__(self, regionsetobj, assetmanagerobj, options, outputdir): """Construct a new TileSet object with the given configuration options dictionary. options is a dictionary of configuration parameters (strings mapping to values) that are interpreted by the rendering engine. regionsetobj is the RegionSet object that is used to render the tiles. assetmanagerobj is the AssetManager object that represents the destination directory where we'll put our tiles. outputdir is the absolute path to the tile output directory where the tiles are saved. It is assumed to exist already. TODO: This should probably be relative to the asset manager's output directory to avoid redundancy. Current valid options for the options dictionary are shown below. All the options must be specified unless they are not relevant. If the given options do not conform to the specifications, behavior is undefined (this class does not do any error checking and assumes items are given in the correct form). bgcolor A hex string specifying the background color for jpeg output. e.g.: "#1A1A1A". Not relevant unless rendering jpeg. renderchecks An integer indicating how to determine which tiles need updating and which don't. This is one of three levels: 0 Only render tiles that have chunks with a greater mtime than the last render timestamp (the fastest option) 1 Render all tiles whose chunks have an mtime greater than the mtime of the tile on disk (slower due to stat calls to determine tile mtimes, but safe if the last render was interrupted) 2 Render all tiles unconditionally. This is a "forcerender" and is the slowest, but SHOULD be specified if this is the first render because the scan will forgo tile stat calls. imgformat A string indicating the output format. Must be one of 'png' or 'jpeg' imgquality An integer 1-100 indicating the quality of the jpeg output. Only relevant in jpeg mode. optimizeimg an integer indiating optimizations to perform on png outputs. 0 indicates no optimizations. Only relevant in png mode. 1 indicates pngcrush is run on all output images 2 indicates pngcrush and advdef are run on all output images with advdef -z2 3 indicates pngcrush and advdef are run on all output images with advdef -z4 rendermode Perhaps the most important/relevant option: a string indicating the render mode to render. This rendermode must have already been registered with the C extension module. rerender_prob A floating point number between 0 and 1 indicating the probability that a tile which is not marked for render by any mtime checks will be rendered anyways. 0 disables this option. """ self.options = options self.regionset = regionsetobj self.am = assetmanagerobj # Throughout the class, self.outputdir is an absolute path to the # directory where we output tiles. It is assumed to exist. self.outputdir = os.path.abspath(outputdir) # Set the image format according to the options if self.options['imgformat'] == 'png': self.imgextension = 'png' elif self.options['imgformat'] == 'jpeg': self.imgextension = 'jpg' def do_preprocessing(self): """For the preprocessing step of the Worker interface, this does the chunk scan and stores the resulting tree as a private instance attribute for later use in iterate_work_items() """ # REMEMBER THAT ATTRIBUTES ASSIGNED IN THIS METHOD ARE NOT AVAILABLE IN # THE do_work() METHOD # Calculate the min and max column over all the chunks self._find_chunk_range() bounds = self.bounds # Calculate the depth of the tree for p in xrange(1,33): # max 32 # Will 2^p tiles wide and high suffice? # X has twice as many chunks as tiles, then halved since this is a # radius xradius = 2**p # Y has 4 times as many chunks as tiles, then halved since this is # a radius yradius = 2*2**p if xradius >= bounds.maxcol and -xradius <= bounds.mincol and \ yradius >= bounds.maxrow and -yradius <= bounds.minrow: break if p >= 15: logging.warning("Just letting you know, your map requries %s zoom levels. This is REALLY big!", p) self.treedepth = p self._rearrange_tiles() def get_num_phases(self): """Returns the number of levels in the quadtree, which is equal to the number of phases of work that need to be done. """ return 1 def iterate_work_items(self, phase): """Iterates over the dirty tiles in the tree at level depth-phase. So the first phase iterates over the deepest tiles in the tree, and works its way to the root node of the tree. """ pass def do_work(self, tileobj): """Renders the given tile. """ def get_persistent_data(self): """Returns a dictionary representing the persistent data of this TileSet. Typically this is called by AssetManager """ pass def _find_chunk_range(self): """Finds the chunk range in rows/columns and stores them in self.minrow, self.maxrow, self.mincol, self.maxcol """ minrow = mincol = maxrow = maxcol = 0 for c_x, c_z, _ in self.regionset.iterate_chunks(): # Convert these coordinates to row/col col, row = convert_coords(c_x, c_z) minrow = min(minrow, row) maxrow = max(maxrow, row) mincol = min(mincol, col) maxcol = max(maxcol, col) self.bounds = Bounds(mincol, maxcol, minrow, maxrow) def _rearrange_tiles(self): """If the target size of the tree is not the same as the existing size on disk, do some re-arranging """ try: curdepth = get_dirdepth(self.outputdir) except Exception: logging.critical("Could not determine existing tile tree depth. Does it exist?") raise if self.treedepth != cur_depth: if self.treedepth > curdepth: logging.warning("Your map seems to have expanded beyond its previous bounds.") logging.warning( "Doing some tile re-arrangements... just a sec...") for _ in xrange(self.p-curdepth): self._increase_depth() elif self.p < curdepth: logging.warning("Your map seems to have shrunk. Did you delete some chunks? No problem. Re-arranging tiles, just a sec...") for _ in xrange(curdepth - self.p): self._decrease_depth() def _increase_depth(self): """Moves existing tiles into place for a larger tree""" getpath = functools.partial(os.path.join, self.outputdir) # At top level of the tree: # quadrant 0 is now 0/3 # 1 is now 1/2 # 2 is now 2/1 # 3 is now 3/0 # then all that needs to be done is to regenerate the new top level for dirnum in range(4): newnum = (3,2,1,0)[dirnum] newdir = "new" + str(dirnum) newdirpath = getpath(newdir) files = [str(dirnum)+"."+self.imgextension, str(dirnum)] newfiles = [str(newnum)+"."+self.imgextension, str(newnum)] os.mkdir(newdirpath) for f, newf in zip(files, newfiles): p = getpath(f) if os.path.exists(p): os.rename(p, getpath(newdir, newf)) os.rename(newdirpath, getpath(str(dirnum))) def _decrease_depth(self): """If the map size decreases, or perhaps the user has a depth override in effect, re-arrange existing tiles for a smaller tree""" getpath = functools.partial(os.path.join, self.outputdir) # quadrant 0/3 goes to 0 # 1/2 goes to 1 # 2/1 goes to 2 # 3/0 goes to 3 # Just worry about the directories here, the files at the top two # levels are cheap enough to replace if os.path.exists(getpath("0", "3")): os.rename(getpath("0", "3"), getpath("new0")) shutil.rmtree(getpath("0")) os.rename(getpath("new0"), getpath("0")) if os.path.exists(getpath("1", "2")): os.rename(getpath("1", "2"), getpath("new1")) shutil.rmtree(getpath("1")) os.rename(getpath("new1"), getpath("1")) if os.path.exists(getpath("2", "1")): os.rename(getpath("2", "1"), getpath("new2")) shutil.rmtree(getpath("2")) os.rename(getpath("new2"), getpath("2")) if os.path.exists(getpath("3", "0")): os.rename(getpath("3", "0"), getpath("new3")) shutil.rmtree(getpath("3")) os.rename(getpath("new3"), getpath("3")) # Delete the files in the top directory to make sure they get re-created. files = [str(num)+"."+self.imgextension for num in xrange(4)] + ["base." + self.imgextension] for f in files: try: os.unlink(getpath(f)) except OSError, e: pass # doesn't exist maybe? def get_dirdepth(outputdir): """Returns the current depth of the tree on disk """ # Traverses down the first directory until it reaches one with no # subdirectories. While all paths of the tree may not exist, all paths # of the tree should and are assumed to be the same depth # This function returns a list of all subdirectories of the given # directory. It's slightly more complicated than you'd think it should be # because one must turn directory names returned by os.listdir into # relative/absolute paths before they can be passed to os.path.isdir() getsubdirs = lambda directory: [ abssubdir for abssubdir in (os.path.join(directory,subdir) for subdir in os.listdir(directory)) if os.path.isdir(abssubdir) ] depth = 1 subdirs = getsubdirs(outputdir) while subdirs: subdirs = getsubdirs(subdirs[0]) depth += 1 return depth class DirtyTiles(object): """This tree holds which tiles need rendering. Each instance is a node, and the root of a subtree. Each node knows its "level", which corresponds to the zoom level where 0 is the inner-most (most zoomed in) tiles. Instances hold the clean/dirty state of their children. Leaf nodes are images and do not physically exist in the tree, level 1 nodes keep track of leaf image state. Level 2 nodes keep track of level 1 state, and so fourth. In attempt to keep things memory efficient, subtrees that are completely dirty are collapsed """ __slots__ = ("depth", "children") def __init__(self, depth): """Initialize a new tree with the specified depth. This actually initializes a node, which is the root of a subtree, with `depth` levels beneath it. """ # Stores the depth of the tree according to this node. This is not the # depth of this node, but rather the number of levels below this node # (including this node). self.depth = depth # the self.children array holds the 4 children of this node. This # follows the same quadtree convention as elsewhere: children 0, 1, 2, # 3 are the upper-left, upper-right, lower-left, and lower-right # respectively # Values are: # False # All children down this subtree are clean # True # All children down this subtree are dirty # A DirtyTiles instance # the instance defines which children down that subtree are # clean/dirty. # A node with depth=1 cannot have a DirtyTiles instance in its # children since its leaves are images, not more tree self.children = [False] * 4 def set_dirty(self, path): """Marks the requested leaf node as "dirty". Path is an iterable of integers representing the path to the leaf node that is requested to be marked as dirty. """ path = list(path) assert len(path) == self.depth path.reverse() self._set_dirty_helper(path) def _set_dirty_helper(self, path): """Recursive call for set_dirty() Expects path to be a list in reversed order If *all* the nodes below this one are dirty, this function returns true. Otherwise, returns None. """ if self.depth == 1: # Base case self.children[path[0]] = True # Check to see if all children are dirty if all(self.children): return True else: # Recursive case childnum = path.pop() child = self.children[childnum] if child == False: # Create a new node child = self.__class__(self.depth-1) child._set_dirty_helper(path) self.children[childnum] = child elif child == True: # Every child is already dirty. Nothing to do. return else: # subtree is mixed clean/dirty. Recurse ret = child._set_dirty_helper(path) if ret: # Child says it's completely dirty, so we can purge the # subtree and mark it as dirty. The subtree will be garbage # collected when this method exits. self.children[childnum] = True # Since we've marked an entire sub-tree as dirty, we may be # able to signal to our parent if all(x is True for x in self.children): return True def iterate_dirty(self, level=None): """Returns an iterator over every dirty tile in this subtree. Each item yielded is a sequence of integers representing the quadtree path to the dirty tile. Yielded sequences are of length self.depth. If level is None, iterates over tiles of the highest level, i.e. worldtiles. If level is a value between 0 and the depth of this tree, this method iterates over tiles at that level. Zoom level 0 is zoomed all the way out, zoom level `depth` is all the way in. In other words, specifying level causes the tree to be iterated as if it was only that depth. """ if level is None: todepth = 1 else: if not (level > 0 and level <= self.depth): raise ValueError("Level parameter must be between 1 and %s" % self.depth) todepth = self.depth - level + 1 return (tuple(reversed(rpath)) for rpath in self._iterate_dirty_helper(todepth)) def _iterate_dirty_helper(self, todepth): if self.depth == todepth: # Base case if self.children[0]: yield [0] if self.children[1]: yield [1] if self.children[2]: yield [2] if self.children[3]: yield [3] else: # Higher levels: for c, child in enumerate(self.children): if child == True: # All dirty down this subtree, iterate over every leaf for x in iterate_base4(self.depth-todepth): x = list(x) x.append(c) yield x elif child != False: # Mixed dirty/clean down this subtree, recurse for path in child._iterate_dirty_helper(todepth): path.append(c) yield path def query_path(self, path): """Queries for the state of the given tile in the tree. Returns False for "clean", True for "dirty" """ # Traverse the tree down the given path. If the tree has been # collapsed, then just return what the subtree is. Otherwise, if we # find the specific DirtyTree requested, return its state using the # __nonzero__ call. treenode = self for pathelement in path: treenode = treenode.children[pathelement] if not isinstance(treenode, DirtyTiles): return treenode # If the method has not returned at this point, treenode is the # requested node, but it is an inner node with possibly mixed state # subtrees. If any of the children are True return True. This call # relies on the __nonzero__ method return bool(treenode) def __nonzero__(self): """Returns the boolean context of this particular node. If any descendent of this node is True return True. Otherwise, False. """ # Any chilren that are True or are DirtyTiles that evaluate to True # IDEA: look at all children for True before recursing # Better idea: every node except the root /must/ have a dirty # descendent or it wouldn't exist. This assumption is only valid as # long as an unset_dirty() method or similar does not exist. return any(self.children) def count(self): """Returns the total number of dirty leaf nodes. """ # TODO: Make this more efficient (although for even the largest trees, # this takes only seconds) c = 0 for _ in self.iterate_dirty(): c += 1 return c class Tile(object): """A simple container class that represents a single render-tile. A render-tile is a tile that is rendered, not a tile composed of other tiles (composite-tile). """ __slots__ = ("col", "row", "path") def __init__(self, col, row, path): """Initialize the tile obj with the given parameters. It's probably better to use one of the other constructors though """ self.col = col self.row = row self.path = tuple(path) def __repr__(self): return "%s(%r,%r,%r)" % (self.__class__.__name__, self.col, self.row, self.path) def __eq__(self,other): return self.col == other.col and self.row == other.row and tuple(self.path) == tuple(other.path) def __ne__(self, other): return not self == other def get_filepath(self, tiledir, imgformat): """Returns the path to this file given the directory to the tiles """ # os.path.join would be the proper way to do this path concatenation, # but it is surprisingly slow, probably because it checks each path # element if it begins with a slash. Since we know these components are # all relative, just concatinate with os.path.sep pathcomponents = [tiledir] pathcomponents.extend(str(x) for x in self.path) path = os.path.sep.join(pathcomponents) imgpath = ".".join((path, imgformat)) return imgpath @classmethod def from_path(cls, path): """Constructor that takes a path and computes the col,row address of the tile and constructs a new tile object. """ path = tuple(path) depth = len(path) # Radius of the world in chunk cols/rows # (Diameter in X is 2**depth, divided by 2 for a radius, multiplied by # 2 for 2 chunks per tile. Similarly for Y) xradius = 2**depth yradius = 2*2**depth col = -xradius row = -yradius xsize = xradius ysize = yradius for p in path: if p in (1,3): col += xsize if p in (2,3): row += ysize xsize //= 2 ysize //= 2 return cls(col, row, path) @classmethod def compute_path(cls, col, row, depth): """Constructor that takes a col,row of a tile and computes the path. """ assert col % 2 == 0 assert row % 4 == 0 xradius = 2**depth yradius = 2*2**depth colbounds = [-xradius, xradius] rowbounds = [-yradius, yradius] path = [] for level in xrange(depth): # Strategy: Find the midpoint of this level, and determine which # quadrant this row/col is in. Then set the bounds to that level # and repeat xmid = (colbounds[1] + colbounds[0]) // 2 ymid = (rowbounds[1] + rowbounds[0]) // 2 if col < xmid: if row < ymid: path.append(0) colbounds[1] = xmid rowbounds[1] = ymid else: path.append(2) colbounds[1] = xmid rowbounds[0] = ymid else: if row < ymid: path.append(1) colbounds[0] = xmid rowbounds[1] = ymid else: path.append(3) colbounds[0] = xmid rowbounds[0] = ymid return cls(col, row, path)