diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 01a6f54..7873b89 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -27,7 +27,7 @@ from collections import namedtuple from PIL import Image -from .util import iterate_base4, convert_coords, unconvert_coords +from .util import iterate_base4, convert_coords, unconvert_coords, get_tiles_by_chunk from .optimizeimages import optimize_image import c_overviewer @@ -239,7 +239,7 @@ class TileSet(object): render mode to render. This rendermode must have already been registered with the C extension module. - rerender_prob + rerenderprob A floating point number between 0 and 1 indicating the probability that a tile which is not marked for render by any mtime checks will be rendered anyways. 0 disables this option. @@ -250,6 +250,9 @@ class TileSet(object): self.am = assetmanagerobj self.textures = texturesobj + # XXX TODO: + self.last_rendertime = 0 # TODO + # Throughout the class, self.outputdir is an absolute path to the # directory where we output tiles. It is assumed to exist. self.outputdir = os.path.abspath(outputdir) @@ -308,12 +311,7 @@ class TileSet(object): number of phases of work that need to be done. """ - if self.options['renderchecks'] == 1: - # Layer by layer for this mode - return self.treedepth - else: - # post-traversal does everything in one phase - return 1 + return 1 def get_phase_length(self, phase): """Returns the number of work items in a given phase, or None if there @@ -322,7 +320,7 @@ class TileSet(object): # Yeah functional programming! return { 0: lambda: self.dirtytree.count_all(), - 1: lambda: self.dirtytree.count() if phase == 0 else None, + 1: lambda: None, 2: lambda: self.dirtytree.count_all(), }[self.options['renderchecks']]() @@ -422,6 +420,11 @@ class TileSet(object): logging.critical("Could not determine existing tile tree depth. Does it exist?") raise + if curdepth == 1: + # Skip a depth 1 tree. A depth 1 tree pretty much can't happen, so + # when we detect this it usually means the tree is actually empty + return + logging.debug("Current tree depth was detected to be %s. Target tree depth is %s", curdepth, self.treedepth) if self.treedepth != curdepth: if self.treedepth > curdepth: logging.warning("Your map seems to have expanded beyond its previous bounds.") @@ -533,8 +536,7 @@ class TileSet(object): rerender_prob = self.options['rerenderprob'] - # XXX TODO: - last_rendertime = 0 # TODO + last_rendertime = self.last_rendertime max_chunk_mtime = 0 @@ -554,72 +556,48 @@ class TileSet(object): # Convert to diagonal coordinates chunkcol, chunkrow = convert_coords(chunkx, chunkz) - # find tile coordinates. Remember tiles are identified by the - # address of the chunk in their upper left corner. - tilecol = chunkcol - chunkcol % 2 - tilerow = chunkrow - chunkrow % 4 + for c, r in get_tiles_by_chunk(chunkcol, chunkrow): - # If this chunk is in an /even/ column, then it spans two tiles. - if chunkcol % 2 == 0: - colrange = (tilecol-2, tilecol) - else: - colrange = (tilecol,) + # Make sure the tile is in the boundary we're rendering. + # This can happen when rendering at lower treedepth than + # can contain the entire map, but shouldn't happen if the + # treedepth is correctly calculated. + if ( + c < -xradius or + c >= xradius or + r < -yradius or + r >= yradius + ): + continue - # If this chunk is in a row divisible by 4, then it touches the - # tile above it as well. Also touches the next 4 tiles down (16 - # rows) - if chunkrow % 4 == 0: - rowrange = xrange(tilerow-4, tilerow+16+1, 4) - else: - rowrange = xrange(tilerow, tilerow+16+1, 4) + # Computes the path in the quadtree from the col,row coordinates + tile = RenderTile.compute_path(c, r, depth) - # Loop over all tiles that this chunk potentially touches. - # The tile at tilecol,tilerow obviously contains the chunk, but so - # do the next 4 tiles down because chunks are very tall, and maybe - # the next column over too. - for c in colrange: - for r in rowrange: + if markall: + # markall mode: Skip all other checks, mark tiles + # as dirty unconditionally + dirty.add(tile.path) + continue - # Make sure the tile is in the boundary we're rendering. - # This can happen when rendering at lower treedepth than - # can contain the entire map, but shouldn't happen if the - # treedepth is correctly calculated. - if ( - c < -xradius or - c >= xradius or - r < -yradius or - r >= yradius - ): - continue + # Check if this tile has already been marked dirty. If so, + # no need to do any of the below. + if dirty.query_path(tile.path): + continue - # Computes the path in the quadtree from the col,row coordinates - tile = RenderTile.compute_path(c, r, depth) + # Stochastic check. Since we're scanning by chunks and not + # by tiles, and the tiles get checked multiple times for + # each chunk, this is only an approximation. The given + # probability is for a particular tile that needs + # rendering, but since a tile gets touched up to 32 times + # (once for each chunk in it), divide the probability by + # 32. + if rerender_prob and rerender_prob/32 > random.random(): + dirty.add(tile.path) + continue - if markall: - # markall mode: Skip all other checks, mark tiles - # as dirty unconditionally - dirty.add(tile.path) - continue - - # Check if this tile has already been marked dirty. If so, - # no need to do any of the below. - if dirty.query_path(tile.path): - continue - - # Stochastic check. Since we're scanning by chunks and not - # by tiles, and the tiles get checked multiple times for - # each chunk, this is only an approximation. The given - # probability is for a particular tile that needs - # rendering, but since a tile gets touched up to 32 times - # (once for each chunk in it), divide the probability by - # 32. - if rerender_prob and rerender_prob/32 > random.random(): - dirty.add(tile.path) - continue - - # Check mtimes and conditionally add tile to the set - if chunkmtime > last_rendertime: - dirty.add(tile.path) + # Check mtimes and conditionally add tile to the set + if chunkmtime > last_rendertime: + dirty.add(tile.path) t = int(time.time()-stime) logging.debug("%s finished chunk scan. %s chunks scanned in %s second%s", @@ -929,7 +907,7 @@ class TileSet(object): # Now that we're done with our children and descendents, see if # this tile needs rendering - if needs_rendering: + if render_me: # yes. yes we do. This is set when one of our children needs # rendering yield path, None, True diff --git a/overviewer_core/util.py b/overviewer_core/util.py index 156d4de..3be41ad 100644 --- a/overviewer_core/util.py +++ b/overviewer_core/util.py @@ -124,6 +124,32 @@ def unconvert_coords(col, row): # col - row = chunkx + chunkx => (col - row)/2 = chunkx return ((col - row) / 2, (col + row) / 2) +def get_tiles_by_chunk(chunkcol, chunkrow): + """For the given chunk, returns an iterator over Render Tiles that this + chunk touches. Iterates over (tilecol, tilerow) + + """ + # find tile coordinates. Remember tiles are identified by the + # address of the chunk in their upper left corner. + tilecol = chunkcol - chunkcol % 2 + tilerow = chunkrow - chunkrow % 4 + + # If this chunk is in an /even/ column, then it spans two tiles. + if chunkcol % 2 == 0: + colrange = (tilecol-2, tilecol) + else: + colrange = (tilecol,) + + # If this chunk is in a row divisible by 4, then it touches the + # tile above it as well. Also touches the next 4 tiles down (16 + # rows) + if chunkrow % 4 == 0: + rowrange = xrange(tilerow-4, tilerow+16+1, 4) + else: + rowrange = xrange(tilerow, tilerow+16+1, 4) + + return product(colrange, rowrange) + # Logging related classes are below # Some cool code for colored logging: diff --git a/test/test_all.py b/test/test_all.py index ce8fad0..de80c5b 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -10,6 +10,7 @@ sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) from test_tileobj import TileTest from test_rendertileset import RendertileSetTest from test_settings import SettingsTest +from test_tileset import TilesetTest if __name__ == "__main__": unittest.main() diff --git a/test/test_tileset.py b/test/test_tileset.py new file mode 100644 index 0000000..9751726 --- /dev/null +++ b/test/test_tileset.py @@ -0,0 +1,327 @@ +import unittest +import tempfile +import shutil +from collections import defaultdict +import os +import os.path +import random +import pprint + +from overviewer_core import tileset, util + +# DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING +if 1: + import logging + root = logging.getLogger() + class NullHandler(logging.Handler): + def handle(self, record): + pass + def emit(self, record): + pass + def createLock(self): + self.lock = None + root.addHandler(NullHandler()) +else: + import overviewer + import logging + overviewer.configure_logger(logging.DEBUG, True) + +# Supporing data +# chunks list: chunkx, chunkz mapping to chunkmtime +# In comments: col, row +chunks = { + (0, 0): 5, # 0, 0 + (0, 1): 5, # 1, 1 + (0, 2): 5, # 2, 2 + (0, 3): 5, # 3, 3 + (0, 4): 5, # 4, 4 + (1, 0): 5, # 1, -1 + (1, 1): 5, # 2, 0 + (1, 2): 5, # 3, 1 + (1, 3): 5, # 4, 2 + (1, 4): 5, # 5, 3 + (2, 0): 5, # 2, -2 + (2, 1): 5, # 3, -1 + (2, 2): 5, # 4, 0 + (2, 3): 5, # 5, 1 + (2, 4): 5, # 6, 2 + (3, 0): 5, # 3, -3 + (3, 1): 5, # 4, -2 + (3, 2): 5, # 5, -1 + (3, 3): 5, # 6, 0 + (3, 4): 5, # 7, 1 + (4, 0): 5, # 4, -4 + (4, 1): 5, # 5, -3 + (4, 2): 5, # 6, -2 + (4, 3): 5, # 7, -1 + (4, 4): 5, # 8, 0 + } + +# The resulting tile tree, in the correct post-traversal order. +correct_tiles = [ + (0, 3, 3), + (0, 3), + (0,), + (1, 2, 1), + (1, 2, 2), + (1, 2, 3), + (1, 2), + (1, 3, 0), + (1, 3, 2), + (1, 3, 3), + (1, 3), + (1,), + (2, 1, 1), + (2, 1, 3), + (2, 1), + (2, 3, 1), + (2, 3, 3), + (2, 3), + (2,), + (3, 0, 0), + (3, 0, 1), + (3, 0, 2), + (3, 0, 3), + (3, 0), + (3, 1, 0), + (3, 1, 1), + (3, 1, 2), + (3, 1, 3), + (3, 1), + (3, 2, 0), + (3, 2, 1), + (3, 2, 2), + (3, 2, 3), + (3, 2), + (3, 3, 0), + (3, 3, 1), + (3, 3, 2), + (3, 3, 3), + (3, 3), + (3,), + (), + ] + +# Supporting resources +###################### + +class FakeRegionset(object): + def __init__(self, chunks): + self.chunks = dict(chunks) + + def get_chunk(self, x,z): + return NotImplementedError() + + def iterate_chunks(self): + for (x,z),mtime in self.chunks.iteritems(): + yield x,z,mtime + + def get_chunk_mtime(self, x, z): + try: + return self.chunks[x,z] + except KeyError: + return None + +def get_tile_set(chunks): + """Given the dictionary mapping chunk coordinates their mtimes, returns a + dict mapping the tiles that are to be rendered to their mtimes that are + expected. Useful for passing into the create_fakedir() function. Used by + the compare_iterate_to_expected() method. + """ + tile_set = defaultdict(int) + for (chunkx, chunkz), chunkmtime in chunks.iteritems(): + + col, row = util.convert_coords(chunkx, chunkz) + + for tilec, tiler in util.get_tiles_by_chunk(col, row): + tile = tileset.RenderTile.compute_path(tilec, tiler, 3) + tile_set[tile.path] = max(tile_set[tile.path], chunkmtime) + + # At this point, tile_set holds all the render-tiles + for tile, tile_mtime in tile_set.copy().iteritems(): + # All render-tiles are length 3. Hard-code its upper tiles + tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime) + tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime) + tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime) + return dict(tile_set) + +def create_fakedir(outputdir, tiles): + """Takes a base output directory and a tiles dict mapping tile paths to + tile mtimes as returned by get_tile_set(), creates the "tiles" (empty + files) and sets mtimes appropriately + + """ + for tilepath, tilemtime in tiles.iteritems(): + dirpath = os.path.join(outputdir, *(str(x) for x in tilepath[:-1])) + if len(tilepath) == 0: + imgname = "base.png" + else: + imgname = str(tilepath[-1]) + ".png" + + if not os.path.exists(dirpath): + os.makedirs(dirpath) + finalpath = os.path.join(dirpath, imgname) + open(finalpath, 'w').close() + os.utime(finalpath, (tilemtime, tilemtime)) + +# The test cases +################ +class TilesetTest(unittest.TestCase): + def setUp(self): + # Set up the region set + self.rs = FakeRegionset(chunks) + + self.tempdirs = [] + + # Consistent random numbers + self.r = random.Random(1) + + def tearDown(self): + for d in self.tempdirs: + shutil.rmtree(d) + + def get_outputdir(self): + d = tempfile.mkdtemp(prefix="OVTEST") + self.tempdirs.append(d) + return d + + def get_tileset(self, options, outputdir, preprocess=None): + """Returns a newly created TileSet object and return it. + A set of default options are provided. Any options passed in will + override the defaults. The output directory is passed in and it is + recommended to use a directory from self.get_outputdir() + + preprocess, if given, is a function that takes the tileset object. It + is called before do_preprocessing() + """ + defoptions = { + 'bgcolor': '#000000', + 'imgformat': 'png', + 'optimizeimg': 0, + 'rendermode': 'normal', + 'rerenderprob': 0 + } + defoptions.update(options) + ts = tileset.TileSet(self.rs, None, None, defoptions, outputdir) + if preprocess: + preprocess(ts) + ts.do_preprocessing() + return ts + + def compare_iterate_to_expected(self, ts, chunks): + """Runs iterate_work_items on the tileset object and compares its + output to what we'd expect if it was run with the given chunks + + chunks is a dictionary whose keys are chunkx,chunkz. This method + calculates from that set of chunks the tiles they touch and their + parent tiles, and compares that to the output of ts.iterate_work_items(). + + """ + paths = set(x[0] for x in ts.iterate_work_items(0)) + pprint.pprint(paths) + + # Get what tiles we expect to be returned + expected = get_tile_set(chunks) + pprint.pprint(set(expected.iterkeys())) + + # Check that all paths returned are in the expected list + for tilepath in paths: + self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected)) + + # Now check that all expected tiles were indeed returned + for tilepath in expected.iterkeys(): + self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths)) + + def test_get_phase_length(self): + ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir()) + self.assertEqual(ts.get_num_phases(), 1) + self.assertEqual(ts.get_phase_length(0), 41) + + def test_forcerender_iterate(self): + """Tests that a rendercheck mode 2 iteration returns every render-tile + and upper-tile + """ + ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir()) + self.compare_iterate_to_expected(ts, self.rs.chunks) + + + def test_update_chunk(self): + """Tests that an update in one chunk properly updates just the + necessary tiles for rendercheck mode 0, normal operation. This + shouldn't touch the filesystem at all. + + """ + + # Update one chunk with a newer mtime + updated_chunks = { + (0,0): 6 + } + self.rs.chunks.update(updated_chunks) + + # Create the tileset and set its last render time to 5 + ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(), + lambda ts: setattr(ts, 'last_rendertime', 5)) + + # Now see if the return is what we expect + self.compare_iterate_to_expected(ts, updated_chunks) + + def test_update_chunk2(self): + """Same as above but with a different set of chunks + """ + # Pick 3 random chunks to update + chunks = self.rs.chunks.keys() + self.r.shuffle(chunks) + updated_chunks = {} + for key in chunks[:3]: + updated_chunks[key] = 6 + self.rs.chunks.update(updated_chunks) + ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(), + lambda ts: setattr(ts, 'last_rendertime', 5)) + self.compare_iterate_to_expected(ts, updated_chunks) + + def test_rendercheckmode_1(self): + """Tests that an interrupted render will correctly pick up tiles that + need rendering + + """ + # For this we actually need to set the tile mtimes on disk and have the + # TileSet object figure out from that what it needs to render. + # Strategy: set some tiles on disk to mtime 3, and TileSet needs to + # find them and update them to mtime 5 as reported by the RegionSet + # object. + outdated_tiles = [ + (0,3,3), + (1,2,1), + (2,1), + (3,) + ] + # These are the tiles that we also expect it to return, even though + # they were not outdated, since they depend on the outdated tiles + additional = [ + (0,3), + (0,), + (1,2), + (1,), + (2,), + (), + ] + + outputdir = self.get_outputdir() + # Fill the output dir with tiles + all_tiles = get_tile_set(self.rs.chunks) + all_tiles.update(dict((x,3) for x in outdated_tiles)) + create_fakedir(outputdir, all_tiles) + + # Create the tileset and do the scan + ts = self.get_tileset({'renderchecks': 1}, outputdir) + + # Now see if it's right + paths = set(x[0] for x in ts.iterate_work_items(0)) + expected = set(outdated_tiles) | set(additional) + pprint.pprint(paths) + pprint.pprint(expected) + for tilepath in paths: + self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected)) + + for tilepath in expected: + self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths))