added TileSet tests, fixed several bugs. Still needs work.

overviewer_core/tileset.py

@@ -27,7 +27,7 @@ from collections import namedtuple
 
 from PIL import Image
 
-from .util import iterate_base4, convert_coords, unconvert_coords
+from .util import iterate_base4, convert_coords, unconvert_coords, get_tiles_by_chunk
 from .optimizeimages import optimize_image
 import c_overviewer
 
@@ -239,7 +239,7 @@ class TileSet(object):
             render mode to render. This rendermode must have already been
             registered with the C extension module. 
 
-        rerender_prob
+        rerenderprob
             A floating point number between 0 and 1 indicating the probability
             that a tile which is not marked for render by any mtime checks will
             be rendered anyways. 0 disables this option.
@@ -250,6 +250,9 @@ class TileSet(object):
         self.am = assetmanagerobj
         self.textures = texturesobj
 
+        # XXX TODO:
+        self.last_rendertime = 0 # TODO
+
         # Throughout the class, self.outputdir is an absolute path to the
         # directory where we output tiles. It is assumed to exist.
         self.outputdir = os.path.abspath(outputdir)
@@ -308,11 +311,6 @@ class TileSet(object):
         number of phases of work that need to be done.
 
         """
-        if self.options['renderchecks'] == 1:
-            # Layer by layer for this mode
-            return self.treedepth
-        else:
-            # post-traversal does everything in one phase
         return 1
     
     def get_phase_length(self, phase):
@@ -322,7 +320,7 @@ class TileSet(object):
         # Yeah functional programming!
         return {
                 0: lambda: self.dirtytree.count_all(),
-                1: lambda: self.dirtytree.count() if phase == 0 else None,
+                1: lambda: None,
                 2: lambda: self.dirtytree.count_all(),
                 }[self.options['renderchecks']]()
 
@@ -422,6 +420,11 @@ class TileSet(object):
             logging.critical("Could not determine existing tile tree depth. Does it exist?")
             raise
         
+        if curdepth == 1:
+            # Skip a depth 1 tree. A depth 1 tree pretty much can't happen, so
+            # when we detect this it usually means the tree is actually empty
+            return
+        logging.debug("Current tree depth was detected to be %s. Target tree depth is %s", curdepth, self.treedepth)
         if self.treedepth != curdepth:
             if self.treedepth > curdepth:
                 logging.warning("Your map seems to have expanded beyond its previous bounds.")
@@ -533,8 +536,7 @@ class TileSet(object):
 
         rerender_prob = self.options['rerenderprob']
 
-        # XXX TODO:
+        last_rendertime = self.last_rendertime
-        last_rendertime = 0 # TODO
 
         max_chunk_mtime = 0
 
@@ -554,31 +556,7 @@ class TileSet(object):
             # Convert to diagonal coordinates
             chunkcol, chunkrow = convert_coords(chunkx, chunkz)
 
-            # find tile coordinates. Remember tiles are identified by the
+            for c, r in get_tiles_by_chunk(chunkcol, chunkrow):
-            # address of the chunk in their upper left corner.
-            tilecol = chunkcol - chunkcol % 2
-            tilerow = chunkrow - chunkrow % 4
-
-            # If this chunk is in an /even/ column, then it spans two tiles.
-            if chunkcol % 2 == 0:
-                colrange = (tilecol-2, tilecol)
-            else:
-                colrange = (tilecol,)
-
-            # If this chunk is in a row divisible by 4, then it touches the
-            # tile above it as well. Also touches the next 4 tiles down (16
-            # rows)
-            if chunkrow % 4 == 0:
-                rowrange = xrange(tilerow-4, tilerow+16+1, 4)
-            else:
-                rowrange = xrange(tilerow, tilerow+16+1, 4)
-
-            # Loop over all tiles that this chunk potentially touches.
-            # The tile at tilecol,tilerow obviously contains the chunk, but so
-            # do the next 4 tiles down because chunks are very tall, and maybe
-            # the next column over too.
-            for c in colrange:
-                for r in rowrange:
 
                 # Make sure the tile is in the boundary we're rendering.
                 # This can happen when rendering at lower treedepth than
@@ -929,7 +907,7 @@ class TileSet(object):
 
             # Now that we're done with our children and descendents, see if
             # this tile needs rendering
-            if needs_rendering:
+            if render_me:
                 # yes. yes we do. This is set when one of our children needs
                 # rendering
                 yield path, None, True

overviewer_core/util.py

@@ -124,6 +124,32 @@ def unconvert_coords(col, row):
     # col - row = chunkx + chunkx => (col - row)/2 = chunkx
     return ((col - row) / 2, (col + row) / 2)
 
+def get_tiles_by_chunk(chunkcol, chunkrow):
+    """For the given chunk, returns an iterator over Render Tiles that this
+    chunk touches.  Iterates over (tilecol, tilerow)
+    
+    """
+    # find tile coordinates. Remember tiles are identified by the
+    # address of the chunk in their upper left corner.
+    tilecol = chunkcol - chunkcol % 2
+    tilerow = chunkrow - chunkrow % 4
+
+    # If this chunk is in an /even/ column, then it spans two tiles.
+    if chunkcol % 2 == 0:
+        colrange = (tilecol-2, tilecol)
+    else:
+        colrange = (tilecol,)
+
+    # If this chunk is in a row divisible by 4, then it touches the
+    # tile above it as well. Also touches the next 4 tiles down (16
+    # rows)
+    if chunkrow % 4 == 0:
+        rowrange = xrange(tilerow-4, tilerow+16+1, 4)
+    else:
+        rowrange = xrange(tilerow, tilerow+16+1, 4)
+
+    return product(colrange, rowrange)
+
 # Logging related classes are below
 
 # Some cool code for colored logging:

test/test_all.py

@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.join(os.getcwd(), os.pardir))
 from test_tileobj import TileTest
 from test_rendertileset import RendertileSetTest
 from test_settings import SettingsTest
+from test_tileset import TilesetTest
 
 if __name__ == "__main__":
     unittest.main()

test/test_tileset.py

@@ -0,0 +1,327 @@
+import unittest
+import tempfile
+import shutil
+from collections import defaultdict
+import os
+import os.path
+import random
+import pprint
+
+from overviewer_core import tileset, util
+
+# DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
+if 1:
+    import logging
+    root = logging.getLogger()
+    class NullHandler(logging.Handler):
+        def handle(self, record):
+            pass
+        def emit(self, record):
+            pass
+        def createLock(self):
+            self.lock = None
+    root.addHandler(NullHandler())
+else:
+    import overviewer
+    import logging
+    overviewer.configure_logger(logging.DEBUG, True)
+
+# Supporing data
+# chunks list: chunkx, chunkz mapping to chunkmtime
+# In comments: col, row
+chunks = {
+        (0, 0): 5, # 0, 0
+        (0, 1): 5, # 1, 1
+        (0, 2): 5, # 2, 2
+        (0, 3): 5, # 3, 3
+        (0, 4): 5, # 4, 4
+        (1, 0): 5, # 1, -1
+        (1, 1): 5, # 2, 0
+        (1, 2): 5, # 3, 1
+        (1, 3): 5, # 4, 2
+        (1, 4): 5, # 5, 3
+        (2, 0): 5, # 2, -2
+        (2, 1): 5, # 3, -1
+        (2, 2): 5, # 4, 0
+        (2, 3): 5, # 5, 1
+        (2, 4): 5, # 6, 2
+        (3, 0): 5, # 3, -3
+        (3, 1): 5, # 4, -2
+        (3, 2): 5, # 5, -1
+        (3, 3): 5, # 6, 0
+        (3, 4): 5, # 7, 1
+        (4, 0): 5, # 4, -4
+        (4, 1): 5, # 5, -3
+        (4, 2): 5, # 6, -2
+        (4, 3): 5, # 7, -1
+        (4, 4): 5, # 8, 0
+        }
+
+# The resulting tile tree, in the correct post-traversal order.
+correct_tiles = [
+        (0, 3, 3),
+        (0, 3),
+        (0,),
+        (1, 2, 1),
+        (1, 2, 2),
+        (1, 2, 3),
+        (1, 2),
+        (1, 3, 0),
+        (1, 3, 2),
+        (1, 3, 3),
+        (1, 3),
+        (1,),
+        (2, 1, 1),
+        (2, 1, 3),
+        (2, 1),
+        (2, 3, 1),
+        (2, 3, 3),
+        (2, 3),
+        (2,),
+        (3, 0, 0),
+        (3, 0, 1),
+        (3, 0, 2),
+        (3, 0, 3),
+        (3, 0),
+        (3, 1, 0),
+        (3, 1, 1),
+        (3, 1, 2),
+        (3, 1, 3),
+        (3, 1),
+        (3, 2, 0),
+        (3, 2, 1),
+        (3, 2, 2),
+        (3, 2, 3),
+        (3, 2),
+        (3, 3, 0),
+        (3, 3, 1),
+        (3, 3, 2),
+        (3, 3, 3),
+        (3, 3),
+        (3,),
+        (),
+        ]
+
+# Supporting resources
+######################
+
+class FakeRegionset(object):
+    def __init__(self, chunks):
+        self.chunks = dict(chunks)
+
+    def get_chunk(self, x,z):
+        return NotImplementedError()
+
+    def iterate_chunks(self):
+        for (x,z),mtime in self.chunks.iteritems():
+            yield x,z,mtime
+
+    def get_chunk_mtime(self, x, z):
+        try:
+            return self.chunks[x,z]
+        except KeyError:
+            return None
+
+def get_tile_set(chunks):
+    """Given the dictionary mapping chunk coordinates their mtimes, returns a
+    dict mapping the tiles that are to be rendered to their mtimes that are
+    expected. Useful for passing into the create_fakedir() function. Used by
+    the compare_iterate_to_expected() method.
+    """
+    tile_set = defaultdict(int)
+    for (chunkx, chunkz), chunkmtime in chunks.iteritems():
+
+        col, row = util.convert_coords(chunkx, chunkz)
+
+        for tilec, tiler in util.get_tiles_by_chunk(col, row):
+            tile = tileset.RenderTile.compute_path(tilec, tiler, 3)
+            tile_set[tile.path] = max(tile_set[tile.path], chunkmtime)
+
+    # At this point, tile_set holds all the render-tiles
+    for tile, tile_mtime in tile_set.copy().iteritems():
+        # All render-tiles are length 3. Hard-code its upper tiles
+        tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime)
+        tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime)
+        tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime)
+    return dict(tile_set)
+
+def create_fakedir(outputdir, tiles):
+    """Takes a base output directory and a tiles dict mapping tile paths to
+    tile mtimes as returned by get_tile_set(), creates the "tiles" (empty
+    files) and sets mtimes appropriately
+
+    """
+    for tilepath, tilemtime in tiles.iteritems():
+        dirpath = os.path.join(outputdir, *(str(x) for x in tilepath[:-1]))
+        if len(tilepath) == 0:
+            imgname = "base.png"
+        else:
+            imgname = str(tilepath[-1]) + ".png"
+
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+        finalpath = os.path.join(dirpath, imgname)
+        open(finalpath, 'w').close()
+        os.utime(finalpath, (tilemtime, tilemtime))
+
+# The test cases
+################
+class TilesetTest(unittest.TestCase):
+    def setUp(self):
+        # Set up the region set
+        self.rs = FakeRegionset(chunks)
+
+        self.tempdirs = []
+
+        # Consistent random numbers
+        self.r = random.Random(1)
+
+    def tearDown(self):
+        for d in self.tempdirs:
+            shutil.rmtree(d)
+
+    def get_outputdir(self):
+        d = tempfile.mkdtemp(prefix="OVTEST")
+        self.tempdirs.append(d)
+        return d
+
+    def get_tileset(self, options, outputdir, preprocess=None):
+        """Returns a newly created TileSet object and return it.
+        A set of default options are provided. Any options passed in will
+        override the defaults. The output directory is passed in and it is
+        recommended to use a directory from self.get_outputdir()
+
+        preprocess, if given, is a function that takes the tileset object. It
+        is called before do_preprocessing()
+        """
+        defoptions = {
+                'bgcolor': '#000000',
+                'imgformat': 'png',
+                'optimizeimg': 0,
+                'rendermode': 'normal',
+                'rerenderprob': 0
+                }
+        defoptions.update(options)
+        ts = tileset.TileSet(self.rs, None, None, defoptions, outputdir)
+        if preprocess:
+            preprocess(ts)
+        ts.do_preprocessing()
+        return ts
+
+    def compare_iterate_to_expected(self, ts, chunks):
+        """Runs iterate_work_items on the tileset object and compares its
+        output to what we'd expect if it was run with the given chunks
+
+        chunks is a dictionary whose keys are chunkx,chunkz. This method
+        calculates from that set of chunks the tiles they touch and their
+        parent tiles, and compares that to the output of ts.iterate_work_items().
+
+        """
+        paths = set(x[0] for x in ts.iterate_work_items(0))
+        pprint.pprint(paths)
+
+        # Get what tiles we expect to be returned
+        expected = get_tile_set(chunks)
+        pprint.pprint(set(expected.iterkeys()))
+
+        # Check that all paths returned are in the expected list
+        for tilepath in paths:
+            self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected))
+
+        # Now check that all expected tiles were indeed returned
+        for tilepath in expected.iterkeys():
+            self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths))
+
+    def test_get_phase_length(self):
+        ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
+        self.assertEqual(ts.get_num_phases(), 1)
+        self.assertEqual(ts.get_phase_length(0), 41)
+
+    def test_forcerender_iterate(self):
+        """Tests that a rendercheck mode 2 iteration returns every render-tile
+        and upper-tile
+        """
+        ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
+        self.compare_iterate_to_expected(ts, self.rs.chunks)
+
+
+    def test_update_chunk(self):
+        """Tests that an update in one chunk properly updates just the
+        necessary tiles for rendercheck mode 0, normal operation. This
+        shouldn't touch the filesystem at all.
+
+        """
+
+        # Update one chunk with a newer mtime
+        updated_chunks = {
+                (0,0): 6
+                }
+        self.rs.chunks.update(updated_chunks)
+
+        # Create the tileset and set its last render time to 5
+        ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(),
+                lambda ts: setattr(ts, 'last_rendertime', 5))
+
+        # Now see if the return is what we expect
+        self.compare_iterate_to_expected(ts, updated_chunks)
+
+    def test_update_chunk2(self):
+        """Same as above but with a different set of chunks
+        """
+        # Pick 3 random chunks to update
+        chunks = self.rs.chunks.keys()
+        self.r.shuffle(chunks)
+        updated_chunks = {}
+        for key in chunks[:3]:
+            updated_chunks[key] = 6
+        self.rs.chunks.update(updated_chunks)
+        ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(),
+                lambda ts: setattr(ts, 'last_rendertime', 5))
+        self.compare_iterate_to_expected(ts, updated_chunks)
+
+    def test_rendercheckmode_1(self):
+        """Tests that an interrupted render will correctly pick up tiles that
+        need rendering
+
+        """
+        # For this we actually need to set the tile mtimes on disk and have the
+        # TileSet object figure out from that what it needs to render.
+        # Strategy: set some tiles on disk to mtime 3, and TileSet needs to
+        # find them and update them to mtime 5 as reported by the RegionSet
+        # object.
+        outdated_tiles = [
+                (0,3,3),
+                (1,2,1),
+                (2,1),
+                (3,)
+                ]
+        # These are the tiles that we also expect it to return, even though
+        # they were not outdated, since they depend on the outdated tiles
+        additional = [
+                (0,3),
+                (0,),
+                (1,2),
+                (1,),
+                (2,),
+                (),
+                ]
+
+        outputdir = self.get_outputdir()
+        # Fill the output dir with tiles
+        all_tiles = get_tile_set(self.rs.chunks)
+        all_tiles.update(dict((x,3) for x in outdated_tiles))
+        create_fakedir(outputdir, all_tiles)
+
+        # Create the tileset and do the scan
+        ts = self.get_tileset({'renderchecks': 1}, outputdir)
+
+        # Now see if it's right
+        paths = set(x[0] for x in ts.iterate_work_items(0))
+        expected = set(outdated_tiles) | set(additional)
+        pprint.pprint(paths)
+        pprint.pprint(expected)
+        for tilepath in paths:
+            self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected))
+
+        for tilepath in expected:
+            self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths))