setup rendercheck mode 1 to go layer-by-layer

Doesn't work yet.

overviewer_core/tileset.py

@@ -131,28 +131,12 @@ Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow"))
 # In mode 1, things are most complicated. The mode 2 chunk scan checks
 # every render tile's mtime for each chunk that touches it, so it can
 # determine accurately which tiles need rendering regardless of the
-# state on disk. The chunk scan also builds a RendertileSet of *every*
+# state on disk.
-# render-tile that exists.
 
-# The mode 1 render iteration then manually iterates over the set of
+# The mode 1 render iteration falls back to the old layer-by-layer instead of a
-# all render-tiles in a post-traversal order. When it visits a
+# post-traversal iteration order. This uses the phases feature of the worker
-# render-node, it does the following:
+# API. A post-traversal is theoretically possible, but the implementation was
-# * Checks the set of dirty render-tiles to see if the node needs
+# significantly more complicated and I have decided it not to be worth it.
-#   rendering, and if so, renders it
-# * If the tile was rendered, set the mtime using os.utime() to the max
-#   of the chunk mtimes.
-# * If the tile was rendered, return (True, mtime).
-# * If the tile was not rendered, return (False, mtime)
-#
-# Then, for upper-tiles, it does the following:
-# * Gathers the return values of each child call.
-# * If any child returned True, render this tile.
-# * Otherwise, check this tile's mtime. If any child's mtime is greater
-#   than this tile's mtime, render this tile.
-# * If the tile was rendered, set the mtime using os.utime() to the max
-#   of the child mtimes.
-# * If the tile was rendered, return (True, mtime).
-# * If the tile was not rendered, return (False, mtime)
 
 __all__ = ["TileSet"]
 class TileSet(object):
@@ -271,7 +255,8 @@ class TileSet(object):
 
         """
         # REMEMBER THAT ATTRIBUTES ASSIGNED IN THIS METHOD ARE NOT AVAILABLE IN
-        # THE do_work() METHOD
+        # THE do_work() METHOD (because this is only called in the main process
+        # not the workers)
 
         # Calculate the min and max column over all the chunks.
         # This sets self.bounds to a Bounds namedtuple
@@ -308,13 +293,23 @@ class TileSet(object):
         number of phases of work that need to be done.
 
         """
+        if self.options['renderchecks'] == 1:
+            # Layer by layer for this mode
+            return self.treedepth
+        else:
+            # post-traversal does everything in one phase
             return 1
     
     def get_phase_length(self, phase):
         """Returns the number of work items in a given phase, or None if there
         is no good estimate.
         """
-        return None
+        # Yeah functional programming!
+        return {
+                0: lambda: self.dirtytree.count_all(),
+                1: lambda: self.dirtytree.count() if phase == 0 else None,
+                2: lambda: self.dirtytree.count_all(),
+                }[self.options['renderchecks']]()
 
     def iterate_work_items(self, phase):
         """Iterates over the dirty tiles in the tree and return them in the
@@ -341,32 +336,21 @@ class TileSet(object):
                 yield tilepath, dependencies
 
         else:
-            # I hope this is kosher. I couldn't think of any other good way to
+            raise NotImplementedError() # TODO
-            # use a complex recursive routine as one big generator/iterator
-            torender = Queue.Queue(1)
-            thread = threading.Thread(target=self._find_dirty_tiles, args=(torender,))
-            thread.start()
-            item = torender.get()
-            while item is not None:
-                dependencies = []
-                for i in range(4):
-                    dependencies.append( "%s/%s" % (item, i) )
-                yield item, dependencies
-
-                item = torender.get()
             
 
     def do_work(self, tileobj):
         """Renders the given tile.
 
         """
+        pass # TODO
 
     def get_persistent_data(self):
         """Returns a dictionary representing the persistent data of this
         TileSet. Typically this is called by AssetManager
 
         """
-        pass
+        return None
 
     def _find_chunk_range(self):
         """Finds the chunk range in rows/columns and stores them in
@@ -492,9 +476,6 @@ class TileSet(object):
         depth = self.treedepth
 
         dirty = RendertileSet(depth)
-        build_fulltileset = self.options['renderchecks'] == 1
-        if build_fulltileset:
-            fulltileset = RendertileSet(depth)
 
         chunkcount = 0
         stime = time.time()
@@ -574,9 +555,6 @@ class TileSet(object):
                     # Computes the path in the quadtree from the col,row coordinates
                     tile = RenderTile.compute_path(c, r, depth)
 
-                    if build_fulltileset:
-                        fulltileset.add(tile.path)
-
                     if rendercheck == 2:
                         # Skip all other checks, mark tiles as dirty unconditionally
                         dirty.add(tile.path)
@@ -607,44 +585,11 @@ class TileSet(object):
                 self, chunkcount, t,
                 "s" if t != 1 else "")
 
-        if build_fulltileset:
-            self.fulltileset = fulltileset
         return dirty
 
     def __str__(self):
         return "<TileSet for %s>" % os.basename(self.outputdir)
 
-    def _find_dirty_tiles(self, renderqueue):
-        """Entry point for the tile iteration thread. This pushes a number of
-        tile paths onto the renderqueue Queue object, and then pushes a
-        sentinel None and exits
-
-        """
-        self._find_dirty_tiles_helper(renderqueue, [], self.fulltileset)
-        renderqueue.push(None)
-        return
-
-    def _find_dirty_tiles_helper(self, renderqueue, path, treenode):
-        """Helper recursion method for _find_dirty_tiles
-
-        This method takes two arguments:
-        * a path, a list of integers. Methods should either not mutate it or
-          make sure it is back as it was when it exits.
-        * treenode: a RendertileSet object corresponding to the node in
-          self.fulltileset corresponding to the above path, or None for
-          rendertiles
-
-        This method returns two things:
-        * A boolean indicating whether this tile was rendered or not
-        * An mtime indicating this tile's mtime. The parent should be rendered
-          if it is older than this value
-
-        For an explanation of what this method does, see the comments at the
-        top of this file.
-        """
-        if treenode.depth == 1:
-            # This call corresponds to the layer /above/ a render-tile
-
 
 def get_dirdepth(outputdir):
     """Returns the current depth of the tree on disk
@@ -918,6 +863,18 @@ class RendertileSet(object):
             c += 1
         return c
 
+    def count_all(self):
+        """Returns the total number of render-tiles plus implicitly marked
+        upper-tiles in this set
+
+        """
+        # TODO: Optimize this too with its own recursive method that avoids
+        # some of the overheads of posttraversal()
+        c = 0
+        for _ in self.posttraversal():
+            c += 1
+        return c
+
 def post_traversal_complete_subtree_recursion_helper(depth):
     """Fakes the recursive calls for RendertileSet.posttraversal() for the case
     that a subtree is collapsed.

test/test_rendertileset.py

@@ -197,6 +197,12 @@ class RendertileSetTest(unittest.TestCase):
 
         self.assertRaises(StopIteration, next, iterator)
 
+    def test_count_all(self):
+        """Tests getting a count of all tiles (render tiles plus upper tiles)
+
+        """
+        c = self.tree.count_all()
+        self.assertEqual(c, 35)
 
 if __name__ == "__main__":
     unittest.main()