Merge remote-tracking branch 'ion1/master'

.travis.yml

@@ -4,7 +4,7 @@ python:
   - "2.7"
   # - "3.2"
 env:
-  - MC_VERSION=1.7.2
+  - MC_VERSION=1.7.4
 before_install:
   - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/Imaging.h
   - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/ImPlatform.h
@@ -16,9 +16,16 @@ before_script:
   - git clone git://github.com/overviewer/Minecraft-Overviewer-Addons.git ~/mcoa/
   - wget -N https://s3.amazonaws.com/Minecraft.Download/versions/${MC_VERSION}/${MC_VERSION}.jar -P ~/.minecraft/versions/${MC_VERSION}/
 script:
+  - PYTHONPATH=. python test/test_all.py
   - python overviewer.py ~/mcoa/exmaple ~/test-output --rendermodes=smooth-lighting -p1
 notifications:
   email: false
+  irc:
+    channels:
+      - "irc.freenode.org#overviewer"
+    skip_join: true
+    template:
+      - "\x0313Minecraft-Overviewer\x03/\x0306%{branch}\x03 \x0314%{commit}\x03 %{build_url} %{message}"
 # matrix:
   # allow_failures:
     # - python: "3.2"

overviewer_core/tileset.py

@@ -25,7 +25,7 @@ import time
 import errno
 import stat
 from collections import namedtuple
-from itertools import product, izip
+from itertools import product, izip, chain
 
 from PIL import Image
 
@@ -377,7 +377,7 @@ class TileSet(object):
 
         # This warning goes here so it's only shown once
         if self.treedepth >= 15:
-            logging.warning("Just letting you know, your map requries %s zoom levels. This is REALLY big!",
+            logging.warning("Just letting you know, your map requires %s zoom levels. This is REALLY big!",
                     self.treedepth)
 
         # Do any tile re-arranging if necessary. Skip if there was no config
@@ -446,7 +446,7 @@ class TileSet(object):
         # render. Iterate over the tiles in using the posttraversal() method.
         # Yield each item. Easy.
         if self.options['renderchecks'] in (0,2):
-            for tilepath in self.dirtytree.posttraversal():
+            for tilepath in self.dirtytree.posttraversal(robin=True):
                 dependencies = []
                 # These tiles may or may not exist, but the dispatcher won't
                 # care according to the worker interface protocol It will only
@@ -1268,21 +1268,19 @@ class RendertileSet(object):
     It is typically used to hold tiles that need rendering. This implementation
     collapses subtrees that are completely in or out of the set to save memory.
 
-    Each instance of this class is a node in the tree, and therefore each
+    An instance of this class holds a full tree.
-    instance is the root of a subtree.
 
-    Each node knows its "level", which corresponds to the zoom level where 0 is
+    The instance knows its "level", which corresponds to the zoom level where 1
-    the inner-most (most zoomed in) tiles.
+    is the inner-most (most zoomed in) tiles.
 
     Instances hold the state of their children (in or out of the set). Leaf
     nodes are images and do not physically exist in the tree as objects, but
     are represented as booleans held by the objects at the second-to-last
     level; level 1 nodes keep track of leaf image state. Level 2 nodes keep
-    track of level 1 state, and so fourth.
+    track of level 1 state, and so forth.
-
 
     """
-    __slots__ = ("depth", "children")
+    __slots__ = ("depth", "children", "num_tiles", "num_tiles_all")
     def __init__(self, depth):
         """Initialize a new tree with the specified depth. This actually
         initializes a node, which is the root of a subtree, with `depth` levels
@@ -1303,52 +1301,15 @@ class RendertileSet(object):
         #   All children down this subtree are not in the set
         # True
         #   All children down this subtree are in the set
-        # A RendertileSet instance
+        # An array of the same format
-        #   the instance defines which children down that subtree are in the
+        #   The array defines which children down that subtree are in the set
-        #   set.
         # A node with depth=1 cannot have a RendertileSet instance in its
         # children since its children are leaves, representing images, not more
         # tree
         self.children = [False] * 4
 
-    def posttraversal(self):
+        self.num_tiles     = 0
-        """Returns an iterator over tile paths for every tile in the
+        self.num_tiles_all = 0
-        set, including the explictly marked render-tiles, as well as the
-        implicitly marked ancestors of those render-tiles. Returns in
-        post-traversal order, so that tiles with dependencies will always be
-        yielded after their dependencies.
-
-        """
-        return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper())
-
-    def _posttraversal_helper(self):
-        """Each node returns an iterator over lists of reversed paths"""
-        if self.depth == 1:
-            # Base case
-            if self.children[0]: yield [0]
-            if self.children[1]: yield [1]
-            if self.children[2]: yield [2]
-            if self.children[3]: yield [3]
-        else:
-            for childnum, child in enumerate(self.children):
-                if child == True:
-                    for path in post_traversal_complete_subtree_recursion_helper(self.depth-1):
-                        path.append(childnum)
-                        yield path
-
-                elif child == False:
-                    pass # do nothing
-
-                else:
-                    # Recurse
-                    for path in child._posttraversal_helper():
-                        path.append(childnum)
-                        yield path
-
-        # Now do this node itself
-        if bool(self):
-            yield []
-
 
     def add(self, path):
         """Marks the requested leaf node as in this set
@@ -1359,72 +1320,66 @@ class RendertileSet(object):
         """
         path = list(path)
         assert len(path) == self.depth
-        path.reverse()
-        self._set_add_helper(path)
 
-    def _set_add_helper(self, path):
+        if self.num_tiles == 0:
+            # The first child is being added. A root composite tile will be
+            # rendered.
+            self.num_tiles_all += 1
+
+        self._add_helper(self.children, list(reversed(path)))
+
+    def _add_helper(self, children, path):
         """Recursive helper for add()
-
-        Expects path to be a list in reversed order
-
-        If *all* the nodes below this one are in the set, this function returns
-        true. Otherwise, returns None.
-
         """
 
-        if self.depth == 1:
-            # Base case
-            self.children[path[0]] = True
-
-            # Check to see if all children are in the set
-            if all(self.children):
-                return True
-        else:
-            # Recursive case
-
         childnum = path.pop()
-            child = self.children[childnum]
 
-            if child == False:
+        if path:
-                # Create a new node and recurse.
+            # We are not at the leaf, recurse.
-                # (The use of __class__ is so possible subclasses of this class
+
-                # work as expected)
+            if children[childnum] == True:
-                child = self.__class__(self.depth-1)
+                # The child is already in the tree.
-                child._set_add_helper(path)
-                self.children[childnum] = child
-            elif child == True:
-                # Every child is already in the set and the subtree is already
-                # collapsed. Nothing to do.
                 return
-            else:
+            elif children[childnum] == False:
-                # subtree is mixed. Recurse to the already existing child node
+                # Expand all-false.
-                ret = child._set_add_helper(path)
+                children[childnum] = [False]*4
-                if ret:
-                    # Child says every descendent is in the set, so we can
-                    # purge the subtree and mark it as such. The subtree will
-                    # be garbage collected when this method exits.
-                    self.children[childnum] = True
 
-                    # Since we've marked an entire sub-tree as in the set, we
+                # This also means an additional composite tile.
-                    # may be able to signal to our parent to do the same
+                self.num_tiles_all += 1
-                    if all(x is True for x in self.children):
+
-                        return True
+            self._add_helper(children[childnum], path)
+
+            if children[childnum] == [True]*4:
+                # Collapse all-true.
+                children[childnum] = True
+
+        else:
+            # We are at the leaf.
+            if not children[childnum]:
+                self.num_tiles     += 1
+                self.num_tiles_all += 1
+
+            children[childnum] = True
 
     def __iter__(self):
         return self.iterate()
-    def iterate(self, level=None):
+
+    def iterate(self, level=None, robin=False, offset=(0,0)):
         """Returns an iterator over every tile in this set. Each item yielded
         is a sequence of integers representing the quadtree path to the tiles
         in the set. Yielded sequences are of length self.depth.
 
         If level is None, iterates over tiles of the highest level, i.e.
-        worldtiles. If level is a value between 0 and the depth of this tree,
+        worldtiles. If level is a value between 1 and the depth of this tree,
-        this method iterates over tiles at that level. Zoom level 0 is zoomed
+        this method iterates over tiles at that level. Zoom level 1 is zoomed
         all the way out, zoom level `depth` is all the way in.
 
         In other words, specifying level causes the tree to be iterated as if
         it was only that depth.
 
+        If the `robin` parameter is True, recurses to the four top-level
+        subtrees simultaneously in a round-robin manner.
+
         """
         if level is None:
             todepth = 1
@@ -1433,30 +1388,47 @@ class RendertileSet(object):
                 raise ValueError("Level parameter must be between 1 and %s" % self.depth)
             todepth = self.depth - level + 1
 
-        return (tuple(reversed(rpath)) for rpath in self._iterate_helper(todepth))
+        return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, robin=robin, offset=offset))
 
-    def _iterate_helper(self, todepth):
+    def posttraversal(self, robin=False, offset=(0,0)):
-        if self.depth == todepth:
+        """Returns an iterator over tile paths for every tile in the
+        set, including the explictly marked render-tiles, as well as the
+        implicitly marked ancestors of those render-tiles. Returns in
+        post-traversal order, so that tiles with dependencies will always be
+        yielded after their dependencies.
+
+        If the `robin` parameter is True, recurses to the four top-level
+        subtrees simultaneously in a round-robin manner.
+
+        """
+        return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, robin=robin, offset=offset))
+
+    def _iterate_helper(self, path, children, depth, onlydepth=None, robin=False, offset=(0,0)):
+        """Returns an iterator over tile paths for every tile in the set."""
+
+        # A variant of children with a collapsed False/True expanded to a list.
+        children_list = [children] * 4 if isinstance(children, bool) else children
+
+        targetdepth = 1 if onlydepth is None else onlydepth
+
+        if depth == targetdepth:
             # Base case
-            if self.children[0]: yield [0]
+            for (childnum, child), _ in distance_sort(enumerate(children_list), offset):
-            if self.children[1]: yield [1]
+                if child:
-            if self.children[2]: yield [2]
+                    yield path + [childnum]
-            if self.children[3]: yield [3]
-
         else:
-            # Higher levels:
+            gens = []
-            for c, child in enumerate(self.children):
+            for (childnum_, child), childoffset_ in distance_sort(enumerate(children_list), offset):
-                if child == True:
+                if child:
-                    # All render-tiles are in the set down this subtree,
+                    def go(childnum, childoffset):
-                    # iterate over every leaf using iterate_base4
+                        for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset):
-                    for x in iterate_base4(self.depth-todepth):
+                            yield p
-                        x = list(x)
+                    gens.append(go(childnum_, childoffset_))
-                        x.append(c)
+
-                        yield x
+            for p in roundrobin(gens) if robin else chain(*gens):
-                elif child != False:
+                yield p
-                    # Mixed in/out of the set down this subtree, recurse
+
-                    for path in child._iterate_helper(todepth):
+        if onlydepth is None and children:
-                        path.append(c)
             yield path
 
     def query_path(self, path):
@@ -1471,74 +1443,46 @@ class RendertileSet(object):
         # collapsed, then just return the stored boolean. Otherwise, if we find
         # the specific tree node requested, return its state using the
         # __nonzero__ call.
-        treenode = self
+        treenode = self.children
         for pathelement in path:
-            treenode = treenode.children[pathelement]
+            treenode = treenode[pathelement]
-            if not isinstance(treenode, RendertileSet):
+            if isinstance(treenode, bool):
                 return treenode
 
         # If the method has not returned at this point, treenode is the
-        # requested node, but it is an inner node with possibly mixed state
+        # requested node, but it is an inner node. That will only happen if one
-        # subtrees. If any of the children are True return True. This call
+        # or more of the children down the tree are True.
-        # relies on the __nonzero__ method
+        return True
-        return bool(treenode)
 
     def __nonzero__(self):
         """Returns the boolean context of this particular node. If any
         descendent of this node is True return True. Otherwise, False.
 
         """
-        # Any chilren that are True or are a RendertileSet that evaluate to
+        # Any children that are True or are a list evaluate to True.
-        # True
-        # IDEA: look at all children for True before recursing
-        # Better idea: every node except the root /must/ have a descendent in
-        # the set or it wouldn't exist. This assumption is only valid as long
-        # as there is no method to remove a tile from the set. So this should
-        # check to see if any children are not False.
         return any(self.children)
 
     def count(self):
         """Returns the total number of render-tiles in this set.
 
         """
-        # TODO: Make this more efficient (although for even the largest trees,
+        return self.num_tiles
-        # this takes only seconds)
-        c = 0
-        for _ in self.iterate():
-            c += 1
-        return c
 
     def count_all(self):
         """Returns the total number of render-tiles plus implicitly marked
         upper-tiles in this set
 
         """
-        # TODO: Optimize this too with its own recursive method that avoids
+        return self.num_tiles_all
-        # some of the overheads of posttraversal()
-        c = 0
-        for _ in self.posttraversal():
-            c += 1
-        return c
 
-def post_traversal_complete_subtree_recursion_helper(depth):
+def distance_sort(children, (off_x, off_y)):
-    """Fakes the recursive calls for RendertileSet.posttraversal() for the case
+    order = []
-    that a subtree is collapsed, so that items are still yielded in the correct
+    for child, (dx, dy) in izip(children, [(-1,-1), (1,-1), (-1,1), (1,1)]):
-    order.
+        x = off_x*2 + dx
+        y = off_y*2 + dy
+        order.append((child, (x,y)))
 
-    """
+    return sorted(order, key=lambda (_, (x,y)): x*x + y*y)
-    if depth == 1:
-        # Base case
-        yield [0]
-        yield [1]
-        yield [2]
-        yield [3]
-    else:
-        for childnum in xrange(4):
-            for item in post_traversal_complete_subtree_recursion_helper(depth-1):
-                item.append(childnum)
-                yield item
-
-    yield []
 
 class RenderTile(object):
     """A simple container class that represents a single render-tile.

test/test_all.py

@@ -2,7 +2,7 @@
 import unittest
 
 # For convenience
-import sys,os
+import sys,os,logging
 sys.path.insert(0, os.getcwd())
 sys.path.insert(0, os.path.join(os.getcwd(), os.pardir))
 
@@ -15,7 +15,6 @@ from test_cache import TestLRU
 
 # DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
 if 0:
-    import logging
     root = logging.getLogger()
     class NullHandler(logging.Handler):
         def handle(self, record):

test/test_rendertileset.py

@@ -1,6 +1,9 @@
 import unittest
 
+from itertools import chain, izip
+
 from overviewer_core.tileset import iterate_base4, RendertileSet
+from overviewer_core.util import roundrobin
 
 class RendertileSetTest(unittest.TestCase):
     # If you change this definition, you must also change the hard-coded
@@ -34,6 +37,61 @@ class RendertileSetTest(unittest.TestCase):
             (2,3,3),
             # Nothing under quadrant 3
             ])
+    # The paths as yielded by posttraversal, in an expanding-from-the-center
+    # order.
+    tile_paths_posttraversal_lists = [
+        [
+            (0,0,3),
+            (0,0,1),
+            (0,0,2),
+            (0,0,0),
+            (0,0),
+            (0,),
+        ],
+        [
+            (1,2,0),
+            (1,2),
+
+            (1,0,3),
+            (1,0),
+
+            (1,1,3),
+            (1,1),
+            (1,),
+        ],
+        [
+            (2,1,1),
+            (2,1,0),
+            (2,1,3),
+            (2,1,2),
+            (2,1),
+
+            (2,0,1),
+            (2,0,3),
+            (2,0,0),
+            (2,0,2),
+            (2,0),
+
+            (2,3,1),
+            (2,3,0),
+            (2,3,3),
+            (2,3,2),
+            (2,3),
+
+            (2,2,1),
+            (2,2,0),
+            (2,2,3),
+            (2,2,2),
+            (2,2),
+            (2,),
+        ],
+    ]
+    # Non-round robin post-traversal: finish the first top-level quadrant
+    # before moving to the second etc.
+    tile_paths_posttraversal       = list(chain(*tile_paths_posttraversal_lists))     + [()]
+    # Round-robin post-traversal: start rendering to all directions from the
+    # center.
+    tile_paths_posttraversal_robin = list(roundrobin(tile_paths_posttraversal_lists)) + [()]
 
     def setUp(self):
         self.tree = RendertileSet(3)
@@ -142,56 +200,18 @@ class RendertileSetTest(unittest.TestCase):
 
     def test_posttraverse(self):
         """Test a post-traversal of the tree's dirty tiles"""
-        # Expect these results in this proper order.
+        # Expect the results in this proper order.
-        expected_list = [
-            (0,0,0),
-            (0,0,1),
-            (0,0,2),
-            (0,0,3),
-            (0,0),
-            (0,),
-
-            (1,0,3),
-            (1,0),
-
-            (1,1,3),
-            (1,1),
-
-            (1,2,0),
-            (1,2),
-            (1,),
-
-            (2,0,0),
-            (2,0,1),
-            (2,0,2),
-            (2,0,3),
-            (2,0),
-
-            (2,1,0),
-            (2,1,1),
-            (2,1,2),
-            (2,1,3),
-            (2,1),
-
-            (2,2,0),
-            (2,2,1),
-            (2,2,2),
-            (2,2,3),
-            (2,2),
-
-            (2,3,0),
-            (2,3,1),
-            (2,3,2),
-            (2,3,3),
-            (2,3),
-            (2,),
-
-            # We should expect the root tile to need rendering too.
-            (),
-            ]
         iterator = iter(self.tree.posttraversal())
-        from itertools import izip
+        for expected, actual in izip(self.tile_paths_posttraversal, iterator):
-        for expected, actual in izip(expected_list, iterator):
+            self.assertEqual(actual, expected)
+
+        self.assertRaises(StopIteration, next, iterator)
+
+    def test_posttraverse_roundrobin(self):
+        """Test a round-robin post-traversal of the tree's dirty tiles"""
+        # Expect the results in this proper order.
+        iterator = iter(self.tree.posttraversal(robin=True))
+        for expected, actual in izip(self.tile_paths_posttraversal_robin, iterator):
             self.assertEqual(actual, expected)
 
         self.assertRaises(StopIteration, next, iterator)

test/test_settings.py

@@ -6,6 +6,8 @@ from overviewer_core.settingsValidators import ValidationException
 from overviewer_core import world
 from overviewer_core import rendermodes
 
+from overviewer_core.util import OrderedDict
+
 class SettingsTest(unittest.TestCase):
     
     def setUp(self):
@@ -13,7 +15,7 @@ class SettingsTest(unittest.TestCase):
     
     def test_missing(self):
         "Validates that a non-existant settings.py causes an exception"
-        self.assertRaises(ValueError, self.s.parse, "doesnotexist.py")
+        self.assertRaises(configParser.MissingConfigException, self.s.parse, "doesnotexist.py")
 
     def test_existing_file(self):
         self.s.parse("test/data/settings/settings_test_1.py")
@@ -45,21 +47,21 @@ class SettingsTest(unittest.TestCase):
         self.s.set_config_item("worlds", {
             'test': "test/data/settings/test_world",
             })
-        self.s.set_config_item("renders", {
+        self.s.set_config_item("renders", OrderedDict([
-                "myworld": { 
+                ("myworld", {
                     "title": "myworld title",
                     "world": "test",
                     "rendermode": rendermodes.normal,
                     "northdirection": "upper-left",
-                },
+                }),
 
-                "otherworld": {
+                ("otherworld", {
                     "title": "otherworld title",
                     "world": "test",
                     "rendermode": rendermodes.normal,
                     "bgcolor": "#ffffff"
-                },
+                }),
-            })
+            ]))
         self.s.set_config_item("outputdir", "/tmp/fictional/outputdir")
         self.assertEquals(fromfile.get_validated_config(), self.s.get_validated_config())
 

test/test_tileset.py

@@ -39,51 +39,6 @@ chunks = {
         (4, 4): 5, # 8, 0
         }
 
-# The resulting tile tree, in the correct post-traversal order.
-correct_tiles = [
-        (0, 3, 3),
-        (0, 3),
-        (0,),
-        (1, 2, 1),
-        (1, 2, 2),
-        (1, 2, 3),
-        (1, 2),
-        (1, 3, 0),
-        (1, 3, 2),
-        (1, 3, 3),
-        (1, 3),
-        (1,),
-        (2, 1, 1),
-        (2, 1, 3),
-        (2, 1),
-        (2, 3, 1),
-        (2, 3, 3),
-        (2, 3),
-        (2,),
-        (3, 0, 0),
-        (3, 0, 1),
-        (3, 0, 2),
-        (3, 0, 3),
-        (3, 0),
-        (3, 1, 0),
-        (3, 1, 1),
-        (3, 1, 2),
-        (3, 1, 3),
-        (3, 1),
-        (3, 2, 0),
-        (3, 2, 1),
-        (3, 2, 2),
-        (3, 2, 3),
-        (3, 2),
-        (3, 3, 0),
-        (3, 3, 1),
-        (3, 3, 2),
-        (3, 3, 3),
-        (3, 3),
-        (3,),
-        (),
-        ]
-
 # Supporting resources
 ######################
 
@@ -123,15 +78,14 @@ def get_tile_set(chunks):
         col, row = tileset.convert_coords(chunkx, chunkz)
 
         for tilec, tiler in tileset.get_tiles_by_chunk(col, row):
-            tile = tileset.RenderTile.compute_path(tilec, tiler, 3)
+            tile = tileset.RenderTile.compute_path(tilec, tiler, 5)
             tile_set[tile.path] = max(tile_set[tile.path], chunkmtime)
 
     # At this point, tile_set holds all the render-tiles
     for tile, tile_mtime in tile_set.copy().iteritems():
-        # All render-tiles are length 3. Hard-code its upper tiles
+        # All render-tiles are length 5. Hard-code its upper tiles
-        tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime)
+        for i in reversed(xrange(5)):
-        tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime)
+            tile_set[tile[:i]] = max(tile_set[tile[:i]], tile_mtime)
-        tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime)
     return dict(tile_set)
 
 def create_fakedir(outputdir, tiles):
@@ -184,6 +138,7 @@ class TilesetTest(unittest.TestCase):
         is called before do_preprocessing()
         """
         defoptions = {
+                'name': 'world name',
                 'bgcolor': '#000000',
                 'imgformat': 'png',
                 'optimizeimg': 0,
@@ -191,7 +146,7 @@ class TilesetTest(unittest.TestCase):
                 'rerenderprob': 0
                 }
         defoptions.update(options)
-        ts = tileset.TileSet(self.rs, FakeAssetmanager(0), None, defoptions, outputdir)
+        ts = tileset.TileSet(None, self.rs, FakeAssetmanager(0), None, defoptions, outputdir)
         if preprocess:
             preprocess(ts)
         ts.do_preprocessing()
@@ -222,7 +177,7 @@ class TilesetTest(unittest.TestCase):
     def test_get_phase_length(self):
         ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
         self.assertEqual(ts.get_num_phases(), 1)
-        self.assertEqual(ts.get_phase_length(0), 41)
+        self.assertEqual(ts.get_phase_length(0), len(get_tile_set(chunks)))
 
     def test_forcerender_iterate(self):
         """Tests that a rendercheck mode 2 iteration returns every render-tile
@@ -278,18 +233,23 @@ class TilesetTest(unittest.TestCase):
         # object.
         # Chosen at random:
         outdated_tiles = [
-                (0,3,3),
+                (0,3,3,3,3),
-                (1,2,1),
+                (1,2,2,2,1),
-                (2,1),
+                (2,1,1),
                 (3,)
                 ]
         # These are the tiles that we also expect it to return, even though
         # they were not outdated, since they depend on the outdated tiles
         additional = [
+                (0,3,3,3),
+                (0,3,3),
                 (0,3),
                 (0,),
+                (1,2,2,2),
+                (1,2,2),
                 (1,2),
                 (1,),
+                (2,1),
                 (2,),
                 (),
                 ]