From a788065cfb9b0382572a096985c96918a43ee624 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Tue, 24 Dec 2013 21:47:27 +0200 Subject: [PATCH 01/18] tileset: Typo fix --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index fe05064..87061ef 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -377,7 +377,7 @@ class TileSet(object): # This warning goes here so it's only shown once if self.treedepth >= 15: - logging.warning("Just letting you know, your map requries %s zoom levels. This is REALLY big!", + logging.warning("Just letting you know, your map requires %s zoom levels. This is REALLY big!", self.treedepth) # Do any tile re-arranging if necessary. Skip if there was no config From f94e3de03f03487ae9f8b3df1923e675900e011e Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Wed, 25 Dec 2013 04:33:59 +0200 Subject: [PATCH 02/18] tileset: Render in expanding order from center --- overviewer_core/tileset.py | 40 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 87061ef..ebf1785 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -1311,7 +1311,7 @@ class RendertileSet(object): # tree self.children = [False] * 4 - def posttraversal(self): + def posttraversal(self, offset=(0,0)): """Returns an iterator over tile paths for every tile in the set, including the explictly marked render-tiles, as well as the implicitly marked ancestors of those render-tiles. Returns in @@ -1319,20 +1319,19 @@ class RendertileSet(object): yielded after their dependencies. """ - return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper()) + return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper(offset=offset)) - def _posttraversal_helper(self): + def _posttraversal_helper(self, offset=(0,0)): """Each node returns an iterator over lists of reversed paths""" if self.depth == 1: # Base case - if self.children[0]: yield [0] - if self.children[1]: yield [1] - if self.children[2]: yield [2] - if self.children[3]: yield [3] + for childnum, _ in distance_sort(xrange(4), offset): + if self.children[childnum]: + yield [childnum] else: - for childnum, child in enumerate(self.children): + for (childnum, child), childoffset in distance_sort(enumerate(self.children), offset): if child == True: - for path in post_traversal_complete_subtree_recursion_helper(self.depth-1): + for path in post_traversal_complete_subtree_recursion_helper(self.depth-1, offset=childoffset): path.append(childnum) yield path @@ -1341,7 +1340,7 @@ class RendertileSet(object): else: # Recurse - for path in child._posttraversal_helper(): + for path in child._posttraversal_helper(offset=childoffset): path.append(childnum) yield path @@ -1520,7 +1519,7 @@ class RendertileSet(object): c += 1 return c -def post_traversal_complete_subtree_recursion_helper(depth): +def post_traversal_complete_subtree_recursion_helper(depth, offset=(0,0)): """Fakes the recursive calls for RendertileSet.posttraversal() for the case that a subtree is collapsed, so that items are still yielded in the correct order. @@ -1528,18 +1527,25 @@ def post_traversal_complete_subtree_recursion_helper(depth): """ if depth == 1: # Base case - yield [0] - yield [1] - yield [2] - yield [3] + for childnum, _ in distance_sort(xrange(4), offset): + yield [childnum] else: - for childnum in xrange(4): - for item in post_traversal_complete_subtree_recursion_helper(depth-1): + for childnum, childoffset in distance_sort(xrange(4), offset): + for item in post_traversal_complete_subtree_recursion_helper(depth-1, offset=childoffset): item.append(childnum) yield item yield [] +def distance_sort(children, (off_x, off_y)): + order = [] + for child, (dx, dy) in izip(children, [(-1,-1), (1,-1), (-1,1), (1,1)]): + x = off_x*2 + dx + y = off_y*2 + dy + order.append((child, (x,y))) + + return sorted(order, key=lambda (_, (x,y)): x*x + y*y) + class RenderTile(object): """A simple container class that represents a single render-tile. From 66fb4afb5c00a2f0ac21a27c95bbf6935b711c08 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Wed, 25 Dec 2013 04:46:56 +0200 Subject: [PATCH 03/18] tileset: Unify _posttraversal_helper and _iterate_helper --- overviewer_core/tileset.py | 47 +++++++++++--------------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index ebf1785..8f22d83 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -1321,9 +1321,11 @@ class RendertileSet(object): """ return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper(offset=offset)) - def _posttraversal_helper(self, offset=(0,0)): + def _posttraversal_helper(self, onlydepth=None, offset=(0,0)): """Each node returns an iterator over lists of reversed paths""" - if self.depth == 1: + targetdepth = 1 if onlydepth is None else onlydepth + + if self.depth == targetdepth: # Base case for childnum, _ in distance_sort(xrange(4), offset): if self.children[childnum]: @@ -1331,7 +1333,7 @@ class RendertileSet(object): else: for (childnum, child), childoffset in distance_sort(enumerate(self.children), offset): if child == True: - for path in post_traversal_complete_subtree_recursion_helper(self.depth-1, offset=childoffset): + for path in post_traversal_complete_subtree_recursion_helper(self.depth-targetdepth, onlydepth=onlydepth, offset=childoffset): path.append(childnum) yield path @@ -1340,12 +1342,12 @@ class RendertileSet(object): else: # Recurse - for path in child._posttraversal_helper(offset=childoffset): + for path in child._posttraversal_helper(onlydepth=onlydepth, offset=childoffset): path.append(childnum) yield path # Now do this node itself - if bool(self): + if onlydepth is None and bool(self): yield [] @@ -1411,7 +1413,7 @@ class RendertileSet(object): def __iter__(self): return self.iterate() - def iterate(self, level=None): + def iterate(self, level=None, offset=(0,0)): """Returns an iterator over every tile in this set. Each item yielded is a sequence of integers representing the quadtree path to the tiles in the set. Yielded sequences are of length self.depth. @@ -1432,31 +1434,7 @@ class RendertileSet(object): raise ValueError("Level parameter must be between 1 and %s" % self.depth) todepth = self.depth - level + 1 - return (tuple(reversed(rpath)) for rpath in self._iterate_helper(todepth)) - - def _iterate_helper(self, todepth): - if self.depth == todepth: - # Base case - if self.children[0]: yield [0] - if self.children[1]: yield [1] - if self.children[2]: yield [2] - if self.children[3]: yield [3] - - else: - # Higher levels: - for c, child in enumerate(self.children): - if child == True: - # All render-tiles are in the set down this subtree, - # iterate over every leaf using iterate_base4 - for x in iterate_base4(self.depth-todepth): - x = list(x) - x.append(c) - yield x - elif child != False: - # Mixed in/out of the set down this subtree, recurse - for path in child._iterate_helper(todepth): - path.append(c) - yield path + return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper(onlydepth=todepth, offset=offset)) def query_path(self, path): """Queries for the state of the given tile in the tree. @@ -1519,7 +1497,7 @@ class RendertileSet(object): c += 1 return c -def post_traversal_complete_subtree_recursion_helper(depth, offset=(0,0)): +def post_traversal_complete_subtree_recursion_helper(depth, onlydepth=None, offset=(0,0)): """Fakes the recursive calls for RendertileSet.posttraversal() for the case that a subtree is collapsed, so that items are still yielded in the correct order. @@ -1531,11 +1509,12 @@ def post_traversal_complete_subtree_recursion_helper(depth, offset=(0,0)): yield [childnum] else: for childnum, childoffset in distance_sort(xrange(4), offset): - for item in post_traversal_complete_subtree_recursion_helper(depth-1, offset=childoffset): + for item in post_traversal_complete_subtree_recursion_helper(depth-1, onlydepth=onlydepth, offset=childoffset): item.append(childnum) yield item - yield [] + if onlydepth is None: + yield [] def distance_sort(children, (off_x, off_y)): order = [] From 2252b1e69b4f6b13896728112bafe5683bcc414e Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Wed, 25 Dec 2013 06:12:18 +0200 Subject: [PATCH 04/18] Update test_posttraverse to match expanding order --- test/test_rendertileset.py | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/test_rendertileset.py b/test/test_rendertileset.py index ea2e017..d19bda4 100644 --- a/test/test_rendertileset.py +++ b/test/test_rendertileset.py @@ -144,46 +144,46 @@ class RendertileSetTest(unittest.TestCase): """Test a post-traversal of the tree's dirty tiles""" # Expect these results in this proper order. expected_list = [ - (0,0,0), + (0,0,3), (0,0,1), (0,0,2), - (0,0,3), + (0,0,0), (0,0), (0,), + (1,2,0), + (1,2), + (1,0,3), (1,0), (1,1,3), (1,1), - - (1,2,0), - (1,2), (1,), - (2,0,0), - (2,0,1), - (2,0,2), - (2,0,3), - (2,0), - - (2,1,0), (2,1,1), - (2,1,2), + (2,1,0), (2,1,3), + (2,1,2), (2,1), - (2,2,0), - (2,2,1), - (2,2,2), - (2,2,3), - (2,2), + (2,0,1), + (2,0,3), + (2,0,0), + (2,0,2), + (2,0), - (2,3,0), (2,3,1), - (2,3,2), + (2,3,0), (2,3,3), + (2,3,2), (2,3), + + (2,2,1), + (2,2,0), + (2,2,3), + (2,2,2), + (2,2), (2,), # We should expect the root tile to need rendering too. From 37ab11899dc1c382893796132e09502fb532e31c Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Wed, 25 Dec 2013 06:14:01 +0200 Subject: [PATCH 05/18] test_all: import logging --- test/test_all.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_all.py b/test/test_all.py index 21ae591..5905bab 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -2,7 +2,7 @@ import unittest # For convenience -import sys,os +import sys,os,logging sys.path.insert(0, os.getcwd()) sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) @@ -15,7 +15,6 @@ from test_cache import TestLRU # DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING if 0: - import logging root = logging.getLogger() class NullHandler(logging.Handler): def handle(self, record): From b1bd77d9622a19b7cc97f5d71d5777b594054602 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 03:24:08 +0200 Subject: [PATCH 06/18] =?UTF-8?q?test=5Fsettings:=20ValueError=20=E2=86=92?= =?UTF-8?q?=20MissingConfigException?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_settings.py b/test/test_settings.py index 8072706..56430ef 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -13,7 +13,7 @@ class SettingsTest(unittest.TestCase): def test_missing(self): "Validates that a non-existant settings.py causes an exception" - self.assertRaises(ValueError, self.s.parse, "doesnotexist.py") + self.assertRaises(configParser.MissingConfigException, self.s.parse, "doesnotexist.py") def test_existing_file(self): self.s.parse("test/data/settings/settings_test_1.py") From c32abc00e0ecc3be544e12a621460aa1e5d9c6f9 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 03:46:03 +0200 Subject: [PATCH 07/18] test_settings/test_manual: Match renders order with file --- test/test_settings.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_settings.py b/test/test_settings.py index 56430ef..3f44ec8 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -6,6 +6,8 @@ from overviewer_core.settingsValidators import ValidationException from overviewer_core import world from overviewer_core import rendermodes +from overviewer_core.util import OrderedDict + class SettingsTest(unittest.TestCase): def setUp(self): @@ -45,21 +47,21 @@ class SettingsTest(unittest.TestCase): self.s.set_config_item("worlds", { 'test': "test/data/settings/test_world", }) - self.s.set_config_item("renders", { - "myworld": { + self.s.set_config_item("renders", OrderedDict([ + ("myworld", { "title": "myworld title", "world": "test", "rendermode": rendermodes.normal, "northdirection": "upper-left", - }, + }), - "otherworld": { + ("otherworld", { "title": "otherworld title", "world": "test", "rendermode": rendermodes.normal, "bgcolor": "#ffffff" - }, - }) + }), + ])) self.s.set_config_item("outputdir", "/tmp/fictional/outputdir") self.assertEquals(fromfile.get_validated_config(), self.s.get_validated_config()) From b5dec03cfff29562b3ee84e91f4b7924dbe2003f Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 03:46:36 +0200 Subject: [PATCH 08/18] test_tileset: Add missing world parameter --- test/test_tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_tileset.py b/test/test_tileset.py index db23d80..9f9b862 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -191,7 +191,7 @@ class TilesetTest(unittest.TestCase): 'rerenderprob': 0 } defoptions.update(options) - ts = tileset.TileSet(self.rs, FakeAssetmanager(0), None, defoptions, outputdir) + ts = tileset.TileSet(None, self.rs, FakeAssetmanager(0), None, defoptions, outputdir) if preprocess: preprocess(ts) ts.do_preprocessing() From 2adebabc1998944d7234a5487fb75be5f927d9af Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 03:46:59 +0200 Subject: [PATCH 09/18] test_tileset: Add required name item to defoptions --- test/test_tileset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_tileset.py b/test/test_tileset.py index 9f9b862..9cde942 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -184,6 +184,7 @@ class TilesetTest(unittest.TestCase): is called before do_preprocessing() """ defoptions = { + 'name': 'world name', 'bgcolor': '#000000', 'imgformat': 'png', 'optimizeimg': 0, From 9d669f4bbf8bf90c83dd33f82de749b70beb7cee Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 04:36:35 +0200 Subject: [PATCH 10/18] test_tileset: Drop unused correct_tiles list --- test/test_tileset.py | 45 -------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/test/test_tileset.py b/test/test_tileset.py index 9cde942..b7ff663 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -39,51 +39,6 @@ chunks = { (4, 4): 5, # 8, 0 } -# The resulting tile tree, in the correct post-traversal order. -correct_tiles = [ - (0, 3, 3), - (0, 3), - (0,), - (1, 2, 1), - (1, 2, 2), - (1, 2, 3), - (1, 2), - (1, 3, 0), - (1, 3, 2), - (1, 3, 3), - (1, 3), - (1,), - (2, 1, 1), - (2, 1, 3), - (2, 1), - (2, 3, 1), - (2, 3, 3), - (2, 3), - (2,), - (3, 0, 0), - (3, 0, 1), - (3, 0, 2), - (3, 0, 3), - (3, 0), - (3, 1, 0), - (3, 1, 1), - (3, 1, 2), - (3, 1, 3), - (3, 1), - (3, 2, 0), - (3, 2, 1), - (3, 2, 2), - (3, 2, 3), - (3, 2), - (3, 3, 0), - (3, 3, 1), - (3, 3, 2), - (3, 3, 3), - (3, 3), - (3,), - (), - ] - # Supporting resources ###################### From 754487389d8a159d733b901221d2ff6e564afd0f Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 04:37:35 +0200 Subject: [PATCH 11/18] =?UTF-8?q?test=5Ftileset:=20Depth=20is=20now=205=20?= =?UTF-8?q?because=20of=20=E2=80=98maxrow=20+=2032=E2=80=99=20in=20tileset?= =?UTF-8?q?.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_tileset.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/test_tileset.py b/test/test_tileset.py index b7ff663..f555eac 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -78,15 +78,14 @@ def get_tile_set(chunks): col, row = tileset.convert_coords(chunkx, chunkz) for tilec, tiler in tileset.get_tiles_by_chunk(col, row): - tile = tileset.RenderTile.compute_path(tilec, tiler, 3) + tile = tileset.RenderTile.compute_path(tilec, tiler, 5) tile_set[tile.path] = max(tile_set[tile.path], chunkmtime) # At this point, tile_set holds all the render-tiles for tile, tile_mtime in tile_set.copy().iteritems(): - # All render-tiles are length 3. Hard-code its upper tiles - tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime) - tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime) - tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime) + # All render-tiles are length 5. Hard-code its upper tiles + for i in reversed(xrange(5)): + tile_set[tile[:i]] = max(tile_set[tile[:i]], tile_mtime) return dict(tile_set) def create_fakedir(outputdir, tiles): @@ -178,7 +177,7 @@ class TilesetTest(unittest.TestCase): def test_get_phase_length(self): ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir()) self.assertEqual(ts.get_num_phases(), 1) - self.assertEqual(ts.get_phase_length(0), 41) + self.assertEqual(ts.get_phase_length(0), len(get_tile_set(chunks))) def test_forcerender_iterate(self): """Tests that a rendercheck mode 2 iteration returns every render-tile @@ -234,18 +233,23 @@ class TilesetTest(unittest.TestCase): # object. # Chosen at random: outdated_tiles = [ - (0,3,3), - (1,2,1), - (2,1), + (0,3,3,3,3), + (1,2,2,2,1), + (2,1,1), (3,) ] # These are the tiles that we also expect it to return, even though # they were not outdated, since they depend on the outdated tiles additional = [ + (0,3,3,3), + (0,3,3), (0,3), (0,), + (1,2,2,2), + (1,2,2), (1,2), (1,), + (2,1), (2,), (), ] From 4a8fb12d63d0cb7086659a5bc0291eb48630f9d0 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 04:39:11 +0200 Subject: [PATCH 12/18] .travis.yml: Run the test suite --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c5992bc..a57e096 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ before_script: - git clone git://github.com/overviewer/Minecraft-Overviewer-Addons.git ~/mcoa/ - wget -N https://s3.amazonaws.com/Minecraft.Download/versions/${MC_VERSION}/${MC_VERSION}.jar -P ~/.minecraft/versions/${MC_VERSION}/ script: + - PYTHONPATH=. python test/test_all.py - python overviewer.py ~/mcoa/exmaple ~/test-output --rendermodes=smooth-lighting -p1 notifications: email: false From 24994b3471e6db29cf55c33ccc6811e974ca697b Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 04:39:48 +0200 Subject: [PATCH 13/18] .travis.yml: MC_VERSION=1.7.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a57e096..efea1ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "2.7" # - "3.2" env: - - MC_VERSION=1.7.2 + - MC_VERSION=1.7.4 before_install: - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/Imaging.h - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/ImPlatform.h From 010ae6af408af1ca0c173f7a942cf0ca595b44c9 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 04:42:45 +0200 Subject: [PATCH 14/18] .travis.yml: Add IRC notifications --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index efea1ad..382f1dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,12 @@ script: - python overviewer.py ~/mcoa/exmaple ~/test-output --rendermodes=smooth-lighting -p1 notifications: email: false + irc: + channels: + - "irc.freenode.org#overviewer" + skip_join: true + template: + - "\x0313Minecraft-Overviewer\x03/\x0306%{branch}\x03 \x0314%{commit}\x03 %{build_url} %{message}" # matrix: # allow_failures: # - python: "3.2" From e4638467ef36e866c5030d1adf6fb43bcfdbd41c Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 08:20:06 +0200 Subject: [PATCH 15/18] Simplify RendertileSet code --- overviewer_core/tileset.py | 195 +++++++++++++------------------------ 1 file changed, 68 insertions(+), 127 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 8f22d83..6aad5c0 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -1268,18 +1268,16 @@ class RendertileSet(object): It is typically used to hold tiles that need rendering. This implementation collapses subtrees that are completely in or out of the set to save memory. - Each instance of this class is a node in the tree, and therefore each - instance is the root of a subtree. + An instance of this class holds a full tree. - Each node knows its "level", which corresponds to the zoom level where 0 is - the inner-most (most zoomed in) tiles. + The instance knows its "level", which corresponds to the zoom level where 1 + is the inner-most (most zoomed in) tiles. Instances hold the state of their children (in or out of the set). Leaf nodes are images and do not physically exist in the tree as objects, but are represented as booleans held by the objects at the second-to-last level; level 1 nodes keep track of leaf image state. Level 2 nodes keep - track of level 1 state, and so fourth. - + track of level 1 state, and so forth. """ __slots__ = ("depth", "children") @@ -1303,54 +1301,13 @@ class RendertileSet(object): # All children down this subtree are not in the set # True # All children down this subtree are in the set - # A RendertileSet instance - # the instance defines which children down that subtree are in the - # set. + # An array of the same format + # The array defines which children down that subtree are in the set # A node with depth=1 cannot have a RendertileSet instance in its # children since its children are leaves, representing images, not more # tree self.children = [False] * 4 - def posttraversal(self, offset=(0,0)): - """Returns an iterator over tile paths for every tile in the - set, including the explictly marked render-tiles, as well as the - implicitly marked ancestors of those render-tiles. Returns in - post-traversal order, so that tiles with dependencies will always be - yielded after their dependencies. - - """ - return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper(offset=offset)) - - def _posttraversal_helper(self, onlydepth=None, offset=(0,0)): - """Each node returns an iterator over lists of reversed paths""" - targetdepth = 1 if onlydepth is None else onlydepth - - if self.depth == targetdepth: - # Base case - for childnum, _ in distance_sort(xrange(4), offset): - if self.children[childnum]: - yield [childnum] - else: - for (childnum, child), childoffset in distance_sort(enumerate(self.children), offset): - if child == True: - for path in post_traversal_complete_subtree_recursion_helper(self.depth-targetdepth, onlydepth=onlydepth, offset=childoffset): - path.append(childnum) - yield path - - elif child == False: - pass # do nothing - - else: - # Recurse - for path in child._posttraversal_helper(onlydepth=onlydepth, offset=childoffset): - path.append(childnum) - yield path - - # Now do this node itself - if onlydepth is None and bool(self): - yield [] - - def add(self, path): """Marks the requested leaf node as in this set @@ -1360,67 +1317,46 @@ class RendertileSet(object): """ path = list(path) assert len(path) == self.depth - path.reverse() - self._set_add_helper(path) - def _set_add_helper(self, path): + self._add_helper(self.children, list(reversed(path))) + + def _add_helper(self, children, path): """Recursive helper for add() - - Expects path to be a list in reversed order - - If *all* the nodes below this one are in the set, this function returns - true. Otherwise, returns None. - """ - if self.depth == 1: - # Base case - self.children[path[0]] = True + childnum = path.pop() - # Check to see if all children are in the set - if all(self.children): - return True - else: - # Recursive case + if path: + # We are not at the leaf, recurse. - childnum = path.pop() - child = self.children[childnum] - - if child == False: - # Create a new node and recurse. - # (The use of __class__ is so possible subclasses of this class - # work as expected) - child = self.__class__(self.depth-1) - child._set_add_helper(path) - self.children[childnum] = child - elif child == True: - # Every child is already in the set and the subtree is already - # collapsed. Nothing to do. + if children[childnum] == True: + # The child is already in the tree. return - else: - # subtree is mixed. Recurse to the already existing child node - ret = child._set_add_helper(path) - if ret: - # Child says every descendent is in the set, so we can - # purge the subtree and mark it as such. The subtree will - # be garbage collected when this method exits. - self.children[childnum] = True + elif children[childnum] == False: + # Expand all-false. + children[childnum] = [False]*4 - # Since we've marked an entire sub-tree as in the set, we - # may be able to signal to our parent to do the same - if all(x is True for x in self.children): - return True + self._add_helper(children[childnum], path) + + if children[childnum] == [True]*4: + # Collapse all-true. + children[childnum] = True + + else: + # We are at the leaf. + children[childnum] = True def __iter__(self): return self.iterate() + def iterate(self, level=None, offset=(0,0)): """Returns an iterator over every tile in this set. Each item yielded is a sequence of integers representing the quadtree path to the tiles in the set. Yielded sequences are of length self.depth. If level is None, iterates over tiles of the highest level, i.e. - worldtiles. If level is a value between 0 and the depth of this tree, - this method iterates over tiles at that level. Zoom level 0 is zoomed + worldtiles. If level is a value between 1 and the depth of this tree, + this method iterates over tiles at that level. Zoom level 1 is zoomed all the way out, zoom level `depth` is all the way in. In other words, specifying level causes the tree to be iterated as if @@ -1434,7 +1370,38 @@ class RendertileSet(object): raise ValueError("Level parameter must be between 1 and %s" % self.depth) todepth = self.depth - level + 1 - return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper(onlydepth=todepth, offset=offset)) + return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, offset=offset)) + + def posttraversal(self, offset=(0,0)): + """Returns an iterator over tile paths for every tile in the + set, including the explictly marked render-tiles, as well as the + implicitly marked ancestors of those render-tiles. Returns in + post-traversal order, so that tiles with dependencies will always be + yielded after their dependencies. + """ + return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, offset=offset)) + + def _iterate_helper(self, path, children, depth, onlydepth=None, offset=(0,0)): + """Returns an iterator over tile paths for every tile in the set.""" + + # A variant of children with a collapsed False/True expanded to a list. + children_list = [children] * 4 if isinstance(children, bool) else children + + targetdepth = 1 if onlydepth is None else onlydepth + + if depth == targetdepth: + # Base case + for (childnum, child), _ in distance_sort(enumerate(children_list), offset): + if child: + yield path + [childnum] + else: + for (childnum, child), childoffset in distance_sort(enumerate(children_list), offset): + if child: + for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset): + yield p + + if onlydepth is None and children: + yield path def query_path(self, path): """Queries for the state of the given tile in the tree. @@ -1448,30 +1415,23 @@ class RendertileSet(object): # collapsed, then just return the stored boolean. Otherwise, if we find # the specific tree node requested, return its state using the # __nonzero__ call. - treenode = self + treenode = self.children for pathelement in path: - treenode = treenode.children[pathelement] - if not isinstance(treenode, RendertileSet): + treenode = treenode[pathelement] + if isinstance(treenode, bool): return treenode # If the method has not returned at this point, treenode is the - # requested node, but it is an inner node with possibly mixed state - # subtrees. If any of the children are True return True. This call - # relies on the __nonzero__ method - return bool(treenode) + # requested node, but it is an inner node. That will only happen if one + # or more of the children down the tree are True. + return True def __nonzero__(self): """Returns the boolean context of this particular node. If any descendent of this node is True return True. Otherwise, False. """ - # Any chilren that are True or are a RendertileSet that evaluate to - # True - # IDEA: look at all children for True before recursing - # Better idea: every node except the root /must/ have a descendent in - # the set or it wouldn't exist. This assumption is only valid as long - # as there is no method to remove a tile from the set. So this should - # check to see if any children are not False. + # Any children that are True or are a list evaluate to True. return any(self.children) def count(self): @@ -1497,25 +1457,6 @@ class RendertileSet(object): c += 1 return c -def post_traversal_complete_subtree_recursion_helper(depth, onlydepth=None, offset=(0,0)): - """Fakes the recursive calls for RendertileSet.posttraversal() for the case - that a subtree is collapsed, so that items are still yielded in the correct - order. - - """ - if depth == 1: - # Base case - for childnum, _ in distance_sort(xrange(4), offset): - yield [childnum] - else: - for childnum, childoffset in distance_sort(xrange(4), offset): - for item in post_traversal_complete_subtree_recursion_helper(depth-1, onlydepth=onlydepth, offset=childoffset): - item.append(childnum) - yield item - - if onlydepth is None: - yield [] - def distance_sort(children, (off_x, off_y)): order = [] for child, (dx, dy) in izip(children, [(-1,-1), (1,-1), (-1,1), (1,1)]): From c49fdd19e17341fd869b899fad179e771d385401 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 08:36:12 +0200 Subject: [PATCH 16/18] RendertileSet: Round-robin the four top-level subtrees --- overviewer_core/tileset.py | 35 +++++++++++++++------- test/test_rendertileset.py | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 6aad5c0..c958919 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -25,7 +25,7 @@ import time import errno import stat from collections import namedtuple -from itertools import product, izip +from itertools import product, izip, chain from PIL import Image @@ -446,7 +446,7 @@ class TileSet(object): # render. Iterate over the tiles in using the posttraversal() method. # Yield each item. Easy. if self.options['renderchecks'] in (0,2): - for tilepath in self.dirtytree.posttraversal(): + for tilepath in self.dirtytree.posttraversal(robin=True): dependencies = [] # These tiles may or may not exist, but the dispatcher won't # care according to the worker interface protocol It will only @@ -1349,7 +1349,7 @@ class RendertileSet(object): def __iter__(self): return self.iterate() - def iterate(self, level=None, offset=(0,0)): + def iterate(self, level=None, robin=False, offset=(0,0)): """Returns an iterator over every tile in this set. Each item yielded is a sequence of integers representing the quadtree path to the tiles in the set. Yielded sequences are of length self.depth. @@ -1362,6 +1362,9 @@ class RendertileSet(object): In other words, specifying level causes the tree to be iterated as if it was only that depth. + If the `robin` parameter is True, recurses to the four top-level + subtrees simultaneously in a round-robin manner. + """ if level is None: todepth = 1 @@ -1370,18 +1373,22 @@ class RendertileSet(object): raise ValueError("Level parameter must be between 1 and %s" % self.depth) todepth = self.depth - level + 1 - return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, offset=offset)) + return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, robin=robin, offset=offset)) - def posttraversal(self, offset=(0,0)): + def posttraversal(self, robin=False, offset=(0,0)): """Returns an iterator over tile paths for every tile in the set, including the explictly marked render-tiles, as well as the implicitly marked ancestors of those render-tiles. Returns in post-traversal order, so that tiles with dependencies will always be yielded after their dependencies. - """ - return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, offset=offset)) - def _iterate_helper(self, path, children, depth, onlydepth=None, offset=(0,0)): + If the `robin` parameter is True, recurses to the four top-level + subtrees simultaneously in a round-robin manner. + + """ + return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, robin=robin, offset=offset)) + + def _iterate_helper(self, path, children, depth, onlydepth=None, robin=False, offset=(0,0)): """Returns an iterator over tile paths for every tile in the set.""" # A variant of children with a collapsed False/True expanded to a list. @@ -1395,10 +1402,16 @@ class RendertileSet(object): if child: yield path + [childnum] else: - for (childnum, child), childoffset in distance_sort(enumerate(children_list), offset): + gens = [] + for (childnum_, child), childoffset_ in distance_sort(enumerate(children_list), offset): if child: - for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset): - yield p + def go(childnum, childoffset): + for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset): + yield p + gens.append(go(childnum_, childoffset_)) + + for p in roundrobin(gens) if robin else chain(*gens): + yield p if onlydepth is None and children: yield path diff --git a/test/test_rendertileset.py b/test/test_rendertileset.py index d19bda4..943fdfa 100644 --- a/test/test_rendertileset.py +++ b/test/test_rendertileset.py @@ -1,6 +1,7 @@ import unittest from overviewer_core.tileset import iterate_base4, RendertileSet +from overviewer_core.util import roundrobin class RendertileSetTest(unittest.TestCase): # If you change this definition, you must also change the hard-coded @@ -196,6 +197,65 @@ class RendertileSetTest(unittest.TestCase): self.assertRaises(StopIteration, next, iterator) + def test_posttraverse_roundrobin(self): + """Test a round-robin post-traversal of the tree's dirty tiles""" + # Expect these results in this proper order. + expected_lists = [ + [ + (0,0,3), + (0,0,1), + (0,0,2), + (0,0,0), + (0,0), + (0,), + ], + [ + (1,2,0), + (1,2), + + (1,0,3), + (1,0), + + (1,1,3), + (1,1), + (1,), + ], + [ + (2,1,1), + (2,1,0), + (2,1,3), + (2,1,2), + (2,1), + + (2,0,1), + (2,0,3), + (2,0,0), + (2,0,2), + (2,0), + + (2,3,1), + (2,3,0), + (2,3,3), + (2,3,2), + (2,3), + + (2,2,1), + (2,2,0), + (2,2,3), + (2,2,2), + (2,2), + (2,), + ], + ] + expected_list = list(roundrobin(expected_lists)) + [ () ] + + iterator = iter(self.tree.posttraversal(robin=True)) + from itertools import izip + for expected, actual in izip(expected_list, iterator): + self.assertEqual(actual, expected) + + self.assertRaises(StopIteration, next, iterator) + def test_count_all(self): """Tests getting a count of all tiles (render tiles plus upper tiles) From 71e3441cb14936c5ef59c5ad2e36fa6162c6fe88 Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Fri, 27 Dec 2013 08:49:18 +0200 Subject: [PATCH 17/18] RendertileSet: Cache the number of tiles --- overviewer_core/tileset.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index c958919..5c0c73c 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -1280,7 +1280,7 @@ class RendertileSet(object): track of level 1 state, and so forth. """ - __slots__ = ("depth", "children") + __slots__ = ("depth", "children", "num_tiles", "num_tiles_all") def __init__(self, depth): """Initialize a new tree with the specified depth. This actually initializes a node, which is the root of a subtree, with `depth` levels @@ -1308,6 +1308,9 @@ class RendertileSet(object): # tree self.children = [False] * 4 + self.num_tiles = 0 + self.num_tiles_all = 0 + def add(self, path): """Marks the requested leaf node as in this set @@ -1318,6 +1321,11 @@ class RendertileSet(object): path = list(path) assert len(path) == self.depth + if self.num_tiles == 0: + # The first child is being added. A root composite tile will be + # rendered. + self.num_tiles_all += 1 + self._add_helper(self.children, list(reversed(path))) def _add_helper(self, children, path): @@ -1336,6 +1344,9 @@ class RendertileSet(object): # Expand all-false. children[childnum] = [False]*4 + # This also means an additional composite tile. + self.num_tiles_all += 1 + self._add_helper(children[childnum], path) if children[childnum] == [True]*4: @@ -1344,6 +1355,10 @@ class RendertileSet(object): else: # We are at the leaf. + if not children[childnum]: + self.num_tiles += 1 + self.num_tiles_all += 1 + children[childnum] = True def __iter__(self): @@ -1451,24 +1466,14 @@ class RendertileSet(object): """Returns the total number of render-tiles in this set. """ - # TODO: Make this more efficient (although for even the largest trees, - # this takes only seconds) - c = 0 - for _ in self.iterate(): - c += 1 - return c + return self.num_tiles def count_all(self): """Returns the total number of render-tiles plus implicitly marked upper-tiles in this set """ - # TODO: Optimize this too with its own recursive method that avoids - # some of the overheads of posttraversal() - c = 0 - for _ in self.posttraversal(): - c += 1 - return c + return self.num_tiles_all def distance_sort(children, (off_x, off_y)): order = [] From 6692d10af00ddd112e79b3df0acec263fe6a928b Mon Sep 17 00:00:00 2001 From: Johan Kiviniemi Date: Sat, 28 Dec 2013 06:03:35 +0200 Subject: [PATCH 18/18] test_posttraverse{,_roundrobin}: Unduplicate expected list --- test/test_rendertileset.py | 162 ++++++++++++++----------------------- 1 file changed, 61 insertions(+), 101 deletions(-) diff --git a/test/test_rendertileset.py b/test/test_rendertileset.py index 943fdfa..28153cb 100644 --- a/test/test_rendertileset.py +++ b/test/test_rendertileset.py @@ -1,5 +1,7 @@ import unittest +from itertools import chain, izip + from overviewer_core.tileset import iterate_base4, RendertileSet from overviewer_core.util import roundrobin @@ -35,6 +37,61 @@ class RendertileSetTest(unittest.TestCase): (2,3,3), # Nothing under quadrant 3 ]) + # The paths as yielded by posttraversal, in an expanding-from-the-center + # order. + tile_paths_posttraversal_lists = [ + [ + (0,0,3), + (0,0,1), + (0,0,2), + (0,0,0), + (0,0), + (0,), + ], + [ + (1,2,0), + (1,2), + + (1,0,3), + (1,0), + + (1,1,3), + (1,1), + (1,), + ], + [ + (2,1,1), + (2,1,0), + (2,1,3), + (2,1,2), + (2,1), + + (2,0,1), + (2,0,3), + (2,0,0), + (2,0,2), + (2,0), + + (2,3,1), + (2,3,0), + (2,3,3), + (2,3,2), + (2,3), + + (2,2,1), + (2,2,0), + (2,2,3), + (2,2,2), + (2,2), + (2,), + ], + ] + # Non-round robin post-traversal: finish the first top-level quadrant + # before moving to the second etc. + tile_paths_posttraversal = list(chain(*tile_paths_posttraversal_lists)) + [()] + # Round-robin post-traversal: start rendering to all directions from the + # center. + tile_paths_posttraversal_robin = list(roundrobin(tile_paths_posttraversal_lists)) + [()] def setUp(self): self.tree = RendertileSet(3) @@ -143,115 +200,18 @@ class RendertileSetTest(unittest.TestCase): def test_posttraverse(self): """Test a post-traversal of the tree's dirty tiles""" - # Expect these results in this proper order. - expected_list = [ - (0,0,3), - (0,0,1), - (0,0,2), - (0,0,0), - (0,0), - (0,), - - (1,2,0), - (1,2), - - (1,0,3), - (1,0), - - (1,1,3), - (1,1), - (1,), - - (2,1,1), - (2,1,0), - (2,1,3), - (2,1,2), - (2,1), - - (2,0,1), - (2,0,3), - (2,0,0), - (2,0,2), - (2,0), - - (2,3,1), - (2,3,0), - (2,3,3), - (2,3,2), - (2,3), - - (2,2,1), - (2,2,0), - (2,2,3), - (2,2,2), - (2,2), - (2,), - - # We should expect the root tile to need rendering too. - (), - ] + # Expect the results in this proper order. iterator = iter(self.tree.posttraversal()) - from itertools import izip - for expected, actual in izip(expected_list, iterator): + for expected, actual in izip(self.tile_paths_posttraversal, iterator): self.assertEqual(actual, expected) self.assertRaises(StopIteration, next, iterator) def test_posttraverse_roundrobin(self): """Test a round-robin post-traversal of the tree's dirty tiles""" - # Expect these results in this proper order. - expected_lists = [ - [ - (0,0,3), - (0,0,1), - (0,0,2), - (0,0,0), - (0,0), - (0,), - ], - [ - (1,2,0), - (1,2), - - (1,0,3), - (1,0), - - (1,1,3), - (1,1), - (1,), - ], - [ - (2,1,1), - (2,1,0), - (2,1,3), - (2,1,2), - (2,1), - - (2,0,1), - (2,0,3), - (2,0,0), - (2,0,2), - (2,0), - - (2,3,1), - (2,3,0), - (2,3,3), - (2,3,2), - (2,3), - - (2,2,1), - (2,2,0), - (2,2,3), - (2,2,2), - (2,2), - (2,), - ], - ] - expected_list = list(roundrobin(expected_lists)) + [ () ] - + # Expect the results in this proper order. iterator = iter(self.tree.posttraversal(robin=True)) - from itertools import izip - for expected, actual in izip(expected_list, iterator): + for expected, actual in izip(self.tile_paths_posttraversal_robin, iterator): self.assertEqual(actual, expected) self.assertRaises(StopIteration, next, iterator)