LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Merge remote-tracking branch 'ion1/master'

Browse Source
This commit is contained in:
Aaron Griffith
2013-12-28 01:12:46 -05:00
parent 28a08631ee 6692d10af0
commit da6d3614bc
6 changed files with 208 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.travis.yml
View File

@@ -4,7 +4,7 @@ python:
- "2.7"
# - "3.2"
env:
- MC_VERSION=1.7.2
- MC_VERSION=1.7.4
before_install:
- wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/Imaging.h
- wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/ImPlatform.h
@@ -16,9 +16,16 @@ before_script:
- git clone git://github.com/overviewer/Minecraft-Overviewer-Addons.git ~/mcoa/
- wget -N https://s3.amazonaws.com/Minecraft.Download/versions/${MC_VERSION}/${MC_VERSION}.jar -P ~/.minecraft/versions/${MC_VERSION}/
script:
- PYTHONPATH=. python test/test_all.py
- python overviewer.py ~/mcoa/exmaple ~/test-output --rendermodes=smooth-lighting -p1
notifications:
email: false
irc:
channels:
- "irc.freenode.org#overviewer"
skip_join: true
template:
- "\x0313Minecraft-Overviewer\x03/\x0306%{branch}\x03 \x0314%{commit}\x03 %{build_url} %{message}"
# matrix:
# allow_failures:
# - python: "3.2"

266
overviewer_core/tileset.py
View File

@@ -25,7 +25,7 @@ import time
import errno
import stat
from collections import namedtuple
from itertools import product, izip
from itertools import product, izip, chain
from PIL import Image
@@ -377,7 +377,7 @@ class TileSet(object):
# This warning goes here so it's only shown once
if self.treedepth >= 15:
logging.warning("Just letting you know, your map requries %s zoom levels. This is REALLY big!",
logging.warning("Just letting you know, your map requires %s zoom levels. This is REALLY big!",
self.treedepth)
# Do any tile re-arranging if necessary. Skip if there was no config
@@ -446,7 +446,7 @@ class TileSet(object):
# render. Iterate over the tiles in using the posttraversal() method.
# Yield each item. Easy.
if self.options['renderchecks'] in (0,2):
for tilepath in self.dirtytree.posttraversal():
for tilepath in self.dirtytree.posttraversal(robin=True):
dependencies = []
# These tiles may or may not exist, but the dispatcher won't
# care according to the worker interface protocol It will only
@@ -1268,21 +1268,19 @@ class RendertileSet(object):
It is typically used to hold tiles that need rendering. This implementation
collapses subtrees that are completely in or out of the set to save memory.
Each instance of this class is a node in the tree, and therefore each
instance is the root of a subtree.
An instance of this class holds a full tree.
Each node knows its "level", which corresponds to the zoom level where 0 is
the inner-most (most zoomed in) tiles.
The instance knows its "level", which corresponds to the zoom level where 1
is the inner-most (most zoomed in) tiles.
Instances hold the state of their children (in or out of the set). Leaf
nodes are images and do not physically exist in the tree as objects, but
are represented as booleans held by the objects at the second-to-last
level; level 1 nodes keep track of leaf image state. Level 2 nodes keep
track of level 1 state, and so fourth.
track of level 1 state, and so forth.
"""
__slots__ = ("depth", "children")
__slots__ = ("depth", "children", "num_tiles", "num_tiles_all")
def __init__(self, depth):
"""Initialize a new tree with the specified depth. This actually
initializes a node, which is the root of a subtree, with `depth` levels
@@ -1303,52 +1301,15 @@ class RendertileSet(object):
# All children down this subtree are not in the set
# True
# All children down this subtree are in the set
# A RendertileSet instance
# the instance defines which children down that subtree are in the
# set.
# An array of the same format
# The array defines which children down that subtree are in the set
# A node with depth=1 cannot have a RendertileSet instance in its
# children since its children are leaves, representing images, not more
# tree
self.children = [False] * 4
def posttraversal(self):
"""Returns an iterator over tile paths for every tile in the
set, including the explictly marked render-tiles, as well as the
implicitly marked ancestors of those render-tiles. Returns in
post-traversal order, so that tiles with dependencies will always be
yielded after their dependencies.
"""
return (tuple(reversed(rpath)) for rpath in self._posttraversal_helper())
def _posttraversal_helper(self):
"""Each node returns an iterator over lists of reversed paths"""
if self.depth == 1:
# Base case
if self.children[0]: yield [0]
if self.children[1]: yield [1]
if self.children[2]: yield [2]
if self.children[3]: yield [3]
else:
for childnum, child in enumerate(self.children):
if child == True:
for path in post_traversal_complete_subtree_recursion_helper(self.depth-1):
path.append(childnum)
yield path
elif child == False:
pass # do nothing
else:
# Recurse
for path in child._posttraversal_helper():
path.append(childnum)
yield path
# Now do this node itself
if bool(self):
yield []
self.num_tiles = 0
self.num_tiles_all = 0
def add(self, path):
"""Marks the requested leaf node as in this set
@@ -1359,72 +1320,66 @@ class RendertileSet(object):
"""
path = list(path)
assert len(path) == self.depth
path.reverse()
self._set_add_helper(path)
def _set_add_helper(self, path):
if self.num_tiles == 0:
# The first child is being added. A root composite tile will be
# rendered.
self.num_tiles_all += 1
self._add_helper(self.children, list(reversed(path)))
def _add_helper(self, children, path):
"""Recursive helper for add()
Expects path to be a list in reversed order
If *all* the nodes below this one are in the set, this function returns
true. Otherwise, returns None.
"""
if self.depth == 1:
# Base case
self.children[path[0]] = True
# Check to see if all children are in the set
if all(self.children):
return True
else:
# Recursive case
childnum = path.pop()
child = self.children[childnum]
if child == False:
# Create a new node and recurse.
# (The use of __class__ is so possible subclasses of this class
# work as expected)
child = self.__class__(self.depth-1)
child._set_add_helper(path)
self.children[childnum] = child
elif child == True:
# Every child is already in the set and the subtree is already
# collapsed. Nothing to do.
if path:
# We are not at the leaf, recurse.
if children[childnum] == True:
# The child is already in the tree.
return
else:
# subtree is mixed. Recurse to the already existing child node
ret = child._set_add_helper(path)
if ret:
# Child says every descendent is in the set, so we can
# purge the subtree and mark it as such. The subtree will
# be garbage collected when this method exits.
self.children[childnum] = True
elif children[childnum] == False:
# Expand all-false.
children[childnum] = [False]*4
# Since we've marked an entire sub-tree as in the set, we
# may be able to signal to our parent to do the same
if all(x is True for x in self.children):
return True
# This also means an additional composite tile.
self.num_tiles_all += 1
self._add_helper(children[childnum], path)
if children[childnum] == [True]*4:
# Collapse all-true.
children[childnum] = True
else:
# We are at the leaf.
if not children[childnum]:
self.num_tiles += 1
self.num_tiles_all += 1
children[childnum] = True
def __iter__(self):
return self.iterate()
def iterate(self, level=None):
def iterate(self, level=None, robin=False, offset=(0,0)):
"""Returns an iterator over every tile in this set. Each item yielded
is a sequence of integers representing the quadtree path to the tiles
in the set. Yielded sequences are of length self.depth.
If level is None, iterates over tiles of the highest level, i.e.
worldtiles. If level is a value between 0 and the depth of this tree,
this method iterates over tiles at that level. Zoom level 0 is zoomed
worldtiles. If level is a value between 1 and the depth of this tree,
this method iterates over tiles at that level. Zoom level 1 is zoomed
all the way out, zoom level `depth` is all the way in.
In other words, specifying level causes the tree to be iterated as if
it was only that depth.
If the `robin` parameter is True, recurses to the four top-level
subtrees simultaneously in a round-robin manner.
"""
if level is None:
todepth = 1
@@ -1433,30 +1388,47 @@ class RendertileSet(object):
raise ValueError("Level parameter must be between 1 and %s" % self.depth)
todepth = self.depth - level + 1
return (tuple(reversed(rpath)) for rpath in self._iterate_helper(todepth))
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, robin=robin, offset=offset))
def _iterate_helper(self, todepth):
if self.depth == todepth:
def posttraversal(self, robin=False, offset=(0,0)):
"""Returns an iterator over tile paths for every tile in the
set, including the explictly marked render-tiles, as well as the
implicitly marked ancestors of those render-tiles. Returns in
post-traversal order, so that tiles with dependencies will always be
yielded after their dependencies.
If the `robin` parameter is True, recurses to the four top-level
subtrees simultaneously in a round-robin manner.
"""
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, robin=robin, offset=offset))
def _iterate_helper(self, path, children, depth, onlydepth=None, robin=False, offset=(0,0)):
"""Returns an iterator over tile paths for every tile in the set."""
# A variant of children with a collapsed False/True expanded to a list.
children_list = [children] * 4 if isinstance(children, bool) else children
targetdepth = 1 if onlydepth is None else onlydepth
if depth == targetdepth:
# Base case
if self.children[0]: yield [0]
if self.children[1]: yield [1]
if self.children[2]: yield [2]
if self.children[3]: yield [3]
for (childnum, child), _ in distance_sort(enumerate(children_list), offset):
if child:
yield path + [childnum]
else:
# Higher levels:
for c, child in enumerate(self.children):
if child == True:
# All render-tiles are in the set down this subtree,
# iterate over every leaf using iterate_base4
for x in iterate_base4(self.depth-todepth):
x = list(x)
x.append(c)
yield x
elif child != False:
# Mixed in/out of the set down this subtree, recurse
for path in child._iterate_helper(todepth):
path.append(c)
gens = []
for (childnum_, child), childoffset_ in distance_sort(enumerate(children_list), offset):
if child:
def go(childnum, childoffset):
for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset):
yield p
gens.append(go(childnum_, childoffset_))
for p in roundrobin(gens) if robin else chain(*gens):
yield p
if onlydepth is None and children:
yield path
def query_path(self, path):
@@ -1471,74 +1443,46 @@ class RendertileSet(object):
# collapsed, then just return the stored boolean. Otherwise, if we find
# the specific tree node requested, return its state using the
# __nonzero__ call.
treenode = self
treenode = self.children
for pathelement in path:
treenode = treenode.children[pathelement]
if not isinstance(treenode, RendertileSet):
treenode = treenode[pathelement]
if isinstance(treenode, bool):
return treenode
# If the method has not returned at this point, treenode is the
# requested node, but it is an inner node with possibly mixed state
# subtrees. If any of the children are True return True. This call
# relies on the __nonzero__ method
return bool(treenode)
# requested node, but it is an inner node. That will only happen if one
# or more of the children down the tree are True.
return True
def __nonzero__(self):
"""Returns the boolean context of this particular node. If any
descendent of this node is True return True. Otherwise, False.
"""
# Any chilren that are True or are a RendertileSet that evaluate to
# True
# IDEA: look at all children for True before recursing
# Better idea: every node except the root /must/ have a descendent in
# the set or it wouldn't exist. This assumption is only valid as long
# as there is no method to remove a tile from the set. So this should
# check to see if any children are not False.
# Any children that are True or are a list evaluate to True.
return any(self.children)
def count(self):
"""Returns the total number of render-tiles in this set.
"""
# TODO: Make this more efficient (although for even the largest trees,
# this takes only seconds)
c = 0
for _ in self.iterate():
c += 1
return c
return self.num_tiles
def count_all(self):
"""Returns the total number of render-tiles plus implicitly marked
upper-tiles in this set
"""
# TODO: Optimize this too with its own recursive method that avoids
# some of the overheads of posttraversal()
c = 0
for _ in self.posttraversal():
c += 1
return c
return self.num_tiles_all
def post_traversal_complete_subtree_recursion_helper(depth):
"""Fakes the recursive calls for RendertileSet.posttraversal() for the case
that a subtree is collapsed, so that items are still yielded in the correct
order.
def distance_sort(children, (off_x, off_y)):
order = []
for child, (dx, dy) in izip(children, [(-1,-1), (1,-1), (-1,1), (1,1)]):
x = off_x*2 + dx
y = off_y*2 + dy
order.append((child, (x,y)))
"""
if depth == 1:
# Base case
yield [0]
yield [1]
yield [2]
yield [3]
else:
for childnum in xrange(4):
for item in post_traversal_complete_subtree_recursion_helper(depth-1):
item.append(childnum)
yield item
yield []
return sorted(order, key=lambda (_, (x,y)): x*x + y*y)
class RenderTile(object):
"""A simple container class that represents a single render-tile.

3
test/test_all.py
View File

@@ -2,7 +2,7 @@
import unittest
# For convenience
import sys,os
import sys,os,logging
sys.path.insert(0, os.getcwd())
sys.path.insert(0, os.path.join(os.getcwd(), os.pardir))
@@ -15,7 +15,6 @@ from test_cache import TestLRU
# DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
if 0:
import logging
root = logging.getLogger()
class NullHandler(logging.Handler):
def handle(self, record):

118
test/test_rendertileset.py
View File

@@ -1,6 +1,9 @@
import unittest
from itertools import chain, izip
from overviewer_core.tileset import iterate_base4, RendertileSet
from overviewer_core.util import roundrobin
class RendertileSetTest(unittest.TestCase):
# If you change this definition, you must also change the hard-coded
@@ -34,6 +37,61 @@ class RendertileSetTest(unittest.TestCase):
(2,3,3),
# Nothing under quadrant 3
])
# The paths as yielded by posttraversal, in an expanding-from-the-center
# order.
tile_paths_posttraversal_lists = [
[
(0,0,3),
(0,0,1),
(0,0,2),
(0,0,0),
(0,0),
(0,),
],
[
(1,2,0),
(1,2),
(1,0,3),
(1,0),
(1,1,3),
(1,1),
(1,),
],
[
(2,1,1),
(2,1,0),
(2,1,3),
(2,1,2),
(2,1),
(2,0,1),
(2,0,3),
(2,0,0),
(2,0,2),
(2,0),
(2,3,1),
(2,3,0),
(2,3,3),
(2,3,2),
(2,3),
(2,2,1),
(2,2,0),
(2,2,3),
(2,2,2),
(2,2),
(2,),
],
]
# Non-round robin post-traversal: finish the first top-level quadrant
# before moving to the second etc.
tile_paths_posttraversal = list(chain(*tile_paths_posttraversal_lists)) + [()]
# Round-robin post-traversal: start rendering to all directions from the
# center.
tile_paths_posttraversal_robin = list(roundrobin(tile_paths_posttraversal_lists)) + [()]
def setUp(self):
self.tree = RendertileSet(3)
@@ -142,56 +200,18 @@ class RendertileSetTest(unittest.TestCase):
def test_posttraverse(self):
"""Test a post-traversal of the tree's dirty tiles"""
# Expect these results in this proper order.
expected_list = [
(0,0,0),
(0,0,1),
(0,0,2),
(0,0,3),
(0,0),
(0,),
(1,0,3),
(1,0),
(1,1,3),
(1,1),
(1,2,0),
(1,2),
(1,),
(2,0,0),
(2,0,1),
(2,0,2),
(2,0,3),
(2,0),
(2,1,0),
(2,1,1),
(2,1,2),
(2,1,3),
(2,1),
(2,2,0),
(2,2,1),
(2,2,2),
(2,2,3),
(2,2),
(2,3,0),
(2,3,1),
(2,3,2),
(2,3,3),
(2,3),
(2,),
# We should expect the root tile to need rendering too.
(),
]
# Expect the results in this proper order.
iterator = iter(self.tree.posttraversal())
from itertools import izip
for expected, actual in izip(expected_list, iterator):
for expected, actual in izip(self.tile_paths_posttraversal, iterator):
self.assertEqual(actual, expected)
self.assertRaises(StopIteration, next, iterator)
def test_posttraverse_roundrobin(self):
"""Test a round-robin post-traversal of the tree's dirty tiles"""
# Expect the results in this proper order.
iterator = iter(self.tree.posttraversal(robin=True))
for expected, actual in izip(self.tile_paths_posttraversal_robin, iterator):
self.assertEqual(actual, expected)
self.assertRaises(StopIteration, next, iterator)

16
test/test_settings.py
View File

@@ -6,6 +6,8 @@ from overviewer_core.settingsValidators import ValidationException
from overviewer_core import world
from overviewer_core import rendermodes
from overviewer_core.util import OrderedDict
class SettingsTest(unittest.TestCase):
def setUp(self):
@@ -13,7 +15,7 @@ class SettingsTest(unittest.TestCase):
def test_missing(self):
"Validates that a non-existant settings.py causes an exception"
self.assertRaises(ValueError, self.s.parse, "doesnotexist.py")
self.assertRaises(configParser.MissingConfigException, self.s.parse, "doesnotexist.py")
def test_existing_file(self):
self.s.parse("test/data/settings/settings_test_1.py")
@@ -45,21 +47,21 @@ class SettingsTest(unittest.TestCase):
self.s.set_config_item("worlds", {
'test': "test/data/settings/test_world",
})
self.s.set_config_item("renders", {
"myworld": {
self.s.set_config_item("renders", OrderedDict([
("myworld", {
"title": "myworld title",
"world": "test",
"rendermode": rendermodes.normal,
"northdirection": "upper-left",
},
}),
"otherworld": {
("otherworld", {
"title": "otherworld title",
"world": "test",
"rendermode": rendermodes.normal,
"bgcolor": "#ffffff"
},
})
}),
]))
self.s.set_config_item("outputdir", "/tmp/fictional/outputdir")
self.assertEquals(fromfile.get_validated_config(), self.s.get_validated_config())

70
test/test_tileset.py
View File

@@ -39,51 +39,6 @@ chunks = {
(4, 4): 5, # 8, 0
}
# The resulting tile tree, in the correct post-traversal order.
correct_tiles = [
(0, 3, 3),
(0, 3),
(0,),
(1, 2, 1),
(1, 2, 2),
(1, 2, 3),
(1, 2),
(1, 3, 0),
(1, 3, 2),
(1, 3, 3),
(1, 3),
(1,),
(2, 1, 1),
(2, 1, 3),
(2, 1),
(2, 3, 1),
(2, 3, 3),
(2, 3),
(2,),
(3, 0, 0),
(3, 0, 1),
(3, 0, 2),
(3, 0, 3),
(3, 0),
(3, 1, 0),
(3, 1, 1),
(3, 1, 2),
(3, 1, 3),
(3, 1),
(3, 2, 0),
(3, 2, 1),
(3, 2, 2),
(3, 2, 3),
(3, 2),
(3, 3, 0),
(3, 3, 1),
(3, 3, 2),
(3, 3, 3),
(3, 3),
(3,),
(),
]
# Supporting resources
######################
@@ -123,15 +78,14 @@ def get_tile_set(chunks):
col, row = tileset.convert_coords(chunkx, chunkz)
for tilec, tiler in tileset.get_tiles_by_chunk(col, row):
tile = tileset.RenderTile.compute_path(tilec, tiler, 3)
tile = tileset.RenderTile.compute_path(tilec, tiler, 5)
tile_set[tile.path] = max(tile_set[tile.path], chunkmtime)
# At this point, tile_set holds all the render-tiles
for tile, tile_mtime in tile_set.copy().iteritems():
# All render-tiles are length 3. Hard-code its upper tiles
tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime)
tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime)
tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime)
# All render-tiles are length 5. Hard-code its upper tiles
for i in reversed(xrange(5)):
tile_set[tile[:i]] = max(tile_set[tile[:i]], tile_mtime)
return dict(tile_set)
def create_fakedir(outputdir, tiles):
@@ -184,6 +138,7 @@ class TilesetTest(unittest.TestCase):
is called before do_preprocessing()
"""
defoptions = {
'name': 'world name',
'bgcolor': '#000000',
'imgformat': 'png',
'optimizeimg': 0,
@@ -191,7 +146,7 @@ class TilesetTest(unittest.TestCase):
'rerenderprob': 0
}
defoptions.update(options)
ts = tileset.TileSet(self.rs, FakeAssetmanager(0), None, defoptions, outputdir)
ts = tileset.TileSet(None, self.rs, FakeAssetmanager(0), None, defoptions, outputdir)
if preprocess:
preprocess(ts)
ts.do_preprocessing()
@@ -222,7 +177,7 @@ class TilesetTest(unittest.TestCase):
def test_get_phase_length(self):
ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
self.assertEqual(ts.get_num_phases(), 1)
self.assertEqual(ts.get_phase_length(0), 41)
self.assertEqual(ts.get_phase_length(0), len(get_tile_set(chunks)))
def test_forcerender_iterate(self):
"""Tests that a rendercheck mode 2 iteration returns every render-tile
@@ -278,18 +233,23 @@ class TilesetTest(unittest.TestCase):
# object.
# Chosen at random:
outdated_tiles = [
(0,3,3),
(1,2,1),
(2,1),
(0,3,3,3,3),
(1,2,2,2,1),
(2,1,1),
(3,)
]
# These are the tiles that we also expect it to return, even though
# they were not outdated, since they depend on the outdated tiles
additional = [
(0,3,3,3),
(0,3,3),
(0,3),
(0,),
(1,2,2,2),
(1,2,2),
(1,2),
(1,),
(2,1),
(2,),
(),
]