LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

added TileSet tests, fixed several bugs. Still needs work.

Browse Source
This commit is contained in:
Andrew Brown
2012-01-08 02:51:06 -05:00
parent fb0a7b161e
commit 3995a06c97
4 changed files with 404 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
overviewer_core/tileset.py
View File

@@ -27,7 +27,7 @@ from collections import namedtuple
from PIL import Image
from .util import iterate_base4, convert_coords, unconvert_coords
from .util import iterate_base4, convert_coords, unconvert_coords, get_tiles_by_chunk
from .optimizeimages import optimize_image
import c_overviewer
@@ -239,7 +239,7 @@ class TileSet(object):
render mode to render. This rendermode must have already been
registered with the C extension module.
rerender_prob
rerenderprob
A floating point number between 0 and 1 indicating the probability
that a tile which is not marked for render by any mtime checks will
be rendered anyways. 0 disables this option.
@@ -250,6 +250,9 @@ class TileSet(object):
self.am = assetmanagerobj
self.textures = texturesobj
# XXX TODO:
self.last_rendertime = 0 # TODO
# Throughout the class, self.outputdir is an absolute path to the
# directory where we output tiles. It is assumed to exist.
self.outputdir = os.path.abspath(outputdir)
@@ -308,11 +311,6 @@ class TileSet(object):
number of phases of work that need to be done.
"""
if self.options['renderchecks'] == 1:
# Layer by layer for this mode
return self.treedepth
else:
# post-traversal does everything in one phase
return 1
def get_phase_length(self, phase):
@@ -322,7 +320,7 @@ class TileSet(object):
# Yeah functional programming!
return {
0: lambda: self.dirtytree.count_all(),
1: lambda: self.dirtytree.count() if phase == 0 else None,
1: lambda: None,
2: lambda: self.dirtytree.count_all(),
}[self.options['renderchecks']]()
@@ -422,6 +420,11 @@ class TileSet(object):
logging.critical("Could not determine existing tile tree depth. Does it exist?")
raise
if curdepth == 1:
# Skip a depth 1 tree. A depth 1 tree pretty much can't happen, so
# when we detect this it usually means the tree is actually empty
return
logging.debug("Current tree depth was detected to be %s. Target tree depth is %s", curdepth, self.treedepth)
if self.treedepth != curdepth:
if self.treedepth > curdepth:
logging.warning("Your map seems to have expanded beyond its previous bounds.")
@@ -533,8 +536,7 @@ class TileSet(object):
rerender_prob = self.options['rerenderprob']
# XXX TODO:
last_rendertime = 0 # TODO
last_rendertime = self.last_rendertime
max_chunk_mtime = 0
@@ -554,31 +556,7 @@ class TileSet(object):
# Convert to diagonal coordinates
chunkcol, chunkrow = convert_coords(chunkx, chunkz)
# find tile coordinates. Remember tiles are identified by the
# address of the chunk in their upper left corner.
tilecol = chunkcol - chunkcol % 2
tilerow = chunkrow - chunkrow % 4
# If this chunk is in an /even/ column, then it spans two tiles.
if chunkcol % 2 == 0:
colrange = (tilecol-2, tilecol)
else:
colrange = (tilecol,)
# If this chunk is in a row divisible by 4, then it touches the
# tile above it as well. Also touches the next 4 tiles down (16
# rows)
if chunkrow % 4 == 0:
rowrange = xrange(tilerow-4, tilerow+16+1, 4)
else:
rowrange = xrange(tilerow, tilerow+16+1, 4)
# Loop over all tiles that this chunk potentially touches.
# The tile at tilecol,tilerow obviously contains the chunk, but so
# do the next 4 tiles down because chunks are very tall, and maybe
# the next column over too.
for c in colrange:
for r in rowrange:
for c, r in get_tiles_by_chunk(chunkcol, chunkrow):
# Make sure the tile is in the boundary we're rendering.
# This can happen when rendering at lower treedepth than
@@ -929,7 +907,7 @@ class TileSet(object):
# Now that we're done with our children and descendents, see if
# this tile needs rendering
if needs_rendering:
if render_me:
# yes. yes we do. This is set when one of our children needs
# rendering
yield path, None, True

26
overviewer_core/util.py
View File

@@ -124,6 +124,32 @@ def unconvert_coords(col, row):
# col - row = chunkx + chunkx => (col - row)/2 = chunkx
return ((col - row) / 2, (col + row) / 2)
def get_tiles_by_chunk(chunkcol, chunkrow):
"""For the given chunk, returns an iterator over Render Tiles that this
chunk touches. Iterates over (tilecol, tilerow)
"""
# find tile coordinates. Remember tiles are identified by the
# address of the chunk in their upper left corner.
tilecol = chunkcol - chunkcol % 2
tilerow = chunkrow - chunkrow % 4
# If this chunk is in an /even/ column, then it spans two tiles.
if chunkcol % 2 == 0:
colrange = (tilecol-2, tilecol)
else:
colrange = (tilecol,)
# If this chunk is in a row divisible by 4, then it touches the
# tile above it as well. Also touches the next 4 tiles down (16
# rows)
if chunkrow % 4 == 0:
rowrange = xrange(tilerow-4, tilerow+16+1, 4)
else:
rowrange = xrange(tilerow, tilerow+16+1, 4)
return product(colrange, rowrange)
# Logging related classes are below
# Some cool code for colored logging:

1
test/test_all.py
View File

@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.join(os.getcwd(), os.pardir))
from test_tileobj import TileTest
from test_rendertileset import RendertileSetTest
from test_settings import SettingsTest
from test_tileset import TilesetTest
if __name__ == "__main__":
unittest.main()

327
test/test_tileset.py Normal file
View File

@@ -0,0 +1,327 @@
import unittest
import tempfile
import shutil
from collections import defaultdict
import os
import os.path
import random
import pprint
from overviewer_core import tileset, util
# DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
if 1:
import logging
root = logging.getLogger()
class NullHandler(logging.Handler):
def handle(self, record):
pass
def emit(self, record):
pass
def createLock(self):
self.lock = None
root.addHandler(NullHandler())
else:
import overviewer
import logging
overviewer.configure_logger(logging.DEBUG, True)
# Supporing data
# chunks list: chunkx, chunkz mapping to chunkmtime
# In comments: col, row
chunks = {
(0, 0): 5, # 0, 0
(0, 1): 5, # 1, 1
(0, 2): 5, # 2, 2
(0, 3): 5, # 3, 3
(0, 4): 5, # 4, 4
(1, 0): 5, # 1, -1
(1, 1): 5, # 2, 0
(1, 2): 5, # 3, 1
(1, 3): 5, # 4, 2
(1, 4): 5, # 5, 3
(2, 0): 5, # 2, -2
(2, 1): 5, # 3, -1
(2, 2): 5, # 4, 0
(2, 3): 5, # 5, 1
(2, 4): 5, # 6, 2
(3, 0): 5, # 3, -3
(3, 1): 5, # 4, -2
(3, 2): 5, # 5, -1
(3, 3): 5, # 6, 0
(3, 4): 5, # 7, 1
(4, 0): 5, # 4, -4
(4, 1): 5, # 5, -3
(4, 2): 5, # 6, -2
(4, 3): 5, # 7, -1
(4, 4): 5, # 8, 0
}
# The resulting tile tree, in the correct post-traversal order.
correct_tiles = [
(0, 3, 3),
(0, 3),
(0,),
(1, 2, 1),
(1, 2, 2),
(1, 2, 3),
(1, 2),
(1, 3, 0),
(1, 3, 2),
(1, 3, 3),
(1, 3),
(1,),
(2, 1, 1),
(2, 1, 3),
(2, 1),
(2, 3, 1),
(2, 3, 3),
(2, 3),
(2,),
(3, 0, 0),
(3, 0, 1),
(3, 0, 2),
(3, 0, 3),
(3, 0),
(3, 1, 0),
(3, 1, 1),
(3, 1, 2),
(3, 1, 3),
(3, 1),
(3, 2, 0),
(3, 2, 1),
(3, 2, 2),
(3, 2, 3),
(3, 2),
(3, 3, 0),
(3, 3, 1),
(3, 3, 2),
(3, 3, 3),
(3, 3),
(3,),
(),
]
# Supporting resources
######################
class FakeRegionset(object):
def __init__(self, chunks):
self.chunks = dict(chunks)
def get_chunk(self, x,z):
return NotImplementedError()
def iterate_chunks(self):
for (x,z),mtime in self.chunks.iteritems():
yield x,z,mtime
def get_chunk_mtime(self, x, z):
try:
return self.chunks[x,z]
except KeyError:
return None
def get_tile_set(chunks):
"""Given the dictionary mapping chunk coordinates their mtimes, returns a
dict mapping the tiles that are to be rendered to their mtimes that are
expected. Useful for passing into the create_fakedir() function. Used by
the compare_iterate_to_expected() method.
"""
tile_set = defaultdict(int)
for (chunkx, chunkz), chunkmtime in chunks.iteritems():
col, row = util.convert_coords(chunkx, chunkz)
for tilec, tiler in util.get_tiles_by_chunk(col, row):
tile = tileset.RenderTile.compute_path(tilec, tiler, 3)
tile_set[tile.path] = max(tile_set[tile.path], chunkmtime)
# At this point, tile_set holds all the render-tiles
for tile, tile_mtime in tile_set.copy().iteritems():
# All render-tiles are length 3. Hard-code its upper tiles
tile_set[tile[:2]] = max(tile_set[tile[:2]], tile_mtime)
tile_set[tile[:1]] = max(tile_set[tile[:1]], tile_mtime)
tile_set[tile[:0]] = max(tile_set[tile[:0]], tile_mtime)
return dict(tile_set)
def create_fakedir(outputdir, tiles):
"""Takes a base output directory and a tiles dict mapping tile paths to
tile mtimes as returned by get_tile_set(), creates the "tiles" (empty
files) and sets mtimes appropriately
"""
for tilepath, tilemtime in tiles.iteritems():
dirpath = os.path.join(outputdir, *(str(x) for x in tilepath[:-1]))
if len(tilepath) == 0:
imgname = "base.png"
else:
imgname = str(tilepath[-1]) + ".png"
if not os.path.exists(dirpath):
os.makedirs(dirpath)
finalpath = os.path.join(dirpath, imgname)
open(finalpath, 'w').close()
os.utime(finalpath, (tilemtime, tilemtime))
# The test cases
################
class TilesetTest(unittest.TestCase):
def setUp(self):
# Set up the region set
self.rs = FakeRegionset(chunks)
self.tempdirs = []
# Consistent random numbers
self.r = random.Random(1)
def tearDown(self):
for d in self.tempdirs:
shutil.rmtree(d)
def get_outputdir(self):
d = tempfile.mkdtemp(prefix="OVTEST")
self.tempdirs.append(d)
return d
def get_tileset(self, options, outputdir, preprocess=None):
"""Returns a newly created TileSet object and return it.
A set of default options are provided. Any options passed in will
override the defaults. The output directory is passed in and it is
recommended to use a directory from self.get_outputdir()
preprocess, if given, is a function that takes the tileset object. It
is called before do_preprocessing()
"""
defoptions = {
'bgcolor': '#000000',
'imgformat': 'png',
'optimizeimg': 0,
'rendermode': 'normal',
'rerenderprob': 0
}
defoptions.update(options)
ts = tileset.TileSet(self.rs, None, None, defoptions, outputdir)
if preprocess:
preprocess(ts)
ts.do_preprocessing()
return ts
def compare_iterate_to_expected(self, ts, chunks):
"""Runs iterate_work_items on the tileset object and compares its
output to what we'd expect if it was run with the given chunks
chunks is a dictionary whose keys are chunkx,chunkz. This method
calculates from that set of chunks the tiles they touch and their
parent tiles, and compares that to the output of ts.iterate_work_items().
"""
paths = set(x[0] for x in ts.iterate_work_items(0))
pprint.pprint(paths)
# Get what tiles we expect to be returned
expected = get_tile_set(chunks)
pprint.pprint(set(expected.iterkeys()))
# Check that all paths returned are in the expected list
for tilepath in paths:
self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected))
# Now check that all expected tiles were indeed returned
for tilepath in expected.iterkeys():
self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths))
def test_get_phase_length(self):
ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
self.assertEqual(ts.get_num_phases(), 1)
self.assertEqual(ts.get_phase_length(0), 41)
def test_forcerender_iterate(self):
"""Tests that a rendercheck mode 2 iteration returns every render-tile
and upper-tile
"""
ts = self.get_tileset({'renderchecks': 2}, self.get_outputdir())
self.compare_iterate_to_expected(ts, self.rs.chunks)
def test_update_chunk(self):
"""Tests that an update in one chunk properly updates just the
necessary tiles for rendercheck mode 0, normal operation. This
shouldn't touch the filesystem at all.
"""
# Update one chunk with a newer mtime
updated_chunks = {
(0,0): 6
}
self.rs.chunks.update(updated_chunks)
# Create the tileset and set its last render time to 5
ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(),
lambda ts: setattr(ts, 'last_rendertime', 5))
# Now see if the return is what we expect
self.compare_iterate_to_expected(ts, updated_chunks)
def test_update_chunk2(self):
"""Same as above but with a different set of chunks
"""
# Pick 3 random chunks to update
chunks = self.rs.chunks.keys()
self.r.shuffle(chunks)
updated_chunks = {}
for key in chunks[:3]:
updated_chunks[key] = 6
self.rs.chunks.update(updated_chunks)
ts = self.get_tileset({'renderchecks': 0}, self.get_outputdir(),
lambda ts: setattr(ts, 'last_rendertime', 5))
self.compare_iterate_to_expected(ts, updated_chunks)
def test_rendercheckmode_1(self):
"""Tests that an interrupted render will correctly pick up tiles that
need rendering
"""
# For this we actually need to set the tile mtimes on disk and have the
# TileSet object figure out from that what it needs to render.
# Strategy: set some tiles on disk to mtime 3, and TileSet needs to
# find them and update them to mtime 5 as reported by the RegionSet
# object.
outdated_tiles = [
(0,3,3),
(1,2,1),
(2,1),
(3,)
]
# These are the tiles that we also expect it to return, even though
# they were not outdated, since they depend on the outdated tiles
additional = [
(0,3),
(0,),
(1,2),
(1,),
(2,),
(),
]
outputdir = self.get_outputdir()
# Fill the output dir with tiles
all_tiles = get_tile_set(self.rs.chunks)
all_tiles.update(dict((x,3) for x in outdated_tiles))
create_fakedir(outputdir, all_tiles)
# Create the tileset and do the scan
ts = self.get_tileset({'renderchecks': 1}, outputdir)
# Now see if it's right
paths = set(x[0] for x in ts.iterate_work_items(0))
expected = set(outdated_tiles) | set(additional)
pprint.pprint(paths)
pprint.pprint(expected)
for tilepath in paths:
self.assertTrue(tilepath in expected, "%s was not expected to be returned. Expected %s" % (tilepath, expected))
for tilepath in expected:
self.assertTrue(tilepath in paths, "%s was expected to be returned but wasn't: %s" % (tilepath, paths))