LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Merge branch 'anvil'

Browse Source
This commit is contained in:
Andrew Brown
2012-03-04 00:02:17 -05:00
parent 4a9c4fd150 b852e8b1e3
commit 3bb14f01e4
6 changed files with 216 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
overviewer.py
View File

@@ -38,6 +38,7 @@ from overviewer_core import util
from overviewer_core import textures
from overviewer_core import optimizeimages, world
from overviewer_core import configParser, tileset, assetmanager, dispatcher
from overviewer_core import cache
helptext = """
%prog [--rendermodes=...] [options] <World> <Output Dir>
@@ -346,16 +347,21 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
# same for textures
texcache = {}
# Set up the cache objects to use
caches = []
caches.append(cache.LRUCache())
# TODO: optionally more caching layers here
renders = config['renders']
for render_name, render in renders.iteritems():
logging.debug("Found the following render thing: %r", render)
# find or create the world object
if (render['world'] not in worldcache):
try:
w = worldcache[render['world']]
except KeyError:
w = world.World(render['world'])
worldcache[render['world']] = w
else:
w = worldcache[render['world']]
# find or create the textures object
texopts = util.dict_subset(render, ["texturepath", "bgcolor", "northdirection"])
@@ -372,6 +378,15 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
logging.error("Sorry, you requested dimension '%s' for %s, but I couldn't find it", render['dimension'], render_name)
return 1
#################
# Apply any regionset transformations here
# Insert a layer of caching above the real regionset. Any world
# tranformations will pull from this cache, but their results will not
# be cached by this layer. This uses a common pool of caches; each
# regionset cache pulls from the same underlying cache object.
rset = world.CachedRegionSet(rset, caches)
# If a crop is requested, wrap the regionset here
if "crop" in render:
rset = world.CroppedRegionSet(rset, *render['crop'])
@@ -382,6 +397,9 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
rset = world.RotatedRegionSet(rset, render['northdirection'])
logging.debug("Using RegionSet %r", rset)
###############################
# Do the final prep and create the TileSet object
# create our TileSet from this RegionSet
tileset_dir = os.path.abspath(os.path.join(destdir, render_name))
if not os.path.exists(tileset_dir):
@@ -420,6 +438,9 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
dispatch.close()
assetMrg.finalize(tilesets)
logging.debug("Final cache stats:")
for c in caches:
logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses)
return 0
def list_worlds():

107
overviewer_core/cache.py
View File

@@ -15,48 +15,97 @@
"""This module has supporting functions for the caching logic used in world.py.
Each cache class should implement the standard container type interface
(__getitem__ and __setitem__, as well as provide a "hits" and "misses"
attribute.
"""
import functools
import logging
def lru_cache(max_size=100):
"""A quick-and-dirty LRU implementation.
Uses a dict to store mappings, and a list to store orderings.
class LRUCache(object):
"""A simple in-memory LRU cache.
Only supports positional arguments
An ordered dict type would simplify this implementation a bit, but we want
Python 2.6 compatibility and the standard library ordereddict was added in
2.7. It's probably okay because this implementation can be tuned for
exactly what we need and nothing more.
This implementation keeps a linked-list of cache keys and values, ordered
in least-recently-used order. A dictionary maps keys to linked-list nodes.
On cache hit, the link is moved to the end of the list. On cache miss, the
first item of the list is evicted. All operations have constant time
complexity (dict lookups are worst case O(n) time)
"""
def lru_decorator(fun):
class _LinkNode(object):
__slots__ = ['left', 'right', 'key', 'value']
def __init__(self,l=None,r=None,k=None,v=None):
self.left = l
self.right = r
self.key = k
self.value = v
cache = {}
lru_ordering = []
def __init__(self, size=100):
self.cache = {}
@functools.wraps(fun)
def new_fun(*args):
self.listhead = LRUCache._LinkNode()
self.listtail = LRUCache._LinkNode()
# Two sentinel nodes at the ends of the linked list simplify boundary
# conditions in the code below.
self.listhead.right = self.listtail
self.listtail.left = self.listhead
self.hits = 0
self.misses = 0
self.size = size
# Initialize an empty cache of the same size for worker processes
def __getstate__(self):
return self.size
def __setstate__(self, size):
self.__init__(size)
def __getitem__(self, key):
try:
result = cache[args]
link = self.cache[key]
except KeyError:
# cache miss =(
new_fun.miss += 1
result = fun(*args)
self.misses += 1
raise
# Insert into cache
cache[args] = result
lru_ordering.append(args)
# Disconnect the link from where it is
link.left.right = link.right
link.right.left = link.left
if len(cache) > max_size:
# Evict an item
del cache[ lru_ordering.pop(0) ]
# Insert the link at the end of the list
tail = self.listtail
link.left = tail.left
link.right = tail
tail.left.right = link
tail.left = link
else:
# Move the result item to the end of the list
new_fun.hits += 1
position = lru_ordering.index(args)
lru_ordering.append(lru_ordering.pop(position))
self.hits += 1
return link.value
return result
def __setitem__(self, key, value):
cache = self.cache
if key in cache:
raise KeyError("That key already exists in the cache!")
if len(cache) >= self.size:
# Evict a node
link = self.listhead.right
del cache[link.key]
link.left.right = link.right
link.right.left = link.left
del link
new_fun.hits = 0
new_fun.miss = 0
return new_fun
# The node doesn't exist already, and we have room for it. Let's do this.
tail = self.listtail
link = LRUCache._LinkNode(tail.left, tail,key,value)
tail.left.right = link
tail.left = link
cache[key] = link
return lru_decorator

1
overviewer_core/settingsValidators.py
View File

@@ -55,6 +55,7 @@ def validateRenderMode(mode):
# make sure that mode is a list of things that are all rendermode primative
if isinstance(mode, str):
# Try and find an item named "mode" in the rendermodes module
mode = mode.lower().replace("-","_")
try:
mode = getattr(rendermodes, mode)
except AttributeError:

59
overviewer_core/world.py
View File

@@ -208,7 +208,7 @@ class RegionSet(object):
"""
def __init__(self, regiondir, cachesize=16):
def __init__(self, regiondir):
"""Initialize a new RegionSet to access the region files in the given
directory.
@@ -232,10 +232,6 @@ class RegionSet(object):
self.empty_chunk = [None,None]
logging.debug("Done scanning regions")
# Caching implementaiton: a simple LRU cache
# Decorate the getter methods with the cache decorator
self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk)
# Re-initialize upon unpickling
def __getstate__(self):
return self.regiondir
@@ -258,7 +254,6 @@ class RegionSet(object):
else:
raise Exception("Woah, what kind of dimension is this! %r" % self.regiondir)
# this is decorated with cache.lru_cache in __init__(). Be aware!
@log_other_exceptions
def get_chunk(self, x, z):
"""Returns a dictionary object representing the "Level" NBT Compound
@@ -547,6 +542,58 @@ class CroppedRegionSet(RegionSetWrapper):
else:
return None
class CachedRegionSet(RegionSetWrapper):
"""A regionset wrapper that implements caching of the results from
get_chunk()
"""
def __init__(self, rsetobj, cacheobjects):
"""Initialize this wrapper around the given regionset object and with
the given list of cache objects. The cache objects may be shared among
other CachedRegionSet objects.
"""
super(CachedRegionSet, self).__init__(rsetobj)
self.caches = cacheobjects
# Construct a key from the sequence of transformations and the real
# RegionSet object, so that items we place in the cache don't conflict
# with other worlds/transformation combinations.
obj = self._r
s = ""
while isinstance(obj, RegionSetWrapper):
s += obj.__class__.__name__ + "."
obj = obj._r
# obj should now be the actual RegionSet object
s += obj.regiondir
logging.debug("Initializing a cache with key '%s'", s)
if len(s) > 32:
import hashlib
s = hashlib.md5(s).hexdigest()
self.key = s
def get_chunk(self, x, z):
key = (self.key, x, z)
for i, cache in enumerate(self.caches):
try:
retval = cache[key]
# This did have it, no need to re-add it to this cache, just
# the ones before it
i -= 1
break
except KeyError:
pass
else:
retval = super(CachedRegionSet, self).get_chunk(x,z)
# Now add retval to all the caches that didn't have it, all the caches
# up to and including index i
for cache in self.caches[:i+1]:
cache[key] = retval
return retval
def get_save_dir():

1
test/test_all.py
View File

@@ -11,6 +11,7 @@ from test_tileobj import TileTest
from test_rendertileset import RendertileSetTest
from test_settings import SettingsTest
from test_tileset import TilesetTest
from test_cache import TestLRU
# DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
if 0:

56
test/test_cache.py Normal file
View File

@@ -0,0 +1,56 @@
import unittest
from overviewer_core import cache
class TestLRU(unittest.TestCase):
def setUp(self):
self.lru = cache.LRUCache(size=5)
def test_single_insert(self):
self.lru[1] = 2
self.assertEquals(self.lru[1], 2)
def test_multiple_insert(self):
self.lru[1] = 2
self.lru[3] = 4
self.lru[5] = 6
self.assertEquals(self.lru[1], 2)
self.assertEquals(self.lru[3], 4)
self.assertEquals(self.lru[5], 6)
def test_full(self):
self.lru[1] = 'asdf'
self.lru[2] = 'asdf'
self.lru[3] = 'asdf'
self.lru[4] = 'asdf'
self.lru[5] = 'asdf'
self.lru[6] = 'asdf'
self.assertRaises(KeyError, self.lru.__getitem__, 1)
self.assertEquals(self.lru[2], 'asdf')
self.assertEquals(self.lru[3], 'asdf')
self.assertEquals(self.lru[4], 'asdf')
self.assertEquals(self.lru[5], 'asdf')
self.assertEquals(self.lru[6], 'asdf')
def test_lru(self):
self.lru[1] = 'asdf'
self.lru[2] = 'asdf'
self.lru[3] = 'asdf'
self.lru[4] = 'asdf'
self.lru[5] = 'asdf'
self.assertEquals(self.lru[1], 'asdf')
self.assertEquals(self.lru[2], 'asdf')
self.assertEquals(self.lru[4], 'asdf')
self.assertEquals(self.lru[5], 'asdf')
# 3 should be evicted now
self.lru[6] = 'asdf'
self.assertRaises(KeyError, self.lru.__getitem__, 3)
self.assertEquals(self.lru[1], 'asdf')
self.assertEquals(self.lru[2], 'asdf')
self.assertEquals(self.lru[4], 'asdf')
self.assertEquals(self.lru[5], 'asdf')
self.assertEquals(self.lru[6], 'asdf')