Merge branch 'anvil'

overviewer.py

@@ -38,6 +38,7 @@ from overviewer_core import util
 from overviewer_core import textures
 from overviewer_core import optimizeimages, world
 from overviewer_core import configParser, tileset, assetmanager, dispatcher
+from overviewer_core import cache
 
 helptext = """
 %prog [--rendermodes=...] [options] <World> <Output Dir>
@@ -346,16 +347,21 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
     # same for textures
     texcache = {}
 
+    # Set up the cache objects to use
+    caches = []
+    caches.append(cache.LRUCache())
+    # TODO: optionally more caching layers here
+
     renders = config['renders']
     for render_name, render in renders.iteritems():
         logging.debug("Found the following render thing: %r", render)
 
         # find or create the world object
-        if (render['world'] not in worldcache):
+        try:
+            w = worldcache[render['world']]
+        except KeyError:
             w = world.World(render['world'])
             worldcache[render['world']] = w
-        else:
-            w = worldcache[render['world']]
         
         # find or create the textures object
         texopts = util.dict_subset(render, ["texturepath", "bgcolor", "northdirection"])
@@ -372,6 +378,15 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
             logging.error("Sorry, you requested dimension '%s' for %s, but I couldn't find it", render['dimension'], render_name)
             return 1
 
+        #################
+        # Apply any regionset transformations here
+
+        # Insert a layer of caching above the real regionset. Any world
+        # tranformations will pull from this cache, but their results will not
+        # be cached by this layer. This uses a common pool of caches; each
+        # regionset cache pulls from the same underlying cache object.
+        rset = world.CachedRegionSet(rset, caches)
+
         # If a crop is requested, wrap the regionset here
         if "crop" in render:
             rset = world.CroppedRegionSet(rset, *render['crop'])
@@ -382,6 +397,9 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
             rset = world.RotatedRegionSet(rset, render['northdirection'])
         logging.debug("Using RegionSet %r", rset) 
 
+        ###############################
+        # Do the final prep and create the TileSet object
+
         # create our TileSet from this RegionSet
         tileset_dir = os.path.abspath(os.path.join(destdir, render_name))
         if not os.path.exists(tileset_dir):
@@ -420,6 +438,9 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
     dispatch.close()
 
     assetMrg.finalize(tilesets)
+    logging.debug("Final cache stats:")
+    for c in caches:
+        logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses)
     return 0
 
 def list_worlds():

overviewer_core/cache.py

@@ -15,48 +15,97 @@
 
 """This module has supporting functions for the caching logic used in world.py.
 
+Each cache class should implement the standard container type interface
+(__getitem__ and __setitem__, as well as provide a "hits" and "misses"
+attribute.
+
 """
 import functools
+import logging
 
-def lru_cache(max_size=100):
+class LRUCache(object):
-    """A quick-and-dirty LRU implementation.
+    """A simple in-memory LRU cache.
-    Uses a dict to store mappings, and a list to store orderings.
 
-    Only supports positional arguments
+    An ordered dict type would simplify this implementation a bit, but we want
+    Python 2.6 compatibility and the standard library ordereddict was added in
+    2.7. It's probably okay because this implementation can be tuned for
+    exactly what we need and nothing more.
+
+    This implementation keeps a linked-list of cache keys and values, ordered
+    in least-recently-used order. A dictionary maps keys to linked-list nodes.
+
+    On cache hit, the link is moved to the end of the list. On cache miss, the
+    first item of the list is evicted. All operations have constant time
+    complexity (dict lookups are worst case O(n) time)
 
     """
-    def lru_decorator(fun):
+    class _LinkNode(object):
+        __slots__ = ['left', 'right', 'key', 'value']
+        def __init__(self,l=None,r=None,k=None,v=None):
+            self.left = l
+            self.right = r
+            self.key = k
+            self.value = v
 
-        cache = {}
+    def __init__(self, size=100):
-        lru_ordering = []
+        self.cache = {}
-        
-        @functools.wraps(fun)
-        def new_fun(*args):
-            try:
-                result = cache[args]
-            except KeyError:
-                # cache miss =(
-                new_fun.miss += 1
-                result = fun(*args)
 
-                # Insert into cache
+        self.listhead = LRUCache._LinkNode()
-                cache[args] = result
+        self.listtail = LRUCache._LinkNode()
-                lru_ordering.append(args)
+        # Two sentinel nodes at the ends of the linked list simplify boundary
+        # conditions in the code below.
+        self.listhead.right = self.listtail
+        self.listtail.left = self.listhead
 
-                if len(cache) > max_size:
+        self.hits = 0
-                    # Evict an item
+        self.misses = 0
-                    del cache[ lru_ordering.pop(0) ]
 
-            else:
+        self.size = size
-                # Move the result item to the end of the list
-                new_fun.hits += 1
-                position = lru_ordering.index(args)
-                lru_ordering.append(lru_ordering.pop(position))
 
-            return result
+    # Initialize an empty cache of the same size for worker processes
+    def __getstate__(self):
+        return self.size
+    def __setstate__(self, size):
+        self.__init__(size)
 
-        new_fun.hits = 0
+    def __getitem__(self, key):
-        new_fun.miss = 0
+        try:
-        return new_fun
+            link = self.cache[key]
+        except KeyError:
+            self.misses += 1
+            raise
+
+        # Disconnect the link from where it is
+        link.left.right = link.right
+        link.right.left = link.left
+
+        # Insert the link at the end of the list
+        tail = self.listtail
+        link.left = tail.left
+        link.right = tail
+        tail.left.right = link
+        tail.left = link
+
+        self.hits += 1
+        return link.value
+
+    def __setitem__(self, key, value):
+        cache = self.cache
+        if key in cache:
+            raise KeyError("That key already exists in the cache!")
+        if len(cache) >= self.size:
+            # Evict a node
+            link = self.listhead.right
+            del cache[link.key]
+            link.left.right = link.right
+            link.right.left = link.left
+            del link
+
+        # The node doesn't exist already, and we have room for it. Let's do this.
+        tail = self.listtail
+        link = LRUCache._LinkNode(tail.left, tail,key,value)
+        tail.left.right = link
+        tail.left = link
+
+        cache[key] = link
 
-    return lru_decorator

overviewer_core/settingsValidators.py

@@ -55,6 +55,7 @@ def validateRenderMode(mode):
     # make sure that mode is a list of things that are all rendermode primative
     if isinstance(mode, str):
         # Try and find an item named "mode" in the rendermodes module
+        mode = mode.lower().replace("-","_")
         try:
             mode = getattr(rendermodes, mode)
         except AttributeError:

overviewer_core/world.py

@@ -208,7 +208,7 @@ class RegionSet(object):
 
     """
 
-    def __init__(self, regiondir, cachesize=16):
+    def __init__(self, regiondir):
         """Initialize a new RegionSet to access the region files in the given
         directory.
 
@@ -232,10 +232,6 @@ class RegionSet(object):
         self.empty_chunk = [None,None]
         logging.debug("Done scanning regions")
 
-        # Caching implementaiton: a simple LRU cache
-        # Decorate the getter methods with the cache decorator
-        self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk)
-
     # Re-initialize upon unpickling
     def __getstate__(self):
         return self.regiondir
@@ -258,7 +254,6 @@ class RegionSet(object):
         else:
             raise Exception("Woah, what kind of dimension is this! %r" % self.regiondir)
     
-    # this is decorated with cache.lru_cache in __init__(). Be aware!
     @log_other_exceptions
     def get_chunk(self, x, z):
         """Returns a dictionary object representing the "Level" NBT Compound
@@ -547,7 +542,59 @@ class CroppedRegionSet(RegionSetWrapper):
         else:
             return None
 
+class CachedRegionSet(RegionSetWrapper):
+    """A regionset wrapper that implements caching of the results from
+    get_chunk()
 
+    """
+    def __init__(self, rsetobj, cacheobjects):
+        """Initialize this wrapper around the given regionset object and with
+        the given list of cache objects. The cache objects may be shared among
+        other CachedRegionSet objects.
+
+        """
+        super(CachedRegionSet, self).__init__(rsetobj)
+        self.caches = cacheobjects
+
+        # Construct a key from the sequence of transformations and the real
+        # RegionSet object, so that items we place in the cache don't conflict
+        # with other worlds/transformation combinations.
+        obj = self._r
+        s = ""
+        while isinstance(obj, RegionSetWrapper):
+            s += obj.__class__.__name__ + "."
+            obj = obj._r
+        # obj should now be the actual RegionSet object
+        s += obj.regiondir
+
+        logging.debug("Initializing a cache with key '%s'", s)
+        if len(s) > 32:
+            import hashlib
+            s = hashlib.md5(s).hexdigest()
+
+        self.key = s
+
+    def get_chunk(self, x, z):
+        key = (self.key, x, z)
+        for i, cache in enumerate(self.caches):
+            try:
+                retval = cache[key]
+                # This did have it, no need to re-add it to this cache, just
+                # the ones before it
+                i -= 1
+                break
+            except KeyError:
+                pass
+        else:
+            retval = super(CachedRegionSet, self).get_chunk(x,z)
+
+        # Now add retval to all the caches that didn't have it, all the caches
+        # up to and including index i
+        for cache in self.caches[:i+1]:
+            cache[key] = retval
+
+        return retval
+        
 
 def get_save_dir():
     """Returns the path to the local saves directory

test/test_all.py

@@ -11,6 +11,7 @@ from test_tileobj import TileTest
 from test_rendertileset import RendertileSetTest
 from test_settings import SettingsTest
 from test_tileset import TilesetTest
+from test_cache import TestLRU
 
 # DISABLE THIS BLOCK TO GET LOG OUTPUT FROM TILESET FOR DEBUGGING
 if 0:

test/test_cache.py

@@ -0,0 +1,56 @@
+import unittest
+
+from overviewer_core import cache
+
+class TestLRU(unittest.TestCase):
+
+    def setUp(self):
+        self.lru = cache.LRUCache(size=5)
+
+    def test_single_insert(self):
+        self.lru[1] = 2
+        self.assertEquals(self.lru[1], 2)
+
+    def test_multiple_insert(self):
+        self.lru[1] = 2
+        self.lru[3] = 4
+        self.lru[5] = 6
+        self.assertEquals(self.lru[1], 2)
+        self.assertEquals(self.lru[3], 4)
+        self.assertEquals(self.lru[5], 6)
+
+    def test_full(self):
+        self.lru[1] = 'asdf'
+        self.lru[2] = 'asdf'
+        self.lru[3] = 'asdf'
+        self.lru[4] = 'asdf'
+        self.lru[5] = 'asdf'
+        self.lru[6] = 'asdf'
+        self.assertRaises(KeyError, self.lru.__getitem__, 1)
+        self.assertEquals(self.lru[2], 'asdf')
+        self.assertEquals(self.lru[3], 'asdf')
+        self.assertEquals(self.lru[4], 'asdf')
+        self.assertEquals(self.lru[5], 'asdf')
+        self.assertEquals(self.lru[6], 'asdf')
+
+    def test_lru(self):
+        self.lru[1] = 'asdf'
+        self.lru[2] = 'asdf'
+        self.lru[3] = 'asdf'
+        self.lru[4] = 'asdf'
+        self.lru[5] = 'asdf'
+
+        self.assertEquals(self.lru[1], 'asdf')
+        self.assertEquals(self.lru[2], 'asdf')
+        self.assertEquals(self.lru[4], 'asdf')
+        self.assertEquals(self.lru[5], 'asdf')
+
+        # 3 should be evicted now
+        self.lru[6] = 'asdf'
+
+        self.assertRaises(KeyError, self.lru.__getitem__, 3)
+        self.assertEquals(self.lru[1], 'asdf')
+        self.assertEquals(self.lru[2], 'asdf')
+        self.assertEquals(self.lru[4], 'asdf')
+        self.assertEquals(self.lru[5], 'asdf')
+        self.assertEquals(self.lru[6], 'asdf')