Added a cache for open regionfile objects

overviewer.py

@@ -349,7 +349,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
 
     # Set up the cache objects to use
     caches = []
-    caches.append(cache.LRUCache())
+    caches.append(cache.LRUCache(size=100))
     # TODO: optionally more caching layers here
 
     renders = config['renders']
@@ -434,14 +434,17 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
         else:
             percent = int(100* completed/total)
             logging.info("Rendered %d of %d.  %d%% complete", completed, total, percent)
+
     dispatch.render_all(tilesets, print_status)
     dispatch.close()
 
     assetMrg.finalize(tilesets)
+
     if config['processes'] == 1:
         logging.debug("Final cache stats:")
         for c in caches:
             logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses)
+
     return 0
 
 def list_worlds():

overviewer_core/cache.py

@@ -16,7 +16,7 @@
 """This module has supporting functions for the caching logic used in world.py.
 
 Each cache class should implement the standard container type interface
-(__getitem__ and __setitem__, as well as provide a "hits" and "misses"
+(__getitem__ and __setitem__), as well as provide a "hits" and "misses"
 attribute.
 
 """
@@ -24,7 +24,8 @@ import functools
 import logging
 
 class LRUCache(object):
-    """A simple in-memory LRU cache.
+    """A simple, generic, in-memory LRU cache that implements the standard
+    python container interface.
 
     An ordered dict type would simplify this implementation a bit, but we want
     Python 2.6 compatibility and the standard library ordereddict was added in
@@ -47,7 +48,14 @@ class LRUCache(object):
             self.key = k
             self.value = v
 
-    def __init__(self, size=100):
+    def __init__(self, size=100, destructor=None):
+        """Initialize a new LRU cache with the given size.
+
+        destructor, if given, is a callable that is called upon an item being
+        evicted from the cache. It takes one argument, the value stored in the
+        cache.
+
+        """
         self.cache = {}
 
         self.listhead = LRUCache._LinkNode()
@@ -62,6 +70,8 @@ class LRUCache(object):
 
         self.size = size
 
+        self.destructor = destructor
+
     # Initialize an empty cache of the same size for worker processes
     def __getstate__(self):
         return self.size
@@ -92,13 +102,18 @@ class LRUCache(object):
     def __setitem__(self, key, value):
         cache = self.cache
         if key in cache:
-            raise KeyError("That key already exists in the cache!")
+            # Shortcut this case
+            cache[key].value = value
+            return
         if len(cache) >= self.size:
             # Evict a node
             link = self.listhead.right
             del cache[link.key]
             link.left.right = link.right
             link.right.left = link.left
+            d = self.destructor
+            if d:
+                d(link.value)
             del link
 
         # The node doesn't exist already, and we have room for it. Let's do this.

overviewer_core/world.py

@@ -21,8 +21,8 @@ import logging
 
 import numpy
 
-import nbt
+from . import nbt
-import cache
+from . import cache
 
 """
 This module has routines for extracting information about available worlds
@@ -225,6 +225,9 @@ class RegionSet(object):
         # This is populated below. It is a mapping from (x,y) region coords to filename
         self.regionfiles = {}
 
+        # This holds up to 16 open regionfile objects
+        self.regioncache = cache.LRUCache(size=16, destructor=lambda regionobj: regionobj.close())
+        
         for x, y, regionfile in self._iterate_regionfiles():
             # regionfile is a pathname
             self.regionfiles[(x,y)] = regionfile
@@ -252,7 +255,15 @@ class RegionSet(object):
         elif self.regiondir.endswith(os.path.normpath("/region")):
             return "overworld"
         else:
-            raise Exception("Woah, what kind of dimension is this! %r" % self.regiondir)
+            raise Exception("Woah, what kind of dimension is this?! %r" % self.regiondir)
+
+    def _get_regionobj(self, regionfilename):
+        try:
+            return self.regioncache[regionfilename]
+        except KeyError:
+            region = nbt.load_region(regionfilename)
+            self.regioncache[regionfilename] = region
+            return region
     
     @log_other_exceptions
     def get_chunk(self, x, z):
@@ -281,14 +292,12 @@ class RegionSet(object):
         modified, lest it affect the return values of future calls for the same
         chunk.
         """
-
         regionfile = self._get_region_path(x, z)
         if regionfile is None:
             raise ChunkDoesntExist("Chunk %s,%s doesn't exist (and neither does its region)" % (x,z))
 
-        region = nbt.load_region(regionfile)
+        region = self._get_regionobj(regionfile)
         data = region.load_chunk(x, z)
-        region.close()
         if data is None:
             raise ChunkDoesntExist("Chunk %s,%s doesn't exist" % (x,z))
 
@@ -362,7 +371,7 @@ class RegionSet(object):
         """
 
         for (regionx, regiony), regionfile in self.regionfiles.iteritems():
-            mcr = nbt.load_region(regionfile)
+            mcr = self._get_regionobj(regionfile)
             for chunkx, chunky in mcr.get_chunks():
                 yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky)
 
@@ -376,8 +385,7 @@ class RegionSet(object):
         regionfile = self._get_region_path(x,z)
         if regionfile is None:
             return None
-
+        data = self._get_regionobj(regionfile)
-        data = nbt.load_region(regionfile)
         if data.chunk_exists(x,z):
             return data.get_chunk_timestamp(x,z)
         return None