diff --git a/overviewer.py b/overviewer.py index af960e8..e90f386 100755 --- a/overviewer.py +++ b/overviewer.py @@ -349,7 +349,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # Set up the cache objects to use caches = [] - caches.append(cache.LRUCache()) + caches.append(cache.LRUCache(size=100)) # TODO: optionally more caching layers here renders = config['renders'] @@ -434,13 +434,17 @@ dir but you forgot to put quotes around the directory, since it contains spaces. else: percent = int(100* completed/total) logging.info("Rendered %d of %d. %d%% complete", completed, total, percent) + dispatch.render_all(tilesets, print_status) dispatch.close() assetMrg.finalize(tilesets) - logging.debug("Final cache stats:") - for c in caches: - logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses) + + if config['processes'] == 1: + logging.debug("Final cache stats:") + for c in caches: + logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses) + return 0 def list_worlds(): diff --git a/overviewer_core/cache.py b/overviewer_core/cache.py index b93eb42..b2f867d 100644 --- a/overviewer_core/cache.py +++ b/overviewer_core/cache.py @@ -16,7 +16,7 @@ """This module has supporting functions for the caching logic used in world.py. Each cache class should implement the standard container type interface -(__getitem__ and __setitem__, as well as provide a "hits" and "misses" +(__getitem__ and __setitem__), as well as provide a "hits" and "misses" attribute. """ @@ -24,7 +24,8 @@ import functools import logging class LRUCache(object): - """A simple in-memory LRU cache. + """A simple, generic, in-memory LRU cache that implements the standard + python container interface. An ordered dict type would simplify this implementation a bit, but we want Python 2.6 compatibility and the standard library ordereddict was added in @@ -47,7 +48,14 @@ class LRUCache(object): self.key = k self.value = v - def __init__(self, size=100): + def __init__(self, size=100, destructor=None): + """Initialize a new LRU cache with the given size. + + destructor, if given, is a callable that is called upon an item being + evicted from the cache. It takes one argument, the value stored in the + cache. + + """ self.cache = {} self.listhead = LRUCache._LinkNode() @@ -62,6 +70,8 @@ class LRUCache(object): self.size = size + self.destructor = destructor + # Initialize an empty cache of the same size for worker processes def __getstate__(self): return self.size @@ -92,13 +102,18 @@ class LRUCache(object): def __setitem__(self, key, value): cache = self.cache if key in cache: - raise KeyError("That key already exists in the cache!") + # Shortcut this case + cache[key].value = value + return if len(cache) >= self.size: # Evict a node link = self.listhead.right del cache[link.key] link.left.right = link.right link.right.left = link.left + d = self.destructor + if d: + d(link.value) del link # The node doesn't exist already, and we have room for it. Let's do this. diff --git a/overviewer_core/nbt.py b/overviewer_core/nbt.py index 11caf85..4420673 100644 --- a/overviewer_core/nbt.py +++ b/overviewer_core/nbt.py @@ -215,7 +215,7 @@ class MCRFileReader(object): # turn this data into a useful list self._locations = self._table_format.unpack(location_data) self._timestamps = self._table_format.unpack(timestamp_data) - + def close(self): """Close the region file and free any resources associated with keeping it open. Using this object after closing it diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 7575701..43b11d8 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -21,8 +21,8 @@ import logging import numpy -import nbt -import cache +from . import nbt +from . import cache """ This module has routines for extracting information about available worlds @@ -224,6 +224,9 @@ class RegionSet(object): # This is populated below. It is a mapping from (x,y) region coords to filename self.regionfiles = {} + + # This holds up to 16 open regionfile objects + self.regioncache = cache.LRUCache(size=16, destructor=lambda regionobj: regionobj.close()) for x, y, regionfile in self._iterate_regionfiles(): # regionfile is a pathname @@ -252,7 +255,15 @@ class RegionSet(object): elif self.regiondir.endswith(os.path.normpath("/region")): return "overworld" else: - raise Exception("Woah, what kind of dimension is this! %r" % self.regiondir) + raise Exception("Woah, what kind of dimension is this?! %r" % self.regiondir) + + def _get_regionobj(self, regionfilename): + try: + return self.regioncache[regionfilename] + except KeyError: + region = nbt.load_region(regionfilename) + self.regioncache[regionfilename] = region + return region @log_other_exceptions def get_chunk(self, x, z): @@ -281,14 +292,12 @@ class RegionSet(object): modified, lest it affect the return values of future calls for the same chunk. """ - regionfile = self._get_region_path(x, z) if regionfile is None: raise ChunkDoesntExist("Chunk %s,%s doesn't exist (and neither does its region)" % (x,z)) - region = nbt.load_region(regionfile) + region = self._get_regionobj(regionfile) data = region.load_chunk(x, z) - region.close() if data is None: raise ChunkDoesntExist("Chunk %s,%s doesn't exist" % (x,z)) @@ -366,7 +375,7 @@ class RegionSet(object): """ for (regionx, regiony), regionfile in self.regionfiles.iteritems(): - mcr = nbt.load_region(regionfile) + mcr = self._get_regionobj(regionfile) for chunkx, chunky in mcr.get_chunks(): yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) @@ -380,8 +389,7 @@ class RegionSet(object): regionfile = self._get_region_path(x,z) if regionfile is None: return None - - data = nbt.load_region(regionfile) + data = self._get_regionobj(regionfile) if data.chunk_exists(x,z): return data.get_chunk_timestamp(x,z) return None