From 30b224d5b1d1c28ed68285d6ef4cbfd45bacc673 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sun, 4 Mar 2012 00:20:22 -0500 Subject: [PATCH 1/2] only output cache stats with -p1 --- overviewer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/overviewer.py b/overviewer.py index af960e8..71a4f36 100755 --- a/overviewer.py +++ b/overviewer.py @@ -438,9 +438,10 @@ dir but you forgot to put quotes around the directory, since it contains spaces. dispatch.close() assetMrg.finalize(tilesets) - logging.debug("Final cache stats:") - for c in caches: - logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses) + if config['processes'] == 1: + logging.debug("Final cache stats:") + for c in caches: + logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses) return 0 def list_worlds(): From 8206272fc87a451d935948c3cce7075ba54b0213 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sun, 4 Mar 2012 18:46:02 -0500 Subject: [PATCH 2/2] Added a cache for open regionfile objects --- overviewer.py | 5 ++++- overviewer_core/cache.py | 23 +++++++++++++++++++---- overviewer_core/nbt.py | 2 +- overviewer_core/world.py | 26 +++++++++++++++++--------- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/overviewer.py b/overviewer.py index 71a4f36..e90f386 100755 --- a/overviewer.py +++ b/overviewer.py @@ -349,7 +349,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # Set up the cache objects to use caches = [] - caches.append(cache.LRUCache()) + caches.append(cache.LRUCache(size=100)) # TODO: optionally more caching layers here renders = config['renders'] @@ -434,14 +434,17 @@ dir but you forgot to put quotes around the directory, since it contains spaces. else: percent = int(100* completed/total) logging.info("Rendered %d of %d. %d%% complete", completed, total, percent) + dispatch.render_all(tilesets, print_status) dispatch.close() assetMrg.finalize(tilesets) + if config['processes'] == 1: logging.debug("Final cache stats:") for c in caches: logging.debug("\t%s: %s hits, %s misses", c.__class__.__name__, c.hits, c.misses) + return 0 def list_worlds(): diff --git a/overviewer_core/cache.py b/overviewer_core/cache.py index b93eb42..b2f867d 100644 --- a/overviewer_core/cache.py +++ b/overviewer_core/cache.py @@ -16,7 +16,7 @@ """This module has supporting functions for the caching logic used in world.py. Each cache class should implement the standard container type interface -(__getitem__ and __setitem__, as well as provide a "hits" and "misses" +(__getitem__ and __setitem__), as well as provide a "hits" and "misses" attribute. """ @@ -24,7 +24,8 @@ import functools import logging class LRUCache(object): - """A simple in-memory LRU cache. + """A simple, generic, in-memory LRU cache that implements the standard + python container interface. An ordered dict type would simplify this implementation a bit, but we want Python 2.6 compatibility and the standard library ordereddict was added in @@ -47,7 +48,14 @@ class LRUCache(object): self.key = k self.value = v - def __init__(self, size=100): + def __init__(self, size=100, destructor=None): + """Initialize a new LRU cache with the given size. + + destructor, if given, is a callable that is called upon an item being + evicted from the cache. It takes one argument, the value stored in the + cache. + + """ self.cache = {} self.listhead = LRUCache._LinkNode() @@ -62,6 +70,8 @@ class LRUCache(object): self.size = size + self.destructor = destructor + # Initialize an empty cache of the same size for worker processes def __getstate__(self): return self.size @@ -92,13 +102,18 @@ class LRUCache(object): def __setitem__(self, key, value): cache = self.cache if key in cache: - raise KeyError("That key already exists in the cache!") + # Shortcut this case + cache[key].value = value + return if len(cache) >= self.size: # Evict a node link = self.listhead.right del cache[link.key] link.left.right = link.right link.right.left = link.left + d = self.destructor + if d: + d(link.value) del link # The node doesn't exist already, and we have room for it. Let's do this. diff --git a/overviewer_core/nbt.py b/overviewer_core/nbt.py index 11caf85..4420673 100644 --- a/overviewer_core/nbt.py +++ b/overviewer_core/nbt.py @@ -215,7 +215,7 @@ class MCRFileReader(object): # turn this data into a useful list self._locations = self._table_format.unpack(location_data) self._timestamps = self._table_format.unpack(timestamp_data) - + def close(self): """Close the region file and free any resources associated with keeping it open. Using this object after closing it diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 8532924..554f6ce 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -21,8 +21,8 @@ import logging import numpy -import nbt -import cache +from . import nbt +from . import cache """ This module has routines for extracting information about available worlds @@ -224,6 +224,9 @@ class RegionSet(object): # This is populated below. It is a mapping from (x,y) region coords to filename self.regionfiles = {} + + # This holds up to 16 open regionfile objects + self.regioncache = cache.LRUCache(size=16, destructor=lambda regionobj: regionobj.close()) for x, y, regionfile in self._iterate_regionfiles(): # regionfile is a pathname @@ -252,7 +255,15 @@ class RegionSet(object): elif self.regiondir.endswith(os.path.normpath("/region")): return "overworld" else: - raise Exception("Woah, what kind of dimension is this! %r" % self.regiondir) + raise Exception("Woah, what kind of dimension is this?! %r" % self.regiondir) + + def _get_regionobj(self, regionfilename): + try: + return self.regioncache[regionfilename] + except KeyError: + region = nbt.load_region(regionfilename) + self.regioncache[regionfilename] = region + return region @log_other_exceptions def get_chunk(self, x, z): @@ -281,14 +292,12 @@ class RegionSet(object): modified, lest it affect the return values of future calls for the same chunk. """ - regionfile = self._get_region_path(x, z) if regionfile is None: raise ChunkDoesntExist("Chunk %s,%s doesn't exist (and neither does its region)" % (x,z)) - region = nbt.load_region(regionfile) + region = self._get_regionobj(regionfile) data = region.load_chunk(x, z) - region.close() if data is None: raise ChunkDoesntExist("Chunk %s,%s doesn't exist" % (x,z)) @@ -362,7 +371,7 @@ class RegionSet(object): """ for (regionx, regiony), regionfile in self.regionfiles.iteritems(): - mcr = nbt.load_region(regionfile) + mcr = self._get_regionobj(regionfile) for chunkx, chunky in mcr.get_chunks(): yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) @@ -376,8 +385,7 @@ class RegionSet(object): regionfile = self._get_region_path(x,z) if regionfile is None: return None - - data = nbt.load_region(regionfile) + data = self._get_regionobj(regionfile) if data.chunk_exists(x,z): return data.get_chunk_timestamp(x,z) return None