From ff3bfceef7816c784741102ecf59029e225c7d47 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Tue, 17 Jan 2012 21:52:01 -0500 Subject: [PATCH] more C code fixes. Activated caching. Also removed some code that I accidentially left in. Also added a traceback printing decorator around get_chunk() because the C code can potentially swallow those exceptions. --- overviewer_core/src/iterate.c | 23 ++--- overviewer_core/src/overviewer.h | 5 +- overviewer_core/src/primitives/cave.c | 22 +++-- overviewer_core/src/primitives/lighting.c | 23 +++-- overviewer_core/src/rendermode-spawn.c | 4 +- overviewer_core/world.py | 101 +++++----------------- 6 files changed, 58 insertions(+), 120 deletions(-) diff --git a/overviewer_core/src/iterate.c b/overviewer_core/src/iterate.c index 8f10771..1513d0d 100644 --- a/overviewer_core/src/iterate.c +++ b/overviewer_core/src/iterate.c @@ -99,8 +99,11 @@ PyObject *init_chunk_render(void) { /* * Returns the requested chunk data from the requested chunk. * Returns NULL with an exception set if the requested chunk doesn't exist. + * If clearexception is true, clears the exception before returning NULL (for + * soft failures) */ -PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type) { +PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type, + unsigned char clearexception) { int x = state->chunkx; int z = state->chunkz; PyObject *chunk = NULL; @@ -127,6 +130,9 @@ PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDa if (chunk == NULL) { // An exception is already set. RegionSet.get_chunk sets // ChunkDoesntExist + if (clearexception) { + PyErr_Clear(); + } return NULL; } @@ -439,28 +445,25 @@ chunk_render(PyObject *self, PyObject *args) { Py_DECREF(imgsize1_py); /* get the block data directly from numpy: */ - blocks_py = get_chunk_data(&state, CURRENT, BLOCKS); + blocks_py = get_chunk_data(&state, CURRENT, BLOCKS, 0); state.blocks = blocks_py; if (blocks_py == NULL) { return NULL; } - state.blockdatas = get_chunk_data(&state, CURRENT, BLOCKDATA); + state.blockdatas = get_chunk_data(&state, CURRENT, BLOCKDATA, 1); - left_blocks_py = get_chunk_data(&state, DOWN_LEFT, BLOCKS); + left_blocks_py = get_chunk_data(&state, DOWN_LEFT, BLOCKS, 1); state.left_blocks = left_blocks_py; - right_blocks_py = get_chunk_data(&state, DOWN_RIGHT, BLOCKS); + right_blocks_py = get_chunk_data(&state, DOWN_RIGHT, BLOCKS, 1); state.right_blocks = right_blocks_py; - up_left_blocks_py = get_chunk_data(&state, UP_LEFT, BLOCKS); + up_left_blocks_py = get_chunk_data(&state, UP_LEFT, BLOCKS, 1); state.up_left_blocks = up_left_blocks_py; - up_right_blocks_py = get_chunk_data(&state, UP_RIGHT, BLOCKS); + up_right_blocks_py = get_chunk_data(&state, UP_RIGHT, BLOCKS, 1); state.up_right_blocks = up_right_blocks_py; - - // Clear any error that was set by the above calls - PyErr_Clear(); /* set up the random number generator again for each chunk so tallgrass is in the same place, no matter what mode is used */ diff --git a/overviewer_core/src/overviewer.h b/overviewer_core/src/overviewer.h index c5c416c..b935ef4 100644 --- a/overviewer_core/src/overviewer.h +++ b/overviewer_core/src/overviewer.h @@ -26,7 +26,7 @@ // increment this value if you've made a change to the c extesion // and want to force users to rebuild -#define OVERVIEWER_EXTENSION_VERSION 15 +#define OVERVIEWER_EXTENSION_VERSION 16 /* Python PIL, and numpy headers */ #include @@ -137,7 +137,8 @@ typedef enum UP_RIGHT, /* +1, 0 */ UP_LEFT, /* 0, -1 */ } ChunkNeighborName; -PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type); +PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type, + unsigned char clearexception); /* pull in the rendermode info */ #include "rendermodes.h" diff --git a/overviewer_core/src/primitives/cave.c b/overviewer_core/src/primitives/cave.c index 6cd6453..b518f0f 100644 --- a/overviewer_core/src/primitives/cave.c +++ b/overviewer_core/src/primitives/cave.c @@ -214,21 +214,19 @@ cave_start(void *data, RenderState *state, PyObject *support) { return 1; /* if there's skylight we are in the surface! */ - self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT); - self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT); - self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT); - self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT); - self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT); + self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1); + self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT, 1); + self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT, 1); + self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT, 1); + self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT, 1); if (self->only_lit) { - self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT); - self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT); - self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT); - self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT); - self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT); + self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1); + self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT, 1); + self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT, 1); + self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT, 1); + self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT, 1); } - // Non-existant neighboring blocks is not an error - PyErr_Clear(); return 0; } diff --git a/overviewer_core/src/primitives/lighting.c b/overviewer_core/src/primitives/lighting.c index 7509578..7f7b478 100644 --- a/overviewer_core/src/primitives/lighting.c +++ b/overviewer_core/src/primitives/lighting.c @@ -359,20 +359,17 @@ lighting_start(void *data, RenderState *state, PyObject *support) { self->facemasks[1] = PyTuple_GetItem(self->facemasks_py, 1); self->facemasks[2] = PyTuple_GetItem(self->facemasks_py, 2); - self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT); - self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT); - self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT); - self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT); - self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT); - self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT); - self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT); - self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT); - self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT); - self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT); + self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1); + self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1); + self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT, 1); + self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT, 1); + self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT, 1); + self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT, 1); + self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT, 1); + self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT, 1); + self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT, 1); + self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT, 1); - // Non-existant neighbor block is not an error - PyErr_Clear(); - if (self->night) { self->calculate_light_color = calculate_light_color_night; } else { diff --git a/overviewer_core/src/rendermode-spawn.c b/overviewer_core/src/rendermode-spawn.c index 6f57085..0c8027b 100644 --- a/overviewer_core/src/rendermode-spawn.c +++ b/overviewer_core/src/rendermode-spawn.c @@ -69,8 +69,8 @@ rendermode_spawn_start(void *data, RenderState *state, PyObject *options) { /* now do custom initializations */ self = (RenderModeSpawn *)data; - self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT); - self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT); + self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1); + self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1); /* setup custom color */ self->parent.get_color = get_color; diff --git a/overviewer_core/world.py b/overviewer_core/world.py index ff21f96..82c3a69 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -55,6 +55,24 @@ def base36encode(number, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'): return "-" + base36 return base36 +def log_other_exceptions(func): + """A decorator that prints out any errors that are not ChunkDoesntExist + errors. This decorates get_chunk because the C code is likely to swallow + exceptions, so this will at least make them visible. + + """ + functools.wraps(func) + def newfunc(*args): + try: + return func(*args) + except ChunkDoesntExist: + raise + except Exception, e: + logging.exception("%s raised this exception", func.func_name) + raise + return newfunc + + class World(object): """Encapsulates the concept of a Minecraft "world". A Minecraft world is a level.dat file, a players directory with info about each player, a data @@ -237,7 +255,7 @@ class RegionSet(object): # Caching implementaiton: a simple LRU cache # Decorate the get_chunk method with the cache decorator - #self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk) + self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk) # Re-initialize upon unpickling def __getstate__(self): @@ -247,6 +265,7 @@ class RegionSet(object): def __repr__(self): return "" % self.regiondir + @log_other_exceptions def get_chunk(self, x, z): """Returns a dictionary object representing the "Level" NBT Compound structure for a chunk given its x, z coordinates. The coordinates are @@ -434,83 +453,3 @@ def get_worlds(): ret[info['Data']['LevelName']] = info['Data'] return ret - - -def lru_cache(maxsize=100): - '''Generalized Least-recently-used cache decorator. - - Arguments to the cached function must be hashable. - Cache performance statistics stored in f.hits and f.misses. - Clear the cache with f.clear(). - http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used - - This snippet is from - http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators/ - - ''' - maxqueue = maxsize * 10 - def decorating_function(user_function, - len=len, iter=iter, tuple=tuple, sorted=sorted, KeyError=KeyError): - cache = {} # mapping of args to results - queue = collections.deque() # order that keys have been used - refcount = collections.defaultdict(int)# times each key is in the queue - sentinel = object() # marker for looping around the queue - kwd_mark = object() # separate positional and keyword args - - # lookup optimizations (ugly but fast) - queue_append, queue_popleft = queue.append, queue.popleft - queue_appendleft, queue_pop = queue.appendleft, queue.pop - - @functools.wraps(user_function) - def wrapper(*args, **kwds): - # cache key records both positional and keyword args - key = args - if kwds: - key += (kwd_mark,) + tuple(sorted(kwds.items())) - - # record recent use of this key - queue_append(key) - refcount[key] += 1 - - # get cache entry or compute if not found - try: - result = cache[key] - wrapper.hits += 1 - except KeyError: - result = user_function(*args, **kwds) - cache[key] = result - wrapper.misses += 1 - - # purge least recently used cache entry - if len(cache) > maxsize: - key = queue_popleft() - refcount[key] -= 1 - while refcount[key]: - key = queue_popleft() - refcount[key] -= 1 - del cache[key], refcount[key] - - # periodically compact the queue by eliminating duplicate keys - # while preserving order of most recent access - if len(queue) > maxqueue: - refcount.clear() - queue_appendleft(sentinel) - for key in ifilterfalse(refcount.__contains__, - iter(queue_pop, sentinel)): - queue_appendleft(key) - refcount[key] = 1 - - - return result - - def clear(): - cache.clear() - queue.clear() - refcount.clear() - wrapper.hits = wrapper.misses = 0 - - wrapper.hits = wrapper.misses = 0 - wrapper.clear = clear - return wrapper - return decorating_function -