more C code fixes. Activated caching.

Also removed some code that I accidentially left in.
Also added a traceback printing decorator around get_chunk() because the
C code can potentially swallow those exceptions.

overviewer_core/src/iterate.c

@@ -99,8 +99,11 @@ PyObject *init_chunk_render(void) {
 /*
  * Returns the requested chunk data from the requested chunk.
  * Returns NULL with an exception set if the requested chunk doesn't exist.
+ * If clearexception is true, clears the exception before returning NULL (for
+ * soft failures)
  */
-PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type) {
+PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type,
+        unsigned char clearexception) {
     int x = state->chunkx;
     int z = state->chunkz;
     PyObject *chunk = NULL;
@@ -127,6 +130,9 @@ PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDa
     if (chunk == NULL) {
         // An exception is already set. RegionSet.get_chunk sets
         // ChunkDoesntExist
+        if (clearexception) {
+            PyErr_Clear();
+        }
         return NULL;
     }
     
@@ -439,28 +445,25 @@ chunk_render(PyObject *self, PyObject *args) {
     Py_DECREF(imgsize1_py);
 
     /* get the block data directly from numpy: */
-    blocks_py = get_chunk_data(&state, CURRENT, BLOCKS);
+    blocks_py = get_chunk_data(&state, CURRENT, BLOCKS, 0);
     state.blocks = blocks_py;
     if (blocks_py == NULL) {
         return NULL;
     }
     
-    state.blockdatas = get_chunk_data(&state, CURRENT, BLOCKDATA);
+    state.blockdatas = get_chunk_data(&state, CURRENT, BLOCKDATA, 1);
 
-    left_blocks_py = get_chunk_data(&state, DOWN_LEFT, BLOCKS);
+    left_blocks_py = get_chunk_data(&state, DOWN_LEFT, BLOCKS, 1);
     state.left_blocks = left_blocks_py;
 
-    right_blocks_py = get_chunk_data(&state, DOWN_RIGHT, BLOCKS);
+    right_blocks_py = get_chunk_data(&state, DOWN_RIGHT, BLOCKS, 1);
     state.right_blocks = right_blocks_py;
 
-    up_left_blocks_py = get_chunk_data(&state, UP_LEFT, BLOCKS);
+    up_left_blocks_py = get_chunk_data(&state, UP_LEFT, BLOCKS, 1);
     state.up_left_blocks = up_left_blocks_py;
 
-    up_right_blocks_py = get_chunk_data(&state, UP_RIGHT, BLOCKS);
+    up_right_blocks_py = get_chunk_data(&state, UP_RIGHT, BLOCKS, 1);
     state.up_right_blocks = up_right_blocks_py;
-
-    // Clear any error that was set by the above calls
-    PyErr_Clear();
     
     /* set up the random number generator again for each chunk
        so tallgrass is in the same place, no matter what mode is used */

overviewer_core/src/overviewer.h

@@ -26,7 +26,7 @@
 
 // increment this value if you've made a change to the c extesion
 // and want to force users to rebuild
-#define OVERVIEWER_EXTENSION_VERSION 15
+#define OVERVIEWER_EXTENSION_VERSION 16
 
 /* Python PIL, and numpy headers */
 #include <Python.h>
@@ -137,7 +137,8 @@ typedef enum
     UP_RIGHT,   /* +1,  0 */
     UP_LEFT,    /*  0, -1 */
 } ChunkNeighborName;
-PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type);
+PyObject *get_chunk_data(RenderState *state, ChunkNeighborName neighbor, ChunkDataType type,
+        unsigned char clearexception);
 
 /* pull in the rendermode info */
 #include "rendermodes.h"

overviewer_core/src/primitives/cave.c

@@ -214,21 +214,19 @@ cave_start(void *data, RenderState *state, PyObject *support) {
         return 1;
     
     /* if there's skylight we are in the surface! */
-    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT);
+    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1);
-    self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT);
+    self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT, 1);
-    self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT);
+    self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT, 1);
-    self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT);
+    self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT, 1);
-    self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT);
+    self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT, 1);
     
     if (self->only_lit) {
-        self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT);
+        self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1);
-        self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT);
+        self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT, 1);
-        self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT);
+        self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT, 1);
-        self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT);
+        self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT, 1);
-        self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT);
+        self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT, 1);
     }
-    // Non-existant neighboring blocks is not an error
-    PyErr_Clear();
 
     return 0;
 }

overviewer_core/src/primitives/lighting.c

@@ -359,20 +359,17 @@ lighting_start(void *data, RenderState *state, PyObject *support) {
     self->facemasks[1] = PyTuple_GetItem(self->facemasks_py, 1);
     self->facemasks[2] = PyTuple_GetItem(self->facemasks_py, 2);
     
-    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT);
+    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1);
-    self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT);
+    self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1);
-    self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT);
+    self->left_skylight = get_chunk_data(state, DOWN_LEFT, SKYLIGHT, 1);
-    self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT);
+    self->left_blocklight = get_chunk_data(state, DOWN_LEFT, BLOCKLIGHT, 1);
-    self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT);
+    self->right_skylight = get_chunk_data(state, DOWN_RIGHT, SKYLIGHT, 1);
-    self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT);
+    self->right_blocklight = get_chunk_data(state, DOWN_RIGHT, BLOCKLIGHT, 1);
-    self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT);
+    self->up_left_skylight = get_chunk_data(state, UP_LEFT, SKYLIGHT, 1);
-    self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT);
+    self->up_left_blocklight = get_chunk_data(state, UP_LEFT, BLOCKLIGHT, 1);
-    self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT);
+    self->up_right_skylight = get_chunk_data(state, UP_RIGHT, SKYLIGHT, 1);
-    self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT);
+    self->up_right_blocklight = get_chunk_data(state, UP_RIGHT, BLOCKLIGHT, 1);
 
-    // Non-existant neighbor block is not an error
-    PyErr_Clear();
-    
     if (self->night) {
         self->calculate_light_color = calculate_light_color_night;
     } else {

overviewer_core/src/rendermode-spawn.c

@@ -69,8 +69,8 @@ rendermode_spawn_start(void *data, RenderState *state, PyObject *options) {
     
     /* now do custom initializations */
     self = (RenderModeSpawn *)data;
-    self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT);
+    self->blocklight = get_chunk_data(state, CURRENT, BLOCKLIGHT, 1);
-    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT);
+    self->skylight = get_chunk_data(state, CURRENT, SKYLIGHT, 1);
     
     /* setup custom color */
     self->parent.get_color = get_color;

overviewer_core/world.py

@@ -55,6 +55,24 @@ def base36encode(number, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'):
         return "-" + base36
     return base36
 
+def log_other_exceptions(func):
+    """A decorator that prints out any errors that are not ChunkDoesntExist
+    errors. This decorates get_chunk because the C code is likely to swallow
+    exceptions, so this will at least make them visible.
+
+    """
+    functools.wraps(func)
+    def newfunc(*args):
+        try:
+            return func(*args)
+        except ChunkDoesntExist:
+            raise
+        except Exception, e:
+            logging.exception("%s raised this exception", func.func_name)
+            raise
+    return newfunc
+
+
 class World(object):
     """Encapsulates the concept of a Minecraft "world". A Minecraft world is a
     level.dat file, a players directory with info about each player, a data
@@ -237,7 +255,7 @@ class RegionSet(object):
 
         # Caching implementaiton: a simple LRU cache
         # Decorate the get_chunk method with the cache decorator
-        #self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk)
+        self.get_chunk = cache.lru_cache(cachesize)(self.get_chunk)
 
     # Re-initialize upon unpickling
     def __getstate__(self):
@@ -247,6 +265,7 @@ class RegionSet(object):
     def __repr__(self):
         return "<RegionSet regiondir=%r>" % self.regiondir
 
+    @log_other_exceptions
     def get_chunk(self, x, z):
         """Returns a dictionary object representing the "Level" NBT Compound
         structure for a chunk given its x, z coordinates. The coordinates are
@@ -434,83 +453,3 @@ def get_worlds():
             ret[info['Data']['LevelName']] = info['Data']
 
     return ret
-
-
-def lru_cache(maxsize=100):
-    '''Generalized Least-recently-used cache decorator.
-
-    Arguments to the cached function must be hashable.
-    Cache performance statistics stored in f.hits and f.misses.
-    Clear the cache with f.clear().
-    http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
-
-    This snippet is from
-    http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators/
-
-    '''
-    maxqueue = maxsize * 10
-    def decorating_function(user_function,
-            len=len, iter=iter, tuple=tuple, sorted=sorted, KeyError=KeyError):
-        cache = {}                  # mapping of args to results
-        queue = collections.deque() # order that keys have been used
-        refcount = collections.defaultdict(int)# times each key is in the queue
-        sentinel = object()         # marker for looping around the queue
-        kwd_mark = object()         # separate positional and keyword args
-
-        # lookup optimizations (ugly but fast)
-        queue_append, queue_popleft = queue.append, queue.popleft
-        queue_appendleft, queue_pop = queue.appendleft, queue.pop
-
-        @functools.wraps(user_function)
-        def wrapper(*args, **kwds):
-            # cache key records both positional and keyword args
-            key = args
-            if kwds:
-                key += (kwd_mark,) + tuple(sorted(kwds.items()))
-
-            # record recent use of this key
-            queue_append(key)
-            refcount[key] += 1
-
-            # get cache entry or compute if not found
-            try:
-                result = cache[key]
-                wrapper.hits += 1
-            except KeyError:
-                result = user_function(*args, **kwds)
-                cache[key] = result
-                wrapper.misses += 1
-
-                # purge least recently used cache entry
-                if len(cache) > maxsize:
-                    key = queue_popleft()
-                    refcount[key] -= 1
-                    while refcount[key]:
-                        key = queue_popleft()
-                        refcount[key] -= 1
-                    del cache[key], refcount[key]
-
-            # periodically compact the queue by eliminating duplicate keys
-            # while preserving order of most recent access
-            if len(queue) > maxqueue:
-                refcount.clear()
-                queue_appendleft(sentinel)
-                for key in ifilterfalse(refcount.__contains__,
-                                        iter(queue_pop, sentinel)):
-                    queue_appendleft(key)
-                    refcount[key] = 1
-
-
-            return result
-
-        def clear():
-            cache.clear()
-            queue.clear()
-            refcount.clear()
-            wrapper.hits = wrapper.misses = 0
-
-        wrapper.hits = wrapper.misses = 0
-        wrapper.clear = clear
-        return wrapper
-    return decorating_function
-