Addded intial region caching.

Loads all the offsets & timestamps @ start to share to worker proceses.
From
14609247 function calls (14608852 primitive calls) in 118.278 CPU seconds
to
12232301 function calls (12231906 primitive calls) in 75.825 CPU seconds

chunk.py

@@ -46,12 +46,12 @@ image
 # alpha_over extension, BUT this extension may fall back to PIL's
 # paste(), which DOES need the workaround.)
 
-def get_lvldata(filename, x, y):
+def get_lvldata(world,filename, x, y):
     """Takes a filename and chunkcoords and returns the Level struct, which contains all the
     level info"""
     
     try:
-        d =  nbt.load_from_region(filename, x, y)
+        d =  world.load_from_region(filename, x, y)
     except Exception, e:
         logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e)
         raise ChunkCorrupt(str(e))
@@ -64,10 +64,10 @@ def get_blockarray(level):
     Block array, which just contains all the block ids"""
     return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
 
-def get_blockarray_fromfile(filename):
+def get_blockarray_fromfile(world,filename):
     """Same as get_blockarray except takes a filename and uses get_lvldata to
     open it. This is a shortcut"""
-    level = get_lvldata(filename)
+    level = get_lvldata(world,filename)
     return get_blockarray(level)
 
 def get_skylight_array(level):
@@ -196,7 +196,7 @@ class ChunkRenderer(object):
         """Loads and returns the level structure"""
         if not hasattr(self, "_level"):
             try:
-                self._level = get_lvldata(self.regionfile, self.chunkX, self.chunkY)
+                self._level = get_lvldata(self.world,self.regionfile, self.chunkX, self.chunkY)
             except NoSuchChunk, e:
                 logging.debug("Skipping non-existant chunk")
                 raise
@@ -228,7 +228,7 @@ class ChunkRenderer(object):
         """Loads and sets data from lower-left chunk"""
         chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY)
         try:
-            chunk_data = get_lvldata(chunk_path, self.chunkX - 1, self.chunkY)
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX - 1, self.chunkY)
             self._left_skylight = get_skylight_array(chunk_data)
             self._left_blocklight = get_blocklight_array(chunk_data)
             self._left_blocks = get_blockarray(chunk_data)
@@ -262,7 +262,7 @@ class ChunkRenderer(object):
         """Loads and sets data from lower-right chunk"""
         chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1)
         try:
-            chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY + 1)
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY + 1)
             self._right_skylight = get_skylight_array(chunk_data)
             self._right_blocklight = get_blocklight_array(chunk_data)
             self._right_blocks = get_blockarray(chunk_data)
@@ -296,7 +296,7 @@ class ChunkRenderer(object):
         """Loads and sets data from upper-right chunk"""
         chunk_path = self.world.get_region_path(self.chunkX + 1, self.chunkY)
         try:
-            chunk_data = get_lvldata(chunk_path, self.chunkX + 1, self.chunkY)
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX + 1, self.chunkY)
             self._up_right_skylight = get_skylight_array(chunk_data)
             self._up_right_blocklight = get_blocklight_array(chunk_data)
             self._up_right_blocks = get_blockarray(chunk_data)
@@ -316,7 +316,7 @@ class ChunkRenderer(object):
         """Loads and sets data from upper-left chunk"""
         chunk_path = self.world.get_region_path(self.chunkX, self.chunkY - 1)
         try:
-            chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY - 1)
+            chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY - 1)
             self._up_left_skylight = get_skylight_array(chunk_data)
             self._up_left_blocklight = get_blocklight_array(chunk_data)
             self._up_left_blocks = get_blockarray(chunk_data)

nbt.py

@@ -34,14 +34,16 @@ def _file_loader(func):
 def load(fileobj):
     return NBTFileReader(fileobj).read_all()
 
-@_file_loader
+def load_from_region(filename, x, y):
-def load_from_region(fileobj, x, y):
+    nbt = load_region(filename).load_chunk(x, y)
-    nbt = MCRFileReader(fileobj).load_chunk(x, y)
+    if nbt is None:
-    if not nbt:
         return None ## return none.  I think this is who we should indicate missing chunks
         #raise IOError("No such chunk in region: (%i, %i)" % (x, y))     
     return nbt.read_all()
   
+def load_region(filename):            
+    return MCRFileReader(filename)
+  
 class NBTFileReader(object):
     def __init__(self, fileobj, is_gzip=True):
         if is_gzip:
@@ -178,9 +180,9 @@ class MCRFileReader(object):
     chunks (as instances of NBTFileReader), getting chunk timestamps,
     and for listing chunks contained in the file."""
     
-    def __init__(self, fileobj):
+    def __init__(self, filename):
-        self._file = fileobj
+        self._file = None
-        
+        self._filename = filename
         # cache used when the entire header tables are read in get_chunks()
         self._locations = None
         self._timestamps = None
@@ -250,7 +252,7 @@ class MCRFileReader(object):
         
         return timestamp
     
-    def get_chunks(self):
+    def get_chunk_info(self,closeFile = True):
         """Return a list of all chunks contained in this region file,
         as a list of (x, y) coordinate tuples. To load these chunks,
         provide these coordinates to load_chunk()."""
@@ -258,6 +260,9 @@ class MCRFileReader(object):
         if self._chunks:
             return self._chunks
         
+        if self._file is None:
+            self._file = open(self._filename,'rb');
+
         self._chunks = []
         self._locations = []
         self._timestamps = []
@@ -279,6 +284,10 @@ class MCRFileReader(object):
                 timestamp = self._read_chunk_timestamp()
                 self._timestamps.append(timestamp)
 
+        if closeFile:        
+            #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
+            self._file.close()
+            self._file =  None
         return self._chunks
     
     def get_chunk_timestamp(self, x, y):
@@ -289,20 +298,16 @@ class MCRFileReader(object):
         x = x % 32
         y = y % 32        
         if self._timestamps is None:
-            #self.get_chunks()
+            self.get_chunk_info() 
-            return self._read_chunk_timestamp(x, y)
+        return self._timestamps[x + y * 32]   
-        else:     
-            return self._timestamps[x + y * 32]   
     
     def chunkExists(self, x, y):
         """Determines if a chunk exists without triggering loading of the backend data"""
         x = x % 32
         y = y % 32
         if self._locations is None:
-            #self.get_chunks()
+            self.get_chunk_info()
-            location = self._read_chunk_location(x, y)
+        location = self._locations[x + y * 32]
-        else:
-            location = self._locations[x + y * 32]
         return location is not None        
 
     def load_chunk(self, x, y):
@@ -315,13 +320,14 @@ class MCRFileReader(object):
         x = x % 32
         y = y % 32
         if self._locations is None:
-            #self.get_chunks()
+            self.get_chunk_info()   
-            location = self._read_chunk_location(x % 32, y % 32)
+                    
-        else:            
+        location = self._locations[x + y * 32]
-            location = self._locations[x + y * 32]
         if location is None:
             return None
 
+        if self._file is None:
+            self._file = open(self._filename,'rb'); 
         # seek to the data
         self._file.seek(location[0])
         

quadtree.py

@@ -556,13 +556,13 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
 
     imgpath = path + "." + quadtree.imgformat
     
+    world = quadtree.world
     # first, remove chunks from `chunks` that don't actually exist in
     # their region files
     def chunk_exists(chunk):
         _, _, chunkx, chunky, region = chunk
-        with open(region, 'rb') as region:
+        r = world.load_region(region)
-            r = nbt.MCRFileReader(region)
+        return r.chunkExists(chunkx, chunky)            
-            return r.chunkExists(chunkx, chunky)
     chunks = filter(chunk_exists, chunks)
 
     #stat the file, we need to know if it exists or it's mtime
@@ -607,11 +607,9 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
                 continue
             
             # checking chunk mtime
-            with open(regionfile, 'rb') as regionfile:
+            region = world.load_region(regionfile)
-                region = nbt.MCRFileReader(regionfile)
+            if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
-                if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
+                needs_rerender = True
-                    needs_rerender = True
-            if needs_rerender:
                 break
         
         # if after all that, we don't need a rerender, return

world.py

@@ -11,7 +11,7 @@
 #    Public License for more details.
 #
 #    You should have received a copy of the GNU General Public License along
-#    with the Overviewer.  If not, see <http://www.gnu.org/licenses/>.
+#    with the Overviewer.  If not, see <http://wfww.gnu.org/licenses/>.
 
 import functools
 import os
@@ -72,10 +72,16 @@ class World(object):
         self.useBiomeData = useBiomeData
 
         #find region files, or load the region list
+        #this also caches all the region file header info
         regionfiles = {}
+        regions = {}
         for x, y, regionfile in self._iterate_regionfiles():            
+            mcr = nbt.MCRFileReader(regionfile)
+            mcr.get_chunk_info()
+            regions[regionfile] = mcr
             regionfiles[(x,y)]	= (x,y,regionfile)
         self.regionfiles = regionfiles	
+        self.regions = regions
         
         # figure out chunk format is in use
         # if not mcregion, error out early
@@ -111,6 +117,22 @@ class World(object):
         _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
         return regionfile
     
+    
+    
+    def load_from_region(self,filename, x, y):
+        nbt = self.load_region(filename).load_chunk(x, y)
+        if nbt is None:
+            return None ## return none.  I think this is who we should indicate missing chunks
+            #raise IOError("No such chunk in region: (%i, %i)" % (x, y))     
+        return nbt.read_all()
+      
+      
+    #filo region cache
+    def load_region(self,filename):                
+        #return nbt.MCRFileReader(filename)    
+        return self.regions[filename]
+        
+        
     def convert_coords(self, chunkx, chunky):
         """Takes a coordinate (chunkx, chunky) where chunkx and chunky are
         in the chunk coordinate system, and figures out the row and column