LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Addded intial region caching.

Browse Source 
Loads all the offsets & timestamps @ start to share to worker proceses.
From
14609247 function calls (14608852 primitive calls) in 118.278 CPU seconds
to
12232301 function calls (12231906 primitive calls) in 75.825 CPU seconds
This commit is contained in:
Xon
2011-03-19 13:36:40 +08:00
parent 1ac922983f
commit 1afb2b3d53
4 changed files with 69 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
chunk.py
View File

@@ -46,12 +46,12 @@ image
# alpha_over extension, BUT this extension may fall back to PIL's
# paste(), which DOES need the workaround.)
def get_lvldata(filename, x, y):
def get_lvldata(world,filename, x, y):
"""Takes a filename and chunkcoords and returns the Level struct, which contains all the
level info"""
try:
d = nbt.load_from_region(filename, x, y)
d = world.load_from_region(filename, x, y)
except Exception, e:
logging.warning("Error opening chunk (%i, %i) in %s. It may be corrupt. %s", x, y, filename, e)
raise ChunkCorrupt(str(e))
@@ -64,10 +64,10 @@ def get_blockarray(level):
Block array, which just contains all the block ids"""
return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
def get_blockarray_fromfile(filename):
def get_blockarray_fromfile(world,filename):
"""Same as get_blockarray except takes a filename and uses get_lvldata to
open it. This is a shortcut"""
level = get_lvldata(filename)
level = get_lvldata(world,filename)
return get_blockarray(level)
def get_skylight_array(level):
@@ -196,7 +196,7 @@ class ChunkRenderer(object):
"""Loads and returns the level structure"""
if not hasattr(self, "_level"):
try:
self._level = get_lvldata(self.regionfile, self.chunkX, self.chunkY)
self._level = get_lvldata(self.world,self.regionfile, self.chunkX, self.chunkY)
except NoSuchChunk, e:
logging.debug("Skipping non-existant chunk")
raise
@@ -228,7 +228,7 @@ class ChunkRenderer(object):
"""Loads and sets data from lower-left chunk"""
chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY)
try:
chunk_data = get_lvldata(chunk_path, self.chunkX - 1, self.chunkY)
chunk_data = get_lvldata(self.world,chunk_path, self.chunkX - 1, self.chunkY)
self._left_skylight = get_skylight_array(chunk_data)
self._left_blocklight = get_blocklight_array(chunk_data)
self._left_blocks = get_blockarray(chunk_data)
@@ -262,7 +262,7 @@ class ChunkRenderer(object):
"""Loads and sets data from lower-right chunk"""
chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1)
try:
chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY + 1)
chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY + 1)
self._right_skylight = get_skylight_array(chunk_data)
self._right_blocklight = get_blocklight_array(chunk_data)
self._right_blocks = get_blockarray(chunk_data)
@@ -296,7 +296,7 @@ class ChunkRenderer(object):
"""Loads and sets data from upper-right chunk"""
chunk_path = self.world.get_region_path(self.chunkX + 1, self.chunkY)
try:
chunk_data = get_lvldata(chunk_path, self.chunkX + 1, self.chunkY)
chunk_data = get_lvldata(self.world,chunk_path, self.chunkX + 1, self.chunkY)
self._up_right_skylight = get_skylight_array(chunk_data)
self._up_right_blocklight = get_blocklight_array(chunk_data)
self._up_right_blocks = get_blockarray(chunk_data)
@@ -316,7 +316,7 @@ class ChunkRenderer(object):
"""Loads and sets data from upper-left chunk"""
chunk_path = self.world.get_region_path(self.chunkX, self.chunkY - 1)
try:
chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY - 1)
chunk_data = get_lvldata(self.world,chunk_path, self.chunkX, self.chunkY - 1)
self._up_left_skylight = get_skylight_array(chunk_data)
self._up_left_blocklight = get_blocklight_array(chunk_data)
self._up_left_blocks = get_blockarray(chunk_data)

40
nbt.py
View File

@@ -34,14 +34,16 @@ def _file_loader(func):
def load(fileobj):
return NBTFileReader(fileobj).read_all()
@_file_loader
def load_from_region(fileobj, x, y):
nbt = MCRFileReader(fileobj).load_chunk(x, y)
if not nbt:
def load_from_region(filename, x, y):
nbt = load_region(filename).load_chunk(x, y)
if nbt is None:
return None ## return none. I think this is who we should indicate missing chunks
#raise IOError("No such chunk in region: (%i, %i)" % (x, y))
return nbt.read_all()
def load_region(filename):
return MCRFileReader(filename)
class NBTFileReader(object):
def __init__(self, fileobj, is_gzip=True):
if is_gzip:
@@ -178,9 +180,9 @@ class MCRFileReader(object):
chunks (as instances of NBTFileReader), getting chunk timestamps,
and for listing chunks contained in the file."""
def __init__(self, fileobj):
self._file = fileobj
def __init__(self, filename):
self._file = None
self._filename = filename
# cache used when the entire header tables are read in get_chunks()
self._locations = None
self._timestamps = None
@@ -250,7 +252,7 @@ class MCRFileReader(object):
return timestamp
def get_chunks(self):
def get_chunk_info(self,closeFile = True):
"""Return a list of all chunks contained in this region file,
as a list of (x, y) coordinate tuples. To load these chunks,
provide these coordinates to load_chunk()."""
@@ -258,6 +260,9 @@ class MCRFileReader(object):
if self._chunks:
return self._chunks
if self._file is None:
self._file = open(self._filename,'rb');
self._chunks = []
self._locations = []
self._timestamps = []
@@ -279,6 +284,10 @@ class MCRFileReader(object):
timestamp = self._read_chunk_timestamp()
self._timestamps.append(timestamp)
if closeFile:
#free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
self._file.close()
self._file = None
return self._chunks
def get_chunk_timestamp(self, x, y):
@@ -289,9 +298,7 @@ class MCRFileReader(object):
x = x % 32
y = y % 32
if self._timestamps is None:
#self.get_chunks()
return self._read_chunk_timestamp(x, y)
else:
self.get_chunk_info()
return self._timestamps[x + y * 32]
def chunkExists(self, x, y):
@@ -299,9 +306,7 @@ class MCRFileReader(object):
x = x % 32
y = y % 32
if self._locations is None:
#self.get_chunks()
location = self._read_chunk_location(x, y)
else:
self.get_chunk_info()
location = self._locations[x + y * 32]
return location is not None
@@ -315,13 +320,14 @@ class MCRFileReader(object):
x = x % 32
y = y % 32
if self._locations is None:
#self.get_chunks()
location = self._read_chunk_location(x % 32, y % 32)
else:
self.get_chunk_info()
location = self._locations[x + y * 32]
if location is None:
return None
if self._file is None:
self._file = open(self._filename,'rb');
# seek to the data
self._file.seek(location[0])

8
quadtree.py
View File

@@ -556,12 +556,12 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
imgpath = path + "." + quadtree.imgformat
world = quadtree.world
# first, remove chunks from `chunks` that don't actually exist in
# their region files
def chunk_exists(chunk):
_, _, chunkx, chunky, region = chunk
with open(region, 'rb') as region:
r = nbt.MCRFileReader(region)
r = world.load_region(region)
return r.chunkExists(chunkx, chunky)
chunks = filter(chunk_exists, chunks)
@@ -607,11 +607,9 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
continue
# checking chunk mtime
with open(regionfile, 'rb') as regionfile:
region = nbt.MCRFileReader(regionfile)
region = world.load_region(regionfile)
if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
needs_rerender = True
if needs_rerender:
break
# if after all that, we don't need a rerender, return

24
world.py
View File

@@ -11,7 +11,7 @@
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the Overviewer. If not, see <http://www.gnu.org/licenses/>.
# with the Overviewer. If not, see <http://wfww.gnu.org/licenses/>.
import functools
import os
@@ -72,10 +72,16 @@ class World(object):
self.useBiomeData = useBiomeData
#find region files, or load the region list
#this also caches all the region file header info
regionfiles = {}
regions = {}
for x, y, regionfile in self._iterate_regionfiles():
mcr = nbt.MCRFileReader(regionfile)
mcr.get_chunk_info()
regions[regionfile] = mcr
regionfiles[(x,y)] = (x,y,regionfile)
self.regionfiles = regionfiles
self.regions = regions
# figure out chunk format is in use
# if not mcregion, error out early
@@ -111,6 +117,22 @@ class World(object):
_, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
return regionfile
def load_from_region(self,filename, x, y):
nbt = self.load_region(filename).load_chunk(x, y)
if nbt is None:
return None ## return none. I think this is who we should indicate missing chunks
#raise IOError("No such chunk in region: (%i, %i)" % (x, y))
return nbt.read_all()
#filo region cache
def load_region(self,filename):
#return nbt.MCRFileReader(filename)
return self.regions[filename]
def convert_coords(self, chunkx, chunky):
"""Takes a coordinate (chunkx, chunky) where chunkx and chunky are
in the chunk coordinate system, and figures out the row and column