LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Working rendering with the new McRegion format.

Browse Source 
Notes:
 * Currently only works with -p 1
 * Caching is mostly compatible with existing caches, but not completly.
   This needs more testing and more code reviews
 * There are probably many code paths that will throw exceptions.
 * Not ready for general use yet, but is OK for testing
This commit is contained in:
Andrew Chin
2011-02-23 22:13:35 -05:00
parent 5f5bda948e
commit 05ea562060
4 changed files with 142 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
chunk.py
View File

@@ -20,6 +20,7 @@ import hashlib
import logging import logging
import time import time
import math import math
import sys
import nbt import nbt
import textures import textures
@@ -45,10 +46,13 @@ image
# alpha_over extension, BUT this extension may fall back to PIL's # alpha_over extension, BUT this extension may fall back to PIL's
# paste(), which DOES need the workaround.) # paste(), which DOES need the workaround.)
def get_lvldata(filename): def get_lvldata(filename, x, y):
"""Takes a filename and returns the Level struct, which contains all the """Takes a filename and chunkcoords and returns the Level struct, which contains all the
level info""" level info"""
return nbt.load(filename)[1]['Level']
d = nbt.load_from_region(filename, x, y)
if not d: raise NoSuchChunk(x,y)
return d[1]['Level']
def get_blockarray(level): def get_blockarray(level):
"""Takes the level struct as returned from get_lvldata, and returns the """Takes the level struct as returned from get_lvldata, and returns the
@@ -124,14 +128,13 @@ fluid_blocks = set([8,9,10,11])
# (glass, half blocks) # (glass, half blocks)
nospawn_blocks = set([20,44]) nospawn_blocks = set([20,44])
def find_oldimage(chunkfile, cached, cave): def find_oldimage(chunkXY, cached, cave):
destdir, filename = os.path.split(chunkfile) # TODO update this
filename_split = filename.split(".") blockid = "%d.%d" % chunkXY
blockid = ".".join(filename_split[1:3])
# Get the name of the existing image. # Get the name of the existing image.
moredirs, dir2 = os.path.split(destdir) dir1 = world.base36encode(chunkXY[0]%64)
dir1 = os.path.basename(moredirs) dir2 = world.base36encode(chunkXY[1]%64)
cachename = '/'.join((dir1, dir2)) cachename = '/'.join((dir1, dir2))
oldimg = oldimg_path = None oldimg = oldimg_path = None
@@ -150,12 +153,16 @@ def check_cache(chunkfile, oldimg):
except OSError: except OSError:
return False return False
def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=None): # chunkcoords should be the coordinates of a possible chunk. it may not exist
def render_and_save(chunkcoords, cachedir, worldobj, oldimg, cave=False, queue=None):
"""Used as the entry point for the multiprocessing workers (since processes """Used as the entry point for the multiprocessing workers (since processes
can't target bound methods) or to easily render and save one chunk can't target bound methods) or to easily render and save one chunk
Returns the image file location""" chunkcoords is a tuple: (chunkX, chunkY)
a = ChunkRenderer(chunkfile, cachedir, worldobj, oldimg, queue)
If the chunk doesn't exist, return None.
Else, returns the image file location"""
a = ChunkRenderer(chunkcoords, cachedir, worldobj, oldimg, queue)
try: try:
return a.render_and_save(cave) return a.render_and_save(cave)
except ChunkCorrupt: except ChunkCorrupt:
@@ -177,36 +184,53 @@ def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=Non
class ChunkCorrupt(Exception): class ChunkCorrupt(Exception):
pass pass
class NoSuchChunk(Exception):
pass
class ChunkRenderer(object): class ChunkRenderer(object):
def __init__(self, chunkfile, cachedir, worldobj, oldimg, queue): def __init__(self, chunkcoords, cachedir, worldobj, oldimg, queue):
"""Make a new chunk renderer for the given chunkfile. """Make a new chunk renderer for the given chunk coordinates.
chunkfile should be a full path to the .dat file to process chunkcoors should be a tuple: (chunkX, chunkY)
cachedir is a directory to save the resulting chunk images to cachedir is a directory to save the resulting chunk images to
""" """
self.queue = queue self.queue = queue
# derive based on worlddir and chunkcoords
self.regionfile = os.path.join(worldobj.worlddir, "region",
"r.%d.%d.mcr" % (chunkcoords[0] // 32, chunkcoords[1]//32))
if not os.path.exists(chunkfile): if not os.path.exists(self.regionfile):
raise ValueError("Could not find chunkfile") raise ValueError("Could not find regionfile: %s" % self.regionfile)
self.chunkfile = chunkfile
destdir, filename = os.path.split(self.chunkfile) ## TODO TODO all of this class
filename_split = filename.split(".")
chunkcoords = filename_split[1:3] #destdir, filename = os.path.split(self.chunkfile)
#filename_split = filename.split(".")
#chunkcoords = filename_split[1:3]
self.coords = map(world.base36decode, chunkcoords) #self.coords = map(world.base36decode, chunkcoords)
self.blockid = ".".join(chunkcoords) self.blockid = "%d.%d" % chunkcoords
# chunk coordinates (useful to converting local block coords to # chunk coordinates (useful to converting local block coords to
# global block coords) # global block coords)
self.chunkX = int(filename_split[1], base=36) self.chunkX = chunkcoords[0]
self.chunkY = int(filename_split[2], base=36) self.chunkY = chunkcoords[1]
self.world = worldobj self.world = worldobj
# Cachedir here is the base directory of the caches. We need to go 2 # Cachedir here is the base directory of the caches. We need to go 2
# levels deeper according to the chunk file. Get the last 2 components # levels deeper according to the chunk file. Get the last 2 components
# of destdir and use that # of destdir and use that
moredirs, dir2 = os.path.split(destdir) ##moredirs, dir2 = os.path.split(destdir)
_, dir1 = os.path.split(moredirs) ##_, dir1 = os.path.split(moredirs)
self.cachedir = os.path.join(cachedir, dir1, dir2) self.cachedir = os.path.join(cachedir,
world.base36encode(self.chunkX%64),
world.base36encode(self.chunkY%64))
#logging.debug("cache location for this chunk: %s", self.cachedir)
self.oldimg, self.oldimg_path = oldimg self.oldimg, self.oldimg_path = oldimg
@@ -229,9 +253,12 @@ class ChunkRenderer(object):
"""Loads and returns the level structure""" """Loads and returns the level structure"""
if not hasattr(self, "_level"): if not hasattr(self, "_level"):
try: try:
self._level = get_lvldata(self.chunkfile) self._level = get_lvldata(self.regionfile, self.chunkX, self.chunkY)
except NoSuchChunk, e:
#logging.debug("Skipping non-existant chunk")
raise
except Exception, e: except Exception, e:
logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.chunkfile, e) logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.regionfile, e)
raise ChunkCorrupt(str(e)) raise ChunkCorrupt(str(e))
return self._level return self._level
level = property(_load_level) level = property(_load_level)
@@ -259,13 +286,13 @@ class ChunkRenderer(object):
def _load_left(self): def _load_left(self):
"""Loads and sets data from lower-left chunk""" """Loads and sets data from lower-left chunk"""
chunk_path = self.world.get_chunk_path(self.coords[0] - 1, self.coords[1]) chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY)
try: try:
chunk_data = get_lvldata(chunk_path) chunk_data = get_lvldata(chunk_path, self.chunkX-1, self.chunkY)
self._left_skylight = get_skylight_array(chunk_data) self._left_skylight = get_skylight_array(chunk_data)
self._left_blocklight = get_blocklight_array(chunk_data) self._left_blocklight = get_blocklight_array(chunk_data)
self._left_blocks = get_blockarray(chunk_data) self._left_blocks = get_blockarray(chunk_data)
except IOError: except NoSuchChunk:
self._left_skylight = None self._left_skylight = None
self._left_blocklight = None self._left_blocklight = None
self._left_blocks = None self._left_blocks = None
@@ -293,13 +320,13 @@ class ChunkRenderer(object):
def _load_right(self): def _load_right(self):
"""Loads and sets data from lower-right chunk""" """Loads and sets data from lower-right chunk"""
chunk_path = self.world.get_chunk_path(self.coords[0], self.coords[1] + 1) chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1)
try: try:
chunk_data = get_lvldata(chunk_path) chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY+1)
self._right_skylight = get_skylight_array(chunk_data) self._right_skylight = get_skylight_array(chunk_data)
self._right_blocklight = get_blocklight_array(chunk_data) self._right_blocklight = get_blocklight_array(chunk_data)
self._right_blocks = get_blockarray(chunk_data) self._right_blocks = get_blockarray(chunk_data)
except IOError: except NoSuchChunk:
self._right_skylight = None self._right_skylight = None
self._right_blocklight = None self._right_blocklight = None
self._right_blocks = None self._right_blocks = None
@@ -448,6 +475,7 @@ class ChunkRenderer(object):
is up to date, this method doesn't render anything. is up to date, this method doesn't render anything.
""" """
blockid = self.blockid blockid = self.blockid
# Reasons for the code to get to this point: # Reasons for the code to get to this point:
# 1) An old image doesn't exist # 1) An old image doesn't exist
@@ -457,13 +485,18 @@ class ChunkRenderer(object):
# the image was invalid and deleted (sort of the same as (1)) # the image was invalid and deleted (sort of the same as (1))
# What /should/ the image be named, go ahead and hash the block array # What /should/ the image be named, go ahead and hash the block array
dest_filename = "img.{0}.{1}.{2}.png".format( try:
dest_filename = "img.{0}.{1}.{2}.png".format(
blockid, blockid,
"cave" if cave else "nocave", "cave" if cave else "nocave",
self._hash_blockarray(), self._hash_blockarray(),
) )
except NoSuchChunk, e:
return None
dest_path = os.path.join(self.cachedir, dest_filename) dest_path = os.path.join(self.cachedir, dest_filename)
#logging.debug("cache filename: %s", dest_path)
if self.oldimg: if self.oldimg:
if dest_filename == self.oldimg: if dest_filename == self.oldimg:
@@ -479,6 +512,7 @@ class ChunkRenderer(object):
# either corrupt or out of date # either corrupt or out of date
os.unlink(self.oldimg_path) os.unlink(self.oldimg_path)
# Render the chunk # Render the chunk
img = self.chunk_render(cave=cave) img = self.chunk_render(cave=cave)
# Save it # Save it
@@ -488,6 +522,7 @@ class ChunkRenderer(object):
os.unlink(dest_path) os.unlink(dest_path)
raise raise
# Return its location # Return its location
#raise Exception("early exit")
return dest_path return dest_path
def calculate_darkness(self, skylight, blocklight): def calculate_darkness(self, skylight, blocklight):

3
nbt.py
View File

@@ -34,7 +34,8 @@ def load(fileobj):
def load_from_region(fileobj, x, y): def load_from_region(fileobj, x, y):
nbt = MCRFileReader(fileobj).load_chunk(x, y) nbt = MCRFileReader(fileobj).load_chunk(x, y)
if not nbt: if not nbt:
raise IOError("No such chunk in region: (%i, %i)" % (x, y)) return None ## return none. I think this is who we should indicate missing chunks
#raise IOError("No such chunk in region: (%i, %i)" % (x, y))
return nbt.read_all() return nbt.read_all()
class NBTFileReader(object): class NBTFileReader(object):

4
quadtree.py
View File

@@ -27,7 +27,7 @@ import logging
import util import util
import cPickle import cPickle
import stat import stat
from time import gmtime, strftime from time import gmtime, strftime, sleep
from PIL import Image from PIL import Image
@@ -297,10 +297,12 @@ class QuadtreeGen(object):
# This image is rendered at: # This image is rendered at:
dest = os.path.join(self.destdir, "tiles", *(str(x) for x in path)) dest = os.path.join(self.destdir, "tiles", *(str(x) for x in path))
#logging.debug("this is rendered at %s", dest)
# And uses these chunks # And uses these chunks
tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, tilechunks = self._get_chunks_in_range(colstart, colend, rowstart,
rowend) rowend)
#logging.debug(" tilechunks: %r", tilechunks)
# Put this in the pool # Put this in the pool
# (even if tilechunks is empty, render_worldtile will delete # (even if tilechunks is empty, render_worldtile will delete

109
world.py
View File

@@ -22,6 +22,7 @@ import sys
import logging import logging
import cPickle import cPickle
import collections import collections
import itertools
import numpy import numpy
@@ -45,7 +46,10 @@ def _convert_coords(chunks):
the image each one should be. the image each one should be.
returns mincol, maxcol, minrow, maxrow, chunks_translated returns mincol, maxcol, minrow, maxrow, chunks_translated
chunks_translated is a list of (col, row, filename) chunks_translated is a list of (col, row, (chunkX, chunkY))
The (chunkX, chunkY) tuple is the chunkCoords, used to identify the
chunk file
""" """
chunks_translated = [] chunks_translated = []
# columns are determined by the sum of the chunk coords, rows are the # columns are determined by the sum of the chunk coords, rows are the
@@ -60,7 +64,7 @@ def _convert_coords(chunks):
row = c[1] - c[0] row = c[1] - c[0]
minrow = min(minrow, row) minrow = min(minrow, row)
maxrow = max(maxrow, row) maxrow = max(maxrow, row)
chunks_translated.append((col, row, c[2])) chunks_translated.append((col, row, (c[0],c[1])))
return mincol, maxcol, minrow, maxrow, chunks_translated return mincol, maxcol, minrow, maxrow, chunks_translated
@@ -118,7 +122,7 @@ class WorldRenderer(object):
#print data #print data
if 'version' in data and data['version'] == 19132: if 'version' in data and data['version'] == 19132:
logging.error("Sorry, Minecraft-Overviewer doesn't yet know how to read McRegion chunks") logging.error("Sorry, Minecraft-Overviewer doesn't yet know how to read McRegion chunks")
sys.exit(1) #sys.exit(1)
if self.useBiomeData: if self.useBiomeData:
textures.prepareBiomeData(worlddir) textures.prepareBiomeData(worlddir)
@@ -127,6 +131,7 @@ class WorldRenderer(object):
# In order to avoid having to look up the cache file names in # In order to avoid having to look up the cache file names in
# ChunkRenderer, get them all and store them here # ChunkRenderer, get them all and store them here
# TODO change how caching works
for root, dirnames, filenames in os.walk(cachedir): for root, dirnames, filenames in os.walk(cachedir):
for filename in filenames: for filename in filenames:
if not filename.endswith('.png') or not filename.startswith("img."): if not filename.endswith('.png') or not filename.startswith("img."):
@@ -138,7 +143,6 @@ class WorldRenderer(object):
bits = '.'.join((x, z, cave)) bits = '.'.join((x, z, cave))
cached[dir][bits] = os.path.join(root, filename) cached[dir][bits] = os.path.join(root, filename)
# stores Points Of Interest to be mapped with markers # stores Points Of Interest to be mapped with markers
# a list of dictionaries, see below for an example # a list of dictionaries, see below for an example
self.POI = [] self.POI = []
@@ -163,6 +167,7 @@ class WorldRenderer(object):
if not self.chunklist: if not self.chunklist:
return None return None
raise Exception("not yet working") ## TODO correctly reimplement this for mcregion
# Get a list of the (chunks, chunky, filename) from the passed in list # Get a list of the (chunks, chunky, filename) from the passed in list
# of filenames # of filenames
chunklist = [] chunklist = []
@@ -190,15 +195,12 @@ class WorldRenderer(object):
return inclusion_set return inclusion_set
def get_chunk_path(self, chunkX, chunkY): def get_region_path(self, chunkX, chunkY):
"""Returns the path to the chunk file at (chunkX, chunkY), if """Returns the path to the region that contains chunk (chunkX, chunkY)
it exists.""" """
chunkFile = "%s/%s/c.%s.%s.dat" % (base36encode(chunkX % 64),
base36encode(chunkY % 64),
base36encode(chunkX),
base36encode(chunkY))
chunkFile = "region/r.%s.%s.mcr" % (chunkX//32, chunkY//32)
return os.path.join(self.worlddir, chunkFile) return os.path.join(self.worlddir, chunkFile)
def findTrueSpawn(self): def findTrueSpawn(self):
@@ -217,9 +219,9 @@ class WorldRenderer(object):
chunkY = spawnZ/16 chunkY = spawnZ/16
## The filename of this chunk ## The filename of this chunk
chunkFile = self.get_chunk_path(chunkX, chunkY) chunkFile = self.get_region_path(chunkX, chunkY)
data=nbt.load(chunkFile)[1] data=nbt.load_from_region(chunkFile, chunkX, chunkY)[1]
level = data['Level'] level = data['Level']
blockArray = numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128)) blockArray = numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
@@ -240,14 +242,17 @@ class WorldRenderer(object):
"""Starts the render. This returns when it is finished""" """Starts the render. This returns when it is finished"""
logging.info("Scanning chunks") logging.info("Scanning chunks")
raw_chunks = self._find_chunkfiles() raw_chunks = self._get_chunklist()
logging.debug("Done scanning chunks") logging.debug("Done scanning chunks")
# Translate chunks to our diagonal coordinate system # Translate chunks to our diagonal coordinate system
# TODO
mincol, maxcol, minrow, maxrow, chunks = _convert_coords(raw_chunks) mincol, maxcol, minrow, maxrow, chunks = _convert_coords(raw_chunks)
del raw_chunks # Free some memory del raw_chunks # Free some memory
self.chunkmap = self._render_chunks_async(chunks, procs) self.chunkmap = self._render_chunks_async(chunks, procs)
logging.debug("world chunkmap has len %d", len(self.chunkmap))
self.mincol = mincol self.mincol = mincol
self.maxcol = maxcol self.maxcol = maxcol
@@ -272,35 +277,47 @@ class WorldRenderer(object):
os.path.join(dirpath, f))) os.path.join(dirpath, f)))
return all_chunks return all_chunks
def _find_chunkfiles(self): def _get_chunklist(self):
"""Returns a list of all the chunk file locations, and the file they """Returns a list of all possible chunk coordinates, based on the
correspond to. available regions files. Note that not all chunk coordinates will
exists. The chunkrender will know how to ignore non-existant chunks
Returns a list of (chunkx, chunky, filename) where chunkx and chunky are
given in chunk coordinates. Use convert_coords() to turn the resulting list returns a list of (chunkx, chunky, regionfile) where regionfile is
into an oblique coordinate system. the region file that contains this chunk
Usually this scans the given worlddir, but will use the chunk list TODO, a --cachedir implemetation should involved thie method
given to the constructor if one was provided."""
"""
all_chunks = [] all_chunks = []
for dirpath, dirnames, filenames in os.walk(self.worlddir): regions = self._find_regionfiles()
if not dirnames and filenames and "DIM-1" not in dirpath: print "found %d regions" % len(regions)
for f in filenames: for region in regions:
if f.startswith("c.") and f.endswith(".dat"): print "region %d, %d --> %s" % region
p = f.split(".") these_chunks = list(itertools.product(
all_chunks.append((base36decode(p[1]), base36decode(p[2]), range(region[0]*32,region[0]*32 + 32),
os.path.join(dirpath, f))) range(region[1]*32,region[1]*32 + 32)
))
print "region %d,%d will go from:"
print " %r" % range(region[0]*32,region[0]*32 + 32)
print " %r" % range(region[1]*32,region[1]*32 + 32)
these_chunks = map(lambda x: (x[0], x[1], region[2]), these_chunks)
assert(len(these_chunks) == 1024)
all_chunks += these_chunks
if not all_chunks: if not all_chunks:
logging.error("Error: No chunks found!") logging.error("Error: No chunks found!")
sys.exit(1) sys.exit(1)
logging.debug("Total possible chunks: %d", len(all_chunks))
return all_chunks return all_chunks
def _render_chunks_async(self, chunks, processes): def _render_chunks_async(self, chunks, processes):
"""Starts up a process pool and renders all the chunks asynchronously. """Starts up a process pool and renders all the chunks asynchronously.
chunks is a list of (col, row, chunkfile) chunks is a list of (col, row, (chunkX, chunkY)). Use chunkX,chunkY
to find the chunk data in a region file
Returns a dictionary mapping (col, row) to the file where that Returns a dictionary mapping (col, row) to the file where that
chunk is rendered as an image chunk is rendered as an image
@@ -318,21 +335,23 @@ class WorldRenderer(object):
if processes == 1: if processes == 1:
# Skip the multiprocessing stuff # Skip the multiprocessing stuff
logging.debug("Rendering chunks synchronously since you requested 1 process") logging.debug("Rendering chunks synchronously since you requested 1 process")
for i, (col, row, chunkfile) in enumerate(chunks): for i, (col, row, chunkXY) in enumerate(chunks):
if inclusion_set and (col, row) not in inclusion_set: ##TODO##/if inclusion_set and (col, row) not in inclusion_set:
# Skip rendering, just find where the existing image is ##TODO##/ # Skip rendering, just find where the existing image is
_, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) ##TODO##/ _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves)
if imgpath: ##TODO##/ if imgpath:
results[(col, row)] = imgpath ##TODO##/ results[(col, row)] = imgpath
continue ##TODO##/ continue
oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) oldimg = chunk.find_oldimage(chunkXY, cached, self.caves)
if chunk.check_cache(chunkfile, oldimg): # TODO remove this shortcircuit
if oldimg[1]:## or chunk.check_cache(chunkfile, oldimg):
result = oldimg[1] result = oldimg[1]
else: else:
result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) result = chunk.render_and_save(chunkXY, self.cachedir, self, oldimg, queue=q)
results[(col, row)] = result if result:
results[(col, row)] = result
if i > 0: if i > 0:
try: try:
item = q.get(block=False) item = q.get(block=False)