diff --git a/chunk.py b/chunk.py index a5bddf8..558ad83 100644 --- a/chunk.py +++ b/chunk.py @@ -20,6 +20,7 @@ import hashlib import logging import time import math +import sys import nbt import textures @@ -45,10 +46,13 @@ image # alpha_over extension, BUT this extension may fall back to PIL's # paste(), which DOES need the workaround.) -def get_lvldata(filename): - """Takes a filename and returns the Level struct, which contains all the +def get_lvldata(filename, x, y): + """Takes a filename and chunkcoords and returns the Level struct, which contains all the level info""" - return nbt.load(filename)[1]['Level'] + + d = nbt.load_from_region(filename, x, y) + if not d: raise NoSuchChunk(x,y) + return d[1]['Level'] def get_blockarray(level): """Takes the level struct as returned from get_lvldata, and returns the @@ -124,14 +128,13 @@ fluid_blocks = set([8,9,10,11]) # (glass, half blocks) nospawn_blocks = set([20,44]) -def find_oldimage(chunkfile, cached, cave): - destdir, filename = os.path.split(chunkfile) - filename_split = filename.split(".") - blockid = ".".join(filename_split[1:3]) +def find_oldimage(chunkXY, cached, cave): +# TODO update this + blockid = "%d.%d" % chunkXY # Get the name of the existing image. - moredirs, dir2 = os.path.split(destdir) - dir1 = os.path.basename(moredirs) + dir1 = world.base36encode(chunkXY[0]%64) + dir2 = world.base36encode(chunkXY[1]%64) cachename = '/'.join((dir1, dir2)) oldimg = oldimg_path = None @@ -150,12 +153,16 @@ def check_cache(chunkfile, oldimg): except OSError: return False -def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=None): +# chunkcoords should be the coordinates of a possible chunk. it may not exist +def render_and_save(chunkcoords, cachedir, worldobj, oldimg, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes can't target bound methods) or to easily render and save one chunk - - Returns the image file location""" - a = ChunkRenderer(chunkfile, cachedir, worldobj, oldimg, queue) + + chunkcoords is a tuple: (chunkX, chunkY) + + If the chunk doesn't exist, return None. + Else, returns the image file location""" + a = ChunkRenderer(chunkcoords, cachedir, worldobj, oldimg, queue) try: return a.render_and_save(cave) except ChunkCorrupt: @@ -177,36 +184,53 @@ def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=Non class ChunkCorrupt(Exception): pass +class NoSuchChunk(Exception): + pass + class ChunkRenderer(object): - def __init__(self, chunkfile, cachedir, worldobj, oldimg, queue): - """Make a new chunk renderer for the given chunkfile. - chunkfile should be a full path to the .dat file to process + def __init__(self, chunkcoords, cachedir, worldobj, oldimg, queue): + """Make a new chunk renderer for the given chunk coordinates. + chunkcoors should be a tuple: (chunkX, chunkY) + cachedir is a directory to save the resulting chunk images to """ self.queue = queue + # derive based on worlddir and chunkcoords + self.regionfile = os.path.join(worldobj.worlddir, "region", + "r.%d.%d.mcr" % (chunkcoords[0] // 32, chunkcoords[1]//32)) - if not os.path.exists(chunkfile): - raise ValueError("Could not find chunkfile") - self.chunkfile = chunkfile - destdir, filename = os.path.split(self.chunkfile) - filename_split = filename.split(".") - chunkcoords = filename_split[1:3] + if not os.path.exists(self.regionfile): + raise ValueError("Could not find regionfile: %s" % self.regionfile) + + ## TODO TODO all of this class + + #destdir, filename = os.path.split(self.chunkfile) + #filename_split = filename.split(".") + #chunkcoords = filename_split[1:3] - self.coords = map(world.base36decode, chunkcoords) - self.blockid = ".".join(chunkcoords) + #self.coords = map(world.base36decode, chunkcoords) + self.blockid = "%d.%d" % chunkcoords # chunk coordinates (useful to converting local block coords to # global block coords) - self.chunkX = int(filename_split[1], base=36) - self.chunkY = int(filename_split[2], base=36) + self.chunkX = chunkcoords[0] + self.chunkY = chunkcoords[1] + + self.world = worldobj + + # Cachedir here is the base directory of the caches. We need to go 2 # levels deeper according to the chunk file. Get the last 2 components # of destdir and use that - moredirs, dir2 = os.path.split(destdir) - _, dir1 = os.path.split(moredirs) - self.cachedir = os.path.join(cachedir, dir1, dir2) + ##moredirs, dir2 = os.path.split(destdir) + ##_, dir1 = os.path.split(moredirs) + self.cachedir = os.path.join(cachedir, + world.base36encode(self.chunkX%64), + world.base36encode(self.chunkY%64)) + + #logging.debug("cache location for this chunk: %s", self.cachedir) self.oldimg, self.oldimg_path = oldimg @@ -229,9 +253,12 @@ class ChunkRenderer(object): """Loads and returns the level structure""" if not hasattr(self, "_level"): try: - self._level = get_lvldata(self.chunkfile) + self._level = get_lvldata(self.regionfile, self.chunkX, self.chunkY) + except NoSuchChunk, e: + #logging.debug("Skipping non-existant chunk") + raise except Exception, e: - logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.chunkfile, e) + logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.regionfile, e) raise ChunkCorrupt(str(e)) return self._level level = property(_load_level) @@ -259,13 +286,13 @@ class ChunkRenderer(object): def _load_left(self): """Loads and sets data from lower-left chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0] - 1, self.coords[1]) + chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX-1, self.chunkY) self._left_skylight = get_skylight_array(chunk_data) self._left_blocklight = get_blocklight_array(chunk_data) self._left_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._left_skylight = None self._left_blocklight = None self._left_blocks = None @@ -293,13 +320,13 @@ class ChunkRenderer(object): def _load_right(self): """Loads and sets data from lower-right chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0], self.coords[1] + 1) + chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY+1) self._right_skylight = get_skylight_array(chunk_data) self._right_blocklight = get_blocklight_array(chunk_data) self._right_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._right_skylight = None self._right_blocklight = None self._right_blocks = None @@ -448,6 +475,7 @@ class ChunkRenderer(object): is up to date, this method doesn't render anything. """ blockid = self.blockid + # Reasons for the code to get to this point: # 1) An old image doesn't exist @@ -457,13 +485,18 @@ class ChunkRenderer(object): # the image was invalid and deleted (sort of the same as (1)) # What /should/ the image be named, go ahead and hash the block array - dest_filename = "img.{0}.{1}.{2}.png".format( + try: + dest_filename = "img.{0}.{1}.{2}.png".format( blockid, "cave" if cave else "nocave", self._hash_blockarray(), ) + except NoSuchChunk, e: + return None + dest_path = os.path.join(self.cachedir, dest_filename) + #logging.debug("cache filename: %s", dest_path) if self.oldimg: if dest_filename == self.oldimg: @@ -479,6 +512,7 @@ class ChunkRenderer(object): # either corrupt or out of date os.unlink(self.oldimg_path) + # Render the chunk img = self.chunk_render(cave=cave) # Save it @@ -488,6 +522,7 @@ class ChunkRenderer(object): os.unlink(dest_path) raise # Return its location + #raise Exception("early exit") return dest_path def calculate_darkness(self, skylight, blocklight): diff --git a/nbt.py b/nbt.py index 3549497..cb9ca41 100644 --- a/nbt.py +++ b/nbt.py @@ -34,7 +34,8 @@ def load(fileobj): def load_from_region(fileobj, x, y): nbt = MCRFileReader(fileobj).load_chunk(x, y) if not nbt: - raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) return nbt.read_all() class NBTFileReader(object): diff --git a/quadtree.py b/quadtree.py index 088f917..15c428b 100644 --- a/quadtree.py +++ b/quadtree.py @@ -27,7 +27,7 @@ import logging import util import cPickle import stat -from time import gmtime, strftime +from time import gmtime, strftime, sleep from PIL import Image @@ -297,10 +297,12 @@ class QuadtreeGen(object): # This image is rendered at: dest = os.path.join(self.destdir, "tiles", *(str(x) for x in path)) + #logging.debug("this is rendered at %s", dest) # And uses these chunks tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, rowend) + #logging.debug(" tilechunks: %r", tilechunks) # Put this in the pool # (even if tilechunks is empty, render_worldtile will delete diff --git a/world.py b/world.py index 80f1611..03d497d 100644 --- a/world.py +++ b/world.py @@ -22,6 +22,7 @@ import sys import logging import cPickle import collections +import itertools import numpy @@ -45,7 +46,10 @@ def _convert_coords(chunks): the image each one should be. returns mincol, maxcol, minrow, maxrow, chunks_translated - chunks_translated is a list of (col, row, filename) + chunks_translated is a list of (col, row, (chunkX, chunkY)) + + The (chunkX, chunkY) tuple is the chunkCoords, used to identify the + chunk file """ chunks_translated = [] # columns are determined by the sum of the chunk coords, rows are the @@ -60,7 +64,7 @@ def _convert_coords(chunks): row = c[1] - c[0] minrow = min(minrow, row) maxrow = max(maxrow, row) - chunks_translated.append((col, row, c[2])) + chunks_translated.append((col, row, (c[0],c[1]))) return mincol, maxcol, minrow, maxrow, chunks_translated @@ -118,7 +122,7 @@ class WorldRenderer(object): #print data if 'version' in data and data['version'] == 19132: logging.error("Sorry, Minecraft-Overviewer doesn't yet know how to read McRegion chunks") - sys.exit(1) + #sys.exit(1) if self.useBiomeData: textures.prepareBiomeData(worlddir) @@ -127,6 +131,7 @@ class WorldRenderer(object): # In order to avoid having to look up the cache file names in # ChunkRenderer, get them all and store them here + # TODO change how caching works for root, dirnames, filenames in os.walk(cachedir): for filename in filenames: if not filename.endswith('.png') or not filename.startswith("img."): @@ -138,7 +143,6 @@ class WorldRenderer(object): bits = '.'.join((x, z, cave)) cached[dir][bits] = os.path.join(root, filename) - # stores Points Of Interest to be mapped with markers # a list of dictionaries, see below for an example self.POI = [] @@ -163,6 +167,7 @@ class WorldRenderer(object): if not self.chunklist: return None + raise Exception("not yet working") ## TODO correctly reimplement this for mcregion # Get a list of the (chunks, chunky, filename) from the passed in list # of filenames chunklist = [] @@ -190,15 +195,12 @@ class WorldRenderer(object): return inclusion_set - def get_chunk_path(self, chunkX, chunkY): - """Returns the path to the chunk file at (chunkX, chunkY), if - it exists.""" - - chunkFile = "%s/%s/c.%s.%s.dat" % (base36encode(chunkX % 64), - base36encode(chunkY % 64), - base36encode(chunkX), - base36encode(chunkY)) + def get_region_path(self, chunkX, chunkY): + """Returns the path to the region that contains chunk (chunkX, chunkY) + """ + chunkFile = "region/r.%s.%s.mcr" % (chunkX//32, chunkY//32) + return os.path.join(self.worlddir, chunkFile) def findTrueSpawn(self): @@ -217,9 +219,9 @@ class WorldRenderer(object): chunkY = spawnZ/16 ## The filename of this chunk - chunkFile = self.get_chunk_path(chunkX, chunkY) + chunkFile = self.get_region_path(chunkX, chunkY) - data=nbt.load(chunkFile)[1] + data=nbt.load_from_region(chunkFile, chunkX, chunkY)[1] level = data['Level'] blockArray = numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128)) @@ -240,14 +242,17 @@ class WorldRenderer(object): """Starts the render. This returns when it is finished""" logging.info("Scanning chunks") - raw_chunks = self._find_chunkfiles() + raw_chunks = self._get_chunklist() logging.debug("Done scanning chunks") # Translate chunks to our diagonal coordinate system + # TODO mincol, maxcol, minrow, maxrow, chunks = _convert_coords(raw_chunks) del raw_chunks # Free some memory self.chunkmap = self._render_chunks_async(chunks, procs) + logging.debug("world chunkmap has len %d", len(self.chunkmap)) + self.mincol = mincol self.maxcol = maxcol @@ -272,35 +277,47 @@ class WorldRenderer(object): os.path.join(dirpath, f))) return all_chunks - def _find_chunkfiles(self): - """Returns a list of all the chunk file locations, and the file they - correspond to. - - Returns a list of (chunkx, chunky, filename) where chunkx and chunky are - given in chunk coordinates. Use convert_coords() to turn the resulting list - into an oblique coordinate system. - - Usually this scans the given worlddir, but will use the chunk list - given to the constructor if one was provided.""" + def _get_chunklist(self): + """Returns a list of all possible chunk coordinates, based on the + available regions files. Note that not all chunk coordinates will + exists. The chunkrender will know how to ignore non-existant chunks + + returns a list of (chunkx, chunky, regionfile) where regionfile is + the region file that contains this chunk + + TODO, a --cachedir implemetation should involved thie method + + """ + all_chunks = [] - for dirpath, dirnames, filenames in os.walk(self.worlddir): - if not dirnames and filenames and "DIM-1" not in dirpath: - for f in filenames: - if f.startswith("c.") and f.endswith(".dat"): - p = f.split(".") - all_chunks.append((base36decode(p[1]), base36decode(p[2]), - os.path.join(dirpath, f))) + regions = self._find_regionfiles() + print "found %d regions" % len(regions) + for region in regions: + print "region %d, %d --> %s" % region + these_chunks = list(itertools.product( + range(region[0]*32,region[0]*32 + 32), + range(region[1]*32,region[1]*32 + 32) + )) + print "region %d,%d will go from:" + print " %r" % range(region[0]*32,region[0]*32 + 32) + print " %r" % range(region[1]*32,region[1]*32 + 32) + these_chunks = map(lambda x: (x[0], x[1], region[2]), these_chunks) + assert(len(these_chunks) == 1024) + all_chunks += these_chunks if not all_chunks: logging.error("Error: No chunks found!") sys.exit(1) + + logging.debug("Total possible chunks: %d", len(all_chunks)) return all_chunks def _render_chunks_async(self, chunks, processes): """Starts up a process pool and renders all the chunks asynchronously. - chunks is a list of (col, row, chunkfile) + chunks is a list of (col, row, (chunkX, chunkY)). Use chunkX,chunkY + to find the chunk data in a region file Returns a dictionary mapping (col, row) to the file where that chunk is rendered as an image @@ -318,21 +335,23 @@ class WorldRenderer(object): if processes == 1: # Skip the multiprocessing stuff logging.debug("Rendering chunks synchronously since you requested 1 process") - for i, (col, row, chunkfile) in enumerate(chunks): - if inclusion_set and (col, row) not in inclusion_set: - # Skip rendering, just find where the existing image is - _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) - if imgpath: - results[(col, row)] = imgpath - continue + for i, (col, row, chunkXY) in enumerate(chunks): + ##TODO##/if inclusion_set and (col, row) not in inclusion_set: + ##TODO##/ # Skip rendering, just find where the existing image is + ##TODO##/ _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) + ##TODO##/ if imgpath: + ##TODO##/ results[(col, row)] = imgpath + ##TODO##/ continue - oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if chunk.check_cache(chunkfile, oldimg): + oldimg = chunk.find_oldimage(chunkXY, cached, self.caves) + # TODO remove this shortcircuit + if oldimg[1]:## or chunk.check_cache(chunkfile, oldimg): result = oldimg[1] else: - result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) - - results[(col, row)] = result + result = chunk.render_and_save(chunkXY, self.cachedir, self, oldimg, queue=q) + + if result: + results[(col, row)] = result if i > 0: try: item = q.get(block=False)