diff --git a/README.rst b/README.rst index 542f2c4..dd77116 100644 --- a/README.rst +++ b/README.rst @@ -123,7 +123,7 @@ Running ------- To generate a set of Google Map tiles, use the gmap.py script like this:: - python gmap.py [OPTIONS] + python gmap.py [OPTIONS] The output directory will be created if it doesn't exist. This will generate a set of image tiles for your world in the directory you choose. When it's done, diff --git a/chunk.py b/chunk.py index 05ccd4b..eeef14b 100644 --- a/chunk.py +++ b/chunk.py @@ -20,6 +20,7 @@ import hashlib import logging import time import math +import sys import nbt import textures @@ -45,10 +46,13 @@ image # alpha_over extension, BUT this extension may fall back to PIL's # paste(), which DOES need the workaround.) -def get_lvldata(filename): - """Takes a filename and returns the Level struct, which contains all the +def get_lvldata(filename, x, y): + """Takes a filename and chunkcoords and returns the Level struct, which contains all the level info""" - return nbt.load(filename)[1]['Level'] + + d = nbt.load_from_region(filename, x, y) + if not d: raise NoSuchChunk(x,y) + return d[1]['Level'] def get_blockarray(level): """Takes the level struct as returned from get_lvldata, and returns the @@ -124,14 +128,12 @@ fluid_blocks = set([8,9,10,11]) # (glass, half blocks) nospawn_blocks = set([20,44]) -def find_oldimage(chunkfile, cached, cave): - destdir, filename = os.path.split(chunkfile) - filename_split = filename.split(".") - blockid = ".".join(filename_split[1:3]) +def find_oldimage(chunkXY, cached, cave): + blockid = "%d.%d" % chunkXY # Get the name of the existing image. - moredirs, dir2 = os.path.split(destdir) - dir1 = os.path.basename(moredirs) + dir1 = world.base36encode(chunkXY[0]%64) + dir2 = world.base36encode(chunkXY[1]%64) cachename = '/'.join((dir1, dir2)) oldimg = oldimg_path = None @@ -139,23 +141,39 @@ def find_oldimage(chunkfile, cached, cave): if key in cached[cachename]: oldimg_path = cached[cachename][key] _, oldimg = os.path.split(oldimg_path) - logging.debug("Found cached image {0}".format(oldimg)) + #logging.debug("Found cached image {0}".format(oldimg)) return oldimg, oldimg_path -def check_cache(chunkfile, oldimg): +def check_cache(world, chunkXY, oldimg): + """Returns True is oldimg is OK to use (i.e. not stale)""" +# TODO read to the region file and get the timestamp?? +# TODO currently, just use the mtime on the region file +# TODO (which will cause a single chunk update to invalidate everything in the region + + if not oldimg[1]: return False + chunkfile = os.path.join(world.worlddir, "region", "r.%d.%d.mcr" % (chunkXY[0]//32, chunkXY[1]//32)) + + with open(chunkfile, "rb") as f: + region = nbt.MCRFileReader(f) + mtime = region.get_chunk_timestamp(chunkXY[0], chunkXY[1]) + #logging.debug("checking cache %s against %s %d", chunkfile, oldimg[1], mtime) try: - if oldimg[1] and os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + if mtime <= os.path.getmtime(oldimg[1]): return True return False except OSError: return False -def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=None): +# chunkcoords should be the coordinates of a possible chunk. it may not exist +def render_and_save(chunkcoords, cachedir, worldobj, oldimg, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes can't target bound methods) or to easily render and save one chunk - - Returns the image file location""" - a = ChunkRenderer(chunkfile, cachedir, worldobj, oldimg, queue) + + chunkcoords is a tuple: (chunkX, chunkY) + + If the chunk doesn't exist, return None. + Else, returns the image file location""" + a = ChunkRenderer(chunkcoords, cachedir, worldobj, oldimg, queue) try: return a.render_and_save(cave) except ChunkCorrupt: @@ -177,36 +195,53 @@ def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=Non class ChunkCorrupt(Exception): pass +class NoSuchChunk(Exception): + pass + class ChunkRenderer(object): - def __init__(self, chunkfile, cachedir, worldobj, oldimg, queue): - """Make a new chunk renderer for the given chunkfile. - chunkfile should be a full path to the .dat file to process + def __init__(self, chunkcoords, cachedir, worldobj, oldimg, queue): + """Make a new chunk renderer for the given chunk coordinates. + chunkcoors should be a tuple: (chunkX, chunkY) + cachedir is a directory to save the resulting chunk images to """ self.queue = queue + # derive based on worlddir and chunkcoords + self.regionfile = os.path.join(worldobj.worlddir, "region", + "r.%d.%d.mcr" % (chunkcoords[0] // 32, chunkcoords[1]//32)) - if not os.path.exists(chunkfile): - raise ValueError("Could not find chunkfile") - self.chunkfile = chunkfile - destdir, filename = os.path.split(self.chunkfile) - filename_split = filename.split(".") - chunkcoords = filename_split[1:3] + if not os.path.exists(self.regionfile): + raise ValueError("Could not find regionfile: %s" % self.regionfile) + + ## TODO TODO all of this class + + #destdir, filename = os.path.split(self.chunkfile) + #filename_split = filename.split(".") + #chunkcoords = filename_split[1:3] - self.coords = map(world.base36decode, chunkcoords) - self.blockid = ".".join(chunkcoords) + #self.coords = map(world.base36decode, chunkcoords) + self.blockid = "%d.%d" % chunkcoords # chunk coordinates (useful to converting local block coords to # global block coords) - self.chunkX = int(filename_split[1], base=36) - self.chunkY = int(filename_split[2], base=36) + self.chunkX = chunkcoords[0] + self.chunkY = chunkcoords[1] + + self.world = worldobj + + # Cachedir here is the base directory of the caches. We need to go 2 # levels deeper according to the chunk file. Get the last 2 components # of destdir and use that - moredirs, dir2 = os.path.split(destdir) - _, dir1 = os.path.split(moredirs) - self.cachedir = os.path.join(cachedir, dir1, dir2) + ##moredirs, dir2 = os.path.split(destdir) + ##_, dir1 = os.path.split(moredirs) + self.cachedir = os.path.join(cachedir, + world.base36encode(self.chunkX%64), + world.base36encode(self.chunkY%64)) + + #logging.debug("cache location for this chunk: %s", self.cachedir) self.oldimg, self.oldimg_path = oldimg @@ -229,9 +264,12 @@ class ChunkRenderer(object): """Loads and returns the level structure""" if not hasattr(self, "_level"): try: - self._level = get_lvldata(self.chunkfile) + self._level = get_lvldata(self.regionfile, self.chunkX, self.chunkY) + except NoSuchChunk, e: + #logging.debug("Skipping non-existant chunk") + raise except Exception, e: - logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.chunkfile, e) + logging.warning("Error opening chunk file %s. It may be corrupt. %s", self.regionfile, e) raise ChunkCorrupt(str(e)) return self._level level = property(_load_level) @@ -259,13 +297,13 @@ class ChunkRenderer(object): def _load_left(self): """Loads and sets data from lower-left chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0] - 1, self.coords[1]) + chunk_path = self.world.get_region_path(self.chunkX - 1, self.chunkY) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX - 1, self.chunkY) self._left_skylight = get_skylight_array(chunk_data) self._left_blocklight = get_blocklight_array(chunk_data) self._left_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._left_skylight = None self._left_blocklight = None self._left_blocks = None @@ -293,13 +331,13 @@ class ChunkRenderer(object): def _load_right(self): """Loads and sets data from lower-right chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0], self.coords[1] + 1) + chunk_path = self.world.get_region_path(self.chunkX, self.chunkY + 1) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY + 1) self._right_skylight = get_skylight_array(chunk_data) self._right_blocklight = get_blocklight_array(chunk_data) self._right_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._right_skylight = None self._right_blocklight = None self._right_blocks = None @@ -327,13 +365,13 @@ class ChunkRenderer(object): def _load_up_right(self): """Loads and sets data from upper-right chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0] + 1, self.coords[1]) + chunk_path = self.world.get_region_path(self.chunkX + 1, self.chunkY) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX + 1, self.chunkY) self._up_right_skylight = get_skylight_array(chunk_data) self._up_right_blocklight = get_blocklight_array(chunk_data) self._up_right_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._up_right_skylight = None self._up_right_blocklight = None self._up_right_blocks = None @@ -347,13 +385,13 @@ class ChunkRenderer(object): def _load_up_left(self): """Loads and sets data from upper-left chunk""" - chunk_path = self.world.get_chunk_path(self.coords[0], self.coords[1] - 1) + chunk_path = self.world.get_region_path(self.chunkX, self.chunkY - 1) try: - chunk_data = get_lvldata(chunk_path) + chunk_data = get_lvldata(chunk_path, self.chunkX, self.chunkY - 1) self._up_left_skylight = get_skylight_array(chunk_data) self._up_left_blocklight = get_blocklight_array(chunk_data) self._up_left_blocks = get_blockarray(chunk_data) - except IOError: + except NoSuchChunk: self._up_left_skylight = None self._up_left_blocklight = None self._up_left_blocks = None @@ -448,6 +486,7 @@ class ChunkRenderer(object): is up to date, this method doesn't render anything. """ blockid = self.blockid + # Reasons for the code to get to this point: # 1) An old image doesn't exist @@ -457,13 +496,17 @@ class ChunkRenderer(object): # the image was invalid and deleted (sort of the same as (1)) # What /should/ the image be named, go ahead and hash the block array - dest_filename = "img.{0}.{1}.{2}.png".format( + try: + dest_filename = "img.{0}.{1}.{2}.png".format( blockid, "cave" if cave else "nocave", self._hash_blockarray(), ) + except NoSuchChunk, e: + return None dest_path = os.path.join(self.cachedir, dest_filename) + #logging.debug("cache filename: %s", dest_path) if self.oldimg: if dest_filename == self.oldimg: @@ -471,14 +514,17 @@ class ChunkRenderer(object): # hashes match. # Before we return it, update its mtime so the next round # doesn't have to check the hash + # TODO confirm hash checking is correct (it should be) os.utime(dest_path, None) - logging.debug("Using cached image") + logging.debug("Using cached image, and updating utime") return dest_path else: # Remove old image for this chunk. Anything already existing is # either corrupt or out of date os.unlink(self.oldimg_path) + + logging.debug("doing a real real render") # Render the chunk img = self.chunk_render(cave=cave) # Save it @@ -488,6 +534,7 @@ class ChunkRenderer(object): os.unlink(dest_path) raise # Return its location + #raise Exception("early exit") return dest_path def calculate_darkness(self, skylight, blocklight): diff --git a/gmap.py b/gmap.py index 749a41b..2cf6d63 100755 --- a/gmap.py +++ b/gmap.py @@ -36,8 +36,8 @@ import world import quadtree helptext = """ -%prog [OPTIONS] -%prog -d [tiles dest dir]""" +%prog [OPTIONS] +%prog -d [tiles dest dir]""" def main(): try: @@ -69,11 +69,23 @@ def main(): worlddir = args[0] if not os.path.exists(worlddir): + # world given is either world number, or name + worlds = world.get_worlds() try: worldnum = int(worlddir) - worlddir = world.get_worlds()[worldnum]['path'] - except (ValueError, KeyError): - print "Invalid world number or directory" + worlddir = worlds[worldnum]['path'] + except ValueError: + # it wasn't a number or path, try using it as a name + try: + worlddir = worlds[worlddir]['path'] + except KeyError: + # it's not a number, name, or path + print "Invalid world name or path" + parser.print_help() + sys.exit(1) + except KeyError: + # it was an invalid number + print "Invalid world number" parser.print_help() sys.exit(1) @@ -176,13 +188,21 @@ def list_worlds(): print 'No world saves found in the usual place' return print "Detected saves:" - for num, info in sorted(worlds.iteritems()): + for name, info in sorted(worlds.iteritems()): + if isinstance(name, basestring) and name.startswith("World") and len(name) == 6: + try: + world_n = int(name[-1]) + # we'll catch this one later, when it shows up as an + # integer key + continue + except ValueError: + pass timestamp = time.strftime("%Y-%m-%d %H:%M", time.localtime(info['LastPlayed'] / 1000)) playtime = info['Time'] / 20 playstamp = '%d:%02d' % (playtime / 3600, playtime / 60 % 60) size = "%.2fMB" % (info['SizeOnDisk'] / 1024. / 1024.) - print "World %s: %s Playtime: %s Modified: %s" % (num, size, playstamp, timestamp) + print "World %s: %s Playtime: %s Modified: %s" % (name, size, playstamp, timestamp) if __name__ == "__main__": diff --git a/nbt.py b/nbt.py index 780acfe..b1bdf4e 100644 --- a/nbt.py +++ b/nbt.py @@ -13,18 +13,44 @@ # You should have received a copy of the GNU General Public License along # with the Overviewer. If not, see . -import gzip +import gzip, zlib import struct +import StringIO +import os +# decorator to handle filename or object as first parameter +def _file_loader(func): + def wrapper(fileobj, *args): + if isinstance(fileobj, basestring): + if not os.path.isfile(fileobj): + return None + + # Is actually a filename + fileobj = open(fileobj, 'rb') + return func(fileobj, *args) + return wrapper + +@_file_loader def load(fileobj): - if isinstance(fileobj, basestring): - # Is actually a filename - fileobj = open(fileobj, 'rb') return NBTFileReader(fileobj).read_all() +@_file_loader +def load_from_region(fileobj, x, y): + nbt = MCRFileReader(fileobj).load_chunk(x, y) + if not nbt: + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + return nbt.read_all() + class NBTFileReader(object): - def __init__(self, fileobj): - self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') + def __init__(self, fileobj, is_gzip=True): + if is_gzip: + self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') + else: + # pure zlib stream -- maybe later replace this with + # a custom zlib file object? + data = zlib.decompress(fileobj.read()) + self._file = StringIO.StringIO(data) # These private methods read the payload only of the following types def _read_tag_end(self): @@ -143,3 +169,162 @@ class NBTFileReader(object): return name, payload + +# For reference, the MCR format is outlined at +# +class MCRFileReader(object): + """A class for reading chunk region files, as introduced in the + Beta 1.3 update. It provides functions for opening individual + chunks (as instances of NBTFileReader), getting chunk timestamps, + and for listing chunks contained in the file.""" + + def __init__(self, fileobj): + self._file = fileobj + + # cache used when the entire header tables are read in get_chunks() + self._locations = None + self._timestamps = None + self._chunks = None + + def _read_24bit_int(self): + """Read in a 24-bit, big-endian int, used in the chunk + location table.""" + + ret = 0 + bytes = self._file.read(3) + for i in xrange(3): + ret = ret << 8 + ret += struct.unpack("B", bytes[i])[0] + + return ret + + def _read_chunk_location(self, x=None, y=None): + """Read and return the (offset, length) of the given chunk + coordinate, or None if the requested chunk doesn't exist. x + and y must be between 0 and 31, or None. If they are None, + then there will be no file seek before doing the read.""" + + if x != None and y != None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._locations: + return self._locations[x + y * 32] + + # go to the correct entry in the chunk location table + self._file.seek(4 * (x + y * 32)) + + # 3-byte offset in 4KiB sectors + offset_sectors = self._read_24bit_int() + + # 1-byte length in 4KiB sectors, rounded up + byte = self._file.read(1) + length_sectors = struct.unpack("B", byte)[0] + + # check for empty chunks + if offset_sectors == 0 or length_sectors == 0: + return None + + return (offset_sectors * 4096, length_sectors * 4096) + + def _read_chunk_timestamp(self, x=None, y=None): + """Read and return the last modification time of the given + chunk coordinate. x and y must be between 0 and 31, or + None. If they are, None, then there will be no file seek + before doing the read.""" + + if x != None and y != None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._timestamps: + return self._timestamps[x + y * 32] + + # go to the correct entry in the chunk timestamp table + self._file.seek(4 * (x + y * 32) + 4096) + + bytes = self._file.read(4) + timestamp = struct.unpack(">I", bytes)[0] + + return timestamp + + def get_chunks(self): + """Return a list of all chunks contained in this region file, + as a list of (x, y) coordinate tuples. To load these chunks, + provide these coordinates to load_chunk().""" + + if self._chunks: + return self._chunks + + self._chunks = [] + self._locations = [] + self._timestamps = [] + + # go to the beginning of the file + self._file.seek(0) + + # read chunk location table + for y in xrange(32): + for x in xrange(32): + location = self._read_chunk_location() + self._locations.append(location) + if location: + self._chunks.append((x, y)) + + # read chunk timestamp table + for y in xrange(32): + for x in xrange(32): + timestamp = self._read_chunk_timestamp() + self._timestamps.append(timestamp) + + return self._chunks + + def get_chunk_timestamp(self, x, y): + """Return the given chunk's modification time. If the given + chunk doesn't exist, this number may be nonsense. Like + load_chunk(), this will wrap x and y into the range [0, 31]. + """ + + return self._read_chunk_timestamp(x % 32, y % 32) + + def load_chunk(self, x, y): + """Return a NBTFileReader instance for the given chunk, or + None if the given chunk doesn't exist in this region file. If + you provide an x or y not between 0 and 31, it will be + modulo'd into this range (x % 32, etc.) This is so you can + provide chunk coordinates in global coordinates, and still + have the chunks load out of regions properly.""" + + location = self._read_chunk_location(x % 32, y % 32) + if not location: + return None + + # seek to the data + self._file.seek(location[0]) + + # read in the chunk data header + bytes = self._file.read(4) + data_length = struct.unpack(">I", bytes)[0] + bytes = self._file.read(1) + compression = struct.unpack("B", bytes)[0] + + # figure out the compression + is_gzip = True + if compression == 1: + # gzip -- not used by the official client, but trivial to support here so... + is_gzip = True + elif compression == 2: + # deflate -- pure zlib stream + is_gzip = False + else: + # unsupported! + raise Exception("Unsupported chunk compression type: %i" % (compression,)) + + # turn the rest of the data into a StringIO object + # (using data_length - 1, as we already read 1 byte for compression) + data = self._file.read(data_length - 1) + data = StringIO.StringIO(data) + + return NBTFileReader(data, is_gzip=is_gzip) diff --git a/quadtree.py b/quadtree.py index 088f917..15c428b 100644 --- a/quadtree.py +++ b/quadtree.py @@ -27,7 +27,7 @@ import logging import util import cPickle import stat -from time import gmtime, strftime +from time import gmtime, strftime, sleep from PIL import Image @@ -297,10 +297,12 @@ class QuadtreeGen(object): # This image is rendered at: dest = os.path.join(self.destdir, "tiles", *(str(x) for x in path)) + #logging.debug("this is rendered at %s", dest) # And uses these chunks tilechunks = self._get_chunks_in_range(colstart, colend, rowstart, rowend) + #logging.debug(" tilechunks: %r", tilechunks) # Put this in the pool # (even if tilechunks is empty, render_worldtile will delete diff --git a/textures.py b/textures.py index bc2eb41..2478260 100644 --- a/textures.py +++ b/textures.py @@ -281,7 +281,7 @@ def _build_blockimages(): # 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 -1, -1, -1, -1, -1, 13, 12, 29, 28, 23, 22, 6, 6, 7, 8, 35, # Gold/iron blocks? Doublestep? TNT from above? # 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 - 36, 37, 80, -1, 65, 4, 25,101, 98, 24, 43, -1, 86, -1, -1, -1, # Torch from above? leaving out fire. Redstone wire? Crops/furnaces handled elsewhere. sign post + 36, 37, 80, -1, 65, 4, 25, -1, 98, 24, 43, -1, 86, -1, -1, -1, # Torch from above? leaving out fire. Redstone wire? Crops/furnaces handled elsewhere. sign post # 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 -1, -1, -1, 16, -1, -1, -1, -1, -1, 51, 51, -1, -1, 1, 66, 67, # door,ladder left out. Minecart rail orientation # 80 81 82 83 84 85 86 87 88 89 90 91 diff --git a/world.py b/world.py index e655273..fabf8c4 100644 --- a/world.py +++ b/world.py @@ -22,6 +22,7 @@ import sys import logging import cPickle import collections +import itertools import numpy @@ -45,7 +46,10 @@ def _convert_coords(chunks): the image each one should be. returns mincol, maxcol, minrow, maxrow, chunks_translated - chunks_translated is a list of (col, row, filename) + chunks_translated is a list of (col, row, (chunkX, chunkY)) + + The (chunkX, chunkY) tuple is the chunkCoords, used to identify the + chunk file """ chunks_translated = [] # columns are determined by the sum of the chunk coords, rows are the @@ -60,7 +64,7 @@ def _convert_coords(chunks): row = c[1] - c[0] minrow = min(minrow, row) maxrow = max(maxrow, row) - chunks_translated.append((col, row, c[2])) + chunks_translated.append((col, row, (c[0],c[1]))) return mincol, maxcol, minrow, maxrow, chunks_translated @@ -112,6 +116,14 @@ class WorldRenderer(object): self.cachedir = cachedir self.useBiomeData = useBiomeData + # figure out chunk format is in use + # if mcregion, error out early until we can add support + data = nbt.load(os.path.join(self.worlddir, "level.dat"))[1]['Data'] + #print data + if not ('version' in data and data['version'] == 19132): + logging.error("Sorry, This version of Minecraft-Overviewer only works with the new McRegion chunk format") + sys.exit(1) + if self.useBiomeData: textures.prepareBiomeData(worlddir) @@ -119,6 +131,7 @@ class WorldRenderer(object): # In order to avoid having to look up the cache file names in # ChunkRenderer, get them all and store them here + # TODO change how caching works for root, dirnames, filenames in os.walk(cachedir): for filename in filenames: if not filename.endswith('.png') or not filename.startswith("img."): @@ -130,7 +143,6 @@ class WorldRenderer(object): bits = '.'.join((x, z, cave)) cached[dir][bits] = os.path.join(root, filename) - # stores Points Of Interest to be mapped with markers # a list of dictionaries, see below for an example self.POI = [] @@ -155,6 +167,7 @@ class WorldRenderer(object): if not self.chunklist: return None + raise Exception("not yet working") ## TODO correctly reimplement this for mcregion # Get a list of the (chunks, chunky, filename) from the passed in list # of filenames chunklist = [] @@ -182,15 +195,12 @@ class WorldRenderer(object): return inclusion_set - def get_chunk_path(self, chunkX, chunkY): - """Returns the path to the chunk file at (chunkX, chunkY), if - it exists.""" - - chunkFile = "%s/%s/c.%s.%s.dat" % (base36encode(chunkX % 64), - base36encode(chunkY % 64), - base36encode(chunkX), - base36encode(chunkY)) + def get_region_path(self, chunkX, chunkY): + """Returns the path to the region that contains chunk (chunkX, chunkY) + """ + chunkFile = "region/r.%s.%s.mcr" % (chunkX//32, chunkY//32) + return os.path.join(self.worlddir, chunkFile) def findTrueSpawn(self): @@ -209,9 +219,9 @@ class WorldRenderer(object): chunkY = spawnZ/16 ## The filename of this chunk - chunkFile = self.get_chunk_path(chunkX, chunkY) + chunkFile = self.get_region_path(chunkX, chunkY) - data=nbt.load(chunkFile)[1] + data=nbt.load_from_region(chunkFile, chunkX, chunkY)[1] level = data['Level'] blockArray = numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128)) @@ -232,14 +242,17 @@ class WorldRenderer(object): """Starts the render. This returns when it is finished""" logging.info("Scanning chunks") - raw_chunks = self._find_chunkfiles() + raw_chunks = self._get_chunklist() logging.debug("Done scanning chunks") # Translate chunks to our diagonal coordinate system + # TODO mincol, maxcol, minrow, maxrow, chunks = _convert_coords(raw_chunks) del raw_chunks # Free some memory self.chunkmap = self._render_chunks_async(chunks, procs) + logging.debug("world chunkmap has len %d", len(self.chunkmap)) + self.mincol = mincol self.maxcol = maxcol @@ -248,35 +261,59 @@ class WorldRenderer(object): self.findTrueSpawn() - def _find_chunkfiles(self): - """Returns a list of all the chunk file locations, and the file they - correspond to. - - Returns a list of (chunkx, chunky, filename) where chunkx and chunky are - given in chunk coordinates. Use convert_coords() to turn the resulting list - into an oblique coordinate system. - - Usually this scans the given worlddir, but will use the chunk list - given to the constructor if one was provided.""" + def _find_regionfiles(self): + """Returns a list of all of the region files, along with their + coordinates + + Returns (regionx, regiony, filename)""" all_chunks = [] - for dirpath, dirnames, filenames in os.walk(self.worlddir): + for dirpath, dirnames, filenames in os.walk(os.path.join(self.worlddir, 'region')): if not dirnames and filenames and "DIM-1" not in dirpath: for f in filenames: - if f.startswith("c.") and f.endswith(".dat"): + if f.startswith("r.") and f.endswith(".mcr"): p = f.split(".") - all_chunks.append((base36decode(p[1]), base36decode(p[2]), + all_chunks.append((int(p[1]), int(p[2]), os.path.join(dirpath, f))) + return all_chunks + + def _get_chunklist(self): + """Returns a list of all possible chunk coordinates, based on the + available regions files. Note that not all chunk coordinates will + exists. The chunkrender will know how to ignore non-existant chunks + + returns a list of (chunkx, chunky, regionfile) where regionfile is + the region file that contains this chunk + + TODO, a --cachedir implemetation should involved thie method + + """ + + all_chunks = [] + + regions = self._find_regionfiles() + logging.debug("Found %d regions",len(regions)) + for region in regions: + these_chunks = list(itertools.product( + range(region[0]*32,region[0]*32 + 32), + range(region[1]*32,region[1]*32 + 32) + )) + these_chunks = map(lambda x: (x[0], x[1], region[2]), these_chunks) + assert(len(these_chunks) == 1024) + all_chunks += these_chunks if not all_chunks: logging.error("Error: No chunks found!") sys.exit(1) + + logging.debug("Total possible chunks: %d", len(all_chunks)) return all_chunks def _render_chunks_async(self, chunks, processes): """Starts up a process pool and renders all the chunks asynchronously. - chunks is a list of (col, row, chunkfile) + chunks is a list of (col, row, (chunkX, chunkY)). Use chunkX,chunkY + to find the chunk data in a region file Returns a dictionary mapping (col, row) to the file where that chunk is rendered as an image @@ -294,21 +331,24 @@ class WorldRenderer(object): if processes == 1: # Skip the multiprocessing stuff logging.debug("Rendering chunks synchronously since you requested 1 process") - for i, (col, row, chunkfile) in enumerate(chunks): - if inclusion_set and (col, row) not in inclusion_set: - # Skip rendering, just find where the existing image is - _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) - if imgpath: - results[(col, row)] = imgpath - continue + for i, (col, row, chunkXY) in enumerate(chunks): + ##TODO##/if inclusion_set and (col, row) not in inclusion_set: + ##TODO##/ # Skip rendering, just find where the existing image is + ##TODO##/ _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) + ##TODO##/ if imgpath: + ##TODO##/ results[(col, row)] = imgpath + ##TODO##/ continue - oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if chunk.check_cache(chunkfile, oldimg): + oldimg = chunk.find_oldimage(chunkXY, cached, self.caves) + # TODO remove this shortcircuit + if chunk.check_cache(self, chunkXY, oldimg): result = oldimg[1] else: - result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) - - results[(col, row)] = result + #logging.debug("check cache failed, need to render (could be ghost chunk)") + result = chunk.render_and_save(chunkXY, self.cachedir, self, oldimg, queue=q) + + if result: + results[(col, row)] = result if i > 0: try: item = q.get(block=False) @@ -324,20 +364,20 @@ class WorldRenderer(object): logging.debug("Rendering chunks in {0} processes".format(processes)) pool = multiprocessing.Pool(processes=processes) asyncresults = [] - for col, row, chunkfile in chunks: - if inclusion_set and (col, row) not in inclusion_set: - # Skip rendering, just find where the existing image is - _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) - if imgpath: - results[(col, row)] = imgpath - continue + for col, row, chunkXY in chunks: + ##TODO/if inclusion_set and (col, row) not in inclusion_set: + ##TODO/ # Skip rendering, just find where the existing image is + ##TODO/ _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) + ##TODO/ if imgpath: + ##TODO/ results[(col, row)] = imgpath + ##TODO/ continue - oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if chunk.check_cache(chunkfile, oldimg): + oldimg = chunk.find_oldimage(chunkXY, cached, self.caves) + if chunk.check_cache(self, chunkXY, oldimg): result = FakeAsyncResult(oldimg[1]) else: result = pool.apply_async(chunk.render_and_save, - args=(chunkfile,self.cachedir,self, oldimg), + args=(chunkXY,self.cachedir,self, oldimg), kwds=dict(cave=self.caves, queue=q)) asyncresults.append((col, row, result)) @@ -384,7 +424,7 @@ def get_save_dir(): return path def get_worlds(): - "Returns {world # : level.dat information}" + "Returns {world # or name : level.dat information}" ret = {} save_dir = get_save_dir() @@ -393,12 +433,17 @@ def get_worlds(): return None for dir in os.listdir(save_dir): + world_dat = os.path.join(save_dir, dir, "level.dat") + if not os.path.exists(world_dat): continue + info = nbt.load(world_dat)[1] + info['Data']['path'] = os.path.join(save_dir, dir) if dir.startswith("World") and len(dir) == 6: - world_n = int(dir[-1]) - world_dat = os.path.join(save_dir, dir, "level.dat") - if not os.path.exists(world_dat): continue - info = nbt.load(world_dat)[1] - info['Data']['path'] = os.path.join(save_dir, dir) - ret[world_n] = info['Data'] + try: + world_n = int(dir[-1]) + ret[world_n] = info['Data'] + except ValueError: + pass + if 'LevelName' in info['Data'].keys(): + ret[info['Data']['LevelName']] = info['Data'] return ret