From 83d7a36ef4e350d8d3025229689d9ff8861b5851 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 1 Mar 2011 13:18:25 -0500 Subject: [PATCH] moved quadtree.py to mtime-based update checking, and added a stub direct-to-tile renderer --- gmap.py | 7 ++- quadtree.py | 169 ++++++++++++++++++++++------------------------------ world.py | 29 +++++---- 3 files changed, 93 insertions(+), 112 deletions(-) diff --git a/gmap.py b/gmap.py index b104e78..da18aa0 100755 --- a/gmap.py +++ b/gmap.py @@ -137,9 +137,10 @@ def main(): w.go(options.procs) # Now generate the tiles - #q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg) - #q.write_html(options.skipjs) - #q.go(options.procs) + # TODO chunklist, render type (night, lighting, spawn) + q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg) + q.write_html(options.skipjs) + q.go(options.procs) def delete_all(tiledir): # Delete all /hash/ files in the tile dir. diff --git a/quadtree.py b/quadtree.py index 15c428b..aa0a46f 100644 --- a/quadtree.py +++ b/quadtree.py @@ -31,6 +31,7 @@ from time import gmtime, strftime, sleep from PIL import Image +import nbt from optimizeimages import optimize_image import composite @@ -424,9 +425,11 @@ class QuadtreeGen(object): chunklist = [] for row in xrange(rowstart-16, rowend+1): for col in xrange(colstart, colend+1): - c = self.world.chunkmap.get((col, row), None) - if c: - chunklist.append((col, row, c)) + # return (col, row, chunkx, chunky, regionpath) + chunkx, chunky = self.world.unconvert_coords(col, row) + c = self.world.get_region_path(chunkx, chunky) + if os.path.exists(c): + chunklist.append((col, row, chunkx, chunky, c)) return chunklist @catch_keyboardinterrupt @@ -436,68 +439,56 @@ def render_innertile(dest, name, imgformat, optimizeimg): os.path.join(dest, name, "{0,1,2,3}.png") """ imgpath = os.path.join(dest, name) + "." + imgformat - hashpath = os.path.join(dest, name) + ".hash" if name == "base": q0path = os.path.join(dest, "0." + imgformat) q1path = os.path.join(dest, "1." + imgformat) q2path = os.path.join(dest, "2." + imgformat) q3path = os.path.join(dest, "3." + imgformat) - q0hash = os.path.join(dest, "0.hash") - q1hash = os.path.join(dest, "1.hash") - q2hash = os.path.join(dest, "2.hash") - q3hash = os.path.join(dest, "3.hash") else: q0path = os.path.join(dest, name, "0." + imgformat) q1path = os.path.join(dest, name, "1." + imgformat) q2path = os.path.join(dest, name, "2." + imgformat) q3path = os.path.join(dest, name, "3." + imgformat) - q0hash = os.path.join(dest, name, "0.hash") - q1hash = os.path.join(dest, name, "1.hash") - q2hash = os.path.join(dest, name, "2.hash") - q3hash = os.path.join(dest, name, "3.hash") # Check which ones exist - if not os.path.exists(q0hash): + if not os.path.exists(q0path): q0path = None - q0hash = None - if not os.path.exists(q1hash): + if not os.path.exists(q1path): q1path = None - q1hash = None - if not os.path.exists(q2hash): + if not os.path.exists(q2path): q2path = None - q2hash = None - if not os.path.exists(q3hash): + if not os.path.exists(q3path): q3path = None - q3hash = None # do they all not exist? if not (q0path or q1path or q2path or q3path): if os.path.exists(imgpath): os.unlink(imgpath) - if os.path.exists(hashpath): - os.unlink(hashpath) return - # Now check the hashes - hasher = hashlib.md5() - if q0hash: - hasher.update(open(q0hash, "rb").read()) - if q1hash: - hasher.update(open(q1hash, "rb").read()) - if q2hash: - hasher.update(open(q2hash, "rb").read()) - if q3hash: - hasher.update(open(q3hash, "rb").read()) - if os.path.exists(hashpath): - oldhash = open(hashpath, "rb").read() - else: - oldhash = None - newhash = hasher.digest() - - if newhash == oldhash: - # Nothing to do - return + # check the mtimes + try: + tile_mtime = os.path.getmtime(imgpath) + needs_rerender = False + + # remove non-existant paths + components = [q0path, q1path, q2path, q3path] + components = filter(lambda p: p != None, components) + + for mtime in [os.path.getmtime(path) for path in components]: + if mtime > tile_mtime: + needs_rerender = True + break + + # quit now if we don't need rerender + if not needs_rerender: + return + except OSError: + # one of our mtime calls failed, so we'll continue + pass + + #logging.debug("writing out innertile {0}".format(imgpath)) # Create the actual image now img = Image.new("RGBA", (384, 384), (38,92,255,0)) @@ -538,10 +529,6 @@ def render_innertile(dest, name, imgformat, optimizeimg): if optimizeimg: optimize_image(imgpath, imgformat, optimizeimg) - with open(hashpath, "wb") as hashout: - hashout.write(newhash) - - @catch_keyboardinterrupt def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat, optimizeimg): """Renders just the specified chunks into a tile and save it. Unlike usual @@ -549,8 +536,8 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat chunks around the edges are half-way cut off (so that neighboring tiles will render the other half) - chunks is a list of (col, row, filename) of chunk images that are relevant - to this call + chunks is a list of (col, row, chunkx, chunky, filename) of chunk + images that are relevant to this call (with their associated regions) The image is saved to path+".ext" and a hash is saved to path+".hash" @@ -592,15 +579,20 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat # Before we render any tiles, check the hash of each image in this tile to # see if it's changed. - hashpath = path + ".hash" + # TODO remove hash files? imgpath = path + "." + imgformat + + # first, remove chunks from `chunks` that don't actually exist in + # their region files + def chunk_exists(chunk): + _, _, chunkx, chunky, region = chunk + return nbt.load_from_region(region, chunkx, chunky) != None + chunks = filter(chunk_exists, chunks) if not chunks: # No chunks were found in this tile if os.path.exists(imgpath): os.unlink(imgpath) - if os.path.exists(hashpath): - os.unlink(hashpath) return None # Create the directory if not exists @@ -615,60 +607,44 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat import errno if e.errno != errno.EEXIST: raise - - imghash = hashlib.md5() - for col, row, chunkfile in chunks: - # Get the hash of this image and add it to our hash for this tile - imghash.update( - os.path.basename(chunkfile).split(".")[4] - ) - digest = imghash.digest() - - if os.path.exists(hashpath): - oldhash = open(hashpath, 'rb').read() - else: - oldhash = None - - if digest == oldhash: - # All the chunks for this tile have not changed according to the hash - return + + # check chunk mtimes to see if they are newer + try: + tile_mtime = os.path.getmtime(imgpath) + needs_rerender = False + for col, row, chunkx, chunky, regionfile in chunks: + # check region file mtime first + if os.path.getmtime(regionfile) <= tile_mtime: + continue + + # checking chunk mtime + with open(regionfile, 'rb') as regionfile: + region = nbt.MCRFileReader(regionfile) + if region.get_chunk_timestamp(chunkx, chunky) > time_mtime: + needs_rerender = True + if needs_rerender: + break + + # if after all that, we don't need a rerender, return + if not needs_rerender: + return None + except OSError: + # couldn't get tile mtime, skip check + pass + + #logging.debug("writing out worldtile {0}".format(imgpath)) # Compile this image tileimg = Image.new("RGBA", (width, height), (38,92,255,0)) # col colstart will get drawn on the image starting at x coordinates -(384/2) # row rowstart will get drawn on the image starting at y coordinates -(192/2) - for col, row, chunkfile in chunks: - try: - chunkimg = Image.open(chunkfile) - chunkimg.load() - except Exception, e: - # If for some reason the chunk failed to load (perhaps a previous - # run was canceled and the file was only written half way, - # corrupting it), then this could error. - # Since we have no easy way of determining how this chunk was - # generated, we need to just ignore it. - logging.warning("Could not open chunk '{0}' ({1})".format(chunkfile,e)) - try: - # Remove the file so that the next run will re-generate it. - os.unlink(chunkfile) - except OSError, e: - import errno - # Ignore if file doesn't exist, another task could have already - # removed it. - if e.errno != errno.ENOENT: - logging.warning("Could not remove chunk '{0}'!".format(chunkfile)) - raise - else: - logging.warning("Removed the corrupt file") - - logging.warning("You will need to re-run the Overviewer to fix this chunk") - continue - + for col, row, chunkx, chunky, regionfile in chunks: xpos = -192 + (col-colstart)*192 ypos = -96 + (row-rowstart)*96 - composite.alpha_over(tileimg, chunkimg.convert("RGB"), (xpos, ypos), chunkimg) + # TODO draw chunks! + #composite.alpha_over(tileimg, chunkimg.convert("RGB"), (xpos, ypos), chunkimg) # Save them tileimg.save(imgpath) @@ -676,9 +652,6 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path, imgformat if optimizeimg: optimize_image(imgpath, imgformat, optimizeimg) - with open(hashpath, "wb") as hashout: - hashout.write(digest) - class FakeResult(object): def __init__(self, res): self.res = res diff --git a/world.py b/world.py index 9c47799..826a651 100644 --- a/world.py +++ b/world.py @@ -38,16 +38,6 @@ This module has routines for extracting information about available worlds base36decode = functools.partial(int, base=36) cached = collections.defaultdict(dict) - -def _convert_coords(chunkx, chunky): - """Takes a coordinate (chunkx, chunky) where chunkx and chunky are - in the chunk coordinate system, and figures out the row and column - in the image each one should be. Returns (col, row).""" - - # columns are determined by the sum of the chunk coords, rows are the - # difference (TODO: be able to change direction of north) - return (chunkx + chunky, chunky - chunkx) - def base36encode(number, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'): ''' Convert an integer to a base36 string. @@ -117,6 +107,23 @@ class World(object): return os.path.join(self.worlddir, chunkFile) + def convert_coords(self, chunkx, chunky): + """Takes a coordinate (chunkx, chunky) where chunkx and chunky are + in the chunk coordinate system, and figures out the row and column + in the image each one should be. Returns (col, row).""" + + # columns are determined by the sum of the chunk coords, rows are the + # difference (TODO: be able to change direction of north) + # change this function, and you MUST change unconvert_coords + return (chunkx + chunky, chunky - chunkx) + + def unconvert_coords(self, col, row): + """Undoes what convert_coords does. Returns (chunkx, chunky).""" + + # col + row = chunky + chunky => (col + row)/2 = chunky + # col - row = chunkx + chunkx => (col - row)/2 = chunkx + return ((col - row) / 2, (col + row) / 2) + def findTrueSpawn(self): """Adds the true spawn location to self.POI. The spawn Y coordinate is almost always the default of 64. Find the first air block above @@ -181,7 +188,7 @@ class World(object): # Translate chunks to our diagonal coordinate system mincol = maxcol = minrow = maxrow = 0 for chunkx, chunky in [(minx, miny), (minx, maxy), (maxx, miny), (maxx, maxy)]: - col, row = _convert_coords(chunkx, chunky) + col, row = self.convert_coords(chunkx, chunky) mincol = min(mincol, col) maxcol = max(maxcol, col) minrow = min(minrow, row)