LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Merge commit 'xon/dtt-c-render' into dtt-c-render

Browse Source 
Conflicts:
	nbt.py
	overviewer.py
This commit is contained in:
Aaron Griffith
2011-03-25 22:44:50 -04:00
parent a6baf5b6e8 af3278e3cc
commit 3c51253eba
9 changed files with 657 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
chunk.py
View File

@@ -73,10 +73,10 @@ def get_blockarray(level):
Block array, which just contains all the block ids"""
return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128))
def get_blockarray_fromfile(world,filename):
"""Same as get_blockarray except takes a filename and uses get_lvldata to
open it. This is a shortcut"""
level = get_lvldata(world,filename)
def get_blockarray_fromfile(filename):
"""Same as get_blockarray except takes a filename. This is a shortcut"""
d = nbt.load_from_region(filename, x, y)
level = d[1]['Level']
return get_blockarray(level)
def get_skylight_array(level):

38
contrib/findSigns.py
View File

@@ -33,30 +33,34 @@ if os.path.exists(worlddir):
else:
sys.exit("Bad WorldDir")
matcher = re.compile(r"^c\..*\.dat$")
matcher = re.compile(r"^r\..*\.mcr$")
POI = []
for dirpath, dirnames, filenames in os.walk(worlddir):
for f in filenames:
if matcher.match(f):
print f
full = os.path.join(dirpath, f)
#print "inspecting %s" % full
data = nbt.load(full)[1]['Level']['TileEntities']
for entity in data:
if entity['id'] == 'Sign':
msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])
#print "checking -->%s<--" % msg.strip()
if msg.strip():
newPOI = dict(type="sign",
x= entity['x'],
y= entity['y'],
z= entity['z'],
msg=msg,
chunk= (entity['x']/16, entity['z']/16),
)
POI.append(newPOI)
print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg'])
r = nbt.load_region(full)
chunks = r.get_chunks()
for x,y in chunks:
chunk = r.load_chunk(x,y).read_all()
data = chunk[1]['Level']['TileEntities']
for entity in data:
if entity['id'] == 'Sign':
msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']])
#print "checking -->%s<--" % msg.strip()
if msg.strip():
newPOI = dict(type="sign",
x= entity['x'],
y= entity['y'],
z= entity['z'],
msg=msg,
chunk= (entity['x']/16, entity['z']/16),
)
POI.append(newPOI)
print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg'])

21
contrib/rerenderBlocks.py
View File

@@ -7,8 +7,8 @@ blockID. The output is a chunklist file that is suitable to use with the
Example:
python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> chunklist.txt
python overviewer.py --chunklist=chunklist.txt world/ output_dir/
python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> regionlist.txt
python overviewer.py --regionlist=regionlist.txt world/ output_dir/
This will rerender any chunks that contain either TNT (46), Ice (79), or
a Jack-O-Lantern (91)
@@ -42,15 +42,20 @@ ids = map(lambda x: int(x),options.ids.split(","))
sys.stderr.write("Searching for these blocks: %r...\n" % ids)
matcher = re.compile(r"^c\..*\.dat$")
matcher = re.compile(r"^r\..*\.mcr$")
for dirpath, dirnames, filenames in os.walk(options.world):
for f in filenames:
if matcher.match(f):
full = os.path.join(dirpath, f)
blocks = get_blockarray_fromfile(full)
for i in ids:
if i in blocks:
print full
break
r = nbt.load_region(full)
chunks = r.get_chunks()
for x,y in chunks:
chunk = r.load_chunk(x,y).read_all()
blocks = get_blockarray(chunk[1]['Level'])
for i in ids:
if i in blocks:
print full
break

3
googlemap.py
View File

@@ -54,7 +54,8 @@ class MapGen(object):
def __init__(self, quadtrees, skipjs=False, web_assets_hook=None):
"""Generates a Google Maps interface for the given list of
quadtrees. All of the quadtrees must have the same destdir,
image format, and world."""
image format, and world.
Note:tiledir for each quadtree should be unique. By default the tiledir is determined by the rendermode"""
self.skipjs = skipjs
self.web_assets_hook = web_assets_hook

89
nbt.py
View File

@@ -26,7 +26,7 @@ def _file_loader(func):
return None
# Is actually a filename
fileobj = open(fileobj, 'rb')
fileobj = open(fileobj, 'rb',4096)
return func(fileobj, *args)
return wrapper
@@ -44,6 +44,20 @@ def load_from_region(filename, x, y):
def load_region(filename):
return MCRFileReader(filename)
# compile the unpacker's into a classes
_byte = struct.Struct("b")
_short = struct.Struct(">h")
_int = struct.Struct(">i")
_long = struct.Struct(">q")
_float = struct.Struct(">f")
_double = struct.Struct(">d")
_24bit_int = struct.Struct("B B B")
_unsigned_byte = struct.Struct("B")
_unsigned_int = struct.Struct(">I")
_chunk_header = struct.Struct(">I B")
class NBTFileReader(object):
def __init__(self, fileobj, is_gzip=True):
if is_gzip:
@@ -61,27 +75,32 @@ class NBTFileReader(object):
def _read_tag_byte(self):
byte = self._file.read(1)
return struct.unpack("b", byte)[0]
return _byte.unpack(byte)[0]
def _read_tag_short(self):
bytes = self._file.read(2)
return struct.unpack(">h", bytes)[0]
global _short
return _short.unpack(bytes)[0]
def _read_tag_int(self):
bytes = self._file.read(4)
return struct.unpack(">i", bytes)[0]
global _int
return _int.unpack(bytes)[0]
def _read_tag_long(self):
bytes = self._file.read(8)
return struct.unpack(">q", bytes)[0]
global _long
return _long.unpack(bytes)[0]
def _read_tag_float(self):
bytes = self._file.read(4)
return struct.unpack(">f", bytes)[0]
global _float
return _float.unpack(bytes)[0]
def _read_tag_double(self):
bytes = self._file.read(8)
return struct.unpack(">d", bytes)[0]
global _double
return _double.unpack(bytes)[0]
def _read_tag_byte_array(self):
length = self._read_tag_int()
@@ -194,9 +213,11 @@ class MCRFileReader(object):
ret = 0
bytes = self._file.read(3)
global _24bit_int
bytes = _24bit_int.unpack(bytes)
for i in xrange(3):
ret = ret << 8
ret += struct.unpack("B", bytes[i])[0]
ret += bytes[i]
return ret
@@ -222,8 +243,9 @@ class MCRFileReader(object):
offset_sectors = self._read_24bit_int()
# 1-byte length in 4KiB sectors, rounded up
global _unsigned_byte
byte = self._file.read(1)
length_sectors = struct.unpack("B", byte)[0]
length_sectors = _unsigned_byte.unpack(byte)[0]
except (IndexError, struct.error):
# got a problem somewhere
return None
@@ -253,24 +275,39 @@ class MCRFileReader(object):
try:
bytes = self._file.read(4)
timestamp = struct.unpack(">I", bytes)[0]
global _unsigned_int
timestamp = _unsigned_int.unpack(bytes)[0]
except (IndexError, struct.error):
return 0
return timestamp
def get_chunk_info(self,closeFile = True):
def get_chunks(self):
"""Return a list of all chunks contained in this region file,
as a list of (x, y) coordinate tuples. To load these chunks,
provide these coordinates to load_chunk()."""
if self._chunks:
if self._chunks is not None:
return self._chunks
if self._locations is None:
self.get_chunk_info()
self._chunks = []
for x in xrange(32):
for y in xrange(32):
if self._locations[x + y * 32] is not None:
self._chunks.append((x,y))
return self._chunks
def get_chunk_info(self,closeFile = True):
"""Preloads region header information."""
if self._locations:
return
if self._file is None:
self._file = open(self._filename,'rb');
self._file = open(self._filename,'rb')
self._chunks = []
self._chunks = None
self._locations = []
self._timestamps = []
@@ -278,24 +315,20 @@ class MCRFileReader(object):
self._file.seek(0)
# read chunk location table
for y in xrange(32):
for x in xrange(32):
location = self._read_chunk_location()
self._locations.append(location)
if location:
self._chunks.append((x, y))
locations_append = self._locations.append
for _ in xrange(32*32):
locations_append(self._read_chunk_location())
# read chunk timestamp table
for y in xrange(32):
for x in xrange(32):
timestamp = self._read_chunk_timestamp()
self._timestamps.append(timestamp)
timestamp_append = self._timestamps.append
for _ in xrange(32*32):
timestamp_append(self._read_chunk_timestamp())
if closeFile:
#free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
self._file.close()
self._file = None
return self._chunks
return
def get_chunk_timestamp(self, x, y):
"""Return the given chunk's modification time. If the given
@@ -339,10 +372,8 @@ class MCRFileReader(object):
self._file.seek(location[0])
# read in the chunk data header
bytes = self._file.read(4)
data_length = struct.unpack(">I", bytes)[0]
bytes = self._file.read(1)
compression = struct.unpack("B", bytes)[0]
bytes = self._file.read(5)
data_length,compression = _chunk_header.unpack(bytes)
# figure out the compression
is_gzip = True

19
overviewer.py
View File

@@ -45,6 +45,7 @@ import composite
import world
import quadtree
import googlemap
import rendernode
helptext = """
%prog [OPTIONS] <World # / Name / Path to World> <tiles dest dir>
@@ -179,15 +180,25 @@ def main():
# create the quadtrees
# TODO chunklist
# NOTE: options.rendermode is now a list of rendermodes. for now, always use the first one
q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode[0])
q = []
qtree_args = {'depth' : options.zoom, 'imgformat' : imgformat, 'optimizeimg' : optimizeimg}
for rendermode in options.rendermode:
if rendermode == 'normal':
qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, tiledir='tiles', **qtree_args)
else:
qtree = quadtree.QuadtreeGen(w, destdir, rendermode=rendermode, **qtree_args)
q.append(qtree)
#create the distributed render
r = rendernode.RenderNode(q)
# write out the map and web assets
m = googlemap.MapGen([q,], skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)
m.go(options.procs)
# render the tiles!
q.go(options.procs)
r.go(options.procs)
def delete_all(worlddir, tiledir):
# TODO should we delete tiledir here too?

587
quadtree.py
View File

@@ -27,6 +27,7 @@ import util
import cPickle
import stat
import errno
import time
from time import gmtime, strftime, sleep
from PIL import Image
@@ -46,31 +47,8 @@ def iterate_base4(d):
"""Iterates over a base 4 number with d digits"""
return itertools.product(xrange(4), repeat=d)
def catch_keyboardinterrupt(func):
"""Decorator that catches a keyboardinterrupt and raises a real exception
so that multiprocessing will propagate it properly"""
@functools.wraps(func)
def newfunc(*args, **kwargs):
try:
return func(*args, **kwargs)
except KeyboardInterrupt:
logging.error("Ctrl-C caught!")
raise Exception("Exiting")
except:
import traceback
traceback.print_exc()
raise
return newfunc
child_quadtree = None
def pool_initializer(quadtree):
logging.debug("Child process {0}".format(os.getpid()))
#stash the quadtree object in a global variable after fork() for windows compat.
global child_quadtree
child_quadtree = quadtree
class QuadtreeGen(object):
def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, rendermode="normal"):
def __init__(self, worldobj, destdir, depth=None, tiledir=None, imgformat=None, optimizeimg=None, rendermode="normal"):
"""Generates a quadtree from the world given into the
given dest directory
@@ -84,11 +62,16 @@ class QuadtreeGen(object):
self.imgformat = imgformat
self.optimizeimg = optimizeimg
self.lighting = rendermode in ("lighting", "night", "spawn")
self.night = rendermode in ("night", "spawn")
self.spawn = rendermode in ("spawn",)
self.rendermode = rendermode
# Make the destination dir
if not os.path.exists(destdir):
os.mkdir(destdir)
if tiledir is None:
tiledir = rendermode
self.tiledir = tiledir
if depth is None:
@@ -122,21 +105,7 @@ class QuadtreeGen(object):
self.world = worldobj
self.destdir = destdir
def print_statusline(self, complete, total, level, unconditional=False):
if unconditional:
pass
elif complete < 100:
if not complete % 25 == 0:
return
elif complete < 1000:
if not complete % 100 == 0:
return
else:
if not complete % 1000 == 0:
return
logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
complete, total, level, self.p))
self.full_tiledir = os.path.join(destdir, tiledir)
def _get_cur_depth(self):
"""How deep is the quadtree currently in the destdir? This glances in
@@ -213,59 +182,8 @@ class QuadtreeGen(object):
shutil.rmtree(getpath("3"))
os.rename(getpath("new3"), getpath("3"))
def _apply_render_worldtiles(self, pool,batch_size):
"""Returns an iterator over result objects. Each time a new result is
requested, a new task is added to the pool and a result returned.
"""
batch = []
tiles = 0
for path in iterate_base4(self.p):
# Get the range for this tile
colstart, rowstart = self._get_range_by_path(path)
colend = colstart + 2
rowend = rowstart + 4
# This image is rendered at:
dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path))
#logging.debug("this is rendered at %s", dest)
# Put this in the batch to be submited to the pool
batch.append((colstart, colend, rowstart, rowend, dest))
tiles += 1
if tiles >= batch_size:
tiles = 0
yield pool.apply_async(func=render_worldtile_batch, args= [batch])
batch = []
if tiles > 0:
yield pool.apply_async(func=render_worldtile_batch, args= (batch,))
def _apply_render_inntertile(self, pool, zoom,batch_size):
"""Same as _apply_render_worltiles but for the inntertile routine.
Returns an iterator that yields result objects from tasks that have
been applied to the pool.
"""
batch = []
tiles = 0
for path in iterate_base4(zoom):
# This image is rendered at:
dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1]))
name = str(path[-1])
batch.append((dest, name, self.imgformat, self.optimizeimg))
tiles += 1
if tiles >= batch_size:
tiles = 0
yield pool.apply_async(func=render_innertile_batch, args= [batch])
batch = []
if tiles > 0:
yield pool.apply_async(func=render_innertile_batch, args= [batch])
def go(self, procs):
"""Renders all tiles"""
"""Processing before tile rendering"""
curdepth = self._get_cur_depth()
if curdepth != -1:
@@ -279,71 +197,6 @@ class QuadtreeGen(object):
for _ in xrange(curdepth - self.p):
self._decrease_depth()
logging.debug("Parent process {0}".format(os.getpid()))
# Create a pool
if procs == 1:
pool = FakePool()
global child_quadtree
child_quadtree = self
else:
pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
#warm up the pool so it reports all the worker id's
pool.map(bool,xrange(multiprocessing.cpu_count()),1)
# Render the highest level of tiles from the chunks
results = collections.deque()
complete = 0
total = 4**self.p
logging.info("Rendering highest zoom level of tiles now.")
logging.info("There are {0} tiles to render".format(total))
logging.info("There are {0} total levels to render".format(self.p))
logging.info("Don't worry, each level has only 25% as many tiles as the last.")
logging.info("The others will go faster")
count = 0
batch_size = 10
for result in self._apply_render_worldtiles(pool,batch_size):
results.append(result)
if len(results) > (10000/batch_size):
# Empty the queue before adding any more, so that memory
# required has an upper bound
while len(results) > (500/batch_size):
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
# Wait for the rest of the results
while len(results) > 0:
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
self.print_statusline(complete, total, 1, True)
# Now do the other layers
for zoom in xrange(self.p-1, 0, -1):
level = self.p - zoom + 1
assert len(results) == 0
complete = 0
total = 4**zoom
logging.info("Starting level {0}".format(level))
for result in self._apply_render_inntertile(pool, zoom,batch_size):
results.append(result)
if len(results) > (10000/batch_size):
while len(results) > (500/batch_size):
complete += results.popleft().get()
self.print_statusline(complete, total, level)
# Empty the queue
while len(results) > 0:
complete += results.popleft().get()
self.print_statusline(complete, total, level)
self.print_statusline(complete, total, level, True)
logging.info("Done")
pool.close()
pool.join()
# Do the final one right here:
render_innertile(os.path.join(self.destdir, self.tiledir), "base", self.imgformat, self.optimizeimg)
def _get_range_by_path(self, path):
"""Returns the x, y chunk coordinates of this tile"""
@@ -362,7 +215,7 @@ class QuadtreeGen(object):
return x, y
def _get_chunks_in_range(self, colstart, colend, rowstart, rowend):
def get_chunks_in_range(self, colstart, colend, rowstart, rowend):
"""Get chunks that are relevant to the tile rendering function that's
rendering that range"""
chunklist = []
@@ -380,248 +233,218 @@ class QuadtreeGen(object):
# return (col, row, chunkx, chunky, regionpath)
chunkx, chunky = unconvert_coords(col, row)
#c = get_region_path(chunkx, chunky)
_, _, c = get_region((chunkx//32, chunky//32),(None,None,None));
if c is not None:
_, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None));
if c is not None and mcr.chunkExists(chunkx,chunky):
chunklist.append((col, row, chunkx, chunky, c))
return chunklist
@catch_keyboardinterrupt
def render_innertile_batch(batch):
count = 0
#logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
for job in batch:
count += 1
render_innertile(job[0],job[1],job[2],job[3])
return count
def get_worldtiles(self):
"""Returns an iterator over the tiles of the most detailed layer
"""
for path in iterate_base4(self.p):
# Get the range for this tile
colstart, rowstart = self._get_range_by_path(path)
colend = colstart + 2
rowend = rowstart + 4
def render_innertile(dest, name, imgformat, optimizeimg):
"""
Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
os.path.join(dest, name, "{0,1,2,3}.png")
"""
imgpath = os.path.join(dest, name) + "." + imgformat
# This image is rendered at(relative to the worker's destdir):
tilepath = [str(x) for x in path]
tilepath = os.sep.join(tilepath)
#logging.debug("this is rendered at %s", dest)
if name == "base":
quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]]
else:
quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]]
# Put this in the batch to be submited to the pool
yield [self,colstart, colend, rowstart, rowend, tilepath]
#stat the tile, we need to know if it exists or it's mtime
try:
tile_mtime = os.stat(imgpath)[stat.ST_MTIME];
except OSError, e:
if e.errno != errno.ENOENT:
raise
tile_mtime = None
def get_innertiles(self,zoom):
"""Same as get_worldtiles but for the inntertile routine.
"""
for path in iterate_base4(zoom):
# This image is rendered at(relative to the worker's destdir):
tilepath = [str(x) for x in path[:-1]]
tilepath = os.sep.join(tilepath)
name = str(path[-1])
#check mtimes on each part of the quad, this also checks if they exist
needs_rerender = tile_mtime is None
quadPath_filtered = []
for path in quadPath:
yield [self,tilepath, name]
def render_innertile(self, dest, name):
"""
Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
os.path.join(dest, name, "{0,1,2,3}.png")
"""
imgformat = self.imgformat
imgpath = os.path.join(dest, name) + "." + imgformat
if name == "base":
quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]]
else:
quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]]
#stat the tile, we need to know if it exists or it's mtime
try:
quad_mtime = os.stat(path[1])[stat.ST_MTIME];
quadPath_filtered.append(path)
if quad_mtime > tile_mtime:
needs_rerender = True
except OSError:
# We need to stat all the quad files, so keep looping
pass
# do they all not exist?
if quadPath_filtered == []:
if tile_mtime is not None:
os.unlink(imgpath)
return
# quit now if we don't need rerender
if not needs_rerender:
return
#logging.debug("writing out innertile {0}".format(imgpath))
# Create the actual image now
img = Image.new("RGBA", (384, 384), (38,92,255,0))
# we'll use paste (NOT alpha_over) for quadtree generation because
# this is just straight image stitching, not alpha blending
for path in quadPath_filtered:
try:
quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
img.paste(quad, path[0])
except Exception, e:
logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
# Save it
if imgformat == 'jpg':
img.save(imgpath, quality=95, subsampling=0)
else: # png
img.save(imgpath)
if optimizeimg:
optimize_image(imgpath, imgformat, optimizeimg)
@catch_keyboardinterrupt
def render_worldtile_batch(batch):
global child_quadtree
return render_worldtile_batch_(child_quadtree, batch)
def render_worldtile_batch_(quadtree, batch):
count = 0
_get_chunks_in_range = quadtree._get_chunks_in_range
#logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
for job in batch:
count += 1
colstart = job[0]
colend = job[1]
rowstart = job[2]
rowend = job[3]
path = job[4]
# (even if tilechunks is empty, render_worldtile will delete
# existing images if appropriate)
# And uses these chunks
tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend)
#logging.debug(" tilechunks: %r", tilechunks)
render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path)
return count
def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):
"""Renders just the specified chunks into a tile and save it. Unlike usual
python conventions, rowend and colend are inclusive. Additionally, the
chunks around the edges are half-way cut off (so that neighboring tiles
will render the other half)
chunks is a list of (col, row, chunkx, chunky, filename) of chunk
images that are relevant to this call (with their associated regions)
The image is saved to path+"."+quadtree.imgformat
If there are no chunks, this tile is not saved (if it already exists, it is
deleted)
Standard tile size has colend-colstart=2 and rowend-rowstart=4
There is no return value
"""
# width of one chunk is 384. Each column is half a chunk wide. The total
# width is (384 + 192*(numcols-1)) since the first column contributes full
# width, and each additional one contributes half since they're staggered.
# However, since we want to cut off half a chunk at each end (384 less
# pixels) and since (colend - colstart + 1) is the number of columns
# inclusive, the equation simplifies to:
width = 192 * (colend - colstart)
# Same deal with height
height = 96 * (rowend - rowstart)
# The standard tile size is 3 columns by 5 rows, which works out to 384x384
# pixels for 8 total chunks. (Since the chunks are staggered but the grid
# is not, some grid coordinates do not address chunks) The two chunks on
# the middle column are shown in full, the two chunks in the middle row are
# half cut off, and the four remaining chunks are one quarter shown.
# The above example with cols 0-3 and rows 0-4 has the chunks arranged like this:
# 0,0 2,0
# 1,1
# 0,2 2,2
# 1,3
# 0,4 2,4
# Due to how the tiles fit together, we may need to render chunks way above
# this (since very few chunks actually touch the top of the sky, some tiles
# way above this one are possibly visible in this tile). Render them
# anyways just in case). "chunks" should include up to rowstart-16
imgpath = path + "." + quadtree.imgformat
world = quadtree.world
# first, remove chunks from `chunks` that don't actually exist in
# their region files
def chunk_exists(chunk):
_, _, chunkx, chunky, region = chunk
r = world.load_region(region)
return r.chunkExists(chunkx, chunky)
chunks = filter(chunk_exists, chunks)
#stat the file, we need to know if it exists or it's mtime
try:
tile_mtime = os.stat(imgpath)[stat.ST_MTIME];
except OSError, e:
if e.errno != errno.ENOENT:
raise
tile_mtime = None
if not chunks:
# No chunks were found in this tile
if tile_mtime is not None:
os.unlink(imgpath)
return None
# Create the directory if not exists
dirdest = os.path.dirname(path)
if not os.path.exists(dirdest):
try:
os.makedirs(dirdest)
tile_mtime = os.stat(imgpath)[stat.ST_MTIME];
except OSError, e:
# Ignore errno EEXIST: file exists. Since this is multithreaded,
# two processes could conceivably try and create the same directory
# at the same time.
if e.errno != errno.EEXIST:
if e.errno != errno.ENOENT:
raise
tile_mtime = None
# check chunk mtimes to see if they are newer
try:
needs_rerender = False
for col, row, chunkx, chunky, regionfile in chunks:
# check region file mtime first.
regionMtime = world.get_region_mtime(regionfile)
if regionMtime <= tile_mtime:
continue
# checking chunk mtime
region = world.load_region(regionfile)
if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
needs_rerender = True
break
# if after all that, we don't need a rerender, return
#check mtimes on each part of the quad, this also checks if they exist
needs_rerender = tile_mtime is None
quadPath_filtered = []
for path in quadPath:
try:
quad_mtime = os.stat(path[1])[stat.ST_MTIME];
quadPath_filtered.append(path)
if quad_mtime > tile_mtime:
needs_rerender = True
except OSError:
# We need to stat all the quad files, so keep looping
pass
# do they all not exist?
if quadPath_filtered == []:
if tile_mtime is not None:
os.unlink(imgpath)
return
# quit now if we don't need rerender
if not needs_rerender:
return
#logging.debug("writing out innertile {0}".format(imgpath))
# Create the actual image now
img = Image.new("RGBA", (384, 384), (38,92,255,0))
# we'll use paste (NOT alpha_over) for quadtree generation because
# this is just straight image stitching, not alpha blending
for path in quadPath_filtered:
try:
quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
img.paste(quad, path[0])
except Exception, e:
logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e)
# Save it
if self.imgformat == 'jpg':
img.save(imgpath, quality=95, subsampling=0)
else: # png
img.save(imgpath)
if self.optimizeimg:
optimize_image(imgpath, self.imgformat, self.optimizeimg)
def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path):
"""Renders just the specified chunks into a tile and save it. Unlike usual
python conventions, rowend and colend are inclusive. Additionally, the
chunks around the edges are half-way cut off (so that neighboring tiles
will render the other half)
chunks is a list of (col, row, chunkx, chunky, filename) of chunk
images that are relevant to this call (with their associated regions)
The image is saved to path+"."+self.imgformat
If there are no chunks, this tile is not saved (if it already exists, it is
deleted)
Standard tile size has colend-colstart=2 and rowend-rowstart=4
There is no return value
"""
# width of one chunk is 384. Each column is half a chunk wide. The total
# width is (384 + 192*(numcols-1)) since the first column contributes full
# width, and each additional one contributes half since they're staggered.
# However, since we want to cut off half a chunk at each end (384 less
# pixels) and since (colend - colstart + 1) is the number of columns
# inclusive, the equation simplifies to:
width = 192 * (colend - colstart)
# Same deal with height
height = 96 * (rowend - rowstart)
# The standard tile size is 3 columns by 5 rows, which works out to 384x384
# pixels for 8 total chunks. (Since the chunks are staggered but the grid
# is not, some grid coordinates do not address chunks) The two chunks on
# the middle column are shown in full, the two chunks in the middle row are
# half cut off, and the four remaining chunks are one quarter shown.
# The above example with cols 0-3 and rows 0-4 has the chunks arranged like this:
# 0,0 2,0
# 1,1
# 0,2 2,2
# 1,3
# 0,4 2,4
# Due to how the tiles fit together, we may need to render chunks way above
# this (since very few chunks actually touch the top of the sky, some tiles
# way above this one are possibly visible in this tile). Render them
# anyways just in case). "chunks" should include up to rowstart-16
imgpath = path + "." + self.imgformat
world = self.world
#stat the file, we need to know if it exists or it's mtime
try:
tile_mtime = os.stat(imgpath)[stat.ST_MTIME];
except OSError, e:
if e.errno != errno.ENOENT:
raise
tile_mtime = None
if not chunks:
# No chunks were found in this tile
if tile_mtime is not None:
os.unlink(imgpath)
return None
except OSError:
# couldn't get tile mtime, skip check
pass
#logging.debug("writing out worldtile {0}".format(imgpath))
# Create the directory if not exists
dirdest = os.path.dirname(path)
if not os.path.exists(dirdest):
try:
os.makedirs(dirdest)
except OSError, e:
# Ignore errno EEXIST: file exists. Since this is multithreaded,
# two processes could conceivably try and create the same directory
# at the same time.
if e.errno != errno.EEXIST:
raise
# Compile this image
tileimg = Image.new("RGBA", (width, height), (38,92,255,0))
# check chunk mtimes to see if they are newer
try:
needs_rerender = False
for col, row, chunkx, chunky, regionfile in chunks:
# check region file mtime first.
region,regionMtime = world.get_region_mtime(regionfile)
if regionMtime <= tile_mtime:
continue
# col colstart will get drawn on the image starting at x coordinates -(384/2)
# row rowstart will get drawn on the image starting at y coordinates -(192/2)
for col, row, chunkx, chunky, regionfile in chunks:
xpos = -192 + (col-colstart)*192
ypos = -96 + (row-rowstart)*96
# checking chunk mtime
if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime:
needs_rerender = True
break
# draw the chunk!
# TODO POI queue
chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), quadtree, False, None)
# if after all that, we don't need a rerender, return
if not needs_rerender:
return None
except OSError:
# couldn't get tile mtime, skip check
pass
# Save them
tileimg.save(imgpath)
#logging.debug("writing out worldtile {0}".format(imgpath))
if quadtree.optimizeimg:
optimize_image(imgpath, quadtree.imgformat, quadtree.optimizeimg)
# Compile this image
tileimg = Image.new("RGBA", (width, height), (38,92,255,0))
class FakeResult(object):
def __init__(self, res):
self.res = res
def get(self):
return self.res
class FakePool(object):
"""A fake pool used to render things in sync. Implements a subset of
multiprocessing.Pool"""
def apply_async(self, func, args=(), kwargs=None):
if not kwargs:
kwargs = {}
result = func(*args, **kwargs)
return FakeResult(result)
def close(self):
pass
def join(self):
pass
# col colstart will get drawn on the image starting at x coordinates -(384/2)
# row rowstart will get drawn on the image starting at y coordinates -(192/2)
for col, row, chunkx, chunky, regionfile in chunks:
xpos = -192 + (col-colstart)*192
ypos = -96 + (row-rowstart)*96
# draw the chunk!
# TODO POI queue
chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None)
# Save them
tileimg.save(imgpath)
if self.optimizeimg:
optimize_image(imgpath, self.imgformat, self.optimizeimg)

318
rendernode.py Normal file
View File

@@ -0,0 +1,318 @@
# This file is part of the Minecraft Overviewer.
#
# Minecraft Overviewer is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# Minecraft Overviewer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the Overviewer. If not, see <http://www.gnu.org/licenses/>.
import multiprocessing
import itertools
from itertools import cycle, islice
import os
import os.path
import functools
import re
import shutil
import collections
import json
import logging
import util
import cPickle
import stat
import errno
import time
from time import gmtime, strftime, sleep
"""
This module has routines related to distributing the render job to multipule nodes
"""
def catch_keyboardinterrupt(func):
"""Decorator that catches a keyboardinterrupt and raises a real exception
so that multiprocessing will propagate it properly"""
@functools.wraps(func)
def newfunc(*args, **kwargs):
try:
return func(*args, **kwargs)
except KeyboardInterrupt:
logging.error("Ctrl-C caught!")
raise Exception("Exiting")
except:
import traceback
traceback.print_exc()
raise
return newfunc
child_rendernode = None
def pool_initializer(rendernode):
logging.debug("Child process {0}".format(os.getpid()))
#stash the quadtree objects in a global variable after fork() for windows compat.
global child_rendernode
child_rendernode = rendernode
#http://docs.python.org/library/itertools.html
def roundrobin(iterables):
"roundrobin('ABC', 'D', 'EF') --> A D E B F C"
# Recipe credited to George Sakkis
pending = len(iterables)
nexts = cycle(iter(it).next for it in iterables)
while pending:
try:
for next in nexts:
yield next()
except StopIteration:
pending -= 1
nexts = cycle(islice(nexts, pending))
class RenderNode(object):
def __init__(self, quadtrees):
"""Distributes the rendering of a list of quadtrees."""
if not len(quadtrees) > 0:
raise ValueError("there must be at least one quadtree to work on")
self.quadtrees = quadtrees
#bind an index value to the quadtree so we can find it again
i = 0
for q in quadtrees:
q._render_index = i
i += 1
def print_statusline(self, complete, total, level, unconditional=False):
if unconditional:
pass
elif complete < 100:
if not complete % 25 == 0:
return
elif complete < 1000:
if not complete % 100 == 0:
return
else:
if not complete % 1000 == 0:
return
logging.info("{0}/{1} tiles complete on level {2}/{3}".format(
complete, total, level, self.max_p))
def go(self, procs):
"""Renders all tiles"""
logging.debug("Parent process {0}".format(os.getpid()))
# Create a pool
if procs == 1:
pool = FakePool()
global child_rendernode
child_rendernode = self
else:
pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,))
#warm up the pool so it reports all the worker id's
pool.map(bool,xrange(multiprocessing.cpu_count()),1)
quadtrees = self.quadtrees
# do per-quadtree init.
max_p = 0
total = 0
for q in quadtrees:
total += 4**q.p
if q.p > max_p:
max_p = q.p
q.go(procs)
self.max_p = max_p
# Render the highest level of tiles from the chunks
results = collections.deque()
complete = 0
logging.info("Rendering highest zoom level of tiles now.")
logging.info("Rendering {0} layer{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' ))
logging.info("There are {0} tiles to render".format(total))
logging.info("There are {0} total levels to render".format(self.max_p))
logging.info("Don't worry, each level has only 25% as many tiles as the last.")
logging.info("The others will go faster")
count = 0
batch_size = 4*len(quadtrees)
while batch_size < 10:
batch_size *= 2
timestamp = time.time()
for result in self._apply_render_worldtiles(pool,batch_size):
results.append(result)
# every second drain some of the queue
timestamp2 = time.time()
if timestamp2 >= timestamp + 1:
timestamp = timestamp2
count_to_remove = (1000//batch_size)
if count_to_remove < len(results):
while count_to_remove > 0:
count_to_remove -= 1
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
if len(results) > (10000//batch_size):
# Empty the queue before adding any more, so that memory
# required has an upper bound
while len(results) > (500//batch_size):
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
# Wait for the rest of the results
while len(results) > 0:
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
self.print_statusline(complete, total, 1, True)
# Now do the other layers
for zoom in xrange(self.max_p-1, 0, -1):
level = self.max_p - zoom + 1
assert len(results) == 0
complete = 0
total = 0
for q in quadtrees:
if zoom <= q.p:
total += 4**zoom
logging.info("Starting level {0}".format(level))
timestamp = time.time()
for result in self._apply_render_inntertile(pool, zoom,batch_size):
results.append(result)
# every second drain some of the queue
timestamp2 = time.time()
if timestamp2 >= timestamp + 1:
timestamp = timestamp2
count_to_remove = (1000//batch_size)
if count_to_remove < len(results):
while count_to_remove > 0:
count_to_remove -= 1
complete += results.popleft().get()
self.print_statusline(complete, total, 1)
if len(results) > (10000/batch_size):
while len(results) > (500/batch_size):
complete += results.popleft().get()
self.print_statusline(complete, total, level)
# Empty the queue
while len(results) > 0:
complete += results.popleft().get()
self.print_statusline(complete, total, level)
self.print_statusline(complete, total, level, True)
logging.info("Done")
pool.close()
pool.join()
# Do the final one right here:
for q in quadtrees:
q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
def _apply_render_worldtiles(self, pool,batch_size):
"""Returns an iterator over result objects. Each time a new result is
requested, a new task is added to the pool and a result returned.
"""
if batch_size < len(self.quadtrees):
batch_size = len(self.quadtrees)
batch = []
jobcount = 0
# roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
iterables = [q.get_worldtiles() for q in self.quadtrees]
for job in roundrobin(iterables):
# fixup so the worker knows which quadtree this is
job[0] = job[0]._render_index
# Put this in the batch to be submited to the pool
batch.append(job)
jobcount += 1
if jobcount >= batch_size:
jobcount = 0
yield pool.apply_async(func=render_worldtile_batch, args= [batch])
batch = []
if jobcount > 0:
yield pool.apply_async(func=render_worldtile_batch, args= [batch])
def _apply_render_inntertile(self, pool, zoom,batch_size):
"""Same as _apply_render_worltiles but for the inntertile routine.
Returns an iterator that yields result objects from tasks that have
been applied to the pool.
"""
if batch_size < len(self.quadtrees):
batch_size = len(self.quadtrees)
batch = []
jobcount = 0
# roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p]
for job in roundrobin(iterables):
# fixup so the worker knows which quadtree this is
job[0] = job[0]._render_index
# Put this in the batch to be submited to the pool
batch.append(job)
jobcount += 1
if jobcount >= batch_size:
jobcount = 0
yield pool.apply_async(func=render_innertile_batch, args= [batch])
batch = []
if jobcount > 0:
yield pool.apply_async(func=render_innertile_batch, args= [batch])
@catch_keyboardinterrupt
def render_worldtile_batch(batch):
global child_rendernode
rendernode = child_rendernode
count = 0
#logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
for job in batch:
count += 1
quadtree = rendernode.quadtrees[job[0]]
colstart = job[1]
colend = job[2]
rowstart = job[3]
rowend = job[4]
path = job[5]
path = quadtree.full_tiledir+os.sep+path
# (even if tilechunks is empty, render_worldtile will delete
# existing images if appropriate)
# And uses these chunks
tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend)
#logging.debug(" tilechunks: %r", tilechunks)
quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path)
return count
@catch_keyboardinterrupt
def render_innertile_batch(batch):
global child_rendernode
rendernode = child_rendernode
count = 0
#logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
for job in batch:
count += 1
quadtree = rendernode.quadtrees[job[0]]
dest = quadtree.full_tiledir+os.sep+job[1]
quadtree.render_innertile(dest=dest,name=job[2])
return count
class FakeResult(object):
def __init__(self, res):
self.res = res
def get(self):
return self.res
class FakePool(object):
"""A fake pool used to render things in sync. Implements a subset of
multiprocessing.Pool"""
def apply_async(self, func, args=(), kwargs=None):
if not kwargs:
kwargs = {}
result = func(*args, **kwargs)
return FakeResult(result)
def close(self):
pass
def join(self):
pass

8
world.py
View File

@@ -73,15 +73,17 @@ class World(object):
#find region files, or load the region list
#this also caches all the region file header info
logging.info("Scanning regions")
regionfiles = {}
regions = {}
for x, y, regionfile in self._iterate_regionfiles():
mcr = nbt.MCRFileReader(regionfile)
mcr.get_chunk_info()
regions[regionfile] = (mcr,os.path.getmtime(regionfile))
regionfiles[(x,y)] = (x,y,regionfile)
regionfiles[(x,y)] = (x,y,regionfile,mcr)
self.regionfiles = regionfiles
self.regions = regions
logging.debug("Done scanning regions")
# figure out chunk format is in use
# if not mcregion, error out early
@@ -114,7 +116,7 @@ class World(object):
def get_region_path(self, chunkX, chunkY):
"""Returns the path to the region that contains chunk (chunkX, chunkY)
"""
_, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None));
_, _, regionfile,_ = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None,None));
return regionfile
def load_from_region(self,filename, x, y):
@@ -133,7 +135,7 @@ class World(object):
return self.regions[filename][0]
def get_region_mtime(self,filename):
return self.regions[filename][1]
return self.regions[filename]
def convert_coords(self, chunkx, chunky):
"""Takes a coordinate (chunkx, chunky) where chunkx and chunky are