LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

I believe I now have a usable program again

Browse Source
This commit is contained in:
Andrew Brown
2010-09-14 23:51:05 -04:00
parent 230e6ad480
commit 2d4f0cc082
4 changed files with 31 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
gmap.py
View File

@@ -5,17 +5,22 @@ import sys
import os.path
from optparse import OptionParser
import re
import multiprocessing
import world
import quadtree
helptext = """
%prog [-p PROCS] <Path to World> <tiles dest dir>
"""
def main():
try:
cpus = multiprocessing.cpu_count()
except NotImplementedError:
cpus = 1
parser = OptionParser(usage=helptext)
parser.add_option("-p", "--processes", dest="procs", help="How many chunks to render in parallel. A good number for this is 1 more than the number of cores in your computer. Default 2", default=2, action="store", type="int")
parser.add_option("-c", "--cachelife", dest="cachelife", help="How many minutes a tile will be considered valid by the web browser before it fetches a new copy. Used if you have a crontab or similar running this every once in a while. Default is no expiration.", default=0, action="store", type="int")
parser.add_option("-p", "--processes", dest="procs", help="How many chunks to render in parallel. A good number for this is the number of cores in your computer. Default %s" % cpus, default=cpus, action="store", type="int")
options, args = parser.parse_args()
@@ -29,54 +34,13 @@ def main():
parser.error("Where do you want to save the tiles?")
destdir = args[1]
print "Scanning chunks"
all_chunks = world.find_chunkfiles(worlddir)
# First generate the world's chunk images
w = world.WorldRenderer(worlddir)
w.go(options.procs)
# Translate chunks from diagonal coordinate system
mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks)
print "Rendering chunks"
results = world.render_chunks_async(chunks, False, options.procs)
if options.procs > 1:
for i, (col, row, filename) in enumerate(chunks):
if i > 0:
if 1000 % i == 0 or i % 1000 == 0:
print "{0}/{1} chunks rendered".format(i, len(chunks))
results['pool'].join()
print "Done"
# Compat-change for windows (which can't pass result objects to
# subprocesses)
chunkmap = {}
for col, row, filename in chunks:
chunkmap[col, row] = results[col, row].get()
del results
print "Writing out html file"
if not os.path.exists(destdir):
os.mkdir(destdir)
zoom = world.get_quadtree_depth(mincol, maxcol, minrow, maxrow)
write_html(destdir, zoom+1, options.cachelife)
print "Your map will have {0} zoom levels".format(zoom+1)
print "Generating quad tree. This may take a while and has no progress bar right now, so sit tight."
tiledir = os.path.join(destdir, "tiles")
if not os.path.exists(tiledir):
os.mkdir(tiledir)
world.generate_quadtree(chunkmap, mincol, maxcol, minrow, maxrow, tiledir, options.procs)
print "DONE"
def write_html(path, zoomlevel, cachelife):
templatepath = os.path.join(os.path.split(__file__)[0], "template.html")
html = open(templatepath, 'r').read()
html = html.replace(
"{maxzoom}", str(zoomlevel))
html = html.replace(
"{cachelife}", str(cachelife))
with open(os.path.join(path, "index.html"), 'w') as output:
output.write(html)
# Now generate the tiles
q = quadtree.QuadtreeGen(w, destdir)
q.go(options.procs)
if __name__ == "__main__":
main()

362
quadtree.py
View File

@@ -61,6 +61,16 @@ class QuadtreeGen(object):
self.world = worldobj
self.destdir = destdir
def write_html(self, zoomlevel):
"""Writes out index.html"""
templatepath = os.path.join(os.path.split(__file__)[0], "template.html")
html = open(templatepath, 'r').read()
html = html.replace(
"{maxzoom}", str(zoomlevel))
with open(os.path.join(self.destdir, "index.html"), 'w') as output:
output.write(html)
def go(self, procs):
"""Renders all tiles"""
@@ -69,6 +79,7 @@ class QuadtreeGen(object):
# Render the highest level of tiles from the chunks
print "Computing the tile ranges and starting tile processers for inner-most tiles..."
print "This takes the longest. The other levels will go quicker"
results = []
for path in iterate_base4(self.p+1):
# Get the range for this tile
@@ -97,6 +108,8 @@ class QuadtreeGen(object):
)
)
self.write_html(self.p+1)
# Wait for all results to finish
print "Rendering inner most zoom level tiles now!"
for i, result in enumerate(results):
@@ -117,7 +130,6 @@ class QuadtreeGen(object):
dest = os.path.join(self.destdir, *(str(x) for x in path[:-1]))
name = str(path[-1])
print "Applying", path, dest, name
results.append(
pool.apply_async(func=render_innertile, args=
(dest, name)
@@ -136,6 +148,9 @@ class QuadtreeGen(object):
render_innertile(self.destdir, "base")
print "Done!"
pool.close()
pool.join()
def _get_range_by_path(self, path):
"""Returns the x, y chunk coordinates of this tile"""
x, y = self.mincol, self.minrow
@@ -212,8 +227,6 @@ def render_innertile(dest, name):
os.unlink(imgpath)
if os.path.exists(hashpath):
os.unlink(hashpath)
print "Not generating due to non-existance of subtiles"
print "\t", dest, name
return
# Now check the hashes
@@ -234,8 +247,6 @@ def render_innertile(dest, name):
if newhash == oldhash:
# Nothing to do
print "Not generating due to hash match"
print "\t", dest, name
return
# Create the actual image now
@@ -255,7 +266,6 @@ def render_innertile(dest, name):
img.paste(quad3, (192, 192))
# Save it
print "Saving", imgpath
img.save(imgpath)
with open(hashpath, "wb") as hashout:
hashout.write(newhash)
@@ -372,343 +382,3 @@ def render_worldtile(chunks, colstart, colend, rowstart, rowend, path):
with open(hashpath, "wb") as hashout:
hashout.write(digest)
def get_quadtree_depth(colstart, colend, rowstart, rowend):
"""Determines the zoom depth of a requested quadtree.
Return value is an integer >= 0. Higher integers mean higher resolution
maps. This is one less than the maximum zoom (level 0 is a single tile,
level 1 is 2 tiles wide by 2 tiles high, etc.)
"""
# This determines how many zoom levels we need to encompass the entire map.
# We need to make sure that each recursive call splits both dimensions
# evenly into a power of 2 tiles wide and high, so this function determines
# how many splits to make, and generate_quadtree() uses this to adjust the
# row and column limits so that everything splits just right.
#
# This comment makes more sense if you consider it inlined in its call from
# generate_quadtree()
# Since a single tile has 3 columns of chunks and 5 rows of chunks, this
# split needs to be sized into the void so that it is some number of rows
# in the form 2*2^p. And columns must be in the form 4*2^p
# They need to be the same power
# In other words, I need to find the smallest power p such that
# colmid + 2*2^p >= colend and rowmid + 4*2^p >= rowend
# I hope that makes some sense. I don't know how to explain this very well,
# it was some trial and error.
colmid = (colstart + colend) // 2
rowmid = (rowstart + rowend) // 2
for p in xrange(15): # That should be a high enough upper limit
if colmid + 2*2**p >= colend and rowmid + 4*2**p >= rowend:
break
else:
raise Exception("Your map is waaaay to big")
return p
def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs):
"""Base call for quadtree_recurse. This sets up the recursion and generates
a quadtree given a chunkmap and the ranges.
"""
p = get_quadtree_depth(colstart, colend, rowstart, rowend);
colmid = (colstart + colend) // 2
rowmid = (rowstart + rowend) // 2
# Modify the lower and upper bounds to be sized correctly. See comments in
# get_quadtree_depth()
colstart = colmid - 2*2**p
colend = colmid + 2*2**p
rowstart = rowmid - 4*2**p
rowend = rowmid + 4*2**p
#print " power is", p
#print " new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend)
# procs is -1 here since the main process always runs as well, only spawn
# procs-1 /new/ processes
sem = multiprocessing.BoundedSemaphore(procs-1)
quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem)
def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem):
"""Recursive method that generates a quadtree.
A single call generates, saves, and returns an image with the range
specified by colstart,colend,rowstart, and rowend.
The image is saved as os.path.join(prefix, quadrant+".png")
If the requested range is larger than a certain threshold, this method will
instead make 4 calls to itself to render the 4 quadrants of the image. The
four pieces are then resized and pasted into one image that is saved and
returned.
If the requested range is not too large, it is generated with
render_worldtile()
The path "prefix" should be a directory where this call should save its
image.
quadrant is used in recursion. If it is "base", the image is saved in the
directory named by prefix, and recursive calls will have quadrant set to
"0" "1" "2" or "3" and prefix will remain unchanged.
If quadrant is anything else, the tile will be saved just the same, but for
recursive calls a directory named quadrant will be created (if it doesn't
exist) and prefix will be set to os.path.join(prefix, quadrant)
So the first call will have prefix "tiles" (e.g.) and quadrant "base" and
will save its image as "tiles/base.png"
The second call will have prefix "tiles" and quadrant "0" and will save its
image as "tiles/0.png". It will create the directory "tiles/0/"
The third call will have prefix "tiles/0", quadrant "0" and will save its image as
"tile/0/0.png"
Each tile outputted is always 384 by 384 pixels.
The last parameter, sem, should be a multiprocessing.Semaphore or
BoundedSemaphore object. Before each recursive call, the semaphore is
acquired without blocking. If the acquire is successful, the recursive call
will spawn a new process. If it is not successful, the recursive call is
run in the same thread. The semaphore is passed to each recursive call, so
any call could spawn new processes if another one exits at some point.
The return from this function is (path, hash) where path is the path to the
file saved, and hash is a byte string that depends on the tile's contents.
If the tile is blank, path will be None, but hash will still be valid.
"""
cols = colend - colstart
rows = rowend - rowstart
# Get the tile's existing hash. Maybe it hasn't changed. Whether this
# function invocation is destined to recurse, or whether we end up calling
# render_worldtile(), the hash will help us short circuit a lot of pixel
# copying.
hashpath = os.path.join(prefix, quadrant+".hash")
if os.path.exists(hashpath):
oldhash = open(hashpath, "rb").read()
else:
# This method (should) never actually return None for a hash, this is
# used so it will always compare unequal.
oldhash = None
if cols == 2 and rows == 4:
# base case: just render the image
img, newhash = render_worldtile(chunkmap, colstart, colend, rowstart, rowend, oldhash)
# There are a few cases to handle here:
# 1) img is None: the image doesn't exist (would have been blank, no
# chunks exist for that range.
# 2) img is True: the image hasn't changed according to the hashes. The
# image object is not returned by render_worldtile, but we do need to
# return the path to it.
# 3) img is a PIL.Image.Image object, a new tile was computed, we need
# to save it and its hash (newhash) to disk.
if not img:
# The image returned is blank, there should not be an image here.
# If one does exist, from a previous world or something, it is not
# deleted, but None is returned to indicate to our caller this tile
# is blank.
remove_tile(prefix, quadrant)
return None, newhash
if img is True:
# No image was returned because the hashes matched. Return the path
# to the image that already exists and is up to date according to
# the hash
path = os.path.join(prefix, quadrant+".png")
if not os.path.exists(path):
# Oops, the image doesn't actually exist. User must have
# deleted it, or must be some bug?
raise Exception("Error, this image should have existed according to the hashes, but didn't")
return path, newhash
# If img was not None or True, it is an image object. The image exists
# and the hashes did not match, so it must have changed. Fall through
# to the last part of this function which saves the image and its hash.
assert isinstance(img, Image.Image)
elif cols < 2 or rows < 4:
raise Exception("Something went wrong, this tile is too small. (Please send "
"me the traceback so I can fix this)")
else:
# Recursively generate each quadrant for this tile
# Find the midpoint
colmid = (colstart + colend) // 2
rowmid = (rowstart + rowend) // 2
# Assert that the split in the center still leaves everything sized
# exactly right by checking divisibility by the final row and
# column sizes. This isn't sufficient, but is necessary for
# success. (A better check would make sure the dimensions fit the
# above equations for the same power of 2)
assert (colmid - colstart) % 2 == 0
assert (colend - colmid) % 2 == 0
assert (rowmid - rowstart) % 4 == 0
assert (rowend - rowmid) % 4 == 0
if quadrant == "base":
newprefix = prefix
else:
# Make the directory for the recursive subcalls
newprefix = os.path.join(prefix, quadrant)
if not os.path.exists(newprefix):
os.mkdir(newprefix)
# Keep a hash of the concatenation of each returned hash. If it matches
# oldhash from above, skip rendering this tile
hasher = hashlib.md5()
# Recurse to generate each quadrant of images
if sem.acquire(False):
Procobj = ReturnableProcess
else:
Procobj = FakeProcess
quad0result = Procobj(sem, target=quadtree_recurse,
args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem)
)
if sem.acquire(False):
Procobj = ReturnableProcess
else:
Procobj = FakeProcess
quad1result = Procobj(sem, target=quadtree_recurse,
args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem)
)
if sem.acquire(False):
Procobj = ReturnableProcess
else:
Procobj = FakeProcess
quad2result = Procobj(sem, target=quadtree_recurse,
args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem)
)
# Start the processes. If one is a fakeprocess, it will do the
# processing right here instead.
quad0result.start()
quad1result.start()
quad2result.start()
# 3rd quadrent always runs in this process, no need to spawn a new one
# since we're just going to turn around and wait for it.
quad3file, hash3 = quadtree_recurse(chunkmap,
colmid, colend, rowmid, rowend,
newprefix, "3", sem)
quad0file, hash0 = quad0result.get()
quad1file, hash1 = quad1result.get()
quad2file, hash2 = quad2result.get()
# Check the hashes. This is checked even if the tile files returned
# None, since that could happen if either the tile was blank or it
# hasn't changed. So the hashes returned should tell us whether we need
# to update this tile or not.
hasher.update(hash0)
hasher.update(hash1)
hasher.update(hash2)
hasher.update(hash3)
newhash = hasher.digest()
if newhash == oldhash:
# Nothing left to do, this tile already exists and hasn't changed.
#if dbg: print "hashes match, nothing to do"
return os.path.join(prefix, quadrant+".png"), oldhash
# Check here if this tile is actually blank. If all 4 returned quadrant
# filenames are None, this tile should not be rendered. However, we
# still need to return a valid hash for it, so that's why this check is
# below the hash check.
if not (bool(quad0file) or bool(quad1file) or bool(quad2file) or
bool(quad3file)):
remove_tile(prefix, quadrant)
return None, newhash
img = Image.new("RGBA", (384, 384))
if quad0file:
quad0 = Image.open(quad0file).resize((192,192), Image.ANTIALIAS)
img.paste(quad0, (0,0))
if quad1file:
quad1 = Image.open(quad1file).resize((192,192), Image.ANTIALIAS)
img.paste(quad1, (192,0))
if quad2file:
quad2 = Image.open(quad2file).resize((192,192), Image.ANTIALIAS)
img.paste(quad2, (0, 192))
if quad3file:
quad3 = Image.open(quad3file).resize((192,192), Image.ANTIALIAS)
img.paste(quad3, (192, 192))
# At this point, if the tile hasn't change or is blank, the function should
# have returned by now.
assert bool(img)
# Save the image
path = os.path.join(prefix, quadrant+".png")
img.save(path)
print "Saving image", path
# Save the hash
with open(os.path.join(prefix, quadrant+".hash"), 'wb') as hashout:
hashout.write(newhash)
# Return the location and hash of this tile
return path, newhash
def remove_tile(prefix, quadrent):
"""Called when a tile doesn't exist, this deletes an existing tile if it
does
"""
path = os.path.join(prefix, quadrent)
img = path + ".png"
hash = path + ".hash"
if os.path.exists(img):
print "removing", img
os.unlink(img)
if os.path.exists(hash):
os.unlink(hash)
class ReturnableProcess(multiprocessing.Process):
"""Like the standard multiprocessing.Process class, but the return value of
the target method is available by calling get().
The given semaphore is released when the target finishes running"""
def __init__(self, semaphore, *args, **kwargs):
self.__sem = semaphore
multiprocessing.Process.__init__(self, *args, **kwargs)
def run(self):
try:
results = self._target(*self._args, **self._kwargs)
except BaseException, e:
self._respipe_in.send(e)
else:
self._respipe_in.send(results)
finally:
self.__sem.release()
def get(self):
self.join()
ret = self._respipe_out.recv()
if isinstance(ret, BaseException):
raise ret
return ret
def start(self):
self._respipe_out, self._respipe_in = multiprocessing.Pipe()
multiprocessing.Process.start(self)
class FakeProcess(object):
"""Identical interface to the above class, but runs in the same thread.
Used to make the code simpler in quadtree_recurse
"""
def __init__(self, semaphore, target, args=None, kwargs=None):
self._target = target
self._args = args if args else ()
self._kwargs = kwargs if kwargs else {}
def start(self):
self.ret = self._target(*self._args, **self._kwargs)
def get(self):
return self.ret

2
template.html
View File

@@ -17,7 +17,7 @@
tileSize: 384,
defaultZoom: 1,
maxZoom: {maxzoom},
cacheMinutes: {cachelife},
cacheMinutes: 0, // Change this to have browsers automatically requiest new images every x minutes
debug: false
};

1
world.py
View File

@@ -117,6 +117,7 @@ class WorldRenderer(object):
if 1000 % i == 0 or i % 1000 == 0:
print "{0}/{1} chunks rendered".format(i, len(chunks))
pool.join()
print "Done!"
return results