diff --git a/README.rst b/README.rst
index 64dabf8..658c6e9 100644
--- a/README.rst
+++ b/README.rst
@@ -21,6 +21,9 @@ Features
* Outputs a Google Map powered interface that is memory efficient, both in
generating and viewing.
+* Renders efficiently in parallel, using as many simultaneous processes as you
+ want!
+
* Utilizes 2 levels of caching to speed up subsequent renderings of your world.
* Throw the output directory up on a web server to share your Minecraft world
@@ -34,8 +37,8 @@ This program requires:
* PIL (Python Imaging Library)
* Numpy
-I developed and tested this on Linux. It has been reported to work on Windows
-and Mac, but if something doesn't, let me know.
+I develop and test this on Linux, but need help testing it on Windows and Mac.
+If something doesn't work, let me know.
Using the Google Map Tile Generator
===================================
@@ -77,9 +80,7 @@ greatly speeds up the rendering.
Using more Cores
----------------
Adding the "-p" option will utilize more cores to generate the chunk files.
-This can speed up rendering quite a bit. However, the tile generation routine
-is currently serial and not written to take advantage of multiple cores. This
-option will only affect the chunk generation (which is around half the process)
+This can speed up rendering quite a bit.
Example::
@@ -96,8 +97,33 @@ render for my world from 85M to 67M.
find /path/to/destination -name "*.png" -exec pngcrush {} {}.crush \; -exec mv {}.crush {} \;
-Windows users, you're on your own, but there's probably a way to do this. (If
-someone figures it out, let me know I'll update this README)
+If you're on Windows, I've gotten word that this command line snippet works
+provided pngout is installed and on your path. Note that the % symbols will
+need to be doubled up if this is in a batch file.
+
+::
+
+ FOR /R c:\path\to\tiles\folder %v IN (*.png) DO pngout %v /y
+
+Viewing the Results
+-------------------
+The output is two things: an index.html file, and a directory hierarchy full of
+images. To view your world, simply open index.html in a web browser. Internet
+access is required to load the Google Maps API files, but you otherwise don't
+need anything else.
+
+You can throw these files up to a web server to let others view your map. You
+do not need a Google Maps API key (as was the case with older versions of the
+API), so just copying the directory to your web server should suffice.
+
+Tip: Since Minecraft worlds rarely produce perfectly square worlds, there will
+be blank and non-existent tiles around the borders of your world. The Google
+Maps API has no way of knowing this until it requests them and the web server
+returns a 404 Not Found. If this doesn't bother you, then fine, stop reading.
+Otherwise: you can avoid a lot of 404s to your logs by configuring your web
+server to redirect all 404 requests in that directory to a single 1px
+"blank.png". This may or may not save on bandwidth, but it will probably save
+on log noise.
Using the Large Image Renderer
==============================
diff --git a/chunk.py b/chunk.py
index 3e9db04..d32bc2c 100644
--- a/chunk.py
+++ b/chunk.py
@@ -6,7 +6,6 @@ import hashlib
import nbt
import textures
-from textures import texturemap as txtarray
# General note about pasting transparent image objects onto an image with an
# alpha channel:
@@ -52,6 +51,14 @@ def render_and_save(chunkfile, cave=False):
import traceback
traceback.print_exc()
raise
+ except KeyboardInterrupt:
+ print
+ print "You pressed Ctrl-C. Unfortunately it got caught by a subprocess"
+ print "The program will terminate... eventually, but the main process"
+ print "may take a while to realize something went wrong."
+ print "To exit immediately, you'll need to kill this process some other"
+ print "way"
+ raise Exception()
class ChunkRenderer(object):
def __init__(self, chunkfile):
@@ -236,7 +243,7 @@ class ChunkRenderer(object):
if blockid not in transparent_blocks:
draw = ImageDraw.Draw(img)
if x != 15 and blocks[x+1,y,z] == 0:
- draw.line(((imgx+12,imgy), (imgx+24,imgy+6)), fill=(0,0,0), width=1)
+ draw.line(((imgx+12,imgy), (imgx+22,imgy+5)), fill=(0,0,0), width=1)
if y != 0 and blocks[x,y-1,z] == 0:
draw.line(((imgx,imgy+6), (imgx+12,imgy)), fill=(0,0,0), width=1)
diff --git a/gmap.py b/gmap.py
index 3d04e4d..6eba965 100755
--- a/gmap.py
+++ b/gmap.py
@@ -35,8 +35,11 @@ def main():
# Translate chunks from diagonal coordinate system
mincol, maxcol, minrow, maxrow, chunks = world.convert_coords(all_chunks)
- print "processing chunks in background"
+ print "Rendering chunks"
results = world.render_chunks_async(chunks, False, options.procs)
+ for i, (col, row, filename) in enumerate(chunks):
+ results[col, row].wait()
+ print "{0}/{1} chunks rendered".format(i, len(chunks))
print "Writing out html file"
if not os.path.exists(destdir):
@@ -49,7 +52,7 @@ def main():
tiledir = os.path.join(destdir, "tiles")
if not os.path.exists(tiledir):
os.mkdir(tiledir)
- world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir)
+ world.generate_quadtree(results, mincol, maxcol, minrow, maxrow, tiledir, options.procs)
print "DONE"
diff --git a/textures.py b/textures.py
index b857781..499b1a4 100644
--- a/textures.py
+++ b/textures.py
@@ -8,36 +8,70 @@ import math
import numpy
from PIL import Image, ImageEnhance
-def _get_terrain_image():
- # Check the current directory for terrain.png first:
- if os.path.isfile("terrain.png"):
- return Image.open("terrain.png")
+def _find_file(filename, mode="rb"):
+ """Searches for the given file and returns an open handle to it.
+ This searches the following locations in this order:
+
+ * The program dir (same dir as this file)
+ * On Darwin, in /Applications/Minecraft
+ * Inside minecraft.jar, which is looked for at these locations
- if "darwin" in sys.platform:
- # On Macs, terrain.png could lie at
- # "/Applications/minecraft/terrain.png" for custom terrain. Try this
- # first.
- png = "/Applications/Minecraft/terrain.png"
- if os.access(png, os.F_OK):
- return Image.open(png)
+ * On Windows, at %APPDATA%/.minecraft/bin/minecraft.jar
+ * On Darwin, at $HOME/Library/Application Support/minecraft/bin/minecraft.jar
+ * at $HOME/.minecraft/bin/minecraft.jar
- # Paths on a Mac are a bit different
- minecraftdir = os.path.join(os.environ['HOME'], "Library",
- "Application Support", "minecraft")
- minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, "bin", "minecraft.jar"))
- textures = minecraftjar.open("terrain.png")
+ * The current working directory
+ * The program dir / textures
- else:
- if "win" in sys.platform:
- minecraftdir = os.environ['APPDATA']
- else:
- minecraftdir = os.environ['HOME']
- minecraftjar = zipfile.ZipFile(os.path.join(minecraftdir, ".minecraft",
+ """
+ programdir = os.path.dirname(__file__)
+ path = os.path.join(programdir, filename)
+ if os.path.exists(path):
+ return open(path, mode)
+
+ if sys.platform == "darwin":
+ path = os.path.join("/Applications/Minecraft", filename)
+ if os.path.exists(path):
+ return open(path, mode)
+
+ # Find minecraft.jar.
+ jarpaths = []
+ if "APPDATA" in os.environ:
+ jarpaths.append( os.path.join(os.environ['APPDATA'], ".minecraft",
"bin", "minecraft.jar"))
- textures = minecraftjar.open("terrain.png")
- buffer = StringIO(textures.read())
+ if "HOME" in os.environ:
+ jarpaths.append(os.path.join(os.environ['HOME'], "Library",
+ "Application Support", "minecraft"))
+ jarpaths.append(os.path.join(os.environ['HOME'], ".minecraft", "bin",
+ "minecraft.jar"))
+
+ for jarpath in jarpaths:
+ if os.path.exists(jarpath):
+ jar = zipfile.ZipFile(jarpath)
+ try:
+ return jar.open(filename)
+ except KeyError:
+ pass
+
+ path = filename
+ if os.path.exists(path):
+ return open(path, mode)
+
+ path = os.path.join(programdir, "textures", filename)
+ if os.path.exists(path):
+ return open(path, mode)
+
+ raise IOError("Could not find the file {0}".format(filename))
+
+def _load_image(filename):
+ """Returns an image object"""
+ fileobj = _find_file(filename)
+ buffer = StringIO(fileobj.read())
return Image.open(buffer)
+def _get_terrain_image():
+ return _load_image("terrain.png")
+
def _split_terrain(terrain):
"""Builds and returns a length 256 array of each 16x16 chunk of texture"""
textures = []
@@ -100,102 +134,113 @@ def _transform_image_side(img):
return newimg
-def _build_texturemap():
- """"""
- t = terrain_images
+def _build_block(top, side):
+ """From a top texture and a side texture, build a block image.
+ top and side should be 16x16 image objects. Returns a 24x24 image
- # Notes are for things I've left out or will probably have to make special
- # exception for
- top = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33,
+ """
+ img = Image.new("RGBA", (24,24))
+
+ top = _transform_image(top)
+
+ if not side:
+ img.paste(top, (0,0), top)
+ return img
+
+ side = _transform_image_side(side)
+
+ otherside = side.transpose(Image.FLIP_LEFT_RIGHT)
+
+ # Darken the sides slightly. These methods also affect the alpha layer,
+ # so save them first (we don't want to "darken" the alpha layer making
+ # the block transparent)
+ sidealpha = side.split()[3]
+ side = ImageEnhance.Brightness(side).enhance(0.9)
+ side.putalpha(sidealpha)
+ othersidealpha = otherside.split()[3]
+ otherside = ImageEnhance.Brightness(otherside).enhance(0.8)
+ otherside.putalpha(othersidealpha)
+
+ img.paste(side, (0,6), side)
+ img.paste(otherside, (12,6), otherside)
+ img.paste(top, (0,0), top)
+
+ # Manually touch up 6 pixels that leave a gap because of how the
+ # shearing works out. This makes the blocks perfectly tessellate-able
+ for x,y in [(13,23), (17,21), (21,19)]:
+ # Copy a pixel to x,y from x-1,y
+ img.putpixel((x,y), img.getpixel((x-1,y)))
+ for x,y in [(3,4), (7,2), (11,0)]:
+ # Copy a pixel to x,y from x+1,y
+ img.putpixel((x,y), img.getpixel((x+1,y)))
+
+ return img
+
+
+def _build_blockimages():
+ """Returns a mapping from blockid to an image of that block in perspective
+ The values of the mapping are actually (image in RGB mode, alpha channel).
+ This is not appropriate for all block types, only block types that are
+ proper cubes"""
+
+ # Top textures of all block types. The number here is the index in the
+ # texture array (terrain_images), which comes from terrain.png's cells, left to right top to
+ # bottom.
+ topids = [-1,1,0,2,16,4,15,17,205,205,237,237,18,19,32,33,
34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, # Cloths are left out
-1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35, # Gold/iron blocks? Doublestep? TNT from above?
36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1, # Torch from above? leaving out fire. Redstone wire? Crops left out. sign post
-1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67, # door,ladder left out. Minecart rail orientation
66,69,72,-1,74 # clay?
]
- side = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33,
- 34,21,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+
+ # And side textures of all block types
+ sideids = [-1,1,3,2,16,4,15,17,205,205,237,237,18,19,32,33,
+ 34,20,52,48,49,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,64,64,13,12,29,28,23,22,6,6,7,8,35,
36,37,-1,-1,65,4,25,101,98,24,43,-1,86,1,1,-1,
-1,-1,-1,16,-1,-1,-1,-1,-1,51,51,-1,-1,1,66,67,
66,69,72,-1,74
]
- side[2] = 2
- return (
- [(t[x] if x != -1 else None) for x in top],
- [(_transform_image(t[x]) if x != -1 else None) for x in top],
- [(_transform_image_side(t[x]) if x != -1 else None) for x in side],
- )
-# texturemap maps block ids to a 16x16 image that goes on the top face
-# perspective_texturemap does the same, except the texture is rotated and shrunk
-# shear_texturemap maps block ids to the image that goes on the side of the
-# block, sheared appropriately
-texturemap, perspective_texturemap, shear_texturemap = _build_texturemap()
-
-def _render_sprite(img):
- """Takes a 16x16 sprite image, and returns a 22x22 image to go in the
- blockmap
- This is for rendering things that are sticking out of the ground, like
- flowers and such
- torches are drawn the same way, but torches that attach to walls are
- handled differently
- """
- pass
-
-def _render_ground_image(img):
- """Takes a 16x16 sprite image and skews it to look like it's on the ground.
- This is for things like mine track and such
-
- """
- pass
-
-def _build_blockimages():
- """Returns a mapping from blockid to an image of that block in perspective
- The values of the mapping are actually (image in RGB mode, alpha channel)"""
# This maps block id to the texture that goes on the side of the block
allimages = []
- for top, side in zip(perspective_texturemap, shear_texturemap):
- if not top or not side:
+ for toptextureid, sidetextureid in zip(topids, sideids):
+ if toptextureid == -1 or sidetextureid == -1:
allimages.append(None)
continue
- img = Image.new("RGBA", (24,24))
-
- otherside = side.transpose(Image.FLIP_LEFT_RIGHT)
- # Darken the sides slightly. These methods also affect the alpha layer,
- # so save them first (we don't want to "darken" the alpha layer making
- # the block transparent)
- if 1:
- sidealpha = side.split()[3]
- side = ImageEnhance.Brightness(side).enhance(0.9)
- side.putalpha(sidealpha)
- othersidealpha = otherside.split()[3]
- otherside = ImageEnhance.Brightness(otherside).enhance(0.8)
- otherside.putalpha(othersidealpha)
+ toptexture = terrain_images[toptextureid]
+ sidetexture = terrain_images[sidetextureid]
- # Copy on the left side
- img.paste(side, (0,6), side)
- # Copy on the other side
- img.paste(otherside, (12,6), otherside)
- # Copy on the top piece (last so it's on top)
- img.paste(top, (0,0), top)
-
- # Manually touch up 6 pixels that leave a gap because of how the
- # shearing works out. This makes the blocks perfectly tessellate-able
- for x,y in [(13,23), (17,21), (21,19)]:
- # Copy a pixel to x,y from x-1,y
- img.putpixel((x,y), img.getpixel((x-1,y)))
- for x,y in [(3,4), (7,2), (11,0)]:
- # Copy a pixel to x,y from x+1,y
- img.putpixel((x,y), img.getpixel((x+1,y)))
+ img = _build_block(toptexture, sidetexture)
allimages.append((img.convert("RGB"), img.split()[3]))
- return allimages
-# Maps block images to the appropriate texture on each side. This map is not
-# appropriate for all block types
+ # Future block types:
+ while len(allimages) < 256:
+ allimages.append(None)
+ return allimages
blockmap = _build_blockimages()
-# Future block types:
-while len(blockmap) < 256:
- blockmap.append(None)
+
+def load_water():
+ """Evidentially, the water and lava textures are not loaded from any files
+ in the jar (that I can tell). They must be generated on the fly. While
+ terrain.png does have some water and lava cells, not all texture packs
+ include them. So I load them here from a couple pngs included.
+
+ This mutates the blockmap global list with the new water and lava blocks.
+ Block 9, standing water, is given a block with only the top face showing.
+ Block 8, flowing water, is given a full 3 sided cube."""
+
+ watertexture = _load_image("water.png")
+ w1 = _build_block(watertexture, None)
+ blockmap[9] = w1.convert("RGB"), w1
+ w2 = _build_block(watertexture, watertexture)
+ blockmap[8] = w2.convert("RGB"), w2
+
+ lavatexture = _load_image("lava.png")
+ lavablock = _build_block(lavatexture, lavatexture)
+ blockmap[10] = lavablock.convert("RGB"), lavablock
+ blockmap[11] = blockmap[10]
+load_water()
diff --git a/textures/lava.png b/textures/lava.png
new file mode 100644
index 0000000..b70c6c1
Binary files /dev/null and b/textures/lava.png differ
diff --git a/textures/water.png b/textures/water.png
new file mode 100644
index 0000000..2c0e69f
Binary files /dev/null and b/textures/water.png differ
diff --git a/world.py b/world.py
index 4d36efe..79aed63 100644
--- a/world.py
+++ b/world.py
@@ -74,6 +74,8 @@ def render_chunks_async(chunks, caves, processes):
kwds=dict(cave=caves))
resultsmap[(chunkx, chunky)] = result
+ pool.close()
+
# Stick the pool object in the dict under the key "pool" so it isn't
# garbage collected (which kills the subprocesses)
resultsmap['pool'] = pool
@@ -326,7 +328,7 @@ def get_quadtree_depth(colstart, colend, rowstart, rowend):
return p
-def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix):
+def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix, procs):
"""Base call for quadtree_recurse. This sets up the recursion and generates
a quadtree given a chunkmap and the ranges.
@@ -345,9 +347,12 @@ def generate_quadtree(chunkmap, colstart, colend, rowstart, rowend, prefix):
#print " power is", p
#print " new bounds: {0},{1} {2},{3}".format(colstart, colend, rowstart, rowend)
- quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base")
+ # procs is -1 here since the main process always runs as well, only spawn
+ # procs-1 /new/ processes
+ sem = multiprocessing.BoundedSemaphore(procs-1)
+ quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, "base", sem)
-def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant):
+def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadrant, sem):
"""Recursive method that generates a quadtree.
A single call generates, saves, and returns an image with the range
specified by colstart,colend,rowstart, and rowend.
@@ -382,6 +387,13 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr
Each tile outputted is always 384 by 384 pixels.
+ The last parameter, sem, should be a multiprocessing.Semaphore or
+ BoundedSemaphore object. Before each recursive call, the semaphore is
+ acquired without blocking. If the acquire is successful, the recursive call
+ will spawn a new process. If it is not successful, the recursive call is
+ run in the same thread. The semaphore is passed to each recursive call, so
+ any call could spawn new processes if another one exits at some point.
+
The return from this function is (path, hash) where path is the path to the
file saved, and hash is a byte string that depends on the tile's contents.
If the tile is blank, path will be None, but hash will still be valid.
@@ -476,18 +488,44 @@ def quadtree_recurse(chunkmap, colstart, colend, rowstart, rowend, prefix, quadr
hasher = hashlib.md5()
# Recurse to generate each quadrant of images
- quad0file, hash0 = quadtree_recurse(chunkmap,
- colstart, colmid, rowstart, rowmid,
- newprefix, "0")
- quad1file, hash1 = quadtree_recurse(chunkmap,
- colmid, colend, rowstart, rowmid,
- newprefix, "1")
- quad2file, hash2 = quadtree_recurse(chunkmap,
- colstart, colmid, rowmid, rowend,
- newprefix, "2")
+ # Quadrent 1:
+ if sem.acquire(False):
+ Procobj = ReturnableProcess
+ else:
+ Procobj = FakeProcess
+
+ quad0result = Procobj(sem, target=quadtree_recurse,
+ args=(chunkmap, colstart, colmid, rowstart, rowmid, newprefix, "0", sem)
+ )
+ quad0result.start()
+
+ if sem.acquire(False):
+ Procobj = ReturnableProcess
+ else:
+ Procobj = FakeProcess
+ quad1result = Procobj(sem, target=quadtree_recurse,
+ args=(chunkmap, colmid, colend, rowstart, rowmid, newprefix, "1", sem)
+ )
+ quad1result.start()
+
+ if sem.acquire(False):
+ Procobj = ReturnableProcess
+ else:
+ Procobj = FakeProcess
+ quad2result = Procobj(sem, target=quadtree_recurse,
+ args=(chunkmap, colstart, colmid, rowmid, rowend, newprefix, "2", sem)
+ )
+ quad2result.start()
+
+ # 3rd quadrent always runs in this process, no need to spawn a new one
+ # since we're just going to turn around and wait for it.
quad3file, hash3 = quadtree_recurse(chunkmap,
colmid, colend, rowmid, rowend,
- newprefix, "3")
+ newprefix, "3", sem)
+
+ quad0file, hash0 = quad0result.get()
+ quad1file, hash1 = quad1result.get()
+ quad2file, hash2 = quad2result.get()
#if dbg:
# print quad0file
@@ -567,3 +605,39 @@ def remove_tile(prefix, quadrent):
os.unlink(img)
if os.path.exists(hash):
os.unlink(hash)
+
+class ReturnableProcess(multiprocessing.Process):
+ """Like the standard multiprocessing.Process class, but the return value of
+ the target method is available by calling get().
+
+ The given semaphore is released when the target finishes running"""
+ def __init__(self, semaphore, *args, **kwargs):
+ self.__sem = semaphore
+ multiprocessing.Process.__init__(self, *args, **kwargs)
+
+ def run(self):
+ results = self._target(*self._args, **self._kwargs)
+ self._respipe_in.send(results)
+ self.__sem.release()
+
+ def get(self):
+ self.join()
+ return self._respipe_out.recv()
+
+ def start(self):
+ self._respipe_out, self._respipe_in = multiprocessing.Pipe()
+ multiprocessing.Process.start(self)
+
+class FakeProcess(object):
+ """Identical interface to the above class, but runs in the same thread.
+ Used to make the code simpler in quadtree_recurse
+
+ """
+ def __init__(self, semaphore, target, args=None, kwargs=None):
+ self._target = target
+ self._args = args if args else ()
+ self._kwargs = kwargs if kwargs else {}
+ def start(self):
+ self.ret = self._target(*self._args, **self._kwargs)
+ def get(self):
+ return self.ret