Merge remote branch 'eminence/master'

chunk.py

@@ -86,6 +86,11 @@ def get_blockdata_array(level):
     in a similar manner to skylight data"""
     return numpy.frombuffer(level['Data'], dtype=numpy.uint8).reshape((16,16,64))
 
+def get_tileentity_data(level):
+    """Returns the TileEntities TAG_List from chunk dat file"""
+    data = level['TileEntities']
+    return data
+
 def iterate_chunkblocks(xoff,yoff):
     """Iterates over the 16x16x128 blocks of a chunk in rendering order.
     Yields (x,y,z,imgx,imgy)
@@ -105,12 +110,12 @@ def iterate_chunkblocks(xoff,yoff):
 transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 44, 50, 51, 52, 53,
     59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 83, 85])
 
-def render_and_save(chunkfile, cachedir, worldobj, cave=False):
+def render_and_save(chunkfile, cachedir, worldobj, cave=False, queue=None):
     """Used as the entry point for the multiprocessing workers (since processes
     can't target bound methods) or to easily render and save one chunk
     
     Returns the image file location"""
-    a = ChunkRenderer(chunkfile, cachedir, worldobj)
+    a = ChunkRenderer(chunkfile, cachedir, worldobj, queue)
     try:
         return a.render_and_save(cave)
     except ChunkCorrupt:
@@ -133,21 +138,29 @@ class ChunkCorrupt(Exception):
     pass
 
 class ChunkRenderer(object):
-    def __init__(self, chunkfile, cachedir, worldobj):
+    def __init__(self, chunkfile, cachedir, worldobj, queue):
         """Make a new chunk renderer for the given chunkfile.
         chunkfile should be a full path to the .dat file to process
         cachedir is a directory to save the resulting chunk images to
         """
+        self.queue = queue
+    
         if not os.path.exists(chunkfile):
             raise ValueError("Could not find chunkfile")
         self.chunkfile = chunkfile
         destdir, filename = os.path.split(self.chunkfile)
+        filename_split = filename.split(".")
+        chunkcoords = filename_split[1:3]
         
-        chunkcoords = filename.split(".")[1:3]
         self.coords = map(world.base36decode, chunkcoords)
         self.blockid = ".".join(chunkcoords)
-        self.world = worldobj
 
+        # chunk coordinates (useful to converting local block coords to 
+        # global block coords)
+        self.chunkX = int(filename_split[1], base=36)
+        self.chunkY = int(filename_split[2], base=36)
+
+        self.world = worldobj
         # Cachedir here is the base directory of the caches. We need to go 2
         # levels deeper according to the chunk file. Get the last 2 components
         # of destdir and use that
@@ -298,7 +311,7 @@ class ChunkRenderer(object):
         is up to date, this method doesn't render anything.
         """
         blockid = self.blockid
-
+        
         oldimg, oldimg_path = self.find_oldimage(cave)
 
         if oldimg:
@@ -479,6 +492,8 @@ class ChunkRenderer(object):
         # Odd elements get the upper 4 bits
         blockData_expanded[:,:,1::2] = blockData >> 4
 
+        tileEntities = get_tileentity_data(self.level)
+
 
         # Each block is 24x24
         # The next block on the X axis adds 12px to x and subtracts 6px from y in the image
@@ -509,6 +524,7 @@ class ChunkRenderer(object):
 
             else:
                 t = textures.blockmap[blockid]
+
             if not t:
                 continue
 
@@ -607,6 +623,30 @@ class ChunkRenderer(object):
                 if y != 0 and blocks[x,y-1,z] == 0:
                     draw.line(((imgx,imgy+6+increment), (imgx+12,imgy+increment)), fill=(0,0,0), width=1)
 
+
+        for entity in tileEntities:
+            if entity['id'] == 'Sign':
+
+                # convert the blockID coordinates from local chunk
+                # coordinates to global world coordinates
+                newPOI = dict(type="sign",
+                                x= entity['x'],
+                                y= entity['y'],
+                                z= entity['z'],
+                                msg="%s\n%s\n%s\n%s" %
+                                   (entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']),
+                                chunk= (self.chunkX, self.chunkY),
+                               )
+                self.queue.put(["newpoi", newPOI])
+
+
+        # check to see if there are any signs in the persistentData list that are from this chunk.
+        # if so, remove them from the persistentData list (since they're have been added to the world.POI
+        # list above.
+        self.queue.put(['removePOI', (self.chunkX, self.chunkY)])
+
+            
+
         return img
 
 # Render 3 blending masks for lighting

contrib/findSigns.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+
+'''
+This script will scan through every chunk looking for signs and write out an
+updated overviewer.dat file.  This can be useful if your overviewer.dat file
+is either out-of-date or non-existant.  
+
+To run, simply give a path to your world directory, for example:
+
+    python contrib/findSigns.py ../world.test/
+
+Once that is done, simply re-run the overviewer to generate markers.js:
+
+    python gmap.py ../world.test/ output_dir/
+
+Note: if your cachedir is not the same as your world-dir, you'll need to manually
+move overviewer.dat into the correct location.
+
+'''
+import sys
+import re
+import os
+import cPickle
+
+sys.path.append(".")
+import nbt
+
+from pprint import pprint
+
+worlddir = sys.argv[1]
+if os.path.exists(worlddir):
+    print "Scanning chunks in ", worlddir
+else:
+    sys.exit("Bad WorldDir")
+
+matcher = re.compile(r"^c\..*\.dat$")
+
+POI = []
+
+for dirpath, dirnames, filenames in os.walk(worlddir):
+    for f in filenames:
+        if matcher.match(f):
+            full = os.path.join(dirpath, f)
+            #print "inspecting %s" % full
+            data = nbt.load(full)[1]['Level']['TileEntities']
+            for entity in data:
+                if entity['id'] == 'Sign':
+                    newPOI = dict(type="sign",
+                                    x= entity['x'],
+                                    y= entity['y'],
+                                    z= entity['z'],
+                                    msg="%s\n%s\n%s\n%s" %
+                                       (entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']),
+                                    chunk= (entity['x']/16, entity['z']/16),
+                                   )
+                    POI.append(newPOI)
+                    print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg'])
+
+
+
+pickleFile = os.path.join(worlddir,"overviewer.dat")
+with open(pickleFile,"wb") as f:
+    cPickle.dump(dict(POI=POI), f)
+

gmap.py

@@ -145,6 +145,12 @@ def delete_all(worlddir, tiledir):
                     logging.info("Deleting {0}".format(filepath))
                     os.unlink(filepath)
 
+    # delete the overviewer.dat persistant data file
+    datfile = os.path.join(worlddir,"overviewer.dat")
+    if os.path.exists(datfile):
+        os.unlink(datfile)
+        logging.info("Deleting {0}".format(datfile))
+
 def list_worlds():
     "Prints out a brief summary of saves found in the default directory"
     print 

quadtree.py

@@ -25,6 +25,7 @@ import collections
 import json
 import logging
 import util
+import cPickle
 
 from PIL import Image
 
@@ -144,12 +145,29 @@ class QuadtreeGen(object):
         if not os.path.exists(tileDir): os.mkdir(tileDir)
         blank.save(os.path.join(tileDir, "blank."+self.imgformat))
 
+        # copy web assets into destdir:
+        for root, dirs, files in os.walk(os.path.join(util.get_program_path(), "web_assets")):
+            for f in files:
+                shutil.copy(os.path.join(root, f), self.destdir)
+
         if skipjs:
             return
 
+        # since we will only discover PointsOfInterest in chunks that need to be 
+        # [re]rendered, POIs like signs in unchanged chunks will not be listed
+        # in self.world.POI.  To make sure we don't remove these from markers.js
+        # we need to merge self.world.POI with the persistant data in world.PersistentData
+
+        self.world.POI += filter(lambda x: x['type'] != 'spawn', self.world.persistentData['POI'])
+
         # write out the default marker table
         with open(os.path.join(self.destdir, "markers.js"), 'w') as output:
             output.write("var markerData=%s" % json.dumps(self.world.POI))
+        
+        # save persistent data
+        self.world.persistentData['POI'] = self.world.POI
+        with open(self.world.pickleFile,"wb") as f:
+            cPickle.dump(self.world.persistentData,f)
 
         # write out the default (empty, but documented) region table
         with open(os.path.join(self.destdir, "regions.js"), 'w') as output:

template.html

@@ -2,11 +2,7 @@
 <html>
 <head>
 <meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
-<style type="text/css">
+<link rel="stylesheet" href="style.css" type="text/css" />
-  html { height: 100% }
-  body { height: 100%; margin: 0px; padding: 0px ; background-color: #000; }
-  #mcmap { height: 100% }
-</style>
 <script type="text/javascript" src="markers.js"></script>
 <script type="text/javascript" src="regions.js"></script>
 <script type="text/javascript"
@@ -138,7 +134,19 @@
   var map;
   
   var markersInit = false;
-  
+
+  function prepareSignMarker(marker, item) {
+      
+      var c = "<div class=\"infoWindow\"><img src=\"signpost.png\" /><p>" + item.msg.replace(/\n/g,"<br/>") + "</p></div>";
+      var infowindow = new google.maps.InfoWindow({
+content: c
+});
+google.maps.event.addListener(marker, 'click', function() {
+        infowindow.open(map,marker);
+        });
+
+}
+
   function initMarkers() {
     if (markersInit) { return; }
     
@@ -146,13 +154,26 @@
 
     for (i in markerData) {
       var item = markerData[i];
+
+      // a default:
+      var iconURL = '';
+      
+      if (item.type == 'spawn') { iconURL = 'http://google-maps-icons.googlecode.com/files/home.png';}
+      if (item.type == 'sign') { iconURL = 'signpost_icon.png';}
       
       var converted = fromWorldToLatLng(item.x, item.y, item.z);
        var marker = new google.maps.Marker({
         position: converted,
         map: map,
-        title: item.msg
+        title: item.msg, 
-      });
+        icon: iconURL
+
+        });
+
+       if (item.type == 'sign') {
+           prepareSignMarker(marker, item);
+       }
+
    }
   }
   
@@ -234,6 +255,18 @@
 	// initialize the markers and regions
 	initMarkers();
     initRegions();
+
+    var compassDiv = document.createElement('DIV');
+
+    compassDiv.style.padding = '5px';
+
+    var compassImg = document.createElement('IMG');
+    compassImg.src="compass.png";
+    compassDiv.appendChild(compassImg);
+
+    map.controls[google.maps.ControlPosition.TOP_RIGHT].push(compassDiv);
+
+
   }
 </script>
 </head>

textures.py

@@ -21,7 +21,7 @@ from cStringIO import StringIO
 import math
 
 import numpy
-from PIL import Image, ImageEnhance
+from PIL import Image, ImageEnhance, ImageOps
 
 import util
 import composite
@@ -268,8 +268,8 @@ def _build_blockimages():
                36, 37, 80, -1, 65,  4, 25,101, 98, 24, 43, -1, 86,  1,  1, -1, # Torch from above? leaving out fire. Redstone wire? Crops/furnaces handled elsewhere. sign post
        #       64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79
                -1, -1, -1, 16, -1, -1, -1, -1, -1, 51, 51, -1, -1,  1, 66, 67, # door,ladder left out. Minecart rail orientation
-       #       80  81  82  83  84
+       #       80  81  82  83  84  85  86  87  88  89  90  91
-               66, 69, 72, 73, 74 # clay?
+               66, 69, 72, 73, 74, -1,102,103,104,105,-1, 102 # clay?
         ]
 
     # NOTE: For non-block textures, the sideid is ignored, but can't be -1
@@ -285,8 +285,8 @@ def _build_blockimages():
                 36, 37, 80, -1, 65,  4, 25,101, 98, 24, 43, -1, 86, 44, 61, -1,
        #        64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79
                 -1, -1, -1, 16, -1, -1, -1, -1, -1, 51, 51, -1, -1,  1, 66, 67,
-       #        80  81  82  83  84
+       #        80  81  82  83  84  85  86  87  88  89  90  91
-                66, 69, 72, 73, 74
+                66, 69, 72, 73, 74,-1 ,118,103,104,105, -1, 118
         ]
 
     # This maps block id to the texture that goes on the side of the block
@@ -393,6 +393,19 @@ def generate_special_texture(blockID, data):
         composite.alpha_over(img, side2, (12,6), side2)
         composite.alpha_over(img, top, (0,0), top)
         return (img.convert("RGB"), img.split()[3])
+
+    if blockID in (86,91): # jack-o-lantern
+        top = transform_image(terrain_images[102])
+        frontID = 119 if blockID == 86 else 120
+        side1 = transform_image_side(terrain_images[frontID])
+        side2 = transform_image_side(terrain_images[118]).transpose(Image.FLIP_LEFT_RIGHT)
+
+        img = Image.new("RGBA", (24,24), (38,92,255,0))
+
+        img.paste(side1, (0,6), side1)
+        img.paste(side2, (12,6), side2)
+        img.paste(top, (0,0), top)
+        return (img.convert("RGB"), img.split()[3])
     
     if blockID == 62: # lit furnace
         top = transform_image(terrain_images[1])
@@ -484,13 +497,43 @@ def generate_special_texture(blockID, data):
         
         return (img.convert("RGB"), img.split()[3])
     
+    if blockID == 2: # grass
+        top = transform_image(tintTexture(terrain_images[0],(170,255,50)))
+        side1 = transform_image_side(terrain_images[3])
+        side2 = transform_image_side(terrain_images[3]).transpose(Image.FLIP_LEFT_RIGHT)
+
+        img = Image.new("RGBA", (24,24), (38,92,255,0))
+
+        img.paste(side1, (0,6), side1)
+        img.paste(side2, (12,6), side2)
+        img.paste(top, (0,0), top)
+        return (img.convert("RGB"), img.split()[3])
+    
+    if blockID == 18: # leaves
+        t = tintTexture(terrain_images[52], (170, 255, 50))
+        top = transform_image(t)
+        side1 = transform_image_side(t)
+        side2 = transform_image_side(t).transpose(Image.FLIP_LEFT_RIGHT)
+
+        img = Image.new("RGBA", (24,24), (38,92,255,0))
+
+        img.paste(side1, (0,6), side1)
+        img.paste(side2, (12,6), side2)
+        img.paste(top, (0,0), top)
+        return (img.convert("RGB"), img.split()[3])
+    
 
     return None
 
+def tintTexture(im, c):
+    # apparently converting to grayscale drops the alpha channel?
+    i = ImageOps.colorize(ImageOps.grayscale(im), (0,0,0), c)
+    i.putalpha(im.split()[3]); # copy the alpha band back in. assuming RGBA
+    return i
 
 # This set holds block ids that require special pre-computing.  These are typically
 # things that require ancillary data to render properly (i.e. ladder plus orientation)
-special_blocks = set([66,59,61,62, 65,64,71])
+special_blocks = set([66,59,61,62, 65,64,71,91,86,2,18])
 
 # this is a map of special blockIDs to a list of all 
 # possible values for ancillary data that it might have.
@@ -502,6 +545,18 @@ special_map[62] = (0,)      # burning furnace
 special_map[65] = (2,3,4,5) # ladder
 special_map[64] = range(16) # wooden door
 special_map[71] = range(16) # iron door
+special_map[91] = range(5)  # jack-o-lantern
+special_map[86] = range(5)  # pumpkin
+# apparently pumpkins and jack-o-lanterns have ancillary data, but it's unknown
+# what that data represents.  For now, assume that the range for data is 0 to 5
+# like torches
+special_map[2] = (0,)       # grass
+special_map[18] = range(16) # leaves
+# grass and leaves are now graysacle in terrain.png
+# we treat them as special so we can manually tint them
+# it is unknown how the specific tint (biomes) is calculated
+
+# leaves have ancilary data, but its meaning is unknown (age perhaps?)
 
 specialblockmap = {}
 

web_assets/style.css

@@ -0,0 +1,18 @@
+html { height: 100% }
+body { height: 100%; margin: 0px; padding: 0px ; background-color: #000; }
+#mcmap { height: 100% }
+
+.infoWindow {
+    height: 100px;
+}
+
+.infoWindow>img {
+    width:80px;
+    float: left;
+
+}
+
+.infoWindow>p {
+    text-align: center;
+    font-family: monospace;
+}

world.py

@@ -17,8 +17,10 @@ import functools
 import os
 import os.path
 import multiprocessing
+import Queue
 import sys
 import logging
+import cPickle
 
 import numpy
 
@@ -105,6 +107,20 @@ class WorldRenderer(object):
         #  a list of dictionaries, see below for an example
         self.POI = []
 
+        # if it exists, open overviewer.dat, and read in the data structure
+        # info self.persistentData.  This dictionary can hold any information
+        # that may be needed between runs.
+        # Currently only holds into about POIs (more more details, see quadtree)
+        self.pickleFile = os.path.join(self.cachedir,"overviewer.dat")
+        if os.path.exists(self.pickleFile):
+            with open(self.pickleFile,"rb") as p:
+                self.persistentData = cPickle.load(p)
+        else:
+            # some defaults
+            self.persistentData = dict(POI=[])
+
+
+
     def _get_chunk_renderset(self):
         """Returns a set of (col, row) chunks that should be rendered. Returns
         None if all chunks should be rendered"""
@@ -180,7 +196,8 @@ class WorldRenderer(object):
             spawnY += 1
 
 
-        self.POI.append( dict(x=spawnX, y=spawnY, z=spawnZ, msg="Spawn"))
+        self.POI.append( dict(x=spawnX, y=spawnY, z=spawnZ, 
+                msg="Spawn", type="spawn", chunk=(inChunkX,inChunkZ)))
 
     def go(self, procs):
         """Starts the render. This returns when it is finished"""
@@ -242,6 +259,9 @@ class WorldRenderer(object):
         inclusion_set = self._get_chunk_renderset()
 
         results = {}
+        manager = multiprocessing.Manager()
+        q = manager.Queue()
+
         if processes == 1:
             # Skip the multiprocessing stuff
             logging.debug("Rendering chunks synchronously since you requested 1 process")
@@ -254,9 +274,17 @@ class WorldRenderer(object):
                         results[(col, row)] = imgpath
                         continue
 
-                result = chunk.render_and_save(chunkfile, self.cachedir, self, cave=self.caves)
+                result = chunk.render_and_save(chunkfile, self.cachedir, self, cave=self.caves, queue=q)
                 results[(col, row)] = result
                 if i > 0:
+                    try:
+                        item = q.get(block=False)
+                        if item[0] == "newpoi":
+                            self.POI.append(item[1])
+                        elif item[0] == "removePOI":
+                            self.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.persistentData['POI'])
+                    except Queue.Empty:
+                        pass
                     if 1000 % i == 0 or i % 1000 == 0:
                         logging.info("{0}/{1} chunks rendered".format(i, len(chunks)))
         else:
@@ -274,13 +302,22 @@ class WorldRenderer(object):
 
                 result = pool.apply_async(chunk.render_and_save,
                         args=(chunkfile,self.cachedir,self),
-                        kwds=dict(cave=self.caves))
+                        kwds=dict(cave=self.caves, queue=q))
                 asyncresults.append((col, row, result))
 
             pool.close()
 
             for i, (col, row, result) in enumerate(asyncresults):
                 results[(col, row)] = result.get()
+                try:
+                    item = q.get(block=False)
+                    if item[0] == "newpoi":
+                        self.POI.append(item[1])
+                    elif item[0] == "removePOI":
+                        self.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.persistentData['POI'])
+
+                except Queue.Empty:
+                    pass
                 if i > 0:
                     if 1000 % i == 0 or i % 1000 == 0:
                         logging.info("{0}/{1} chunks rendered".format(i, len(asyncresults)))