Merge pull request #1135 from MasterofJOKers/my_genpoi

addressing genPOI RAM usage

overviewer.py

@@ -114,6 +114,8 @@ def main():
             help="Runs the genPOI script")
     exegroup.add_option("--skip-scan", dest="skipscan", action="store_true",
             help="When running GenPOI, don't scan for entities")
+    exegroup.add_option("--skip-players", dest="skipplayers", action="store_true",
+            help="When running GenPOI, don't get player data")
 
     parser.add_option_group(exegroup)
 

overviewer_core/aux_files/genPOI.py

@@ -21,17 +21,18 @@ import json
 import sys
 import re
 import urllib2
-import Queue
 import multiprocessing
+import itertools
 import gzip
 
-from multiprocessing import Process
+from collections import defaultdict
 from multiprocessing import Pool
 from optparse import OptionParser
 
 from overviewer_core import logger
 from overviewer_core import nbt
 from overviewer_core import configParser, world
+from overviewer_core import rendermodes
 
 UUID_LOOKUP_URL = 'https://sessionserver.mojang.com/session/minecraft/profile/'
 
@@ -43,22 +44,26 @@ def replaceBads(s):
         x = x.replace(bad,"_")
     return x
 
+
 # yes there's a double parenthesis here
 # see below for when this is called, and why we do this
 # a smarter way would be functools.partial, but that's broken on python 2.6
 # when used with multiprocessing
-def parseBucketChunks((bucket, rset)):
+def parseBucketChunks((bucket, rset, filters)):
     pid = multiprocessing.current_process().pid
-    pois = dict(TileEntities=[], Entities=[]);
+    markers = defaultdict(list)
 
     i = 0
     cnt = 0
-    l = len(bucket)
     for b in bucket:
         try:
             data = rset.get_chunk(b[0],b[1])
-            pois['TileEntities'] += data['TileEntities']
+            for poi in itertools.chain(data['TileEntities'], data['Entities']):
-            pois['Entities']     += data['Entities']
+                for name, filter_function in filters:
+                    result = filter_function(poi)
+                    if result:
+                        d = create_marker_from_filter_result(poi, result)
+                        markers[name].append(d)
         except nbt.CorruptChunkError:
             logging.warning("Ignoring POIs in corrupt chunk %d,%d", b[0], b[1])
 
@@ -67,67 +72,75 @@ def parseBucketChunks((bucket, rset)):
         if i == 250:
             i = 0
             cnt = 250 + cnt
-            logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt);
+            logging.info("Found %d markers in thread %d so far at %d chunks", sum(len(v) for v in markers.itervalues()), pid, cnt);
 
-    return pois
+    return markers
 
-def handleEntities(rset, outputdir, render, rname, config):
 
-    # if we're already handled the POIs for this region regionset, do nothing
+def handleEntities(rset, config, filters, markers):
-    if hasattr(rset, "_pois"):
+    """
-        return
+    Add markers for Entities or TileEntities.
 
+    For this every chunk of the regionset is parsed and filtered using multiple
+    processes, if so configured.
+    This function will not return anything, but it will update the parameter
+    `markers`.
+    """
     logging.info("Looking for entities in %r", rset)
 
-    filters = render['markers']
-    rset._pois = dict(TileEntities=[], Entities=[])
-
     numbuckets = config['processes'];
     if numbuckets < 0:
         numbuckets = multiprocessing.cpu_count()
 
     if numbuckets == 1:
-        for (x,z,mtime) in rset.iterate_chunks():
+        for (x, z, mtime) in rset.iterate_chunks():
             try:
-                data = rset.get_chunk(x,z) 
+                data = rset.get_chunk(x, z)
-                rset._pois['TileEntities'] += data['TileEntities']
+                for poi in itertools.chain(data['TileEntities'], data['Entities']):
-                rset._pois['Entities']     += data['Entities']
+                    for name, __, filter_function, __, __, __ in filters:
+                        result = filter_function(poi)
+                        if result:
+                            d = create_marker_from_filter_result(poi, result)
+                            markers[name]['raw'].append(d)
             except nbt.CorruptChunkError:
                 logging.warning("Ignoring POIs in corrupt chunk %d,%d", x,z)
   
     else:
         buckets = [[] for i in range(numbuckets)];
   
-        for (x,z,mtime) in rset.iterate_chunks():
+        for (x, z, mtime) in rset.iterate_chunks():
             i = x / 32 + z / 32
             i = i % numbuckets 
-            buckets[i].append([x,z])
+            buckets[i].append([x, z])
   
         for b in buckets:
             logging.info("Buckets has %d entries", len(b));
   
         # Create a pool of processes and run all the functions
         pool = Pool(processes=numbuckets)
-        results = pool.map(parseBucketChunks, ((buck, rset) for buck in buckets))
+
+        # simplify the filters dict, so pickle doesn't have to do so much
+        filters = [(name, filter_function) for name, __, filter_function, __, __, __ in filters]
+
+        results = pool.map(parseBucketChunks, ((buck, rset, filters) for buck in buckets))
   
         logging.info("All the threads completed")
   
-        # Fix up all the quests in the reset
+        for marker_dict in results:
-        for data in results:
+            for name, marker_list in marker_dict.iteritems():
-            rset._pois['TileEntities'] += data['TileEntities']
+                markers[name]['raw'].extend(marker_list)
-            rset._pois['Entities']     += data['Entities']
 
     logging.info("Done.")
 
+
 class PlayerDict(dict):
     use_uuid = False
     _name = ''
-    uuid_cache = None # A cache the UUID->profile lookups
+    uuid_cache = None # A cache for the UUID->profile lookups
     
     @classmethod
     def load_cache(cls, outputdir):
         cache_file = os.path.join(outputdir, "uuidcache.dat")
-        pid = multiprocessing.current_process().pid
         if os.path.exists(cache_file):
             gz = gzip.GzipFile(cache_file) 
             cls.uuid_cache = json.load(gz)
@@ -135,8 +148,6 @@ class PlayerDict(dict):
         else:
             cls.uuid_cache = {}
             logging.info("Initialized an empty UUID cache")
-            cls.save_cache(outputdir)
-
 
     @classmethod
     def save_cache(cls, outputdir):
@@ -145,7 +156,6 @@ class PlayerDict(dict):
         json.dump(cls.uuid_cache, gz)
         logging.info("Wrote UUID cache with %d entries", len(cls.uuid_cache.keys()))
 
-
     def __getitem__(self, item):
         if item == "EntityId":
             if not super(PlayerDict, self).has_key("EntityId"):
@@ -172,19 +182,16 @@ class PlayerDict(dict):
         except (ValueError, urllib2.URLError):
             logging.warning("Unable to get player name for UUID %s", self._name)
 
-def handlePlayers(rset, render, worldpath, outputdir):
-    if not hasattr(rset, "_pois"):
-        rset._pois = dict(TileEntities=[], Entities=[])
 
-    # only handle this region set once
+def handlePlayers(worldpath, filters, markers):
-    if 'Players' in rset._pois:
+    """
-        return
+    Add markers for players to the list of markers.
-
-    if rset.get_type():
-        dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2))
-    else:
-        dimension = 0
 
+    For this the player files under the given `worldpath` are parsed and
+    filtered.
+    This function will not return anything, but it will update the parameter
+    `markers`.
+    """
     playerdir = os.path.join(worldpath, "playerdata")
     useUUIDs = True
     if not os.path.isdir(playerdir):
@@ -195,13 +202,10 @@ def handlePlayers(rset, render, worldpath, outputdir):
         playerfiles = os.listdir(playerdir)
         playerfiles = [x for x in playerfiles if x.endswith(".dat")]
         isSinglePlayer = False
-
     else:
         playerfiles = [os.path.join(worldpath, "level.dat")]
         isSinglePlayer = True
 
-    rset._pois['Players'] = []
-
     for playerfile in playerfiles:
         try:
             data = PlayerDict(nbt.load(os.path.join(playerdir, playerfile))[1])
@@ -211,14 +215,12 @@ def handlePlayers(rset, render, worldpath, outputdir):
         except IOError:
             logging.warning("Skipping bad player dat file %r", playerfile)
             continue
-        playername = playerfile.split(".")[0]
 
+        playername = playerfile.split(".")[0]
         if isSinglePlayer:
             playername = 'Player'
-
         data._name = playername
 
-        if data['Dimension'] == dimension:
         # Position at last logout
         data['id'] = "Player"
         data['x'] = int(data['Pos'][0])
@@ -226,8 +228,9 @@ def handlePlayers(rset, render, worldpath, outputdir):
         data['z'] = int(data['Pos'][2])
         # Time at last logout, calculated from last time the player's file was modified
         data['time'] = time.localtime(os.path.getmtime(os.path.join(playerdir, playerfile)))
-            rset._pois['Players'].append(data)
+
-        if "SpawnX" in data and dimension == 0:
+        # Spawn position (bed or main spawn)
+        if "SpawnX" in data:
             # Spawn position (bed or main spawn)
             spawn = PlayerDict()
             spawn.use_uuid = useUUIDs
@@ -236,16 +239,94 @@ def handlePlayers(rset, render, worldpath, outputdir):
             spawn["x"] = data['SpawnX']
             spawn["y"] = data['SpawnY']
             spawn["z"] = data['SpawnZ']
-            rset._pois['Players'].append(spawn)
 
-def handleManual(rset, manualpois):
+        for name, __, filter_function, rset, __, __ in filters:
-    if not hasattr(rset, "_pois"):
+            # get the dimension for the filter
-        rset._pois = dict(TileEntities=[], Entities=[])
+            # This has do be done every time, because we have filters for
+            # different regionsets.
 
-    rset._pois['Manual'] = []
+            if rset.get_type():
+                dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2))
+            else:
+                dimension = 0
+
+            if data['Dimension'] == dimension:
+                result = filter_function(data)
+                if result:
+                    d = create_marker_from_filter_result(data, result)
+                    markers[name]['raw'].append(d)
+
+            if dimension == 0 and "SpawnX" in data:
+                result = filter_function(spawn)
+                if result:
+                    d = create_marker_from_filter_result(spawn, result)
+                    markers[name]['raw'].append(d)
+
+
+def handleManual(manualpois, filters, markers):
+    """
+    Add markers for manually defined POIs to the list of markers.
+
+    This function will not return anything, but it will update the parameter
+    `markers`.
+    """
+    for poi in manualpois:
+        for name, __, filter_function, __, __, __ in filters:
+            result = filter_function(poi)
+            if result:
+                d = create_marker_from_filter_result(poi, result)
+                markers[name]['raw'].append(d)
+
+
+def create_marker_from_filter_result(poi, result):
+    """
+    Takes a POI and the return value of a filter function for it and creates a
+    marker dict depending on the type of the returned value.
+    """
+    # every marker has a position either directly via attributes x, y, z or
+    # via tuple attribute Pos
+    if 'Pos' in poi:
+        d = dict((v, poi['Pos'][i]) for i, v in enumerate('xyz'))
+    else:
+        d = dict((v, poi[v]) for v in 'xyz')
+
+    # read some Defaults from POI
+    if "icon" in poi:
+        d["icon"] = poi['icon']
+    if "createInfoWindow" in poi:
+        d["createInfoWindow"] = poi['createInfoWindow']
+
+    # Fill in the rest from result
+    if isinstance(result, basestring):
+        d.update(dict(text=result, hovertext=result))
+    elif isinstance(result, tuple):
+        d.update(dict(text=result[1], hovertext=result[0]))
+    # Dict support to allow more flexible things in the future as well as polylines on the map.
+    elif isinstance(result, dict):
+        d['text'] = result['text']
+
+        # Use custom hovertext if provided...
+        if 'hovertext' in result:
+            d['hovertext'] = unicode(result['hovertext'])
+        else: # ...otherwise default to display text.
+            d['hovertext'] = result['text']
+
+        if 'polyline' in result and hasattr(result['polyline'], '__iter__'):
+            d['polyline'] = []
+            for point in result['polyline']:
+                d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z'])) # point.copy() would work, but this validates better
+            if isinstance(result['color'], basestring):
+                d['strokeColor'] = result['color']
+
+            if "icon" in result:
+                d["icon"] = result['icon']
+            if "createInfoWindow" in result:
+                d["createInfoWindow"] = result['createInfoWindow']
+    else:
+        raise ValueError("got an %s as result for POI with id %s" % (type(result).__name__, poi['id']))
+
+    return d
 
-    if manualpois:
-        rset._pois['Manual'].extend(manualpois)
 
 def main():
 
@@ -262,6 +343,7 @@ def main():
     parser.add_option("-c", "--config", dest="config", action="store", help="Specify the config file to use.")
     parser.add_option("--quiet", dest="quiet", action="count", help="Reduce logging output")
     parser.add_option("--skip-scan", dest="skipscan", action="store_true", help="Skip scanning for entities when using GenPOI")
+    parser.add_option("--skip-players", dest="skipplayers", action="store_true", help="Skip getting player data when using GenPOI")
 
     options, args = parser.parse_args()
     if not options.config:
@@ -284,12 +366,13 @@ def main():
     # saves us from creating the same World object over and over again
     worldcache = {}
 
-    markersets = set()
+    filters = set()
-    markers = dict()
+    marker_groups = defaultdict(list)
-
-    PlayerDict.load_cache(destdir)
 
+    # collect all filters and get regionsets
     for rname, render in config['renders'].iteritems():
+        # Convert render['world'] to the world path, and store the original
+        # in render['worldname_orig']
         try:
             worldpath = config['worlds'][render['world']]
         except KeyError:
@@ -306,148 +389,82 @@ def main():
         else:
             w = worldcache[render['world']]
 
+        # get the regionset for this dimension
         rset = w.get_regionset(render['dimension'][1])
         if rset == None: # indicates no such dimension was found:
             logging.warn("Sorry, you requested dimension '%s' for the render '%s', but I couldn't find it", render['dimension'][0], rname)
             continue
 
+        # find filters for this render
         for f in render['markers']:
-            markersets.add(((f['name'], f['filterFunction']), rset))
+            # internal identifier for this filter
             name = replaceBads(f['name']) + hex(hash(f['filterFunction']))[-4:] + "_" + hex(hash(rset))[-4:]
-            to_append = dict(groupName=name, 
+
+            # we need to make the function pickleable for multiprocessing to
+            # work
+            # We set a custom prefix here to not override any functions there.
+            # These functions are only pickleable if they are bound to a
+            # module. Since rendermodes imports the config, they are bound to
+            # it anyway, but don't end up importable as
+            # `rendermodes.filter_fn`. That's why we set it here explicitly on
+            # the module.
+            f['filterFunction'].__name__ = "custom_filter_" + f['filterFunction'].__name__
+            setattr(rendermodes, f['filterFunction'].__name__, f['filterFunction'])
+
+            # add it to the list of filters
+            filters.add((name, f['name'], f['filterFunction'], rset, worldpath, rname))
+
+            # add an entry in the menu to show markers found by this filter
+            group = dict(groupName=name,
                     displayName = f['name'], 
                     icon=f.get('icon', 'signpost_icon.png'), 
-                    createInfoWindow=f.get('createInfoWindow',True),
+                    createInfoWindow=f.get('createInfoWindow', True),
                     checked = f.get('checked', False))
-            try:
+            marker_groups[rname].append(group)
-                l = markers[rname]
-                l.append(to_append)
-            except KeyError:
-                markers[rname] = [to_append]
 
+    # initialize the structure for the markers
+    markers = dict((name, dict(created=False, raw=[], name=filter_name))
+                    for name, filter_name, __, __, __, __ in filters)
+
+    # apply filters to regionsets
     if not options.skipscan:
-            handleEntities(rset, os.path.join(destdir, rname), render, rname, config)
+        # group filters by rset
+        keyfunc = lambda x: x[3]
+        sfilters = sorted(filters, key=keyfunc)
+        for rset, rset_filters in itertools.groupby(sfilters, keyfunc):
+            handleEntities(rset, config, rset_filters, markers)
 
-        handlePlayers(rset, render, worldpath, destdir)
+    # apply filters to players
-        handleManual(rset, render['manualpois'])
+    if not options.skipplayers:
+        PlayerDict.load_cache(destdir)
+        # group filters by worldpath, so we only search for players once per
+        # world
+        keyfunc = lambda x: x[4]
+        sfilters = sorted(filters, key=keyfunc)
+        for worldpath, worldpath_filters in itertools.groupby(sfilters, keyfunc):
+            handlePlayers(worldpath, worldpath_filters, markers)
+
+    # add manual POIs
+    # group filters by name of the render, because only filter functions for
+    # the current render should be used on the current render's manualpois
+    keyfunc = lambda x: x[5]
+    sfilters = sorted(filters, key=keyfunc)
+    for rname, rname_filters in itertools.groupby(sfilters, keyfunc):
+        manualpois = config['renders'][rname]['manualpois']
+        handleManual(manualpois, rname_filters, markers)
 
     logging.info("Done handling POIs")
     logging.info("Writing out javascript files")
-    markerSetDict = dict()
-    for (flter, rset) in markersets:
-        # generate a unique name for this markerset.  it will not be user visible
-        filter_name =     flter[0]
-        filter_function = flter[1]
-
-        name = replaceBads(filter_name) + hex(hash(filter_function))[-4:] + "_" + hex(hash(rset))[-4:]
-        markerSetDict[name] = dict(created=False, raw=[], name=filter_name)
-        for poi in rset._pois['Entities']:
-            result = filter_function(poi)
-            if result:
-                if isinstance(result, basestring):
-                    d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result, hovertext=result)
-                elif type(result) == tuple:
-                    d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result[1], hovertext=result[0])
-                if "icon" in poi:
-                    d.update({"icon": poi['icon']})
-                if "createInfoWindow" in poi:
-                    d.update({"createInfoWindow": poi['createInfoWindow']})
-                markerSetDict[name]['raw'].append(d)
-        for poi in rset._pois['TileEntities']:
-            result = filter_function(poi)
-            if result:
-                if isinstance(result, basestring):
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result)
-                elif type(result) == tuple:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0])
-                # Dict support to allow more flexible things in the future as well as polylines on the map.
-                elif type(result) == dict:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text'])
-                    # Use custom hovertext if provided...
-                    if 'hovertext' in result and isinstance(result['hovertext'], basestring):
-                        d['hovertext'] = result['hovertext']
-                    else: # ...otherwise default to display text.
-                        d['hovertext'] = result['text']
-                    if 'polyline' in result and type(result['polyline']) == tuple:  #if type(result.get('polyline', '')) == tuple:
-                        d['polyline'] = []
-                        for point in result['polyline']:
-                            # This poor man's validation code almost definately needs improving.
-                            if type(point) == dict:
-                                d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z']))
-                        if isinstance(result['color'], basestring):
-                            d['strokeColor'] = result['color']
-                if "icon" in poi:
-                    d.update({"icon": poi['icon']})
-                if "createInfoWindow" in poi:
-                    d.update({"createInfoWindow": poi['createInfoWindow']})
-                markerSetDict[name]['raw'].append(d)
-        for poi in rset._pois['Players']:
-            result = filter_function(poi)
-            if result:
-                if isinstance(result, basestring):
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result)
-                elif type(result) == tuple:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0])
-                # Dict support to allow more flexible things in the future as well as polylines on the map.
-                elif type(result) == dict:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text'])
-                    # Use custom hovertext if provided...
-                    if 'hovertext' in result and isinstance(result['hovertext'], basestring):
-                        d['hovertext'] = result['hovertext']
-                    else: # ...otherwise default to display text.
-                        d['hovertext'] = result['text']
-                    if 'polyline' in result and type(result['polyline']) == tuple:  #if type(result.get('polyline', '')) == tuple:
-                        d['polyline'] = []
-                        for point in result['polyline']:
-                            # This poor man's validation code almost definately needs improving.
-                            if type(point) == dict:
-                                d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z']))
-                        if isinstance(result['color'], basestring):
-                            d['strokeColor'] = result['color']
-                if "icon" in poi:
-                    d.update({"icon": poi['icon']})
-                if "createInfoWindow" in poi:
-                    d.update({"createInfoWindow": poi['createInfoWindow']})
-                markerSetDict[name]['raw'].append(d)
-        for poi in rset._pois['Manual']:
-            result = filter_function(poi)
-            if result:
-                if isinstance(result, basestring):
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result)
-                elif type(result) == tuple:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0])
-                # Dict support to allow more flexible things in the future as well as polylines on the map.
-                elif type(result) == dict:
-                    d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text'])
-                    # Use custom hovertext if provided...
-                    if 'hovertext' in result and isinstance(result['hovertext'], basestring):
-                        d['hovertext'] = result['hovertext']
-                    else: # ...otherwise default to display text.
-                        d['hovertext'] = result['text']
-                    if 'polyline' in result and type(result['polyline']) == tuple:  #if type(result.get('polyline', '')) == tuple:
-                        d['polyline'] = []
-                        for point in result['polyline']:
-                            # This poor man's validation code almost definately needs improving.
-                            if type(point) == dict:
-                                d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z']))
-                        if isinstance(result['color'], basestring):
-                            d['strokeColor'] = result['color']
-                if "icon" in poi:
-                    d.update({"icon": poi['icon']})
-                if "createInfoWindow" in poi:
-                    d.update({"createInfoWindow": poi['createInfoWindow']})
-                markerSetDict[name]['raw'].append(d)
-    #print markerSetDict
 
     PlayerDict.save_cache(destdir)
 
     with open(os.path.join(destdir, "markersDB.js"), "w") as output:
         output.write("var markersDB=")
-        json.dump(markerSetDict, output, indent=2)
+        json.dump(markers, output, indent=2)
         output.write(";\n");
     with open(os.path.join(destdir, "markers.js"), "w") as output:
         output.write("var markers=")
-        json.dump(markers, output, indent=2)
+        json.dump(marker_groups, output, indent=2)
         output.write(";\n");
     with open(os.path.join(destdir, "baseMarkers.js"), "w") as output:
         output.write("overviewer.util.injectMarkerScript('markersDB.js');\n")