From 61339b1c42fe93481c1f458acba99ce664d875a3 Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Wed, 30 Jul 2014 20:55:12 +0200 Subject: [PATCH 1/9] genPOI: add option --skip-players If one simply wants to add some manual POIs, no player files have to be parsed. --- overviewer.py | 2 ++ overviewer_core/aux_files/genPOI.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/overviewer.py b/overviewer.py index 83bb6de..50c37e8 100755 --- a/overviewer.py +++ b/overviewer.py @@ -104,6 +104,8 @@ def main(): help="Runs the genPOI script") exegroup.add_option("--skip-scan", dest="skipscan", action="store_true", help="When running GenPOI, don't scan for entities") + exegroup.add_option("--skip-players", dest="skipplayers", action="store_true", + help="When running GenPOI, don't get player data") parser.add_option_group(exegroup) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 305ff82..fd5cb36 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -224,6 +224,7 @@ def main(): parser.add_option("-c", "--config", dest="config", action="store", help="Specify the config file to use.") parser.add_option("--quiet", dest="quiet", action="count", help="Reduce logging output") parser.add_option("--skip-scan", dest="skipscan", action="store_true", help="Skip scanning for entities when using GenPOI") + parser.add_option("--skip-players", dest="skipplayers", action="store_true", help="Skip getting player data when using GenPOI") options, args = parser.parse_args() if not options.config: @@ -288,7 +289,11 @@ def main(): if not options.skipscan: handleEntities(rset, os.path.join(destdir, rname), render, rname, config) - handlePlayers(rset, render, worldpath) + if options.skipplayers: + rset._pois['Players'] = [] + else: + handlePlayers(rset, render, worldpath) + handleManual(rset, render['manualpois']) logging.info("Done handling POIs") From d7aa4fd4c22b4f7a76b8a9db977b2af815ac2d3d Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Wed, 30 Jul 2014 21:35:02 +0200 Subject: [PATCH 2/9] genPOI: function for doubled code for marker creation The code creating the actual marker dict out of the entity and the result of the filter function was almost the same for every set of entities. Thus it is now a function. --- overviewer_core/aux_files/genPOI.py | 128 +++++++++++----------------- 1 file changed, 48 insertions(+), 80 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index fd5cb36..98ad1ee 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -209,6 +209,50 @@ def handleManual(rset, manualpois): if manualpois: rset._pois['Manual'].extend(manualpois) + +def create_marker_from_filter_result(poi, result): + """ + Takes a POI and the return value of a filter function for it and creates a + marker dict depending on the type of the returned value. + """ + # every marker has a position either directly via attributes x, y, z or + # via tuple attribute Pos + if 'Pos' in poi: + d = dict((v, poi['Pos'][i]) for i, v in enumerate('xyz')) + else: + d = dict((v, poi[v]) for v in 'xyz') + + if isinstance(result, basestring): + d.update(dict(text=result, hovertext=result)) + elif type(result) == tuple: + d.update(dict(text=result[1], hovertext=result[0])) + # Dict support to allow more flexible things in the future as well as polylines on the map. + elif type(result) == dict: + d['text'] = result['text'] + + # Use custom hovertext if provided... + if 'hovertext' in result and isinstance(result['hovertext'], basestring): + d['hovertext'] = result['hovertext'] + else: # ...otherwise default to display text. + d['hovertext'] = result['text'] + + if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: + d['polyline'] = [] + for point in result['polyline']: + # This poor man's validation code almost definately needs improving. + if type(point) == dict: + d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z'])) + if isinstance(result['color'], basestring): + d['strokeColor'] = result['color'] + + if "icon" in poi: + d.update({"icon": poi['icon']}) + if "createInfoWindow" in poi: + d.update({"createInfoWindow": poi['createInfoWindow']}) + + return d + + def main(): if os.path.basename(sys.argv[0]) == """genPOI.py""": @@ -309,98 +353,22 @@ def main(): for poi in rset._pois['Entities']: result = filter_function(poi) if result: - if isinstance(result, basestring): - d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result[1], hovertext=result[0]) - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) + d = create_marker_from_filter_result(poi, result) markerSetDict[name]['raw'].append(d) for poi in rset._pois['TileEntities']: result = filter_function(poi) if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) + d = create_marker_from_filter_result(poi, result) markerSetDict[name]['raw'].append(d) for poi in rset._pois['Players']: result = filter_function(poi) if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) + d = create_marker_from_filter_result(poi, result) markerSetDict[name]['raw'].append(d) for poi in rset._pois['Manual']: result = filter_function(poi) if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) + d = create_marker_from_filter_result(poi, result) markerSetDict[name]['raw'].append(d) #print markerSetDict From 6c14d47650daf3a44cec2f6a392184dec14fe9cc Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Wed, 30 Jul 2014 21:39:39 +0200 Subject: [PATCH 3/9] genPOI: unite for-loops with itertools Since all the POIs are created from different lists, multiple for loops were used. With itertools.chain these lists can be looped over with only one for loop thus removing doubled code. --- overviewer_core/aux_files/genPOI.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 98ad1ee..cb54fe9 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -22,6 +22,7 @@ import re import urllib2 import Queue import multiprocessing +import itertools from multiprocessing import Process from multiprocessing import Pool @@ -350,22 +351,8 @@ def main(): name = replaceBads(filter_name) + hex(hash(filter_function))[-4:] + "_" + hex(hash(rset))[-4:] markerSetDict[name] = dict(created=False, raw=[], name=filter_name) - for poi in rset._pois['Entities']: - result = filter_function(poi) - if result: - d = create_marker_from_filter_result(poi, result) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['TileEntities']: - result = filter_function(poi) - if result: - d = create_marker_from_filter_result(poi, result) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['Players']: - result = filter_function(poi) - if result: - d = create_marker_from_filter_result(poi, result) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['Manual']: + poi_sets = ['Entities', 'TileEntities', 'Players', 'Manual'] + for poi in itertools.chain(rset._pois[n] for n in poi_sets): result = filter_function(poi) if result: d = create_marker_from_filter_result(poi, result) From e2b6474b2856ed3dff4a97a1638eb559f3001c05 Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Wed, 30 Jul 2014 21:56:06 +0200 Subject: [PATCH 4/9] genPOI: use a defaultdict for markers This is a little faster and make the code a little shorter. --- overviewer_core/aux_files/genPOI.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index cb54fe9..e06e13a 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -24,6 +24,7 @@ import Queue import multiprocessing import itertools +from collections import defaultdict from multiprocessing import Process from multiprocessing import Pool from optparse import OptionParser @@ -293,7 +294,7 @@ def main(): worldcache = {} markersets = set() - markers = dict() + markers = defaultdict(list) for rname, render in config['renders'].iteritems(): try: @@ -325,11 +326,7 @@ def main(): icon=f.get('icon', 'signpost_icon.png'), createInfoWindow=f.get('createInfoWindow',True), checked = f.get('checked', False)) - try: - l = markers[rname] - l.append(to_append) - except KeyError: - markers[rname] = [to_append] + markers[rname].append(to_append) if not options.skipscan: handleEntities(rset, os.path.join(destdir, rname), render, rname, config) From eccad401ca67fa56166629494f9f9a0b839a97e0 Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Wed, 30 Jul 2014 21:58:03 +0200 Subject: [PATCH 5/9] genPOI: generate marker's internal name only once Why should it be duplicated anyway? --- overviewer_core/aux_files/genPOI.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index e06e13a..b52c760 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -319,8 +319,8 @@ def main(): return 1 for f in render['markers']: - markersets.add(((f['name'], f['filterFunction']), rset)) name = replaceBads(f['name']) + hex(hash(f['filterFunction']))[-4:] + "_" + hex(hash(rset))[-4:] + markersets.add((name, (f['name'], f['filterFunction']), rset)) to_append = dict(groupName=name, displayName = f['name'], icon=f.get('icon', 'signpost_icon.png'), @@ -341,12 +341,12 @@ def main(): logging.info("Done handling POIs") logging.info("Writing out javascript files") markerSetDict = dict() - for (flter, rset) in markersets: + + for (name, flter, rset) in markersets: # generate a unique name for this markerset. it will not be user visible filter_name = flter[0] filter_function = flter[1] - name = replaceBads(filter_name) + hex(hash(filter_function))[-4:] + "_" + hex(hash(rset))[-4:] markerSetDict[name] = dict(created=False, raw=[], name=filter_name) poi_sets = ['Entities', 'TileEntities', 'Players', 'Manual'] for poi in itertools.chain(rset._pois[n] for n in poi_sets): From 2271f628ef5cd09d966c3f36a47e30493f21fae0 Mon Sep 17 00:00:00 2001 From: MasterofJOKers Date: Sun, 3 Aug 2014 22:33:51 +0200 Subject: [PATCH 6/9] genPOI: use filter functions on the fly Instead of reading all Players and especially Entities and TileEntities into RAM and than applying the filter functions onto them, filter functions are used after each parsed chunk to get only the markers of this chunk and not all Entities. This reduced the RAM usage dramatically: On a 233 MB map over 200 MB of RAM was used before. Using these changes (and only having two filter functions for signs) not more than 36 MB of RAM was used. --- overviewer_core/aux_files/genPOI.py | 246 +++++++++++++++++----------- 1 file changed, 152 insertions(+), 94 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index b52c760..15a8532 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -20,18 +20,17 @@ import json import sys import re import urllib2 -import Queue import multiprocessing import itertools from collections import defaultdict -from multiprocessing import Process from multiprocessing import Pool from optparse import OptionParser from overviewer_core import logger from overviewer_core import nbt from overviewer_core import configParser, world +from overviewer_core import rendermodes UUID_LOOKUP_URL = 'https://sessionserver.mojang.com/session/minecraft/profile/' @@ -43,22 +42,26 @@ def replaceBads(s): x = x.replace(bad,"_") return x + # yes there's a double parenthesis here # see below for when this is called, and why we do this # a smarter way would be functools.partial, but that's broken on python 2.6 # when used with multiprocessing -def parseBucketChunks((bucket, rset)): +def parseBucketChunks((bucket, rset, filters)): pid = multiprocessing.current_process().pid - pois = dict(TileEntities=[], Entities=[]); - + markers = defaultdict(list) + i = 0 cnt = 0 - l = len(bucket) for b in bucket: try: data = rset.get_chunk(b[0],b[1]) - pois['TileEntities'] += data['TileEntities'] - pois['Entities'] += data['Entities'] + for poi in itertools.chain(data['TileEntities'], data['Entities']): + for name, filter_function in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name].append(d) except nbt.CorruptChunkError: logging.warning("Ignoring POIs in corrupt chunk %d,%d", b[0], b[1]) @@ -67,58 +70,67 @@ def parseBucketChunks((bucket, rset)): if i == 250: i = 0 cnt = 250 + cnt - logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt); + logging.info("Found %d markers in thread %d so far at %d chunks", sum(len(v) for v in markers.itervalues()), pid, cnt); - return pois + return markers -def handleEntities(rset, outputdir, render, rname, config): - # if we're already handled the POIs for this region regionset, do nothing - if hasattr(rset, "_pois"): - return +def handleEntities(rset, config, filters, markers): + """ + Add markers for Entities or TileEntities. + For this every chunk of the regionset is parsed and filtered using multiple + processes, if so configured. + This function will not return anything, but it will update the parameter + `markers`. + """ logging.info("Looking for entities in %r", rset) - filters = render['markers'] - rset._pois = dict(TileEntities=[], Entities=[]) - numbuckets = config['processes']; if numbuckets < 0: numbuckets = multiprocessing.cpu_count() if numbuckets == 1: - for (x,z,mtime) in rset.iterate_chunks(): + for (x, z, mtime) in rset.iterate_chunks(): try: - data = rset.get_chunk(x,z) - rset._pois['TileEntities'] += data['TileEntities'] - rset._pois['Entities'] += data['Entities'] + data = rset.get_chunk(x, z) + for poi in itertools.chain(data['TileEntities'], data['Entities']): + for name, __, filter_function, __, __, __ in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name]['raw'].append(d) except nbt.CorruptChunkError: logging.warning("Ignoring POIs in corrupt chunk %d,%d", x,z) else: buckets = [[] for i in range(numbuckets)]; - for (x,z,mtime) in rset.iterate_chunks(): + for (x, z, mtime) in rset.iterate_chunks(): i = x / 32 + z / 32 i = i % numbuckets - buckets[i].append([x,z]) + buckets[i].append([x, z]) for b in buckets: logging.info("Buckets has %d entries", len(b)); # Create a pool of processes and run all the functions pool = Pool(processes=numbuckets) - results = pool.map(parseBucketChunks, ((buck, rset) for buck in buckets)) + + # simplify the filters dict, so pickle doesn't have to do so much + filters = [(name, filter_function) for name, __, filter_function, __, __, __ in filters] + + results = pool.map(parseBucketChunks, ((buck, rset, filters) for buck in buckets)) logging.info("All the threads completed") - # Fix up all the quests in the reset - for data in results: - rset._pois['TileEntities'] += data['TileEntities'] - rset._pois['Entities'] += data['Entities'] + for marker_dict in results: + for name, marker_list in marker_dict.iteritems(): + markers[name]['raw'].extend(marker_list) logging.info("Done.") + class PlayerDict(dict): use_uuid = False _name = '' @@ -140,19 +152,16 @@ class PlayerDict(dict): except (ValueError, urllib2.URLError): logging.warning("Unable to get player name for UUID %s", self._name) -def handlePlayers(rset, render, worldpath): - if not hasattr(rset, "_pois"): - rset._pois = dict(TileEntities=[], Entities=[]) - # only handle this region set once - if 'Players' in rset._pois: - return - - if rset.get_type(): - dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2)) - else: - dimension = 0 +def handlePlayers(worldpath, filters, markers): + """ + Add markers for players to the list of markers. + For this the player files under the given `worldpath` are parsed and + filtered. + This function will not return anything, but it will update the parameter + `markers`. + """ playerdir = os.path.join(worldpath, "playerdata") useUUIDs = True if not os.path.isdir(playerdir): @@ -163,12 +172,10 @@ def handlePlayers(rset, render, worldpath): playerfiles = os.listdir(playerdir) playerfiles = [x for x in playerfiles if x.endswith(".dat")] isSinglePlayer = False - else: playerfiles = [os.path.join(worldpath, "level.dat")] isSinglePlayer = True - rset._pois['Players'] = [] for playerfile in playerfiles: try: data = PlayerDict(nbt.load(os.path.join(playerdir, playerfile))[1]) @@ -178,38 +185,63 @@ def handlePlayers(rset, render, worldpath): except IOError: logging.warning("Skipping bad player dat file %r", playerfile) continue - playername = playerfile.split(".")[0] + playername = playerfile.split(".")[0] if isSinglePlayer: playername = 'Player' - data._name = playername - if data['Dimension'] == dimension: - # Position at last logout - data['id'] = "Player" - data['x'] = int(data['Pos'][0]) - data['y'] = int(data['Pos'][1]) - data['z'] = int(data['Pos'][2]) - rset._pois['Players'].append(data) - if "SpawnX" in data and dimension == 0: - # Spawn position (bed or main spawn) + # Position at last logout + data['id'] = "Player" + data['x'] = int(data['Pos'][0]) + data['y'] = int(data['Pos'][1]) + data['z'] = int(data['Pos'][2]) + + # Spawn position (bed or main spawn) + if "SpawnX" in data: spawn = PlayerDict() spawn._name = playername spawn["id"] = "PlayerSpawn" spawn["x"] = data['SpawnX'] spawn["y"] = data['SpawnY'] spawn["z"] = data['SpawnZ'] - rset._pois['Players'].append(spawn) -def handleManual(rset, manualpois): - if not hasattr(rset, "_pois"): - rset._pois = dict(TileEntities=[], Entities=[]) - - rset._pois['Manual'] = [] + for name, __, filter_function, rset, __, __ in filters: + # get the dimension for the filter + # This has do be done every time, because we have filters for + # different regionsets. - if manualpois: - rset._pois['Manual'].extend(manualpois) + if rset.get_type(): + dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2)) + else: + dimension = 0 + + if data['Dimension'] == dimension: + result = filter_function(data) + if result: + d = create_marker_from_filter_result(data, result) + markers[name]['raw'].append(d) + + if dimension == 0 and "SpawnX" in data: + result = filter_function(spawn) + if result: + d = create_marker_from_filter_result(spawn, result) + markers[name]['raw'].append(d) + + +def handleManual(manualpois, filters, markers): + """ + Add markers for manually defined POIs to the list of markers. + + This function will not return anything, but it will update the parameter + `markers`. + """ + for poi in manualpois: + for name, __, filter_function, __, __, __ in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name]['raw'].append(d) def create_marker_from_filter_result(poi, result): @@ -293,10 +325,13 @@ def main(): # saves us from creating the same World object over and over again worldcache = {} - markersets = set() - markers = defaultdict(list) + filters = set() + marker_groups = defaultdict(list) + # collect all filters and get regionsets for rname, render in config['renders'].iteritems(): + # Convert render['world'] to the world path, and store the original + # in render['worldname_orig'] try: worldpath = config['worlds'][render['world']] except KeyError: @@ -305,64 +340,87 @@ def main(): return 1 render['worldname_orig'] = render['world'] render['world'] = worldpath - + # find or create the world object if (render['world'] not in worldcache): w = world.World(render['world']) worldcache[render['world']] = w else: w = worldcache[render['world']] - + + # get the regionset for this dimension rset = w.get_regionset(render['dimension'][1]) if rset == None: # indicates no such dimension was found: logging.error("Sorry, you requested dimension '%s' for the render '%s', but I couldn't find it", render['dimension'][0], rname) return 1 - + + # find filters for this render for f in render['markers']: + # internal identifier for this filter name = replaceBads(f['name']) + hex(hash(f['filterFunction']))[-4:] + "_" + hex(hash(rset))[-4:] - markersets.add((name, (f['name'], f['filterFunction']), rset)) - to_append = dict(groupName=name, + + # we need to make the function pickleable for multiprocessing to + # work + # We set a custom prefix here to not override any functions there. + # These functions are only pickleable if they are bound to a + # module. Since rendermodes imports the config, they are bound to + # it anyway, but don't end up importable as + # `rendermodes.filter_fn`. That's why we set it here explicitly on + # the module. + f['filterFunction'].__name__ = "custom_filter_" + f['filterFunction'].__name__ + setattr(rendermodes, f['filterFunction'].__name__, f['filterFunction']) + + # add it to the list of filters + filters.add((name, f['name'], f['filterFunction'], rset, worldpath, rname)) + + # add an entry in the menu to show markers found by this filter + group = dict(groupName=name, displayName = f['name'], icon=f.get('icon', 'signpost_icon.png'), - createInfoWindow=f.get('createInfoWindow',True), + createInfoWindow=f.get('createInfoWindow', True), checked = f.get('checked', False)) - markers[rname].append(to_append) - - if not options.skipscan: - handleEntities(rset, os.path.join(destdir, rname), render, rname, config) + marker_groups[rname].append(group) - if options.skipplayers: - rset._pois['Players'] = [] - else: - handlePlayers(rset, render, worldpath) + # initialize the structure for the markers + markers = dict((name, dict(created=False, raw=[], name=filter_name)) + for name, filter_name, __, __, __, __ in filters) - handleManual(rset, render['manualpois']) + # apply filters to regionsets + if not options.skipscan: + # group filters by rset + keyfunc = lambda x: x[3] + sfilters = sorted(filters, key=keyfunc) + for rset, rset_filters in itertools.groupby(sfilters, keyfunc): + handleEntities(rset, config, rset_filters, markers) + + # apply filters to players + if not options.skipplayers: + # group filters by worldpath, so we only search for players once per + # world + keyfunc = lambda x: x[4] + sfilters = sorted(filters, key=keyfunc) + for worldpath, worldpath_filters in itertools.groupby(sfilters, keyfunc): + handlePlayers(worldpath, worldpath_filters, markers) + + # add manual POIs + # group filters by name of the render, because only filter functions for + # the current render should be used on the current render's manualpois + keyfunc = lambda x: x[5] + sfilters = sorted(filters, key=keyfunc) + for rname, rname_filters in itertools.groupby(sfilters, keyfunc): + manualpois = config['renders'][rname]['manualpois'] + handleManual(manualpois, rname_filters, markers) logging.info("Done handling POIs") logging.info("Writing out javascript files") - markerSetDict = dict() - - for (name, flter, rset) in markersets: - # generate a unique name for this markerset. it will not be user visible - filter_name = flter[0] - filter_function = flter[1] - - markerSetDict[name] = dict(created=False, raw=[], name=filter_name) - poi_sets = ['Entities', 'TileEntities', 'Players', 'Manual'] - for poi in itertools.chain(rset._pois[n] for n in poi_sets): - result = filter_function(poi) - if result: - d = create_marker_from_filter_result(poi, result) - markerSetDict[name]['raw'].append(d) - #print markerSetDict with open(os.path.join(destdir, "markersDB.js"), "w") as output: output.write("var markersDB=") - json.dump(markerSetDict, output, indent=2) + json.dump(markers, output, indent=2) output.write(";\n"); with open(os.path.join(destdir, "markers.js"), "w") as output: output.write("var markers=") - json.dump(markers, output, indent=2) + json.dump(marker_groups, output, indent=2) output.write(";\n"); with open(os.path.join(destdir, "baseMarkers.js"), "w") as output: output.write("overviewer.util.injectMarkerScript('markersDB.js');\n") From b08e34b064728db251b1977c39291cab96d9efc2 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 23 Feb 2015 21:11:56 +0100 Subject: [PATCH 7/9] genPOI: add icon and createInfoWindow support for filters The defaults for "icon" and "createInfoWindow" are read from the POI, but can be overridden by a filter function returning an appripriate dict. --- overviewer_core/aux_files/genPOI.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index c46e143..1b15044 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -290,6 +290,13 @@ def create_marker_from_filter_result(poi, result): else: d = dict((v, poi[v]) for v in 'xyz') + # read some Defaults from POI + if "icon" in poi: + d["icon"] = poi['icon'] + if "createInfoWindow" in poi: + d["createInfoWindow"] = poi['createInfoWindow'] + + # Fill in the rest from result if isinstance(result, basestring): d.update(dict(text=result, hovertext=result)) elif type(result) == tuple: @@ -313,10 +320,10 @@ def create_marker_from_filter_result(poi, result): if isinstance(result['color'], basestring): d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) + if "icon" in result: + d["icon"] = result['icon'] + if "createInfoWindow" in result: + d["createInfoWindow"] = result['createInfoWindow'] return d From 7229fcc3dfd6d07172c1f81cc49ac894cc2a2c84 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 23 Feb 2015 21:14:17 +0100 Subject: [PATCH 8/9] genPOI: use more pythonic ways ... One should use isinstance instead of `type(A) ==` because of inheritance. There should be an exception if a list of `elif`s don't match. Make Polyline not only accept tuples, but any iterable. --- overviewer_core/aux_files/genPOI.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 1b15044..4027d03 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -299,19 +299,19 @@ def create_marker_from_filter_result(poi, result): # Fill in the rest from result if isinstance(result, basestring): d.update(dict(text=result, hovertext=result)) - elif type(result) == tuple: + elif isinstance(result, tuple): d.update(dict(text=result[1], hovertext=result[0])) # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: + elif isinstance(result, dict): d['text'] = result['text'] # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] + if 'hovertext' in result: + d['hovertext'] = unicode(result['hovertext']) else: # ...otherwise default to display text. d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: + if 'polyline' in result and hasattr(result['polyline'], '__iter__'): d['polyline'] = [] for point in result['polyline']: # This poor man's validation code almost definately needs improving. @@ -324,6 +324,8 @@ def create_marker_from_filter_result(poi, result): d["icon"] = result['icon'] if "createInfoWindow" in result: d["createInfoWindow"] = result['createInfoWindow'] + else: + raise ValueError("got an %s as result for POI with id %s" % (type(result).__name__, poi['id'])) return d From 3cd2afb0c4976a9102bbe8b1e0ba9d29d97c4051 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 23 Feb 2015 21:16:49 +0100 Subject: [PATCH 9/9] genPOI: don't ignore invalid polyline point There is no point in ignoring an invalid point, because without an error message the user might wonder, why it's poyline doesn't look the way it should. --- overviewer_core/aux_files/genPOI.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 4027d03..0d0851b 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -314,9 +314,7 @@ def create_marker_from_filter_result(poi, result): if 'polyline' in result and hasattr(result['polyline'], '__iter__'): d['polyline'] = [] for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z'])) + d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z'])) # point.copy() would work, but this validates better if isinstance(result['color'], basestring): d['strokeColor'] = result['color']