diff --git a/overviewer.py b/overviewer.py index 6739f0f..275c0dc 100755 --- a/overviewer.py +++ b/overviewer.py @@ -114,6 +114,8 @@ def main(): help="Runs the genPOI script") exegroup.add_option("--skip-scan", dest="skipscan", action="store_true", help="When running GenPOI, don't scan for entities") + exegroup.add_option("--skip-players", dest="skipplayers", action="store_true", + help="When running GenPOI, don't get player data") parser.add_option_group(exegroup) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 3b17812..0d0851b 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -21,17 +21,18 @@ import json import sys import re import urllib2 -import Queue import multiprocessing +import itertools import gzip -from multiprocessing import Process +from collections import defaultdict from multiprocessing import Pool from optparse import OptionParser from overviewer_core import logger from overviewer_core import nbt from overviewer_core import configParser, world +from overviewer_core import rendermodes UUID_LOOKUP_URL = 'https://sessionserver.mojang.com/session/minecraft/profile/' @@ -43,22 +44,26 @@ def replaceBads(s): x = x.replace(bad,"_") return x + # yes there's a double parenthesis here # see below for when this is called, and why we do this # a smarter way would be functools.partial, but that's broken on python 2.6 # when used with multiprocessing -def parseBucketChunks((bucket, rset)): +def parseBucketChunks((bucket, rset, filters)): pid = multiprocessing.current_process().pid - pois = dict(TileEntities=[], Entities=[]); - + markers = defaultdict(list) + i = 0 cnt = 0 - l = len(bucket) for b in bucket: try: data = rset.get_chunk(b[0],b[1]) - pois['TileEntities'] += data['TileEntities'] - pois['Entities'] += data['Entities'] + for poi in itertools.chain(data['TileEntities'], data['Entities']): + for name, filter_function in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name].append(d) except nbt.CorruptChunkError: logging.warning("Ignoring POIs in corrupt chunk %d,%d", b[0], b[1]) @@ -67,67 +72,75 @@ def parseBucketChunks((bucket, rset)): if i == 250: i = 0 cnt = 250 + cnt - logging.info("Found %d entities and %d tile entities in thread %d so far at %d chunks", len(pois['Entities']), len(pois['TileEntities']), pid, cnt); + logging.info("Found %d markers in thread %d so far at %d chunks", sum(len(v) for v in markers.itervalues()), pid, cnt); - return pois + return markers -def handleEntities(rset, outputdir, render, rname, config): - # if we're already handled the POIs for this region regionset, do nothing - if hasattr(rset, "_pois"): - return +def handleEntities(rset, config, filters, markers): + """ + Add markers for Entities or TileEntities. + For this every chunk of the regionset is parsed and filtered using multiple + processes, if so configured. + This function will not return anything, but it will update the parameter + `markers`. + """ logging.info("Looking for entities in %r", rset) - filters = render['markers'] - rset._pois = dict(TileEntities=[], Entities=[]) - numbuckets = config['processes']; if numbuckets < 0: numbuckets = multiprocessing.cpu_count() if numbuckets == 1: - for (x,z,mtime) in rset.iterate_chunks(): + for (x, z, mtime) in rset.iterate_chunks(): try: - data = rset.get_chunk(x,z) - rset._pois['TileEntities'] += data['TileEntities'] - rset._pois['Entities'] += data['Entities'] + data = rset.get_chunk(x, z) + for poi in itertools.chain(data['TileEntities'], data['Entities']): + for name, __, filter_function, __, __, __ in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name]['raw'].append(d) except nbt.CorruptChunkError: logging.warning("Ignoring POIs in corrupt chunk %d,%d", x,z) else: buckets = [[] for i in range(numbuckets)]; - for (x,z,mtime) in rset.iterate_chunks(): + for (x, z, mtime) in rset.iterate_chunks(): i = x / 32 + z / 32 i = i % numbuckets - buckets[i].append([x,z]) + buckets[i].append([x, z]) for b in buckets: logging.info("Buckets has %d entries", len(b)); # Create a pool of processes and run all the functions pool = Pool(processes=numbuckets) - results = pool.map(parseBucketChunks, ((buck, rset) for buck in buckets)) + + # simplify the filters dict, so pickle doesn't have to do so much + filters = [(name, filter_function) for name, __, filter_function, __, __, __ in filters] + + results = pool.map(parseBucketChunks, ((buck, rset, filters) for buck in buckets)) logging.info("All the threads completed") - # Fix up all the quests in the reset - for data in results: - rset._pois['TileEntities'] += data['TileEntities'] - rset._pois['Entities'] += data['Entities'] + for marker_dict in results: + for name, marker_list in marker_dict.iteritems(): + markers[name]['raw'].extend(marker_list) logging.info("Done.") + class PlayerDict(dict): use_uuid = False _name = '' - uuid_cache = None # A cache the UUID->profile lookups + uuid_cache = None # A cache for the UUID->profile lookups @classmethod def load_cache(cls, outputdir): cache_file = os.path.join(outputdir, "uuidcache.dat") - pid = multiprocessing.current_process().pid if os.path.exists(cache_file): gz = gzip.GzipFile(cache_file) cls.uuid_cache = json.load(gz) @@ -135,8 +148,6 @@ class PlayerDict(dict): else: cls.uuid_cache = {} logging.info("Initialized an empty UUID cache") - cls.save_cache(outputdir) - @classmethod def save_cache(cls, outputdir): @@ -144,7 +155,6 @@ class PlayerDict(dict): gz = gzip.GzipFile(cache_file, "wb") json.dump(cls.uuid_cache, gz) logging.info("Wrote UUID cache with %d entries", len(cls.uuid_cache.keys())) - def __getitem__(self, item): if item == "EntityId": @@ -172,19 +182,16 @@ class PlayerDict(dict): except (ValueError, urllib2.URLError): logging.warning("Unable to get player name for UUID %s", self._name) -def handlePlayers(rset, render, worldpath, outputdir): - if not hasattr(rset, "_pois"): - rset._pois = dict(TileEntities=[], Entities=[]) - # only handle this region set once - if 'Players' in rset._pois: - return - - if rset.get_type(): - dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2)) - else: - dimension = 0 +def handlePlayers(worldpath, filters, markers): + """ + Add markers for players to the list of markers. + For this the player files under the given `worldpath` are parsed and + filtered. + This function will not return anything, but it will update the parameter + `markers`. + """ playerdir = os.path.join(worldpath, "playerdata") useUUIDs = True if not os.path.isdir(playerdir): @@ -195,13 +202,10 @@ def handlePlayers(rset, render, worldpath, outputdir): playerfiles = os.listdir(playerdir) playerfiles = [x for x in playerfiles if x.endswith(".dat")] isSinglePlayer = False - else: playerfiles = [os.path.join(worldpath, "level.dat")] isSinglePlayer = True - rset._pois['Players'] = [] - for playerfile in playerfiles: try: data = PlayerDict(nbt.load(os.path.join(playerdir, playerfile))[1]) @@ -211,23 +215,22 @@ def handlePlayers(rset, render, worldpath, outputdir): except IOError: logging.warning("Skipping bad player dat file %r", playerfile) continue - playername = playerfile.split(".")[0] + playername = playerfile.split(".")[0] if isSinglePlayer: playername = 'Player' - data._name = playername - if data['Dimension'] == dimension: - # Position at last logout - data['id'] = "Player" - data['x'] = int(data['Pos'][0]) - data['y'] = int(data['Pos'][1]) - data['z'] = int(data['Pos'][2]) - # Time at last logout, calculated from last time the player's file was modified - data['time'] = time.localtime(os.path.getmtime(os.path.join(playerdir, playerfile))) - rset._pois['Players'].append(data) - if "SpawnX" in data and dimension == 0: + # Position at last logout + data['id'] = "Player" + data['x'] = int(data['Pos'][0]) + data['y'] = int(data['Pos'][1]) + data['z'] = int(data['Pos'][2]) + # Time at last logout, calculated from last time the player's file was modified + data['time'] = time.localtime(os.path.getmtime(os.path.join(playerdir, playerfile))) + + # Spawn position (bed or main spawn) + if "SpawnX" in data: # Spawn position (bed or main spawn) spawn = PlayerDict() spawn.use_uuid = useUUIDs @@ -236,16 +239,94 @@ def handlePlayers(rset, render, worldpath, outputdir): spawn["x"] = data['SpawnX'] spawn["y"] = data['SpawnY'] spawn["z"] = data['SpawnZ'] - rset._pois['Players'].append(spawn) -def handleManual(rset, manualpois): - if not hasattr(rset, "_pois"): - rset._pois = dict(TileEntities=[], Entities=[]) - - rset._pois['Manual'] = [] + for name, __, filter_function, rset, __, __ in filters: + # get the dimension for the filter + # This has do be done every time, because we have filters for + # different regionsets. + + if rset.get_type(): + dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2)) + else: + dimension = 0 + + if data['Dimension'] == dimension: + result = filter_function(data) + if result: + d = create_marker_from_filter_result(data, result) + markers[name]['raw'].append(d) + + if dimension == 0 and "SpawnX" in data: + result = filter_function(spawn) + if result: + d = create_marker_from_filter_result(spawn, result) + markers[name]['raw'].append(d) + + +def handleManual(manualpois, filters, markers): + """ + Add markers for manually defined POIs to the list of markers. + + This function will not return anything, but it will update the parameter + `markers`. + """ + for poi in manualpois: + for name, __, filter_function, __, __, __ in filters: + result = filter_function(poi) + if result: + d = create_marker_from_filter_result(poi, result) + markers[name]['raw'].append(d) + + +def create_marker_from_filter_result(poi, result): + """ + Takes a POI and the return value of a filter function for it and creates a + marker dict depending on the type of the returned value. + """ + # every marker has a position either directly via attributes x, y, z or + # via tuple attribute Pos + if 'Pos' in poi: + d = dict((v, poi['Pos'][i]) for i, v in enumerate('xyz')) + else: + d = dict((v, poi[v]) for v in 'xyz') + + # read some Defaults from POI + if "icon" in poi: + d["icon"] = poi['icon'] + if "createInfoWindow" in poi: + d["createInfoWindow"] = poi['createInfoWindow'] + + # Fill in the rest from result + if isinstance(result, basestring): + d.update(dict(text=result, hovertext=result)) + elif isinstance(result, tuple): + d.update(dict(text=result[1], hovertext=result[0])) + # Dict support to allow more flexible things in the future as well as polylines on the map. + elif isinstance(result, dict): + d['text'] = result['text'] + + # Use custom hovertext if provided... + if 'hovertext' in result: + d['hovertext'] = unicode(result['hovertext']) + else: # ...otherwise default to display text. + d['hovertext'] = result['text'] + + if 'polyline' in result and hasattr(result['polyline'], '__iter__'): + d['polyline'] = [] + for point in result['polyline']: + d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z'])) # point.copy() would work, but this validates better + if isinstance(result['color'], basestring): + d['strokeColor'] = result['color'] + + if "icon" in result: + d["icon"] = result['icon'] + if "createInfoWindow" in result: + d["createInfoWindow"] = result['createInfoWindow'] + else: + raise ValueError("got an %s as result for POI with id %s" % (type(result).__name__, poi['id'])) + + return d - if manualpois: - rset._pois['Manual'].extend(manualpois) def main(): @@ -262,6 +343,7 @@ def main(): parser.add_option("-c", "--config", dest="config", action="store", help="Specify the config file to use.") parser.add_option("--quiet", dest="quiet", action="count", help="Reduce logging output") parser.add_option("--skip-scan", dest="skipscan", action="store_true", help="Skip scanning for entities when using GenPOI") + parser.add_option("--skip-players", dest="skipplayers", action="store_true", help="Skip getting player data when using GenPOI") options, args = parser.parse_args() if not options.config: @@ -284,12 +366,13 @@ def main(): # saves us from creating the same World object over and over again worldcache = {} - markersets = set() - markers = dict() - - PlayerDict.load_cache(destdir) + filters = set() + marker_groups = defaultdict(list) + # collect all filters and get regionsets for rname, render in config['renders'].iteritems(): + # Convert render['world'] to the world path, and store the original + # in render['worldname_orig'] try: worldpath = config['worlds'][render['world']] except KeyError: @@ -298,156 +381,90 @@ def main(): return 1 render['worldname_orig'] = render['world'] render['world'] = worldpath - + # find or create the world object if (render['world'] not in worldcache): w = world.World(render['world']) worldcache[render['world']] = w else: w = worldcache[render['world']] - + + # get the regionset for this dimension rset = w.get_regionset(render['dimension'][1]) if rset == None: # indicates no such dimension was found: logging.warn("Sorry, you requested dimension '%s' for the render '%s', but I couldn't find it", render['dimension'][0], rname) continue - + + # find filters for this render for f in render['markers']: - markersets.add(((f['name'], f['filterFunction']), rset)) + # internal identifier for this filter name = replaceBads(f['name']) + hex(hash(f['filterFunction']))[-4:] + "_" + hex(hash(rset))[-4:] - to_append = dict(groupName=name, + + # we need to make the function pickleable for multiprocessing to + # work + # We set a custom prefix here to not override any functions there. + # These functions are only pickleable if they are bound to a + # module. Since rendermodes imports the config, they are bound to + # it anyway, but don't end up importable as + # `rendermodes.filter_fn`. That's why we set it here explicitly on + # the module. + f['filterFunction'].__name__ = "custom_filter_" + f['filterFunction'].__name__ + setattr(rendermodes, f['filterFunction'].__name__, f['filterFunction']) + + # add it to the list of filters + filters.add((name, f['name'], f['filterFunction'], rset, worldpath, rname)) + + # add an entry in the menu to show markers found by this filter + group = dict(groupName=name, displayName = f['name'], icon=f.get('icon', 'signpost_icon.png'), - createInfoWindow=f.get('createInfoWindow',True), + createInfoWindow=f.get('createInfoWindow', True), checked = f.get('checked', False)) - try: - l = markers[rname] - l.append(to_append) - except KeyError: - markers[rname] = [to_append] - - if not options.skipscan: - handleEntities(rset, os.path.join(destdir, rname), render, rname, config) + marker_groups[rname].append(group) - handlePlayers(rset, render, worldpath, destdir) - handleManual(rset, render['manualpois']) + # initialize the structure for the markers + markers = dict((name, dict(created=False, raw=[], name=filter_name)) + for name, filter_name, __, __, __, __ in filters) + + # apply filters to regionsets + if not options.skipscan: + # group filters by rset + keyfunc = lambda x: x[3] + sfilters = sorted(filters, key=keyfunc) + for rset, rset_filters in itertools.groupby(sfilters, keyfunc): + handleEntities(rset, config, rset_filters, markers) + + # apply filters to players + if not options.skipplayers: + PlayerDict.load_cache(destdir) + # group filters by worldpath, so we only search for players once per + # world + keyfunc = lambda x: x[4] + sfilters = sorted(filters, key=keyfunc) + for worldpath, worldpath_filters in itertools.groupby(sfilters, keyfunc): + handlePlayers(worldpath, worldpath_filters, markers) + + # add manual POIs + # group filters by name of the render, because only filter functions for + # the current render should be used on the current render's manualpois + keyfunc = lambda x: x[5] + sfilters = sorted(filters, key=keyfunc) + for rname, rname_filters in itertools.groupby(sfilters, keyfunc): + manualpois = config['renders'][rname]['manualpois'] + handleManual(manualpois, rname_filters, markers) logging.info("Done handling POIs") logging.info("Writing out javascript files") - markerSetDict = dict() - for (flter, rset) in markersets: - # generate a unique name for this markerset. it will not be user visible - filter_name = flter[0] - filter_function = flter[1] - - name = replaceBads(filter_name) + hex(hash(filter_function))[-4:] + "_" + hex(hash(rset))[-4:] - markerSetDict[name] = dict(created=False, raw=[], name=filter_name) - for poi in rset._pois['Entities']: - result = filter_function(poi) - if result: - if isinstance(result, basestring): - d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['Pos'][0], y=poi['Pos'][1], z=poi['Pos'][2], text=result[1], hovertext=result[0]) - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['TileEntities']: - result = filter_function(poi) - if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['Players']: - result = filter_function(poi) - if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) - markerSetDict[name]['raw'].append(d) - for poi in rset._pois['Manual']: - result = filter_function(poi) - if result: - if isinstance(result, basestring): - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result, hovertext=result) - elif type(result) == tuple: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result[1], hovertext=result[0]) - # Dict support to allow more flexible things in the future as well as polylines on the map. - elif type(result) == dict: - d = dict(x=poi['x'], y=poi['y'], z=poi['z'], text=result['text']) - # Use custom hovertext if provided... - if 'hovertext' in result and isinstance(result['hovertext'], basestring): - d['hovertext'] = result['hovertext'] - else: # ...otherwise default to display text. - d['hovertext'] = result['text'] - if 'polyline' in result and type(result['polyline']) == tuple: #if type(result.get('polyline', '')) == tuple: - d['polyline'] = [] - for point in result['polyline']: - # This poor man's validation code almost definately needs improving. - if type(point) == dict: - d['polyline'].append(dict(x=point['x'],y=point['y'],z=point['z'])) - if isinstance(result['color'], basestring): - d['strokeColor'] = result['color'] - if "icon" in poi: - d.update({"icon": poi['icon']}) - if "createInfoWindow" in poi: - d.update({"createInfoWindow": poi['createInfoWindow']}) - markerSetDict[name]['raw'].append(d) - #print markerSetDict PlayerDict.save_cache(destdir) with open(os.path.join(destdir, "markersDB.js"), "w") as output: output.write("var markersDB=") - json.dump(markerSetDict, output, indent=2) + json.dump(markers, output, indent=2) output.write(";\n"); with open(os.path.join(destdir, "markers.js"), "w") as output: output.write("var markers=") - json.dump(markers, output, indent=2) + json.dump(marker_groups, output, indent=2) output.write(";\n"); with open(os.path.join(destdir, "baseMarkers.js"), "w") as output: output.write("overviewer.util.injectMarkerScript('markersDB.js');\n")