genPOI: port to Python 3
There's some stuff left to do, like properly do the functools.partial stuff, and check that the uuidcache is actually working. But hey, player markers work at least, and so does --skip-scan. Still needs a port to argparse though.
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/env python2
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
'''
|
'''
|
||||||
genPOI.py
|
genPOI.py
|
||||||
@@ -24,7 +24,8 @@ import os
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib2
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
from multiprocessing import Pool
|
from multiprocessing import Pool
|
||||||
@@ -77,7 +78,7 @@ def jsonText(s):
|
|||||||
bar += foo["text"]
|
bar += foo["text"]
|
||||||
if "extra" in foo:
|
if "extra" in foo:
|
||||||
bar += parseLevel(foo["extra"])
|
bar += parseLevel(foo["extra"])
|
||||||
elif isinstance(foo, basestring):
|
elif isinstance(foo, str):
|
||||||
bar = foo
|
bar = foo
|
||||||
return bar
|
return bar
|
||||||
|
|
||||||
@@ -99,7 +100,7 @@ def initBucketChunks(config_path):
|
|||||||
mw_parser.parse(config_path)
|
mw_parser.parse(config_path)
|
||||||
# ought not to fail since we already did it once
|
# ought not to fail since we already did it once
|
||||||
config = mw_parser.get_validated_config()
|
config = mw_parser.get_validated_config()
|
||||||
for name, render in config['renders'].iteritems():
|
for name, render in config['renders'].items():
|
||||||
for f in render['markers']:
|
for f in render['markers']:
|
||||||
ff = f['filterFunction']
|
ff = f['filterFunction']
|
||||||
bucketChunkFuncs[ff.__name__] = ff
|
bucketChunkFuncs[ff.__name__] = ff
|
||||||
@@ -109,8 +110,11 @@ def initBucketChunks(config_path):
|
|||||||
# see below for when this is called, and why we do this
|
# see below for when this is called, and why we do this
|
||||||
# a smarter way would be functools.partial, but that's broken on python 2.6
|
# a smarter way would be functools.partial, but that's broken on python 2.6
|
||||||
# when used with multiprocessing
|
# when used with multiprocessing
|
||||||
def parseBucketChunks((bucket, rset, filters)):
|
# TODO: Do the smarter way
|
||||||
|
def parseBucketChunks(task_tuple):
|
||||||
global bucketChunkFuncs
|
global bucketChunkFuncs
|
||||||
|
|
||||||
|
bucket, rset, filters = task_tuple
|
||||||
pid = multiprocessing.current_process().pid
|
pid = multiprocessing.current_process().pid
|
||||||
markers = defaultdict(list)
|
markers = defaultdict(list)
|
||||||
|
|
||||||
@@ -139,7 +143,7 @@ def parseBucketChunks((bucket, rset, filters)):
|
|||||||
i = 0
|
i = 0
|
||||||
cnt = 250 + cnt
|
cnt = 250 + cnt
|
||||||
logging.info("Found %d markers in thread %d so far at %d chunks.",
|
logging.info("Found %d markers in thread %d so far at %d chunks.",
|
||||||
sum(len(v) for v in markers.itervalues()), pid, cnt)
|
sum(len(v) for v in markers.values()), pid, cnt)
|
||||||
|
|
||||||
return markers
|
return markers
|
||||||
|
|
||||||
@@ -190,7 +194,7 @@ def handleEntities(rset, config, config_path, filters, markers):
|
|||||||
buckets = [[] for i in range(numbuckets)]
|
buckets = [[] for i in range(numbuckets)]
|
||||||
|
|
||||||
for (x, z, mtime) in rset.iterate_chunks():
|
for (x, z, mtime) in rset.iterate_chunks():
|
||||||
i = x / 32 + z / 32
|
i = x // 32 + z // 32
|
||||||
i = i % numbuckets
|
i = i % numbuckets
|
||||||
buckets[i].append([x, z])
|
buckets[i].append([x, z])
|
||||||
|
|
||||||
@@ -209,7 +213,7 @@ def handleEntities(rset, config, config_path, filters, markers):
|
|||||||
logging.info("All the threads completed.")
|
logging.info("All the threads completed.")
|
||||||
|
|
||||||
for marker_dict in results:
|
for marker_dict in results:
|
||||||
for name, marker_list in marker_dict.iteritems():
|
for name, marker_list in marker_dict.items():
|
||||||
markers[name]['raw'].extend(marker_list)
|
markers[name]['raw'].extend(marker_list)
|
||||||
|
|
||||||
logging.info("Done.")
|
logging.info("Done.")
|
||||||
@@ -252,13 +256,13 @@ class PlayerDict(dict):
|
|||||||
|
|
||||||
with FileReplacer(cache_file, caps) as cache_file_name:
|
with FileReplacer(cache_file, caps) as cache_file_name:
|
||||||
with closing(gzip.GzipFile(cache_file_name, "wb")) as gz:
|
with closing(gzip.GzipFile(cache_file_name, "wb")) as gz:
|
||||||
json.dump(cls.uuid_cache, gz)
|
gz.write(json.dumps(cls.uuid_cache).encode())
|
||||||
logging.info("Wrote UUID cache with %d entries.",
|
logging.info("Wrote UUID cache with %d entries.",
|
||||||
len(cls.uuid_cache.keys()))
|
len(cls.uuid_cache.keys()))
|
||||||
|
|
||||||
def __getitem__(self, item):
|
def __getitem__(self, item):
|
||||||
if item == "EntityId":
|
if item == "EntityId":
|
||||||
if not super(PlayerDict, self).has_key("EntityId"):
|
if "EntityId" not in self:
|
||||||
if self.use_uuid:
|
if self.use_uuid:
|
||||||
super(PlayerDict, self).__setitem__("EntityId", self.get_name_from_uuid())
|
super(PlayerDict, self).__setitem__("EntityId", self.get_name_from_uuid())
|
||||||
else:
|
else:
|
||||||
@@ -275,12 +279,12 @@ class PlayerDict(dict):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + sname).read())
|
profile = json.loads(urllib.request.urlopen(UUID_LOOKUP_URL + sname).read())
|
||||||
if 'name' in profile:
|
if 'name' in profile:
|
||||||
profile['retrievedAt'] = time.mktime(time.localtime())
|
profile['retrievedAt'] = time.mktime(time.localtime())
|
||||||
PlayerDict.uuid_cache[sname] = profile
|
PlayerDict.uuid_cache[sname] = profile
|
||||||
return profile['name']
|
return profile['name']
|
||||||
except (ValueError, urllib2.URLError):
|
except (ValueError, urllib.error.URLError):
|
||||||
logging.warning("Unable to get player name for UUID %s.", self._name)
|
logging.warning("Unable to get player name for UUID %s.", self._name)
|
||||||
|
|
||||||
|
|
||||||
@@ -400,7 +404,7 @@ def create_marker_from_filter_result(poi, result):
|
|||||||
d["createInfoWindow"] = poi['createInfoWindow']
|
d["createInfoWindow"] = poi['createInfoWindow']
|
||||||
|
|
||||||
# Fill in the rest from result
|
# Fill in the rest from result
|
||||||
if isinstance(result, basestring):
|
if isinstance(result, str):
|
||||||
d.update(dict(text=result, hovertext=result))
|
d.update(dict(text=result, hovertext=result))
|
||||||
elif isinstance(result, tuple):
|
elif isinstance(result, tuple):
|
||||||
d.update(dict(text=result[1], hovertext=result[0]))
|
d.update(dict(text=result[1], hovertext=result[0]))
|
||||||
@@ -410,7 +414,7 @@ def create_marker_from_filter_result(poi, result):
|
|||||||
|
|
||||||
# Use custom hovertext if provided...
|
# Use custom hovertext if provided...
|
||||||
if 'hovertext' in result:
|
if 'hovertext' in result:
|
||||||
d['hovertext'] = unicode(result['hovertext'])
|
d['hovertext'] = str(result['hovertext'])
|
||||||
else: # ...otherwise default to display text.
|
else: # ...otherwise default to display text.
|
||||||
d['hovertext'] = result['text']
|
d['hovertext'] = result['text']
|
||||||
|
|
||||||
@@ -419,7 +423,7 @@ def create_marker_from_filter_result(poi, result):
|
|||||||
for point in result['polyline']:
|
for point in result['polyline']:
|
||||||
# point.copy() would work, but this validates better
|
# point.copy() would work, but this validates better
|
||||||
d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z']))
|
d['polyline'].append(dict(x=point['x'], y=point['y'], z=point['z']))
|
||||||
if isinstance(result['color'], basestring):
|
if isinstance(result['color'], str):
|
||||||
d['strokeColor'] = result['color']
|
d['strokeColor'] = result['color']
|
||||||
|
|
||||||
if "icon" in result:
|
if "icon" in result:
|
||||||
@@ -459,7 +463,7 @@ def main():
|
|||||||
parser.print_help()
|
parser.print_help()
|
||||||
return
|
return
|
||||||
|
|
||||||
if options.quiet > 0:
|
if options.quiet and options.quiet > 0:
|
||||||
logger.configure(logging.WARN, False)
|
logger.configure(logging.WARN, False)
|
||||||
|
|
||||||
# Parse the config file
|
# Parse the config file
|
||||||
@@ -479,7 +483,7 @@ def main():
|
|||||||
marker_groups = defaultdict(list)
|
marker_groups = defaultdict(list)
|
||||||
|
|
||||||
# collect all filters and get regionsets
|
# collect all filters and get regionsets
|
||||||
for rname, render in config['renders'].iteritems():
|
for rname, render in config['renders'].items():
|
||||||
# Convert render['world'] to the world path, and store the original
|
# Convert render['world'] to the world path, and store the original
|
||||||
# in render['worldname_orig']
|
# in render['worldname_orig']
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user