From 0a1560431a22e18aa1e958b219cb64820fd3a7c3 Mon Sep 17 00:00:00 2001 From: Cliff Meyers Date: Wed, 15 Apr 2020 07:34:54 -0400 Subject: [PATCH 1/3] fix a bug where multiple renders of the same world led to repeat genPOI scans the code that calls handleEntities in genPOI was building a list of all filter funcs in the config, then used itertools.groupby to walk over each region set, calling handleEntities and passing filters funcs for the rset. this works fine when all of the renders point at a different world, since rsets are sorted via __lt__ using the regiondir. when an rset appears in the config multiple times, groupby can't sort the rsets reliably, so a single rset is passed multiple times to handleEntities with only some of the applicable filters. for a config with just two renders of the same world, handleEntities would frequently be called 5-7 times instead of just 2. this change eliminates group by, and just iterates over all the rsets, each time plucking the list of applicable filters based on rset associated with each filter (simple equality). --- overviewer_core/aux_files/genPOI.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 2e69450..0b009e6 100644 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -502,6 +502,8 @@ def main(): filters = set() marker_groups = defaultdict(list) + logging.info("Searching renders: %s", list(config['renders'])) + # collect all filters and get regionsets for rname, render in config['renders'].items(): # Convert render['world'] to the world path, and store the original @@ -559,14 +561,15 @@ def main(): markers = dict((name, dict(created=False, raw=[], name=filter_name)) for name, filter_name, __, __, __, __ in filters) + all_rsets = set(map(lambda f : f[3], filters)) + logging.info("Will search %s region sets using %s filters", len(all_rsets), len(filters)) + # apply filters to regionsets if not args.skipscan: - # group filters by rset - def keyfunc(x): - return x[3] - sfilters = sorted(filters, key=keyfunc) - for rset, rset_filters in itertools.groupby(sfilters, keyfunc): - handleEntities(rset, config, args.config, list(rset_filters), markers) + for rset in all_rsets: + rset_filters = list(filter(lambda f : f[3] == rset, filters)) + logging.info("Calling handleEntities for %s with %s filters", rset, len(rset_filters)) + handleEntities(rset, config, args.config, rset_filters, markers) # apply filters to players if not args.skipplayers: From 512d6e7583f157007963949fece3e5aeed43037f Mon Sep 17 00:00:00 2001 From: Cliff Meyers Date: Wed, 15 Apr 2020 07:52:34 -0400 Subject: [PATCH 2/3] use logging.debug for detailed reporting of entities found during genPOI --- overviewer_core/aux_files/genPOI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 0b009e6..bbcb94a 100644 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -142,7 +142,7 @@ def parseBucketChunks(task_tuple): if i == 250: i = 0 cnt = 250 + cnt - logging.info("Found %d markers in thread %d so far at %d chunks.", + logging.debug("Found %d markers in thread %d so far at %d chunks.", sum(len(v) for v in markers.values()), pid, cnt) return markers From 61903939a30d39507a275fe40bda67a24aa106bc Mon Sep 17 00:00:00 2001 From: Cliff Meyers Date: Wed, 15 Apr 2020 08:06:12 -0400 Subject: [PATCH 3/3] remove __lt__ from RegionSet & wrapper as it's no longer used in genPOI --- overviewer_core/world.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/overviewer_core/world.py b/overviewer_core/world.py index c96db53..da5e9a6 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -881,13 +881,6 @@ class RegionSet(object): def __repr__(self): return "" % self.regiondir - def __lt__(self, other): - """This garbage is only needed because genPOI wants to use - itertools.groupby, which needs sorted keys, and Python 2 somehow - just sorted objects like ???????? how????? why????? - """ - return self.regiondir < other.regiondir - def _get_block(self, palette_entry): wood_slabs = ('minecraft:oak_slab','minecraft:spruce_slab','minecraft:birch_slab','minecraft:jungle_slab', 'minecraft:acacia_slab','minecraft:dark_oak_slab','minecraft:petrified_oak_slab') @@ -1553,13 +1546,6 @@ class RegionSetWrapper(object): def __init__(self, rsetobj): self._r = rsetobj - def __lt__(self, other): - """This garbage is only needed because genPOI wants to use - itertools.groupby, which needs sorted keys, and Python 2 somehow - just sorted objects like ???????? how????? why????? - """ - return self.regiondir < other.regiondir - @property def regiondir(self): """