multiprocessing-based POI generator no longer crashes on python 2.6
fixes Issue #1041
This commit is contained in:
@@ -24,7 +24,6 @@ import multiprocessing
|
|||||||
|
|
||||||
from multiprocessing import Process
|
from multiprocessing import Process
|
||||||
from multiprocessing import Pool
|
from multiprocessing import Pool
|
||||||
from functools import partial
|
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
|
|
||||||
from overviewer_core import logger
|
from overviewer_core import logger
|
||||||
@@ -39,7 +38,11 @@ def replaceBads(s):
|
|||||||
x = x.replace(bad,"_")
|
x = x.replace(bad,"_")
|
||||||
return x
|
return x
|
||||||
|
|
||||||
def parseBucketChunks(bucket, rset):
|
# yes there's a double parenthesis here
|
||||||
|
# see below for when this is called, and why we do this
|
||||||
|
# a smarter way would be functools.partial, but that's broken on python 2.6
|
||||||
|
# when used with multiprocessing
|
||||||
|
def parseBucketChunks((bucket, rset)):
|
||||||
pid = multiprocessing.current_process().pid
|
pid = multiprocessing.current_process().pid
|
||||||
pois = dict(TileEntities=[], Entities=[]);
|
pois = dict(TileEntities=[], Entities=[]);
|
||||||
|
|
||||||
@@ -98,12 +101,9 @@ def handleEntities(rset, outputdir, render, rname, config):
|
|||||||
for b in buckets:
|
for b in buckets:
|
||||||
logging.info("Buckets has %d entries", len(b));
|
logging.info("Buckets has %d entries", len(b));
|
||||||
|
|
||||||
# create the partial to lock the rset
|
|
||||||
dest = partial(parseBucketChunks, rset=rset)
|
|
||||||
|
|
||||||
# Create a pool of processes and run all the functions
|
# Create a pool of processes and run all the functions
|
||||||
pool = Pool(processes=numbuckets)
|
pool = Pool(processes=numbuckets)
|
||||||
results = pool.map(dest, buckets)
|
results = pool.map(parseBucketChunks, ((buck, rset) for buck in buckets))
|
||||||
|
|
||||||
logging.info("All the threads completed")
|
logging.info("All the threads completed")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user