From e7c19f9309058e37718908289688d4da43722df9 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:10:59 +1000 Subject: [PATCH 01/17] Use a more efficient checker for cached images --- chunk.py | 15 ++++++--------- world.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/chunk.py b/chunk.py index bdfd062..611eb53 100644 --- a/chunk.py +++ b/chunk.py @@ -169,6 +169,7 @@ class ChunkRenderer(object): moredirs, dir2 = os.path.split(destdir) _, dir1 = os.path.split(moredirs) self.cachedir = os.path.join(cachedir, dir1, dir2) + self.dirbits = (dir1, dir2) if self.world.useBiomeData: @@ -301,16 +302,12 @@ class ChunkRenderer(object): return self._digest def find_oldimage(self, cave): - # Get the name of the existing image. No way to do this but to look at - # all the files + # Get the name of the existing image. oldimg = oldimg_path = None - for filename in os.listdir(self.cachedir): - if filename.startswith("img.{0}.{1}.".format(self.blockid, - "cave" if cave else "nocave")) and \ - filename.endswith(".png"): - oldimg = filename - oldimg_path = os.path.join(self.cachedir, oldimg) - break + key = ".".join((dirbits[0], dirbits[1], self.blockid, "cave" if cave else "nocave")) + if key in self.world.cached: + oldimg_path = self.world.cached[key] + _, oldimg = os.path.split(oldimg_path) return oldimg, oldimg_path def render_and_save(self, cave=False): diff --git a/world.py b/world.py index 1165053..8d5c2c5 100644 --- a/world.py +++ b/world.py @@ -21,6 +21,7 @@ import Queue import sys import logging import cPickle +import collections import numpy @@ -107,6 +108,20 @@ class WorldRenderer(object): textures.prepareBiomeData(worlddir) self.chunklist = chunklist + self.cached = collections.defaultdict(dict) + + # In order to avoid having to look up the cache file names in + # ChunkRenderer, get them all and store them here + for root, dirnames, filenames in os.walk('.'): + for filename in filenames: + if not filename.endswith('.png'): + continue + dirname, dir_b = os.path.split(root) + _, dir_a = os.path.split(dirname) + _, x, z, cave, _ = filename.split('.', 4) + bits = '.'.join((dir_a, dir_b, x, z, cave)) + self.cached[bits] = os.path.join(root, filename) + # stores Points Of Interest to be mapped with markers # a list of dictionaries, see below for an example From 2eb85ae9462cd2d393bdae1fdf2582484f689878 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:18:02 +1000 Subject: [PATCH 02/17] Use the proper directory to walk --- world.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/world.py b/world.py index 8d5c2c5..17b9ef6 100644 --- a/world.py +++ b/world.py @@ -112,7 +112,7 @@ class WorldRenderer(object): # In order to avoid having to look up the cache file names in # ChunkRenderer, get them all and store them here - for root, dirnames, filenames in os.walk('.'): + for root, dirnames, filenames in os.walk(cachedir): for filename in filenames: if not filename.endswith('.png'): continue From b94daab5a0e11c5fa2416a4ad7726e04ac3a77f9 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:21:08 +1000 Subject: [PATCH 03/17] Use self.dirbits instead of dirbits --- chunk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunk.py b/chunk.py index 611eb53..c5dec69 100644 --- a/chunk.py +++ b/chunk.py @@ -304,7 +304,7 @@ class ChunkRenderer(object): def find_oldimage(self, cave): # Get the name of the existing image. oldimg = oldimg_path = None - key = ".".join((dirbits[0], dirbits[1], self.blockid, "cave" if cave else "nocave")) + key = ".".join((self.dirbits[0], self.dirbits[1], self.blockid, "cave" if cave else "nocave")) if key in self.world.cached: oldimg_path = self.world.cached[key] _, oldimg = os.path.split(oldimg_path) From dff4f4aea816527bbb502bf094dd22a8bb606932 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:22:58 +1000 Subject: [PATCH 04/17] Add debugging info --- chunk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chunk.py b/chunk.py index c5dec69..32d74b5 100644 --- a/chunk.py +++ b/chunk.py @@ -306,6 +306,7 @@ class ChunkRenderer(object): oldimg = oldimg_path = None key = ".".join((self.dirbits[0], self.dirbits[1], self.blockid, "cave" if cave else "nocave")) if key in self.world.cached: + print "Found cached image {0}".format(key) oldimg_path = self.world.cached[key] _, oldimg = os.path.split(oldimg_path) return oldimg, oldimg_path From b7f4855ec6a03921661e97937ff784b93ef8f678 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:31:20 +1000 Subject: [PATCH 05/17] Use logging instead of print statement --- chunk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunk.py b/chunk.py index 32d74b5..92efe10 100644 --- a/chunk.py +++ b/chunk.py @@ -306,9 +306,9 @@ class ChunkRenderer(object): oldimg = oldimg_path = None key = ".".join((self.dirbits[0], self.dirbits[1], self.blockid, "cave" if cave else "nocave")) if key in self.world.cached: - print "Found cached image {0}".format(key) oldimg_path = self.world.cached[key] _, oldimg = os.path.split(oldimg_path) + logging.debug("Found cached image {0}".format(oldimg)) return oldimg, oldimg_path def render_and_save(self, cave=False): From 211f489f66595bdb855afeb90096bd50ad0899d7 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Sat, 11 Dec 2010 21:32:36 +1000 Subject: [PATCH 06/17] Add logging regarding whether cached image is used or not --- chunk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chunk.py b/chunk.py index 92efe10..6af295c 100644 --- a/chunk.py +++ b/chunk.py @@ -327,6 +327,7 @@ class ChunkRenderer(object): if os.path.getmtime(self.chunkfile) <= os.path.getmtime(oldimg_path): # chunkfile is older than the image, don't even bother checking # the hash + logging.debug("Using cached image") return oldimg_path # Reasons for the code to get to this point: @@ -352,6 +353,7 @@ class ChunkRenderer(object): # Before we return it, update its mtime so the next round # doesn't have to check the hash os.utime(dest_path, None) + logging.debug("Using cached image") return dest_path else: # Remove old image for this chunk. Anything already existing is From 8ec7986353cb16764a0fd3ab2dee54b17c7ddb35 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 20:34:17 +1000 Subject: [PATCH 07/17] Only pass the part of the dict we need to the chunk renderer --- chunk.py | 14 +++++++------- world.py | 19 ++++++++++++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/chunk.py b/chunk.py index 6af295c..8701fc4 100644 --- a/chunk.py +++ b/chunk.py @@ -112,12 +112,12 @@ def iterate_chunkblocks(xoff,yoff): transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 44, 50, 51, 52, 53, 59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 83, 85]) -def render_and_save(chunkfile, cachedir, worldobj, cave=False, queue=None): +def render_and_save(chunkfile, cachedir, worldobj, cached, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes can't target bound methods) or to easily render and save one chunk Returns the image file location""" - a = ChunkRenderer(chunkfile, cachedir, worldobj, queue) + a = ChunkRenderer(chunkfile, cachedir, worldobj, cached, queue) try: return a.render_and_save(cave) except ChunkCorrupt: @@ -140,7 +140,7 @@ class ChunkCorrupt(Exception): pass class ChunkRenderer(object): - def __init__(self, chunkfile, cachedir, worldobj, queue): + def __init__(self, chunkfile, cachedir, cached, worldobj, queue): """Make a new chunk renderer for the given chunkfile. chunkfile should be a full path to the .dat file to process cachedir is a directory to save the resulting chunk images to @@ -169,7 +169,7 @@ class ChunkRenderer(object): moredirs, dir2 = os.path.split(destdir) _, dir1 = os.path.split(moredirs) self.cachedir = os.path.join(cachedir, dir1, dir2) - self.dirbits = (dir1, dir2) + self.cached = cached if self.world.useBiomeData: @@ -304,9 +304,9 @@ class ChunkRenderer(object): def find_oldimage(self, cave): # Get the name of the existing image. oldimg = oldimg_path = None - key = ".".join((self.dirbits[0], self.dirbits[1], self.blockid, "cave" if cave else "nocave")) - if key in self.world.cached: - oldimg_path = self.world.cached[key] + key = ".".join((self.blockid, "cave" if cave else "nocave")) + if key in self.cached: + oldimg_path = self.cached[key] _, oldimg = os.path.split(oldimg_path) logging.debug("Found cached image {0}".format(oldimg)) return oldimg, oldimg_path diff --git a/world.py b/world.py index 17b9ef6..c0aaa41 100644 --- a/world.py +++ b/world.py @@ -36,6 +36,7 @@ and for extracting information about available worlds """ base36decode = functools.partial(int, base=36) +cached = collections.defaultdict(dict) def _convert_coords(chunks): @@ -108,7 +109,6 @@ class WorldRenderer(object): textures.prepareBiomeData(worlddir) self.chunklist = chunklist - self.cached = collections.defaultdict(dict) # In order to avoid having to look up the cache file names in # ChunkRenderer, get them all and store them here @@ -119,8 +119,9 @@ class WorldRenderer(object): dirname, dir_b = os.path.split(root) _, dir_a = os.path.split(dirname) _, x, z, cave, _ = filename.split('.', 4) - bits = '.'.join((dir_a, dir_b, x, z, cave)) - self.cached[bits] = os.path.join(root, filename) + dir = '/'.join((dir_a, dir_b)) + bits = '.'.join((x, z, cave)) + cached[dir][bits] = os.path.join(root, filename) # stores Points Of Interest to be mapped with markers @@ -294,7 +295,11 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue - result = chunk.render_and_save(chunkfile, self.cachedir, self, cave=self.caves, queue=q) + moredirs, dir2 = os.path.split(os.path.dirname(self.chunkfile)) + dir1 = os.path.basename(moredirs) + cachename = '/'.join(dir1, dir2) + + result = chunk.render_and_save(chunkfile, self.cachedir, self, cached[chunkname], queue=q) results[(col, row)] = result if i > 0: try: @@ -320,8 +325,12 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue + moredirs, dir2 = os.path.split(os.path.dirname(self.chunkfile)) + dir1 = os.path.basename(moredirs) + cachename = '/'.join(dir1, dir2) + result = pool.apply_async(chunk.render_and_save, - args=(chunkfile,self.cachedir,self), + args=(chunkfile,self.cachedir,self, cached[chunkname]), kwds=dict(cave=self.caves, queue=q)) asyncresults.append((col, row, result)) From ef82e657ee34666913ac7f2c4a31b6e636f4e735 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 20:38:11 +1000 Subject: [PATCH 08/17] Use the right name --- world.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/world.py b/world.py index c0aaa41..19741bb 100644 --- a/world.py +++ b/world.py @@ -295,7 +295,7 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue - moredirs, dir2 = os.path.split(os.path.dirname(self.chunkfile)) + moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) dir1 = os.path.basename(moredirs) cachename = '/'.join(dir1, dir2) @@ -325,7 +325,7 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue - moredirs, dir2 = os.path.split(os.path.dirname(self.chunkfile)) + moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) dir1 = os.path.basename(moredirs) cachename = '/'.join(dir1, dir2) From ead41141bb420c5162549aa4349199bc3ca1d89f Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 20:39:34 +1000 Subject: [PATCH 09/17] Use a tuple properly, oops --- world.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/world.py b/world.py index 19741bb..b72bd28 100644 --- a/world.py +++ b/world.py @@ -297,7 +297,7 @@ class WorldRenderer(object): moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) dir1 = os.path.basename(moredirs) - cachename = '/'.join(dir1, dir2) + cachename = '/'.join((dir1, dir2)) result = chunk.render_and_save(chunkfile, self.cachedir, self, cached[chunkname], queue=q) results[(col, row)] = result @@ -327,7 +327,7 @@ class WorldRenderer(object): moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) dir1 = os.path.basename(moredirs) - cachename = '/'.join(dir1, dir2) + cachename = '/'.join((dir1, dir2)) result = pool.apply_async(chunk.render_and_save, args=(chunkfile,self.cachedir,self, cached[chunkname]), From c14c6d2bcc9b32fae3c4f60487350b1be5b14656 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 20:40:58 +1000 Subject: [PATCH 10/17] My typing is horrible tonight, use the proper variable name --- world.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/world.py b/world.py index b72bd28..616837e 100644 --- a/world.py +++ b/world.py @@ -299,7 +299,7 @@ class WorldRenderer(object): dir1 = os.path.basename(moredirs) cachename = '/'.join((dir1, dir2)) - result = chunk.render_and_save(chunkfile, self.cachedir, self, cached[chunkname], queue=q) + result = chunk.render_and_save(chunkfile, self.cachedir, self, cached[cachename], queue=q) results[(col, row)] = result if i > 0: try: @@ -330,7 +330,7 @@ class WorldRenderer(object): cachename = '/'.join((dir1, dir2)) result = pool.apply_async(chunk.render_and_save, - args=(chunkfile,self.cachedir,self, cached[chunkname]), + args=(chunkfile,self.cachedir,self, cached[cachename]), kwds=dict(cave=self.caves, queue=q)) asyncresults.append((col, row, result)) From 8def0ffaef6d3b615c7e582c59c52cb6459aaf5f Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 21:16:21 +1000 Subject: [PATCH 11/17] Check cache synchronously, to avoid async overhead. This is starting to do my head in. Apologies if these don't make any sense, but they do to me. :) --- chunk.py | 50 ++++++++++++++++++++++++++++++-------------------- world.py | 23 ++++++++++++----------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/chunk.py b/chunk.py index 8701fc4..08c340a 100644 --- a/chunk.py +++ b/chunk.py @@ -112,6 +112,28 @@ def iterate_chunkblocks(xoff,yoff): transparent_blocks = set([0, 6, 8, 9, 18, 20, 37, 38, 39, 40, 44, 50, 51, 52, 53, 59, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 83, 85]) +def find_oldimage(chunkfile, cached, cave): + destdir, filename = os.path.split(chunkfile) + filename_split = filename.split(".") + blockid = ".".join(filename_split[1:3]) + + # Get the name of the existing image. + moredirs, dir2 = os.path.split(destdir) + dir1 = os.path.basename(moredirs) + cachename = '/'.join((dir1, dir2)) + + oldimg = oldimg_path = None + key = ".".join((blockid, "cave" if cave else "nocave")) + if key in cached[cachename]: + oldimg_path = cached[cachename][key] + _, oldimg = os.path.split(oldimg_path) + logging.debug("Found cached image {0}".format(oldimg)) + return oldimg, oldimg_path + +def check_cache(chunkfile, oldimg): + if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + return self.oldimg_path + def render_and_save(chunkfile, cachedir, worldobj, cached, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes can't target bound methods) or to easily render and save one chunk @@ -140,7 +162,7 @@ class ChunkCorrupt(Exception): pass class ChunkRenderer(object): - def __init__(self, chunkfile, cachedir, cached, worldobj, queue): + def __init__(self, chunkfile, cachedir, oldimg, worldobj, queue): """Make a new chunk renderer for the given chunkfile. chunkfile should be a full path to the .dat file to process cachedir is a directory to save the resulting chunk images to @@ -169,7 +191,7 @@ class ChunkRenderer(object): moredirs, dir2 = os.path.split(destdir) _, dir1 = os.path.split(moredirs) self.cachedir = os.path.join(cachedir, dir1, dir2) - self.cached = cached + self.oldimg, self.oldimg_path = oldimg if self.world.useBiomeData: @@ -301,34 +323,22 @@ class ChunkRenderer(object): self._digest = digest[:6] return self._digest - def find_oldimage(self, cave): - # Get the name of the existing image. - oldimg = oldimg_path = None - key = ".".join((self.blockid, "cave" if cave else "nocave")) - if key in self.cached: - oldimg_path = self.cached[key] - _, oldimg = os.path.split(oldimg_path) - logging.debug("Found cached image {0}".format(oldimg)) - return oldimg, oldimg_path - def render_and_save(self, cave=False): """Render the chunk using chunk_render, and then save it to a file in the same directory as the source image. If the file already exists and is up to date, this method doesn't render anything. """ blockid = self.blockid - - oldimg, oldimg_path = self.find_oldimage(cave) - if oldimg: + if self.oldimg: # An image exists? Instead of checking the hash which is kinda # expensive (for tens of thousands of chunks, yes it is) check if # the mtime of the chunk file is newer than the mtime of oldimg - if os.path.getmtime(self.chunkfile) <= os.path.getmtime(oldimg_path): + if os.path.getmtime(self.chunkfile) <= os.path.getmtime(self.oldimg_path): # chunkfile is older than the image, don't even bother checking # the hash logging.debug("Using cached image") - return oldimg_path + return self.oldimg_path # Reasons for the code to get to this point: # 1) An old image doesn't exist @@ -346,8 +356,8 @@ class ChunkRenderer(object): dest_path = os.path.join(self.cachedir, dest_filename) - if oldimg: - if dest_filename == oldimg: + if self.oldimg: + if dest_filename == self.oldimg: # There is an existing file, the chunk has a newer mtime, but the # hashes match. # Before we return it, update its mtime so the next round @@ -358,7 +368,7 @@ class ChunkRenderer(object): else: # Remove old image for this chunk. Anything already existing is # either corrupt or out of date - os.unlink(oldimg_path) + os.unlink(self.oldimg_path) # Render the chunk img = self.chunk_render(cave=cave) diff --git a/world.py b/world.py index 616837e..816531d 100644 --- a/world.py +++ b/world.py @@ -295,11 +295,12 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue - moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) - dir1 = os.path.basename(moredirs) - cachename = '/'.join((dir1, dir2)) + oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) + if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + result = oldimg[1] + else: + result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) - result = chunk.render_and_save(chunkfile, self.cachedir, self, cached[cachename], queue=q) results[(col, row)] = result if i > 0: try: @@ -325,13 +326,13 @@ class WorldRenderer(object): results[(col, row)] = imgpath continue - moredirs, dir2 = os.path.split(os.path.dirname(chunkfile)) - dir1 = os.path.basename(moredirs) - cachename = '/'.join((dir1, dir2)) - - result = pool.apply_async(chunk.render_and_save, - args=(chunkfile,self.cachedir,self, cached[cachename]), - kwds=dict(cave=self.caves, queue=q)) + oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) + if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + result = oldimg[1] + else: + result = pool.apply_async(chunk.render_and_save, + args=(chunkfile,self.cachedir,self, oldimg), + kwds=dict(cave=self.caves, queue=q)) asyncresults.append((col, row, result)) pool.close() From fcfaffffbb87f250146cc8b1c81b4d0bd96b4241 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 21:21:24 +1000 Subject: [PATCH 12/17] Check if the key exists first --- chunk.py | 10 ---------- world.py | 4 ++-- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/chunk.py b/chunk.py index 08c340a..49b109c 100644 --- a/chunk.py +++ b/chunk.py @@ -330,16 +330,6 @@ class ChunkRenderer(object): """ blockid = self.blockid - if self.oldimg: - # An image exists? Instead of checking the hash which is kinda - # expensive (for tens of thousands of chunks, yes it is) check if - # the mtime of the chunk file is newer than the mtime of oldimg - if os.path.getmtime(self.chunkfile) <= os.path.getmtime(self.oldimg_path): - # chunkfile is older than the image, don't even bother checking - # the hash - logging.debug("Using cached image") - return self.oldimg_path - # Reasons for the code to get to this point: # 1) An old image doesn't exist # 2) An old image exists, but the chunk was more recently modified (the diff --git a/world.py b/world.py index 816531d..522a99d 100644 --- a/world.py +++ b/world.py @@ -296,7 +296,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + if oldimg[1] and (os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1])): result = oldimg[1] else: result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) @@ -327,7 +327,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + if oldimg[1] and (os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1])): result = oldimg[1] else: result = pool.apply_async(chunk.render_and_save, From 810ecaf6db4e587f323fec183ff5e982c76b2a9b Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 21:38:19 +1000 Subject: [PATCH 13/17] Use the proper find_oldimage() --- world.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/world.py b/world.py index 522a99d..919190b 100644 --- a/world.py +++ b/world.py @@ -289,8 +289,7 @@ class WorldRenderer(object): for i, (col, row, chunkfile) in enumerate(chunks): if inclusion_set and (col, row) not in inclusion_set: # Skip rendering, just find where the existing image is - _, imgpath = chunk.ChunkRenderer(chunkfile, - self.cachedir, self, q).find_oldimage(False) + _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) if imgpath: results[(col, row)] = imgpath continue @@ -320,8 +319,7 @@ class WorldRenderer(object): for col, row, chunkfile in chunks: if inclusion_set and (col, row) not in inclusion_set: # Skip rendering, just find where the existing image is - _, imgpath = chunk.ChunkRenderer(chunkfile, - self.cachedir, self, q).find_oldimage(False) + _, imgpath = chunk.find_oldimage(chunkfile, cached, self.caves) if imgpath: results[(col, row)] = imgpath continue From 439231d280e6d065cd581ecd5c345d1290b39ac6 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 21:57:35 +1000 Subject: [PATCH 14/17] Pretend a string is an AsyncResult --- world.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/world.py b/world.py index 919190b..be90a73 100644 --- a/world.py +++ b/world.py @@ -87,6 +87,12 @@ def base36encode(number, alphabet='0123456789abcdefghijklmnopqrstuvwxyz'): return "-" + base36 return base36 +class FakeAsyncResult: + def __init__(self, string): + self.string = string + def get(self): + return self.string + class WorldRenderer(object): """Renders a world's worth of chunks. worlddir is the path to the minecraft world @@ -326,7 +332,7 @@ class WorldRenderer(object): oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) if oldimg[1] and (os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1])): - result = oldimg[1] + result = FakeAsyncResult(oldimg[1]) else: result = pool.apply_async(chunk.render_and_save, args=(chunkfile,self.cachedir,self, oldimg), From 72ec5469fa2b89c025b8acc4eefee1da118abfc1 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 22:04:52 +1000 Subject: [PATCH 15/17] Fix order and naming of parameters --- chunk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chunk.py b/chunk.py index 49b109c..f6042bf 100644 --- a/chunk.py +++ b/chunk.py @@ -134,12 +134,12 @@ def check_cache(chunkfile, oldimg): if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): return self.oldimg_path -def render_and_save(chunkfile, cachedir, worldobj, cached, cave=False, queue=None): +def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes can't target bound methods) or to easily render and save one chunk Returns the image file location""" - a = ChunkRenderer(chunkfile, cachedir, worldobj, cached, queue) + a = ChunkRenderer(chunkfile, cachedir, worldobj, oldimg, queue) try: return a.render_and_save(cave) except ChunkCorrupt: @@ -162,7 +162,7 @@ class ChunkCorrupt(Exception): pass class ChunkRenderer(object): - def __init__(self, chunkfile, cachedir, oldimg, worldobj, queue): + def __init__(self, chunkfile, cachedir, worldobj, oldimg, queue): """Make a new chunk renderer for the given chunkfile. chunkfile should be a full path to the .dat file to process cachedir is a directory to save the resulting chunk images to From b3abb072c8c1dc7104de82c24bb476cba094807d Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Mon, 13 Dec 2010 22:16:35 +1000 Subject: [PATCH 16/17] Use chunk.check_cache --- chunk.py | 8 ++++++-- world.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/chunk.py b/chunk.py index f6042bf..181553e 100644 --- a/chunk.py +++ b/chunk.py @@ -131,8 +131,12 @@ def find_oldimage(chunkfile, cached, cave): return oldimg, oldimg_path def check_cache(chunkfile, oldimg): - if os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): - return self.oldimg_path + try: + if oldimg[1] and os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1]): + return True + return False + except OSError: + return False def render_and_save(chunkfile, cachedir, worldobj, oldimg, cave=False, queue=None): """Used as the entry point for the multiprocessing workers (since processes diff --git a/world.py b/world.py index be90a73..effc46a 100644 --- a/world.py +++ b/world.py @@ -301,7 +301,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if oldimg[1] and (os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1])): + if chunk.check_cache(chunkfile, oldimg[1]): result = oldimg[1] else: result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) @@ -331,7 +331,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if oldimg[1] and (os.path.getmtime(chunkfile) <= os.path.getmtime(oldimg[1])): + if chunk.check_cache(chunkfile, oldimg[1]): result = FakeAsyncResult(oldimg[1]) else: result = pool.apply_async(chunk.render_and_save, From ae0b3dfedf32ddf91ae1543d48ce2e47463fd4d8 Mon Sep 17 00:00:00 2001 From: Ryan McCue Date: Tue, 14 Dec 2010 11:54:26 +1000 Subject: [PATCH 17/17] check_cache() wants a full tuple, not just part --- world.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/world.py b/world.py index effc46a..03a8651 100644 --- a/world.py +++ b/world.py @@ -301,7 +301,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if chunk.check_cache(chunkfile, oldimg[1]): + if chunk.check_cache(chunkfile, oldimg): result = oldimg[1] else: result = chunk.render_and_save(chunkfile, self.cachedir, self, oldimg, queue=q) @@ -331,7 +331,7 @@ class WorldRenderer(object): continue oldimg = chunk.find_oldimage(chunkfile, cached, self.caves) - if chunk.check_cache(chunkfile, oldimg[1]): + if chunk.check_cache(chunkfile, oldimg): result = FakeAsyncResult(oldimg[1]) else: result = pool.apply_async(chunk.render_and_save,