From 4579998663d163b795cc2ccdcd5d0c43ad443a1a Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 15:34:36 -0500 Subject: [PATCH 1/8] Prevent opening and parsing files which haven't been modified since the last time that the render was done --- overviewer_core/tileset.py | 2 +- overviewer_core/world.py | 42 +++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 0f0e1c3..1660348 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -757,7 +757,7 @@ class TileSet(object): # Compare the last modified time of the chunk and tile. If the # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks(): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 9c2bc86..95effe5 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -272,7 +272,7 @@ class RegionSet(object): for x, y, regionfile in self._iterate_regionfiles(): # regionfile is a pathname - self.regionfiles[(x,y)] = regionfile + self.regionfiles[(x,y)] = (regionfile, os.path.getmtime(regionfile)) self.empty_chunk = [None,None] logging.debug("Done scanning regions") @@ -458,7 +458,7 @@ class RegionSet(object): """ - for (regionx, regiony), regionfile in self.regionfiles.iteritems(): + for (regionx, regiony), (regionfile, filemtime) in self.regionfiles.iteritems(): try: mcr = self._get_regionobj(regionfile) except nbt.CorruptRegionError: @@ -467,6 +467,27 @@ class RegionSet(object): for chunkx, chunky in mcr.get_chunks(): yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) + def iterate_newer_chunks(self, mtime): + """Returns an iterator over all chunk metadata in this world. Iterates + over tuples of integers (x,z,mtime) for each chunk. Other chunk data + is not returned here. + + """ + + for (regionx, regiony), (regionfile, filemtime) in self.regionfiles.iteritems(): + """ SKIP LOADING A REGION WHICH HAS NOT BEEN MODIFIED! """ + if (filemtime < mtime): + continue + + try: + mcr = self._get_regionobj(regionfile) + except nbt.CorruptRegionError: + logging.warning("Found a corrupt region file at %s,%s. Skipping it.", regionx, regiony) + continue + + for chunkx, chunky in mcr.get_chunks(): + yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) + def get_chunk_mtime(self, x, z): """Returns a chunk's mtime, or False if the chunk does not exist. This is therefore a dual purpose method. It corrects for the given north @@ -492,7 +513,7 @@ class RegionSet(object): Coords can be either be global chunk coords, or local to a region """ - regionfile = self.regionfiles.get((chunkX//32, chunkY//32),None) + (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),None) return regionfile def _iterate_regionfiles(self): @@ -536,6 +557,8 @@ class RegionSetWrapper(object): return self._r.get_chunk(x,z) def iterate_chunks(self): return self._r.iterate_chunks() + def iterate_newer_chunks(self,filemtime): + return self._r.iterate_newer_chunks(filemtime) def get_chunk_mtime(self, x, z): return self._r.get_chunk_mtime(x,z) @@ -622,6 +645,11 @@ class RotatedRegionSet(RegionSetWrapper): x,z = self.rotate(x,z) yield x,z,mtime + def iterate_newer_chunks(self, filemtime): + for x,z,mtime in super(RotatedRegionSet, self).iterate_newer_chunks(filemtime): + x,z = self.rotate(x,z) + yield x,z,mtime + class CroppedRegionSet(RegionSetWrapper): def __init__(self, rsetobj, xmin, zmin, xmax, zmax): super(CroppedRegionSet, self).__init__(rsetobj) @@ -645,6 +673,14 @@ class CroppedRegionSet(RegionSetWrapper): self.xmin <= x <= self.xmax and self.zmin <= z <= self.zmax ) + + def iterate_newer_chunks(self, filemtime): + return ((x,z,mtime) for (x,z,mtime) in super(CroppedRegionSet,self).iterate_newer_chunks(filemtime) + if + self.xmin <= x <= self.xmax and + self.zmin <= z <= self.zmax + ) + def get_chunk_mtime(self,x,z): if ( self.xmin <= x <= self.xmax and From 8458451044321d8857bcfe9adeea514da3fafe1c Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 15:55:51 -0500 Subject: [PATCH 2/8] Respect the markall parameter --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 1660348..6046ea4 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -757,8 +757,8 @@ class TileSet(object): # Compare the last modified time of the chunk and tile. If the # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if markall else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 2b2d929659bad4429353fd8ea2b608e3c3ffffef Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:04:11 -0500 Subject: [PATCH 3/8] Add the new function to the synthetic test --- test/test_tileset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_tileset.py b/test/test_tileset.py index f555eac..864538f 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -1,3 +1,4 @@ +M import unittest import tempfile import shutil @@ -53,6 +54,10 @@ class FakeRegionset(object): for (x,z),mtime in self.chunks.iteritems(): yield x,z,mtime + def iterate_newer_chunks(self, filemtime): + for (x,z),mtime in self.chunks.iteritems(): + yield x,z,mtime + def get_chunk_mtime(self, x, z): try: return self.chunks[x,z] From a0640e8bdb0ef81173f83962dafe77f8fc39b81e Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:10:48 -0500 Subject: [PATCH 4/8] extra character ? --- test/test_tileset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_tileset.py b/test/test_tileset.py index 864538f..5658d00 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -1,4 +1,3 @@ -M import unittest import tempfile import shutil From c438a37b295a14dbaa08ddfc01a05eb5ec8c97cd Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:30:08 -0500 Subject: [PATCH 5/8] Default value is compatible with the expecteed result in fetching the region for a path --- overviewer_core/world.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 95effe5..f20e317 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -513,7 +513,7 @@ class RegionSet(object): Coords can be either be global chunk coords, or local to a region """ - (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),None) + (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),(None, None)) return regionfile def _iterate_regionfiles(self): From 6ee3eba550c4be1fc52a04095b2508a171d2449d Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:08:19 -0500 Subject: [PATCH 6/8] Do not use the newer than check on windows since apparently minecraft will not force a timestamp update on the file when they are written until it's closed. --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 6046ea4..d14bde2 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -758,7 +758,7 @@ class TileSet(object): # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if markall else self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall || platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 8023b52fdcc20a951349e151a473ff88d991e83e Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:52:53 -0500 Subject: [PATCH 7/8] Pythonic or --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index d14bde2..41a8756 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -758,7 +758,7 @@ class TileSet(object): # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall || platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall or platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 18beae66249833164a151c71bf905cd3eaec0bc4 Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:55:13 -0500 Subject: [PATCH 8/8] Import the platform --- overviewer_core/tileset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 41a8756..8899993 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -24,6 +24,7 @@ import functools import time import errno import stat +import platform from collections import namedtuple from itertools import product, izip, chain