misc cleanup and minor optimizations

overviewer_core/quadtree.py

@@ -23,7 +23,6 @@ import shutil
 import collections
 import json
 import logging
-import util
 import cPickle
 import stat
 import errno 
@@ -33,10 +32,10 @@ from time import gmtime, strftime, sleep
 
 from PIL import Image
 
-import nbt
+from . import nbt
-import chunk
+from . import chunk
+from .optimizeimages import optimize_image
 from c_overviewer import get_render_mode_inheritance
-from optimizeimages import optimize_image
 import composite
 
 
@@ -216,40 +215,50 @@ class QuadtreeGen(object):
                     self._decrease_depth()
     
     
-    def get_chunks_in_range(self, colstart, colend, rowstart, rowend):
+    def get_chunks_for_tile(self, tile):
-        """Get chunks that are relevant to the tile rendering function that's
+        """Get chunks that are relevant to the given tile
-        rendering that range
         
         Returns a list of chunks where each item is
-        (col, row, chunkx, chunky, regionfilename)
+        (col, row, chunkx, chunky, regionobj)
         """
+
         chunklist = []
+
         unconvert_coords = self.world.unconvert_coords
-        #get_region_path = self.world.get_region_path
         get_region = self.world.regionfiles.get
+
+        # Cached region object for consecutive iterations
         regionx = None
         regiony = None
         c = None
         mcr = None
-        for row in xrange(rowstart-16, rowend+1):
-            for col in xrange(colstart, colend+1):
-                # due to how chunks are arranged, we can only allow
-                # even row, even column or odd row, odd column
-                # otherwise, you end up with duplicates!
-                if row % 2 != col % 2:
-                    continue
 
-                chunkx, chunky = unconvert_coords(col, row)
+        rowstart = tile.row
+        rowend = rowstart+4
+        colstart = tile.col
+        colend = colstart+2
 
-                regionx_ = chunkx//32
+        # Start 16 rows up from the actual tile's row, since chunks are that tall.
-                regiony_ = chunky//32
+        # Also, every other tile doesn't exist due to how chunks are arranged. See
-                if regionx_ != regionx or regiony_ != regiony:
+        # http://docs.overviewer.org/en/latest/design/designdoc/#chunk-addressing
-                    regionx = regionx_
+        for row, col in itertools.product(
-                    regiony = regiony_
+                xrange(rowstart-16, rowend+1),
-                    _, _, c, mcr = get_region((regionx, regiony),(None,None,None,None))
+                xrange(colstart, colend+1)
+                ):
+            if row % 2 != col % 2:
+                continue
             
-                if c is not None and mcr.chunkExists(chunkx,chunky):
+            chunkx, chunky = unconvert_coords(col, row)
-                    chunklist.append((col, row, chunkx, chunky, c))
+
+            regionx_ = chunkx//32
+            regiony_ = chunky//32
+            if regionx_ != regionx or regiony_ != regiony:
+                regionx = regionx_
+                regiony = regiony_
+                _, _, fname, mcr = get_region((regionx, regiony),(None,None,None,None))
+                
+            if fname is not None and mcr.chunkExists(chunkx,chunky):
+                chunklist.append((col, row, chunkx, chunky, mcr))
                     
         return chunklist   
         
@@ -362,18 +371,8 @@ class QuadtreeGen(object):
 
         imgpath = tile.get_filepath(self.full_tiledir, self.imgformat)
 
-        # Tiles always involve 3 columns of chunks and 5 rows of tiles (end
-        # ranges are inclusive)
-        colstart = tile.col
-        colend = colstart + 2
-        rowstart = tile.row
-        rowend = rowstart + 4
-        
-        width = 384
-        height = 384
-
         # Calculate which chunks are relevant to this tile
-        chunks = self.get_chunks_in_range(colstart, colend, rowstart, rowend)
+        chunks = self.get_chunks_for_tile(tile)
 
         world = self.world
 
@@ -389,6 +388,8 @@ class QuadtreeGen(object):
             
         if not chunks:
             # No chunks were found in this tile
+            if not check_tile:
+                logging.warning("Tile %s was requested for render, but no chunks found! This may be a bug", tile)
             try:
                 os.unlink(imgpath)
             except OSError, e:
@@ -416,8 +417,7 @@ class QuadtreeGen(object):
                 needs_rerender = False
                 get_region_mtime = world.get_region_mtime
                 
-                for col, row, chunkx, chunky, regionfile in chunks:
+                for col, row, chunkx, chunky, region in chunks:
-                    region, regionMtime = get_region_mtime(regionfile)
 
                     # don't even check if it's not in the regionlist
                     if self.world.regionlist and os.path.abspath(region._filename) not in self.world.regionlist:
@@ -450,12 +450,14 @@ class QuadtreeGen(object):
         #logging.debug("writing out worldtile {0}".format(imgpath))
 
         # Compile this image
-        tileimg = Image.new("RGBA", (width, height), self.bgcolor)
+        tileimg = Image.new("RGBA", (384, 384), self.bgcolor)
 
         rendermode = self.rendermode
+        colstart = tile.col
+        rowstart = tile.row
         # col colstart will get drawn on the image starting at x coordinates -(384/2)
         # row rowstart will get drawn on the image starting at y coordinates -(192/2)
-        for col, row, chunkx, chunky, regionfile in chunks:
+        for col, row, chunkx, chunky, region in chunks:
             xpos = -192 + (col-colstart)*192
             ypos = -96 + (row-rowstart)*96
 

overviewer_core/rendernode.py

@@ -15,25 +15,16 @@
 
 import multiprocessing
 import Queue
-import itertools
-from itertools import cycle, islice
 import os
 import os.path
 import functools
-import re
-import shutil
 import collections
-import json
 import logging
-import util
-import textures
-import c_overviewer
-import cPickle
-import stat
-import errno 
 import time
-from time import gmtime, strftime, sleep
 
+from . import textures
+from . import util
+import c_overviewer
 
 """
 This module has routines related to distributing the render job to multiple nodes
@@ -85,20 +76,6 @@ def pool_initializer(rendernode):
             # only load biome data once
             break
                     
-#http://docs.python.org/library/itertools.html    
-def roundrobin(iterables):
-    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
-    # Recipe credited to George Sakkis
-    pending = len(iterables)
-    nexts = cycle(iter(it).next for it in iterables)
-    while pending:
-        try:
-            for next in nexts:
-                yield next()
-        except StopIteration:
-            pending -= 1
-            nexts = cycle(islice(nexts, pending))
-
             
 class RenderNode(object):
     def __init__(self, quadtrees, options):
@@ -298,8 +275,11 @@ class RenderNode(object):
             q.render_innertile(os.path.join(q.destdir, q.tiledir), "base")
 
     def _apply_render_worldtiles(self, pool,batch_size):
-        """Returns an iterator over result objects. Each time a new result is
+        """Adds tiles to the render queue and dispatch them to the worker pool.
-        requested, a new task is added to the pool and a result returned.
+        
+        Returns an iterator over result objects. Each time a new result is
+        requested, a new batch of tasks are added to the pool and a result
+        object is returned.
         """
         if batch_size < len(self.quadtrees):
             batch_size = len(self.quadtrees)
@@ -307,7 +287,7 @@ class RenderNode(object):
         jobcount = 0
         # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
         iterables = [q.get_worldtiles() for q in self.quadtrees]
-        for job in roundrobin(iterables):
+        for job in util.roundrobin(iterables):
             # fixup so the worker knows which quadtree this is                 
             job[0] = job[0]._render_index
             # Put this in the batch to be submited to the pool  
@@ -332,7 +312,7 @@ class RenderNode(object):
         jobcount = 0
         # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks)
         iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p]
-        for job in roundrobin(iterables):
+        for job in util.roundrobin(iterables):
             # fixup so the worker knows which quadtree this is  
             job[0] = job[0]._render_index
             # Put this in the batch to be submited to the pool  

overviewer_core/util.py

@@ -26,6 +26,7 @@ import logging
 from cStringIO import StringIO
 import ctypes
 import platform
+from itertools import cycle, islice
 
 def get_program_path():
     if hasattr(sys, "frozen") or imp.is_frozen("__main__"):
@@ -79,6 +80,20 @@ def findGitVersion():
         except Exception:
             return "unknown"
 
+# http://docs.python.org/library/itertools.html
+def roundrobin(iterables):
+    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+    # Recipe credited to George Sakkis
+    pending = len(iterables)
+    nexts = cycle(iter(it).next for it in iterables)
+    while pending:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            pending -= 1
+            nexts = cycle(islice(nexts, pending))
+
 
 # Logging related classes are below