diff --git a/overviewer_core/quadtree.py b/overviewer_core/quadtree.py index 903ad5f..cca4a4e 100644 --- a/overviewer_core/quadtree.py +++ b/overviewer_core/quadtree.py @@ -23,7 +23,6 @@ import shutil import collections import json import logging -import util import cPickle import stat import errno @@ -33,10 +32,10 @@ from time import gmtime, strftime, sleep from PIL import Image -import nbt -import chunk +from . import nbt +from . import chunk +from .optimizeimages import optimize_image from c_overviewer import get_render_mode_inheritance -from optimizeimages import optimize_image import composite @@ -216,40 +215,50 @@ class QuadtreeGen(object): self._decrease_depth() - def get_chunks_in_range(self, colstart, colend, rowstart, rowend): - """Get chunks that are relevant to the tile rendering function that's - rendering that range + def get_chunks_for_tile(self, tile): + """Get chunks that are relevant to the given tile Returns a list of chunks where each item is - (col, row, chunkx, chunky, regionfilename) + (col, row, chunkx, chunky, regionobj) """ + chunklist = [] + unconvert_coords = self.world.unconvert_coords - #get_region_path = self.world.get_region_path get_region = self.world.regionfiles.get + + # Cached region object for consecutive iterations regionx = None regiony = None c = None mcr = None - for row in xrange(rowstart-16, rowend+1): - for col in xrange(colstart, colend+1): - # due to how chunks are arranged, we can only allow - # even row, even column or odd row, odd column - # otherwise, you end up with duplicates! - if row % 2 != col % 2: - continue - - chunkx, chunky = unconvert_coords(col, row) - regionx_ = chunkx//32 - regiony_ = chunky//32 - if regionx_ != regionx or regiony_ != regiony: - regionx = regionx_ - regiony = regiony_ - _, _, c, mcr = get_region((regionx, regiony),(None,None,None,None)) - - if c is not None and mcr.chunkExists(chunkx,chunky): - chunklist.append((col, row, chunkx, chunky, c)) + rowstart = tile.row + rowend = rowstart+4 + colstart = tile.col + colend = colstart+2 + + # Start 16 rows up from the actual tile's row, since chunks are that tall. + # Also, every other tile doesn't exist due to how chunks are arranged. See + # http://docs.overviewer.org/en/latest/design/designdoc/#chunk-addressing + for row, col in itertools.product( + xrange(rowstart-16, rowend+1), + xrange(colstart, colend+1) + ): + if row % 2 != col % 2: + continue + + chunkx, chunky = unconvert_coords(col, row) + + regionx_ = chunkx//32 + regiony_ = chunky//32 + if regionx_ != regionx or regiony_ != regiony: + regionx = regionx_ + regiony = regiony_ + _, _, fname, mcr = get_region((regionx, regiony),(None,None,None,None)) + + if fname is not None and mcr.chunkExists(chunkx,chunky): + chunklist.append((col, row, chunkx, chunky, mcr)) return chunklist @@ -362,18 +371,8 @@ class QuadtreeGen(object): imgpath = tile.get_filepath(self.full_tiledir, self.imgformat) - # Tiles always involve 3 columns of chunks and 5 rows of tiles (end - # ranges are inclusive) - colstart = tile.col - colend = colstart + 2 - rowstart = tile.row - rowend = rowstart + 4 - - width = 384 - height = 384 - # Calculate which chunks are relevant to this tile - chunks = self.get_chunks_in_range(colstart, colend, rowstart, rowend) + chunks = self.get_chunks_for_tile(tile) world = self.world @@ -389,6 +388,8 @@ class QuadtreeGen(object): if not chunks: # No chunks were found in this tile + if not check_tile: + logging.warning("Tile %s was requested for render, but no chunks found! This may be a bug", tile) try: os.unlink(imgpath) except OSError, e: @@ -416,8 +417,7 @@ class QuadtreeGen(object): needs_rerender = False get_region_mtime = world.get_region_mtime - for col, row, chunkx, chunky, regionfile in chunks: - region, regionMtime = get_region_mtime(regionfile) + for col, row, chunkx, chunky, region in chunks: # don't even check if it's not in the regionlist if self.world.regionlist and os.path.abspath(region._filename) not in self.world.regionlist: @@ -450,12 +450,14 @@ class QuadtreeGen(object): #logging.debug("writing out worldtile {0}".format(imgpath)) # Compile this image - tileimg = Image.new("RGBA", (width, height), self.bgcolor) + tileimg = Image.new("RGBA", (384, 384), self.bgcolor) rendermode = self.rendermode + colstart = tile.col + rowstart = tile.row # col colstart will get drawn on the image starting at x coordinates -(384/2) # row rowstart will get drawn on the image starting at y coordinates -(192/2) - for col, row, chunkx, chunky, regionfile in chunks: + for col, row, chunkx, chunky, region in chunks: xpos = -192 + (col-colstart)*192 ypos = -96 + (row-rowstart)*96 diff --git a/overviewer_core/rendernode.py b/overviewer_core/rendernode.py index c45214b..c0c24b0 100644 --- a/overviewer_core/rendernode.py +++ b/overviewer_core/rendernode.py @@ -15,25 +15,16 @@ import multiprocessing import Queue -import itertools -from itertools import cycle, islice import os import os.path import functools -import re -import shutil import collections -import json import logging -import util -import textures -import c_overviewer -import cPickle -import stat -import errno import time -from time import gmtime, strftime, sleep +from . import textures +from . import util +import c_overviewer """ This module has routines related to distributing the render job to multiple nodes @@ -85,20 +76,6 @@ def pool_initializer(rendernode): # only load biome data once break -#http://docs.python.org/library/itertools.html -def roundrobin(iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - # Recipe credited to George Sakkis - pending = len(iterables) - nexts = cycle(iter(it).next for it in iterables) - while pending: - try: - for next in nexts: - yield next() - except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) - class RenderNode(object): def __init__(self, quadtrees, options): @@ -298,8 +275,11 @@ class RenderNode(object): q.render_innertile(os.path.join(q.destdir, q.tiledir), "base") def _apply_render_worldtiles(self, pool,batch_size): - """Returns an iterator over result objects. Each time a new result is - requested, a new task is added to the pool and a result returned. + """Adds tiles to the render queue and dispatch them to the worker pool. + + Returns an iterator over result objects. Each time a new result is + requested, a new batch of tasks are added to the pool and a result + object is returned. """ if batch_size < len(self.quadtrees): batch_size = len(self.quadtrees) @@ -307,7 +287,7 @@ class RenderNode(object): jobcount = 0 # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks) iterables = [q.get_worldtiles() for q in self.quadtrees] - for job in roundrobin(iterables): + for job in util.roundrobin(iterables): # fixup so the worker knows which quadtree this is job[0] = job[0]._render_index # Put this in the batch to be submited to the pool @@ -332,7 +312,7 @@ class RenderNode(object): jobcount = 0 # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks) iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p] - for job in roundrobin(iterables): + for job in util.roundrobin(iterables): # fixup so the worker knows which quadtree this is job[0] = job[0]._render_index # Put this in the batch to be submited to the pool diff --git a/overviewer_core/util.py b/overviewer_core/util.py index 8a2947b..56fc257 100644 --- a/overviewer_core/util.py +++ b/overviewer_core/util.py @@ -26,6 +26,7 @@ import logging from cStringIO import StringIO import ctypes import platform +from itertools import cycle, islice def get_program_path(): if hasattr(sys, "frozen") or imp.is_frozen("__main__"): @@ -79,6 +80,20 @@ def findGitVersion(): except Exception: return "unknown" +# http://docs.python.org/library/itertools.html +def roundrobin(iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + # Recipe credited to George Sakkis + pending = len(iterables) + nexts = cycle(iter(it).next for it in iterables) + while pending: + try: + for next in nexts: + yield next() + except StopIteration: + pending -= 1 + nexts = cycle(islice(nexts, pending)) + # Logging related classes are below