diff --git a/chunk.py b/chunk.py index 99019b6..a59d90e 100644 --- a/chunk.py +++ b/chunk.py @@ -177,8 +177,8 @@ class ChunkRenderer(object): self.queue = queue self.regionfile = worldobj.get_region_path(*chunkcoords) - if not os.path.exists(self.regionfile): - raise ValueError("Could not find regionfile: %s" % self.regionfile) + #if not os.path.exists(self.regionfile): + # raise ValueError("Could not find regionfile: %s" % self.regionfile) ## TODO TODO all of this class @@ -187,7 +187,7 @@ class ChunkRenderer(object): #chunkcoords = filename_split[1:3] #self.coords = map(world.base36decode, chunkcoords) - self.blockid = "%d.%d" % chunkcoords + #self.blockid = "%d.%d" % chunkcoords # chunk coordinates (useful to converting local block coords to # global block coords) @@ -197,12 +197,6 @@ class ChunkRenderer(object): self.world = worldobj self.rendermode = rendermode - if self.world.useBiomeData: - # make sure we've at least *tried* to load the color arrays in this process... - textures.prepareBiomeData(self.world.worlddir) - if not textures.grasscolor or not textures.foliagecolor: - raise Exception("Can't find grasscolor.png or foliagecolor.png") - def _load_level(self): """Loads and returns the level structure""" if not hasattr(self, "_level"): @@ -412,22 +406,7 @@ class ChunkRenderer(object): For cave mode, all blocks that have any direct sunlight are not rendered, and blocks are drawn with a color tint depending on their depth.""" - blocks = self.blocks - pseudo_ancildata_blocks = set([85]) - left_blocks = self.left_blocks - right_blocks = self.right_blocks - - if cave: - # Cave mode. Actually go through and 0 out all blocks that are not in a - # cave, so that it only renders caves. - - # Places where the skylight is not 0 (there's some amount of skylight - # touching it) change it to something that won't get rendered, AND - # won't get counted as "transparent". - blocks = blocks.copy() - blocks[self.skylight != 0] = 21 - blockData = get_blockdata_array(self.level) blockData_expanded = numpy.empty((16,16,128), dtype=numpy.uint8) # Even elements get the lower 4 bits @@ -435,7 +414,6 @@ class ChunkRenderer(object): # Odd elements get the upper 4 bits blockData_expanded[:,:,1::2] = blockData >> 4 - tileEntities = get_tileentity_data(self.level) # Each block is 24x24 # The next block on the X axis adds 12px to x and subtracts 6px from y in the image @@ -449,6 +427,7 @@ class ChunkRenderer(object): c_overviewer.render_loop(self, img, xoff, yoff, blockData_expanded) + tileEntities = get_tileentity_data(self.level) for entity in tileEntities: if entity['id'] == 'Sign': msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']]) diff --git a/overviewer.py b/overviewer.py index a161f22..0e012fe 100755 --- a/overviewer.py +++ b/overviewer.py @@ -186,7 +186,7 @@ def main(): q.append(qtree) #create the distributed render - r = rendernode.RenderNode(q, world=w) + r = rendernode.RenderNode(q) # write out the map and web assets m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook) diff --git a/quadtree.py b/quadtree.py index b64333a..2437b33 100644 --- a/quadtree.py +++ b/quadtree.py @@ -434,6 +434,8 @@ class QuadtreeGen(object): # Compile this image tileimg = Image.new("RGBA", (width, height), (38,92,255,0)) + world = self.world + rendermode = self.rendermode # col colstart will get drawn on the image starting at x coordinates -(384/2) # row rowstart will get drawn on the image starting at y coordinates -(192/2) for col, row, chunkx, chunky, regionfile in chunks: @@ -441,8 +443,10 @@ class QuadtreeGen(object): ypos = -96 + (row-rowstart)*96 # draw the chunk! - # TODO POI queue - chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, poi_queue) + a = chunk.ChunkRenderer((chunkx, chunky), world, rendermode, poi_queue) + a.chunk_render(tileimg, xpos, ypos, None) +# chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None) + # Save them tileimg.save(imgpath) diff --git a/rendernode.py b/rendernode.py index 74d7c46..d762b73 100644 --- a/rendernode.py +++ b/rendernode.py @@ -60,7 +60,14 @@ def pool_initializer(rendernode): #stash the quadtree objects in a global variable after fork() for windows compat. global child_rendernode child_rendernode = rendernode - + for quadtree in rendernode.quadtrees: + if quadtree.world.useBiomeData: + import textures + # make sure we've at least *tried* to load the color arrays in this process... + textures.prepareBiomeData(quadtree.world.worlddir) + if not textures.grasscolor or not textures.foliagecolor: + raise Exception("Can't find grasscolor.png or foliagecolor.png") + #http://docs.python.org/library/itertools.html def roundrobin(iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" @@ -77,7 +84,7 @@ def roundrobin(iterables): class RenderNode(object): - def __init__(self, quadtrees, world): + def __init__(self, quadtrees): """Distributes the rendering of a list of quadtrees.""" if not len(quadtrees) > 0: @@ -85,17 +92,21 @@ class RenderNode(object): self.quadtrees = quadtrees #bind an index value to the quadtree so we can find it again + #and figure out which worlds are where i = 0 + self.worlds = [] for q in quadtrees: q._render_index = i - i += 1 + i += 1 + if q.world not in self.worlds: + self.worlds.append(q.world) + manager = multiprocessing.Manager() # queue for receiving interesting events from the renderer # (like the discovery of signs! - self.manager = multiprocessing.Manager() - self.poi_q = self.manager.Queue() - - self.world = world + #stash into the world object like we stash an index into the quadtree + for world in self.worlds: + world.poi_q = manager.Queue() def print_statusline(self, complete, total, level, unconditional=False): @@ -120,12 +131,14 @@ class RenderNode(object): # Create a pool if procs == 1: pool = FakePool() - global child_rendernode - child_rendernode = self + pool_initializer(self) else: pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's - pool.map(bool,xrange(multiprocessing.cpu_count()),1) + if logging.getLogger().level >= 10: + pool.map(bool,xrange(multiprocessing.cpu_count()),1) + else: + pool.map_async(bool,xrange(multiprocessing.cpu_count()),1) quadtrees = self.quadtrees @@ -160,18 +173,19 @@ class RenderNode(object): timestamp = timestamp2 count_to_remove = (1000//batch_size) if count_to_remove < len(results): - try: - while (1): - # an exception will break us out of this loop - item = self.poi_q.get(block=False) - if item[0] == "newpoi": - if item[1] not in self.world.POI: - #print "got an item from the queue!" - self.world.POI.append(item[1]) - elif item[0] == "removePOI": - self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) - except Queue.Empty: - pass + for world in self.worlds: + try: + while (1): + # an exception will break us out of this loop + item = world.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in world.POI: + #print "got an item from the queue!" + world.POI.append(item[1]) + elif item[0] == "removePOI": + world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI']) + except Queue.Empty: + pass while count_to_remove > 0: count_to_remove -= 1 complete += results.popleft().get() @@ -187,18 +201,19 @@ class RenderNode(object): while len(results) > 0: complete += results.popleft().get() self.print_statusline(complete, total, 1) - try: - while (1): - # an exception will break us out of this loop - item = self.poi_q.get(block=False) - if item[0] == "newpoi": - if item[1] not in self.world.POI: - #print "got an item from the queue!" - self.world.POI.append(item[1]) - elif item[0] == "removePOI": - self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) - except Queue.Empty: - pass + for world in self.worlds: + try: + while (1): + # an exception will break us out of this loop + item = world.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in world.POI: + #print "got an item from the queue!" + world.POI.append(item[1]) + elif item[0] == "removePOI": + world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI']) + except Queue.Empty: + pass self.print_statusline(complete, total, 1, True) @@ -263,10 +278,10 @@ class RenderNode(object): jobcount += 1 if jobcount >= batch_size: jobcount = 0 - yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) batch = [] if jobcount > 0: - yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) def _apply_render_inntertile(self, pool, zoom,batch_size): """Same as _apply_render_worltiles but for the inntertile routine. @@ -295,7 +310,7 @@ class RenderNode(object): yield pool.apply_async(func=render_innertile_batch, args= [batch]) @catch_keyboardinterrupt -def render_worldtile_batch(batch, poi_queue): +def render_worldtile_batch(batch): global child_rendernode rendernode = child_rendernode count = 0 @@ -308,6 +323,7 @@ def render_worldtile_batch(batch, poi_queue): rowstart = job[3] rowend = job[4] path = job[5] + poi_queue = quadtree.world.poi_q path = quadtree.full_tiledir+os.sep+path # (even if tilechunks is empty, render_worldtile will delete # existing images if appropriate) diff --git a/world.py b/world.py index 4f1b4cf..665619d 100644 --- a/world.py +++ b/world.py @@ -84,9 +84,10 @@ class World(object): self.regionfiles = regionfiles # set the number of region file handles we will permit open at any time before we start closing them # self.regionlimit = 1000 - # the max number of chunks we will keep before removing them - self.chunklimit = 1024*6 # this should be a multipule of the max chunks per region or things could get wonky ??? + # the max number of chunks we will keep before removing them (includes emptry chunks) + self.chunklimit = 1024 self.chunkcount = 0 + self.empty_chunk = [None,None] logging.debug("Done scanning regions") # figure out chunk format is in use @@ -116,7 +117,8 @@ class World(object): else: # some defaults self.persistentData = dict(POI=[]) - + + def get_region_path(self, chunkX, chunkY): """Returns the path to the region that contains chunk (chunkX, chunkY) """ @@ -131,16 +133,18 @@ class World(object): chunks = regioninfo[2] chunk_data = chunks.get((x,y)) if chunk_data is None: - nbt = self.load_region(filename).load_chunk(x, y) - if nbt is None: - chunks[(x,y)] = [None,None] - return None ## return none. I think this is who we should indicate missing chunks - #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) #prune the cache if required if self.chunkcount > self.chunklimit: #todo: make the emptying the chunk cache slightly less crazy [self.reload_region(regionfile) for regionfile in self.regions if regionfile <> filename] + self.chunkcount = 0 self.chunkcount += 1 + nbt = self.load_region(filename).load_chunk(x, y) + if nbt is None: + chunks[(x,y)] = self.empty_chunk + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + #we cache the transformed data, not it's raw form data = nbt.read_all() level = data[1]['Level']