From d2252acfe6d4a4a24a253fffb98df796dd7ec2f5 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 27 Mar 2011 11:42:11 -0400 Subject: [PATCH 01/16] Signs should be working again --- googlemap.py | 7 +++++-- overviewer.py | 4 +++- quadtree.py | 4 ++-- rendernode.py | 42 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/googlemap.py b/googlemap.py index 840d2ac..65fd8e8 100644 --- a/googlemap.py +++ b/googlemap.py @@ -127,6 +127,11 @@ class MapGen(object): self.web_assets_hook(self) return + + def finalize(self): + if self.skipjs: + return + # since we will only discover PointsOfInterest in chunks that need to be # [re]rendered, POIs like signs in unchanged chunks will not be listed # in self.world.POI. To make sure we don't remove these from markers.js @@ -153,5 +158,3 @@ class MapGen(object): output.write(' // ]},\n') output.write('];') - if self.web_assets_hook: - self.web_assets_hook(self) diff --git a/overviewer.py b/overviewer.py index b2aa275..a161f22 100755 --- a/overviewer.py +++ b/overviewer.py @@ -186,7 +186,7 @@ def main(): q.append(qtree) #create the distributed render - r = rendernode.RenderNode(q) + r = rendernode.RenderNode(q, world=w) # write out the map and web assets m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook) @@ -195,6 +195,8 @@ def main(): # render the tiles! r.go(options.procs) + m.finalize() + def delete_all(worlddir, tiledir): # TODO should we delete tiledir here too? diff --git a/quadtree.py b/quadtree.py index 77fac8b..b64333a 100644 --- a/quadtree.py +++ b/quadtree.py @@ -333,7 +333,7 @@ class QuadtreeGen(object): - def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path): + def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path, poi_queue=None): """Renders just the specified chunks into a tile and save it. Unlike usual python conventions, rowend and colend are inclusive. Additionally, the chunks around the edges are half-way cut off (so that neighboring tiles @@ -442,7 +442,7 @@ class QuadtreeGen(object): # draw the chunk! # TODO POI queue - chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None) + chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, poi_queue) # Save them tileimg.save(imgpath) diff --git a/rendernode.py b/rendernode.py index a3e12b1..43b175d 100644 --- a/rendernode.py +++ b/rendernode.py @@ -14,6 +14,7 @@ # with the Overviewer. If not, see . import multiprocessing +import Queue import itertools from itertools import cycle, islice import os @@ -76,7 +77,7 @@ def roundrobin(iterables): class RenderNode(object): - def __init__(self, quadtrees): + def __init__(self, quadtrees, world): """Distributes the rendering of a list of quadtrees.""" if not len(quadtrees) > 0: @@ -89,6 +90,13 @@ class RenderNode(object): q._render_index = i i += 1 + # queue for receiving interesting events from the renderer + # (like the discovery of signs! + self.poi_q = multiprocessing.Queue() + + self.world = world + + def print_statusline(self, complete, total, level, unconditional=False): if unconditional: pass @@ -151,6 +159,18 @@ class RenderNode(object): timestamp = timestamp2 count_to_remove = (1000//batch_size) if count_to_remove < len(results): + try: + while (1): + # an exception will break us out of this loop + item = self.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in self.world.POI: + #print "got an item from the queue!" + self.world.POI.append(item[1]) + elif item[0] == "removePOI": + self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) + except Queue.Empty: + pass while count_to_remove > 0: count_to_remove -= 1 complete += results.popleft().get() @@ -166,6 +186,18 @@ class RenderNode(object): while len(results) > 0: complete += results.popleft().get() self.print_statusline(complete, total, 1) + try: + while (1): + # an exception will break us out of this loop + item = self.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in self.world.POI: + #print "got an item from the queue!" + self.world.POI.append(item[1]) + elif item[0] == "removePOI": + self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) + except Queue.Empty: + pass self.print_statusline(complete, total, 1, True) @@ -230,10 +262,10 @@ class RenderNode(object): jobcount += 1 if jobcount >= batch_size: jobcount = 0 - yield pool.apply_async(func=render_worldtile_batch, args= [batch]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) batch = [] if jobcount > 0: - yield pool.apply_async(func=render_worldtile_batch, args= [batch]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) def _apply_render_inntertile(self, pool, zoom,batch_size): """Same as _apply_render_worltiles but for the inntertile routine. @@ -262,7 +294,7 @@ class RenderNode(object): yield pool.apply_async(func=render_innertile_batch, args= [batch]) @catch_keyboardinterrupt -def render_worldtile_batch(batch): +def render_worldtile_batch(batch, poi_queue): global child_rendernode rendernode = child_rendernode count = 0 @@ -282,7 +314,7 @@ def render_worldtile_batch(batch): tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend) #logging.debug(" tilechunks: %r", tilechunks) - quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path) + quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path, poi_queue) return count @catch_keyboardinterrupt From d5477275567bcf7d3f4b7c867ce3e3d06e2a94ef Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 28 Mar 2011 12:48:18 +0800 Subject: [PATCH 02/16] Inline render_to_image since it was just creating a ChunkRender object can calling it. Moved biome init code out of chunk.py and into rendernode.py for per-worker initialization --- chunk.py | 12 +++--------- quadtree.py | 8 ++++++-- rendernode.py | 11 ++++++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/chunk.py b/chunk.py index 99019b6..38c4e7e 100644 --- a/chunk.py +++ b/chunk.py @@ -177,8 +177,8 @@ class ChunkRenderer(object): self.queue = queue self.regionfile = worldobj.get_region_path(*chunkcoords) - if not os.path.exists(self.regionfile): - raise ValueError("Could not find regionfile: %s" % self.regionfile) + #if not os.path.exists(self.regionfile): + # raise ValueError("Could not find regionfile: %s" % self.regionfile) ## TODO TODO all of this class @@ -187,7 +187,7 @@ class ChunkRenderer(object): #chunkcoords = filename_split[1:3] #self.coords = map(world.base36decode, chunkcoords) - self.blockid = "%d.%d" % chunkcoords + #self.blockid = "%d.%d" % chunkcoords # chunk coordinates (useful to converting local block coords to # global block coords) @@ -197,12 +197,6 @@ class ChunkRenderer(object): self.world = worldobj self.rendermode = rendermode - if self.world.useBiomeData: - # make sure we've at least *tried* to load the color arrays in this process... - textures.prepareBiomeData(self.world.worlddir) - if not textures.grasscolor or not textures.foliagecolor: - raise Exception("Can't find grasscolor.png or foliagecolor.png") - def _load_level(self): """Loads and returns the level structure""" if not hasattr(self, "_level"): diff --git a/quadtree.py b/quadtree.py index b64333a..2437b33 100644 --- a/quadtree.py +++ b/quadtree.py @@ -434,6 +434,8 @@ class QuadtreeGen(object): # Compile this image tileimg = Image.new("RGBA", (width, height), (38,92,255,0)) + world = self.world + rendermode = self.rendermode # col colstart will get drawn on the image starting at x coordinates -(384/2) # row rowstart will get drawn on the image starting at y coordinates -(192/2) for col, row, chunkx, chunky, regionfile in chunks: @@ -441,8 +443,10 @@ class QuadtreeGen(object): ypos = -96 + (row-rowstart)*96 # draw the chunk! - # TODO POI queue - chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, poi_queue) + a = chunk.ChunkRenderer((chunkx, chunky), world, rendermode, poi_queue) + a.chunk_render(tileimg, xpos, ypos, None) +# chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None) + # Save them tileimg.save(imgpath) diff --git a/rendernode.py b/rendernode.py index 43b175d..006c820 100644 --- a/rendernode.py +++ b/rendernode.py @@ -60,7 +60,13 @@ def pool_initializer(rendernode): #stash the quadtree objects in a global variable after fork() for windows compat. global child_rendernode child_rendernode = rendernode - + for quadtree in rendernode.quadtrees: + if quadtree.world.useBiomeData: + # make sure we've at least *tried* to load the color arrays in this process... + textures.prepareBiomeData(quadtree.world.worlddir) + if not textures.grasscolor or not textures.foliagecolor: + raise Exception("Can't find grasscolor.png or foliagecolor.png") + #http://docs.python.org/library/itertools.html def roundrobin(iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" @@ -119,8 +125,7 @@ class RenderNode(object): # Create a pool if procs == 1: pool = FakePool() - global child_rendernode - child_rendernode = self + pool_initializer(self) else: pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's From bb07e07f26f66ee9c19b7326175f018cd0d0ff1b Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 28 Mar 2011 16:11:04 +0800 Subject: [PATCH 03/16] Cleaned up leftovers in chunk.py from before c_overviewer did rendering --- chunk.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/chunk.py b/chunk.py index 38c4e7e..47d6d02 100644 --- a/chunk.py +++ b/chunk.py @@ -406,22 +406,7 @@ class ChunkRenderer(object): For cave mode, all blocks that have any direct sunlight are not rendered, and blocks are drawn with a color tint depending on their depth.""" - blocks = self.blocks - pseudo_ancildata_blocks = set([85]) - left_blocks = self.left_blocks - right_blocks = self.right_blocks - - if cave: - # Cave mode. Actually go through and 0 out all blocks that are not in a - # cave, so that it only renders caves. - - # Places where the skylight is not 0 (there's some amount of skylight - # touching it) change it to something that won't get rendered, AND - # won't get counted as "transparent". - blocks = blocks.copy() - blocks[self.skylight != 0] = 21 - blockData = get_blockdata_array(self.level) blockData_expanded = numpy.empty((16,16,128), dtype=numpy.uint8) # Even elements get the lower 4 bits @@ -429,7 +414,6 @@ class ChunkRenderer(object): # Odd elements get the upper 4 bits blockData_expanded[:,:,1::2] = blockData >> 4 - tileEntities = get_tileentity_data(self.level) # Each block is 24x24 # The next block on the X axis adds 12px to x and subtracts 6px from y in the image @@ -443,6 +427,8 @@ class ChunkRenderer(object): c_overviewer.render_loop(self, img, xoff, yoff, blockData_expanded) + #tileEntities = get_tileentity_data(self.level) + tileEntities = [] for entity in tileEntities: if entity['id'] == 'Sign': msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']]) From ff7e9d0794d5966a94eee2bf11a30d1ebbdf0486 Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 28 Mar 2011 16:51:13 +0800 Subject: [PATCH 04/16] Pool warmup (to report worker IDs) set to aynsc when not in verbose mode --- rendernode.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rendernode.py b/rendernode.py index 006c820..98a70e1 100644 --- a/rendernode.py +++ b/rendernode.py @@ -129,7 +129,10 @@ class RenderNode(object): else: pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's - pool.map(bool,xrange(multiprocessing.cpu_count()),1) + if logging.getLogger().level >= 10: + pool.map_(bool,xrange(multiprocessing.cpu_count()),1) + else: + pool.map_async(bool,xrange(multiprocessing.cpu_count()),1) quadtrees = self.quadtrees From db7c61a0905eda835a9414d04ca2f4ef2997ef2c Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 28 Mar 2011 19:32:41 +0800 Subject: [PATCH 05/16] Fix missing import --- rendernode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rendernode.py b/rendernode.py index 98a70e1..c993a17 100644 --- a/rendernode.py +++ b/rendernode.py @@ -62,6 +62,7 @@ def pool_initializer(rendernode): child_rendernode = rendernode for quadtree in rendernode.quadtrees: if quadtree.world.useBiomeData: + import textures # make sure we've at least *tried* to load the color arrays in this process... textures.prepareBiomeData(quadtree.world.worlddir) if not textures.grasscolor or not textures.foliagecolor: From 1ff6783e35b56dec67c505fe7e5eddf6fe4fa02a Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Mon, 28 Mar 2011 19:26:48 -0400 Subject: [PATCH 06/16] Use a different Queue object the multiprocessing module is a mystery to me --- rendernode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rendernode.py b/rendernode.py index 43b175d..74d7c46 100644 --- a/rendernode.py +++ b/rendernode.py @@ -92,7 +92,8 @@ class RenderNode(object): # queue for receiving interesting events from the renderer # (like the discovery of signs! - self.poi_q = multiprocessing.Queue() + self.manager = multiprocessing.Manager() + self.poi_q = self.manager.Queue() self.world = world From 0caa033292536828c43cf0456ae167a15ac49338 Mon Sep 17 00:00:00 2001 From: Xon Date: Tue, 29 Mar 2011 07:38:49 +0800 Subject: [PATCH 07/16] Made rendernode.py POI queue world independant --- overviewer.py | 2 +- rendernode.py | 72 ++++++++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/overviewer.py b/overviewer.py index a161f22..0e012fe 100755 --- a/overviewer.py +++ b/overviewer.py @@ -186,7 +186,7 @@ def main(): q.append(qtree) #create the distributed render - r = rendernode.RenderNode(q, world=w) + r = rendernode.RenderNode(q) # write out the map and web assets m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook) diff --git a/rendernode.py b/rendernode.py index c993a17..f61a986 100644 --- a/rendernode.py +++ b/rendernode.py @@ -84,7 +84,7 @@ def roundrobin(iterables): class RenderNode(object): - def __init__(self, quadtrees, world): + def __init__(self, quadtrees): """Distributes the rendering of a list of quadtrees.""" if not len(quadtrees) > 0: @@ -92,16 +92,21 @@ class RenderNode(object): self.quadtrees = quadtrees #bind an index value to the quadtree so we can find it again + #and figure out which worlds are where i = 0 + self.worlds = [] for q in quadtrees: q._render_index = i - i += 1 + i += 1 + if q.world not in self.worlds: + self.worlds.append(q.world) + manager = multiprocessing.Manager() # queue for receiving interesting events from the renderer # (like the discovery of signs! - self.poi_q = multiprocessing.Queue() - - self.world = world + #stash into the world object like we stash an index into the quadtree + for world in self.worlds: + world.poi_q = manager.Queue() def print_statusline(self, complete, total, level, unconditional=False): @@ -168,18 +173,19 @@ class RenderNode(object): timestamp = timestamp2 count_to_remove = (1000//batch_size) if count_to_remove < len(results): - try: - while (1): - # an exception will break us out of this loop - item = self.poi_q.get(block=False) - if item[0] == "newpoi": - if item[1] not in self.world.POI: - #print "got an item from the queue!" - self.world.POI.append(item[1]) - elif item[0] == "removePOI": - self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) - except Queue.Empty: - pass + for world in self.worlds: + try: + while (1): + # an exception will break us out of this loop + item = world.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in world.POI: + #print "got an item from the queue!" + world.POI.append(item[1]) + elif item[0] == "removePOI": + world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI']) + except Queue.Empty: + pass while count_to_remove > 0: count_to_remove -= 1 complete += results.popleft().get() @@ -195,18 +201,19 @@ class RenderNode(object): while len(results) > 0: complete += results.popleft().get() self.print_statusline(complete, total, 1) - try: - while (1): - # an exception will break us out of this loop - item = self.poi_q.get(block=False) - if item[0] == "newpoi": - if item[1] not in self.world.POI: - #print "got an item from the queue!" - self.world.POI.append(item[1]) - elif item[0] == "removePOI": - self.world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], self.world.persistentData['POI']) - except Queue.Empty: - pass + for world in self.worlds: + try: + while (1): + # an exception will break us out of this loop + item = world.poi_q.get(block=False) + if item[0] == "newpoi": + if item[1] not in world.POI: + #print "got an item from the queue!" + world.POI.append(item[1]) + elif item[0] == "removePOI": + world.persistentData['POI'] = filter(lambda x: x['chunk'] != item[1], world.persistentData['POI']) + except Queue.Empty: + pass self.print_statusline(complete, total, 1, True) @@ -271,10 +278,10 @@ class RenderNode(object): jobcount += 1 if jobcount >= batch_size: jobcount = 0 - yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) batch = [] if jobcount > 0: - yield pool.apply_async(func=render_worldtile_batch, args= [batch, self.poi_q]) + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) def _apply_render_inntertile(self, pool, zoom,batch_size): """Same as _apply_render_worltiles but for the inntertile routine. @@ -303,7 +310,7 @@ class RenderNode(object): yield pool.apply_async(func=render_innertile_batch, args= [batch]) @catch_keyboardinterrupt -def render_worldtile_batch(batch, poi_queue): +def render_worldtile_batch(batch): global child_rendernode rendernode = child_rendernode count = 0 @@ -316,6 +323,7 @@ def render_worldtile_batch(batch, poi_queue): rowstart = job[3] rowend = job[4] path = job[5] + poi_queue = quadtree.world.poi_q path = quadtree.full_tiledir+os.sep+path # (even if tilechunks is empty, render_worldtile will delete # existing images if appropriate) From a1f5c9ddaf4be8e07f407c697ba225eb9c2663a7 Mon Sep 17 00:00:00 2001 From: Xon Date: Tue, 29 Mar 2011 09:13:30 +0800 Subject: [PATCH 08/16] Fixed POI code in chunk.py after cleanup --- chunk.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chunk.py b/chunk.py index 47d6d02..a59d90e 100644 --- a/chunk.py +++ b/chunk.py @@ -427,8 +427,7 @@ class ChunkRenderer(object): c_overviewer.render_loop(self, img, xoff, yoff, blockData_expanded) - #tileEntities = get_tileentity_data(self.level) - tileEntities = [] + tileEntities = get_tileentity_data(self.level) for entity in tileEntities: if entity['id'] == 'Sign': msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']]) From 2a234e065960927d16b3d0c625f7948699719223 Mon Sep 17 00:00:00 2001 From: Xon Date: Tue, 29 Mar 2011 09:15:34 +0800 Subject: [PATCH 09/16] Fix typo in pool warmup in non-verbose mode --- rendernode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rendernode.py b/rendernode.py index f61a986..d762b73 100644 --- a/rendernode.py +++ b/rendernode.py @@ -136,7 +136,7 @@ class RenderNode(object): pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) #warm up the pool so it reports all the worker id's if logging.getLogger().level >= 10: - pool.map_(bool,xrange(multiprocessing.cpu_count()),1) + pool.map(bool,xrange(multiprocessing.cpu_count()),1) else: pool.map_async(bool,xrange(multiprocessing.cpu_count()),1) From 346c42d64bf76f79d789c63c536d08beba719f23 Mon Sep 17 00:00:00 2001 From: Xon Date: Tue, 29 Mar 2011 09:56:00 +0800 Subject: [PATCH 10/16] Fixed chunk caching to correctly count empty chunk responses. Tweaked upper bound on number of chunks to cache. --- world.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/world.py b/world.py index 4f1b4cf..665619d 100644 --- a/world.py +++ b/world.py @@ -84,9 +84,10 @@ class World(object): self.regionfiles = regionfiles # set the number of region file handles we will permit open at any time before we start closing them # self.regionlimit = 1000 - # the max number of chunks we will keep before removing them - self.chunklimit = 1024*6 # this should be a multipule of the max chunks per region or things could get wonky ??? + # the max number of chunks we will keep before removing them (includes emptry chunks) + self.chunklimit = 1024 self.chunkcount = 0 + self.empty_chunk = [None,None] logging.debug("Done scanning regions") # figure out chunk format is in use @@ -116,7 +117,8 @@ class World(object): else: # some defaults self.persistentData = dict(POI=[]) - + + def get_region_path(self, chunkX, chunkY): """Returns the path to the region that contains chunk (chunkX, chunkY) """ @@ -131,16 +133,18 @@ class World(object): chunks = regioninfo[2] chunk_data = chunks.get((x,y)) if chunk_data is None: - nbt = self.load_region(filename).load_chunk(x, y) - if nbt is None: - chunks[(x,y)] = [None,None] - return None ## return none. I think this is who we should indicate missing chunks - #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) #prune the cache if required if self.chunkcount > self.chunklimit: #todo: make the emptying the chunk cache slightly less crazy [self.reload_region(regionfile) for regionfile in self.regions if regionfile <> filename] + self.chunkcount = 0 self.chunkcount += 1 + nbt = self.load_region(filename).load_chunk(x, y) + if nbt is None: + chunks[(x,y)] = self.empty_chunk + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + #we cache the transformed data, not it's raw form data = nbt.read_all() level = data[1]['Level'] From d7af3d437b65d3776344c155be465331561082a6 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 04:33:41 -0400 Subject: [PATCH 11/16] changing the C header files now correctly rebuilds whole extension --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 34f646b..62cc86e 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,8 @@ except AttributeError: c_overviewer_files = ['src/main.c', 'src/composite.c', 'src/iterate.c', 'src/endian.c'] c_overviewer_files += ['src/rendermodes.c', 'src/rendermode-normal.c', 'src/rendermode-lighting.c', 'src/rendermode-night.c', 'src/rendermode-spawn.c'] -setup_kwargs['ext_modules'].append(Extension('c_overviewer', c_overviewer_files, include_dirs=['.', numpy_include], extra_link_args=[])) +c_overviewer_includes = ['src/overviewer.h', 'src/rendermodes.h'] +setup_kwargs['ext_modules'].append(Extension('c_overviewer', c_overviewer_files, include_dirs=['.', numpy_include], depends=c_overviewer_includes, extra_link_args=[])) # tell build_ext to build the extension in-place # (NOT in build/) setup_kwargs['options']['build_ext'] = {'inplace' : 1} From 862ee62cb924fc520a936c98f2cfd04aac39fd2c Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 04:46:40 -0400 Subject: [PATCH 12/16] fixed warning during compile about endian.c --- src/endian.c | 2 +- src/overviewer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/endian.c b/src/endian.c index 5b0a105..6434239 100644 --- a/src/endian.c +++ b/src/endian.c @@ -23,7 +23,7 @@ static int endianness = UNKNOWN_ENDIAN; -void init_endian() { +void init_endian(void) { /* figure out what our endianness is! */ short word = 0x0001; char* byte = (char*)(&word); diff --git a/src/overviewer.h b/src/overviewer.h index 3475fdf..dc1c31e 100644 --- a/src/overviewer.h +++ b/src/overviewer.h @@ -79,7 +79,7 @@ PyObject *chunk_render(PyObject *self, PyObject *args); #include "rendermodes.h" /* in endian.c */ -void init_endian(); +void init_endian(void); unsigned short big_endian_ushort(unsigned short in); unsigned int big_endian_uint(unsigned int in); From 5f0766839e71cbfcef69135451ecbbd0e87716bf Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 05:13:25 -0400 Subject: [PATCH 13/16] added metadata to rendermodes, and python bindings for them --- src/main.c | 4 ++ src/rendermode-lighting.c | 1 + src/rendermode-night.c | 1 + src/rendermode-normal.c | 1 + src/rendermode-spawn.c | 1 + src/rendermodes.c | 79 +++++++++++++++++++++++++++++++++++---- src/rendermodes.h | 10 ++++- 7 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/main.c b/src/main.c index a8c6075..e2be7a3 100644 --- a/src/main.c +++ b/src/main.c @@ -22,6 +22,10 @@ static PyMethodDef COverviewerMethods[] = { "alpha over composite function"}, {"render_loop", chunk_render, METH_VARARGS, "Renders stuffs"}, + {"get_render_modes", get_render_modes, METH_VARARGS, + "returns available render modes"}, + {"get_render_mode_info", get_render_mode_info, METH_VARARGS, + "returns info for a particular render mode"}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/src/rendermode-lighting.c b/src/rendermode-lighting.c index 737a11b..628e5c4 100644 --- a/src/rendermode-lighting.c +++ b/src/rendermode-lighting.c @@ -228,6 +228,7 @@ rendermode_lighting_draw(void *data, RenderState *state, PyObject *src, PyObject } RenderModeInterface rendermode_lighting = { + "lighting", "draw shadows from the lighting data", sizeof(RenderModeLighting), rendermode_lighting_start, rendermode_lighting_finish, diff --git a/src/rendermode-night.c b/src/rendermode-night.c index d4b1eb4..1e54f05 100644 --- a/src/rendermode-night.c +++ b/src/rendermode-night.c @@ -60,6 +60,7 @@ rendermode_night_draw(void *data, RenderState *state, PyObject *src, PyObject *m } RenderModeInterface rendermode_night = { + "night", "like \"lighting\", except at night", sizeof(RenderModeNight), rendermode_night_start, rendermode_night_finish, diff --git a/src/rendermode-normal.c b/src/rendermode-normal.c index d7a1367..e38118d 100644 --- a/src/rendermode-normal.c +++ b/src/rendermode-normal.c @@ -164,6 +164,7 @@ rendermode_normal_draw(void *data, RenderState *state, PyObject *src, PyObject * } RenderModeInterface rendermode_normal = { + "normal", "nothing special, just render the blocks", sizeof(RenderModeNormal), rendermode_normal_start, rendermode_normal_finish, diff --git a/src/rendermode-spawn.c b/src/rendermode-spawn.c index 1a7c96a..b19f409 100644 --- a/src/rendermode-spawn.c +++ b/src/rendermode-spawn.c @@ -118,6 +118,7 @@ rendermode_spawn_draw(void *data, RenderState *state, PyObject *src, PyObject *m } RenderModeInterface rendermode_spawn = { + "spawn", "draws red where monsters can spawn at night", sizeof(RenderModeSpawn), rendermode_spawn_start, rendermode_spawn_finish, diff --git a/src/rendermodes.c b/src/rendermodes.c index f4fbe89..0ef5550 100644 --- a/src/rendermodes.c +++ b/src/rendermodes.c @@ -18,21 +18,84 @@ #include "overviewer.h" #include +/* list of all render modes, ending in NULL + all of these will be available to the user, so DON'T include modes + that are only useful as a base for other modes. */ +static RenderModeInterface *render_modes[] = { + &rendermode_normal, + &rendermode_lighting, + &rendermode_night, + &rendermode_spawn, + NULL +}; + /* decides which render mode to use */ RenderModeInterface *get_render_mode(RenderState *state) { - /* default: normal */ - RenderModeInterface *iface = &rendermode_normal; + unsigned int i; + /* default: NULL --> an error */ + RenderModeInterface *iface = NULL; PyObject *rendermode_py = PyObject_GetAttrString(state->self, "rendermode"); const char *rendermode = PyString_AsString(rendermode_py); - if (strcmp(rendermode, "lighting") == 0) { - iface = &rendermode_lighting; - } else if (strcmp(rendermode, "night") == 0) { - iface = &rendermode_night; - } else if (strcmp(rendermode, "spawn") == 0) { - iface = &rendermode_spawn; + for (i = 0; render_modes[i] != NULL; i++) { + if (strcmp(render_modes[i]->name, rendermode) == 0) { + iface = render_modes[i]; + break; + } } Py_DECREF(rendermode_py); return iface; } + +/* bindings for python -- get all the rendermode names */ +PyObject *get_render_modes(PyObject *self, PyObject *args) { + PyObject *modes; + unsigned int i; + if (!PyArg_ParseTuple(args, "")) + return NULL; + + modes = PyList_New(0); + if (modes == NULL) + return NULL; + + for (i = 0; render_modes[i] != NULL; i++) { + PyObject *name = PyString_FromString(render_modes[i]->name); + PyList_Append(modes, name); + Py_DECREF(name); + } + + return modes; +} + +/* more bindings -- return info for a given rendermode name */ +PyObject *get_render_mode_info(PyObject *self, PyObject *args) { + const char* rendermode; + PyObject *info; + unsigned int i; + if (!PyArg_ParseTuple(args, "s", &rendermode)) + return NULL; + + info = PyDict_New(); + if (info == NULL) + return NULL; + + for (i = 0; render_modes[i] != NULL; i++) { + if (strcmp(render_modes[i]->name, rendermode) == 0) { + PyObject *tmp; + + tmp = PyString_FromString(render_modes[i]->name); + PyDict_SetItemString(info, "name", tmp); + Py_DECREF(tmp); + + tmp = PyString_FromString(render_modes[i]->description); + PyDict_SetItemString(info, "description", tmp); + Py_DECREF(tmp); + + return info; + } + } + + Py_DECREF(info); + Py_RETURN_NONE; +} diff --git a/src/rendermodes.h b/src/rendermodes.h index d26091d..3027a7f 100644 --- a/src/rendermodes.h +++ b/src/rendermodes.h @@ -29,7 +29,7 @@ * (see rendermode-night.c for a simple example derived from * the "lighting" mode) * - * * add a condition to get_render_mode() in rendermodes.c + * * add your mode to the list in rendermodes.c */ #ifndef __RENDERMODES_H_INCLUDED__ @@ -39,6 +39,11 @@ /* rendermode interface */ typedef struct { + /* the name of this mode */ + const char* name; + /* the short description of this render mode */ + const char* description; + /* the size of the local storage for this rendermode */ unsigned int data_size; @@ -53,6 +58,9 @@ typedef struct { /* figures out the render mode to use from the given ChunkRenderer */ RenderModeInterface *get_render_mode(RenderState *state); +/* python bindings */ +PyObject *get_render_modes(PyObject *self, PyObject *args); +PyObject *get_render_mode_info(PyObject *self, PyObject *args); /* individual rendermode interface declarations follow */ From 88d4ebe5ae1c949b44a8fbb18e41b56ab17a3be8 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 05:24:25 -0400 Subject: [PATCH 14/16] added --list-rendermodes option --- overviewer.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/overviewer.py b/overviewer.py index 0e012fe..b16f835 100755 --- a/overviewer.py +++ b/overviewer.py @@ -57,14 +57,17 @@ def main(): cpus = multiprocessing.cpu_count() except NotImplementedError: cpus = 1 - + + avail_rendermodes = c_overviewer.get_render_modes() + parser = ConfigOptionParser(usage=helptext, config="settings.py") parser.add_option("-V", "--version", dest="version", help="Displays version information and then exits", action="store_true") parser.add_option("-p", "--processes", dest="procs", help="How many worker processes to start. Default %s" % cpus, default=cpus, action="store", type="int") parser.add_option("-z", "--zoom", dest="zoom", help="Sets the zoom level manually instead of calculating it. This can be useful if you have outlier chunks that make your world too big. This value will make the highest zoom level contain (2**ZOOM)^2 tiles", action="store", type="int", configFileOnly=True) parser.add_option("-d", "--delete", dest="delete", help="Clear all caches. Next time you render your world, it will have to start completely over again. This is probably not a good idea for large worlds. Use this if you change texture packs and want to re-render everything.", action="store_true", commandLineOnly=True) parser.add_option("--chunklist", dest="chunklist", help="A file containing, on each line, a path to a chunkfile to update. Instead of scanning the world directory for chunks, it will just use this list. Normal caching rules still apply.") - parser.add_option("--rendermodes", dest="rendermode", help="Specifies the render type: normal (default), lighting, night, or spawn.", type="choice", choices=["normal", "lighting", "night", "spawn"], required=True, default="normal", listify=True) + parser.add_option("--rendermodes", dest="rendermode", help="Specifies the render types, separated by commas. Use --list-rendermodes to list them all.", type="choice", choices=avail_rendermodes, required=True, default=avail_rendermodes[0], listify=True) + parser.add_option("--list-rendermodes", dest="list_rendermodes", action="store_true", help="List available render modes and exit.", commandLineOnly=True) parser.add_option("--imgformat", dest="imgformat", help="The image output format to use. Currently supported: png(default), jpg. NOTE: png will always be used as the intermediate image format.", configFileOnly=True ) parser.add_option("--optimize-img", dest="optimizeimg", help="If using png, perform image file size optimizations on the output. Specify 1 for pngcrush, 2 for pngcrush+optipng+advdef. This may double (or more) render times, but will produce up to 30% smaller images. NOTE: requires corresponding programs in $PATH or %PATH%", configFileOnly=True) parser.add_option("--web-assets-hook", dest="web_assets_hook", help="If provided, run this function after the web assets have been copied, but before actual tile rendering begins. It should accept a QuadtreeGen object as its only argument.", action="store", metavar="SCRIPT", type="function", configFileOnly=True) @@ -88,6 +91,13 @@ def main(): except: pass sys.exit(0) + + if options.list_rendermodes: + rendermode_info = map(c_overviewer.get_render_mode_info, avail_rendermodes) + name_width = max(map(lambda i: len(i['name']), rendermode_info)) + for info in rendermode_info: + print "{name:{0}} {description}".format(name_width, **info) + sys.exit(0) if len(args) < 1: print "You need to give me your world number or directory" From 9b36d21c37b263252957b81a1bde10fdd29a523f Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 05:29:37 -0400 Subject: [PATCH 15/16] moved non-config, global variables out of config.js --- config.js | 7 ------- web_assets/functions.js | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config.js b/config.js index 9296853..6be0893 100644 --- a/config.js +++ b/config.js @@ -50,10 +50,3 @@ var mapTypeData=[ var mapTypeData = {maptypedata}; -// Please leave the following variables here: -var markerCollection = {}; // holds groups of markers - -var map; - -var markersInit = false; -var regionsInit = false; diff --git a/web_assets/functions.js b/web_assets/functions.js index 174a43a..3487d5b 100644 --- a/web_assets/functions.js +++ b/web_assets/functions.js @@ -1,3 +1,10 @@ +var markerCollection = {}; // holds groups of markers + +var map; + +var markersInit = false; +var regionsInit = false; + var prevInfoWindow = null; function prepareSignMarker(marker, item) { From 9c25c6259cc07b3a09e657e479e729c6511573e4 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Tue, 29 Mar 2011 05:48:40 -0400 Subject: [PATCH 16/16] added center to config.js, lets you set map center in world coords By default, it is set to spawn. Also I changed defaultZoom to 2, which looks better (the world used to show up as *tiny* by default). --- config.js | 5 ++++- googlemap.py | 3 +++ web_assets/functions.js | 8 ++++++-- world.py | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/config.js b/config.js index 6be0893..78dfd7d 100644 --- a/config.js +++ b/config.js @@ -2,8 +2,11 @@ var config = { fileExt: '{imgformat}', tileSize: 384, - defaultZoom: 1, + defaultZoom: 2, maxZoom: {maxzoom}, + // center on this point, in world coordinates, ex: + //center: [0,0,0], + center: {spawn_coords}, cacheMinutes: 0, // Change this to have browsers automatically request new images every x minutes bg_color: '#1A1A1A', debug: false diff --git a/googlemap.py b/googlemap.py index 65fd8e8..2321049 100644 --- a/googlemap.py +++ b/googlemap.py @@ -94,6 +94,9 @@ class MapGen(object): config = config.replace( "{imgformat}", str(imgformat)) + config = config.replace("{spawn_coords}", + json.dumps(list(self.world.spawn))) + # create generated map type data, from given quadtrees maptypedata = map(lambda q: {'label' : q.rendermode.capitalize(), 'path' : q.tiledir}, self.quadtrees) diff --git a/web_assets/functions.js b/web_assets/functions.js index 3487d5b..0ec7eae 100644 --- a/web_assets/functions.js +++ b/web_assets/functions.js @@ -256,8 +256,12 @@ function initialize() { var query = location.search.substring(1); - var lat = 0.5; - var lng = 0.5; + var defaultCenter = fromWorldToLatLng(config.center[0], + config.center[1], + config.center[2]); + var lat = defaultCenter.lat(); + var lng = defaultCenter.lng(); + var zoom = config.defaultZoom; var pairs = query.split("&"); for (var i=0; i