From 059492b3a19e0745cf875fb9df873f2808808acd Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 21 Mar 2011 06:54:15 +0800 Subject: [PATCH 01/11] Drain the processing queue every second by ~1000 to give more consistant feedback and reduce stalls when draining the queue from 10000->500 --- quadtree.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/quadtree.py b/quadtree.py index 5e2ad13..6f9091b 100644 --- a/quadtree.py +++ b/quadtree.py @@ -27,6 +27,7 @@ import util import cPickle import stat import errno +import time from time import gmtime, strftime, sleep from PIL import Image @@ -300,13 +301,24 @@ class QuadtreeGen(object): logging.info("Don't worry, each level has only 25% as many tiles as the last.") logging.info("The others will go faster") count = 0 - batch_size = 10 + batch_size = 8 + timestamp = time.time() for result in self._apply_render_worldtiles(pool,batch_size): - results.append(result) - if len(results) > (10000/batch_size): + results.append(result) + # every second drain some of the queue + timestamp2 = time.time() + if timestamp2 >= timestamp + 1: + timestamp = timestamp2 + count_to_remove = (1000//batch_size) + if count_to_remove < len(results): + while count_to_remove > 0: + count_to_remove -= 1 + complete += results.popleft().get() + self.print_statusline(complete, total, 1) + if len(results) > (10000//batch_size): # Empty the queue before adding any more, so that memory # required has an upper bound - while len(results) > (500/batch_size): + while len(results) > (500//batch_size): complete += results.popleft().get() self.print_statusline(complete, total, 1) @@ -324,8 +336,19 @@ class QuadtreeGen(object): complete = 0 total = 4**zoom logging.info("Starting level {0}".format(level)) + timestamp = time.time() for result in self._apply_render_inntertile(pool, zoom,batch_size): results.append(result) + # every second drain some of the queue + timestamp2 = time.time() + if timestamp2 >= timestamp + 1: + timestamp = timestamp2 + count_to_remove = (1000//batch_size) + if count_to_remove < len(results): + while count_to_remove > 0: + count_to_remove -= 1 + complete += results.popleft().get() + self.print_statusline(complete, total, 1) if len(results) > (10000/batch_size): while len(results) > (500/batch_size): complete += results.popleft().get() From c1b7b12592b6af957c4fb6bafb6c8ba0bec5eb0b Mon Sep 17 00:00:00 2001 From: Xon Date: Mon, 21 Mar 2011 08:03:18 +0800 Subject: [PATCH 02/11] Add reporting of scanning/indexing regions. Shortened paths being sent to the worker processes, and removed os.path.join from _apply_render_worldtiles's inner loop. --- quadtree.py | 35 ++++++++++++++++++++++------------- world.py | 4 +++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/quadtree.py b/quadtree.py index 6f9091b..ce2dab1 100644 --- a/quadtree.py +++ b/quadtree.py @@ -90,8 +90,8 @@ class QuadtreeGen(object): # Make the destination dir if not os.path.exists(destdir): os.mkdir(destdir) - self.tiledir = tiledir - + self.tiledir = tiledir + if depth is None: # Determine quadtree depth (midpoint is always 0,0) for p in xrange(15): @@ -123,6 +123,9 @@ class QuadtreeGen(object): self.world = worldobj self.destdir = destdir + self.full_tiledir = os.path.join(destdir, tiledir) + + def print_statusline(self, complete, total, level, unconditional=False): if unconditional: @@ -227,12 +230,13 @@ class QuadtreeGen(object): colend = colstart + 2 rowend = rowstart + 4 - # This image is rendered at: - dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path)) + # This image is rendered at(relative to the worker's destdir): + tilepath = [str(x) for x in path] + tilepath = os.sep.join(tilepath) #logging.debug("this is rendered at %s", dest) # Put this in the batch to be submited to the pool - batch.append((colstart, colend, rowstart, rowend, dest)) + batch.append((colstart, colend, rowstart, rowend, tilepath)) tiles += 1 if tiles >= batch_size: tiles = 0 @@ -251,11 +255,14 @@ class QuadtreeGen(object): batch = [] tiles = 0 for path in iterate_base4(zoom): - # This image is rendered at: - dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1])) + # This image is rendered at(relative to the worker's destdir): + tilepath = [str(x) for x in path[:-1]] + tilepath = os.sep.join(tilepath) name = str(path[-1]) + - batch.append((dest, name, self.imgformat, self.optimizeimg)) + self.full_tiledir + batch.append((tilepath, name, self.imgformat, self.optimizeimg)) tiles += 1 if tiles >= batch_size: tiles = 0 @@ -410,11 +417,14 @@ class QuadtreeGen(object): @catch_keyboardinterrupt def render_innertile_batch(batch): + global child_quadtree + quadtree = child_quadtree count = 0 #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 - render_innertile(job[0],job[1],job[2],job[3]) + dest = quadtree.full_tiledir+os.sep+job[0] + render_innertile(dest,job[1],job[2],job[3]) return count def render_innertile(dest, name, imgformat, optimizeimg): @@ -481,11 +491,9 @@ def render_innertile(dest, name, imgformat, optimizeimg): optimize_image(imgpath, imgformat, optimizeimg) @catch_keyboardinterrupt -def render_worldtile_batch(batch): +def render_worldtile_batch(batch): global child_quadtree - return render_worldtile_batch_(child_quadtree, batch) - -def render_worldtile_batch_(quadtree, batch): + quadtree = child_quadtree count = 0 _get_chunks_in_range = quadtree._get_chunks_in_range #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) @@ -496,6 +504,7 @@ def render_worldtile_batch_(quadtree, batch): rowstart = job[2] rowend = job[3] path = job[4] + path = quadtree.full_tiledir+os.sep+path # (even if tilechunks is empty, render_worldtile will delete # existing images if appropriate) # And uses these chunks diff --git a/world.py b/world.py index ce869cb..bd4d9a6 100644 --- a/world.py +++ b/world.py @@ -70,9 +70,10 @@ class World(object): def __init__(self, worlddir, useBiomeData=False,regionlist=None): self.worlddir = worlddir self.useBiomeData = useBiomeData - + #find region files, or load the region list #this also caches all the region file header info + logging.info("Scanning regions") regionfiles = {} regions = {} for x, y, regionfile in self._iterate_regionfiles(): @@ -82,6 +83,7 @@ class World(object): regionfiles[(x,y)] = (x,y,regionfile) self.regionfiles = regionfiles self.regions = regions + logging.debug("Done scanning regions") # figure out chunk format is in use # if not mcregion, error out early From dbdd5d0fc87356a42a486128d54fd3900ee9e2cf Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 16:44:27 +0800 Subject: [PATCH 03/11] Switched from struct.unpack (module) -> Struct.unpack (class), it compiles the format string and reduces parsing costs. Coalesced a few unpack calls into a compound unpack call. Moved the functionality to get a list of valid chunks into get_chunks out from get_chunk_info. --- nbt.py | 741 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 385 insertions(+), 356 deletions(-) diff --git a/nbt.py b/nbt.py index 73730a0..5258ffb 100644 --- a/nbt.py +++ b/nbt.py @@ -1,356 +1,385 @@ -# This file is part of the Minecraft Overviewer. -# -# Minecraft Overviewer is free software: you can redistribute it and/or -# modify it under the terms of the GNU General Public License as published -# by the Free Software Foundation, either version 3 of the License, or (at -# your option) any later version. -# -# Minecraft Overviewer is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with the Overviewer. If not, see . - -import gzip, zlib -import struct -import StringIO -import os - -# decorator to handle filename or object as first parameter -def _file_loader(func): - def wrapper(fileobj, *args): - if isinstance(fileobj, basestring): - if not os.path.isfile(fileobj): - return None - - # Is actually a filename - fileobj = open(fileobj, 'rb') - return func(fileobj, *args) - return wrapper - -@_file_loader -def load(fileobj): - return NBTFileReader(fileobj).read_all() - -def load_from_region(filename, x, y): - nbt = load_region(filename).load_chunk(x, y) - if nbt is None: - return None ## return none. I think this is who we should indicate missing chunks - #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) - return nbt.read_all() - -def load_region(filename): - return MCRFileReader(filename) - -class NBTFileReader(object): - def __init__(self, fileobj, is_gzip=True): - if is_gzip: - self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') - else: - # pure zlib stream -- maybe later replace this with - # a custom zlib file object? - data = zlib.decompress(fileobj.read()) - self._file = StringIO.StringIO(data) - - # These private methods read the payload only of the following types - def _read_tag_end(self): - # Nothing to read - return 0 - - def _read_tag_byte(self): - byte = self._file.read(1) - return struct.unpack("b", byte)[0] - - def _read_tag_short(self): - bytes = self._file.read(2) - return struct.unpack(">h", bytes)[0] - - def _read_tag_int(self): - bytes = self._file.read(4) - return struct.unpack(">i", bytes)[0] - - def _read_tag_long(self): - bytes = self._file.read(8) - return struct.unpack(">q", bytes)[0] - - def _read_tag_float(self): - bytes = self._file.read(4) - return struct.unpack(">f", bytes)[0] - - def _read_tag_double(self): - bytes = self._file.read(8) - return struct.unpack(">d", bytes)[0] - - def _read_tag_byte_array(self): - length = self._read_tag_int() - bytes = self._file.read(length) - return bytes - - def _read_tag_string(self): - length = self._read_tag_short() - - # Read the string - string = self._file.read(length) - - # decode it and return - return string.decode("UTF-8") - - def _read_tag_list(self): - tagid = self._read_tag_byte() - length = self._read_tag_int() - - read_tagmap = { - 0: self._read_tag_end, - 1: self._read_tag_byte, - 2: self._read_tag_short, - 3: self._read_tag_int, - 4: self._read_tag_long, - 5: self._read_tag_float, - 6: self._read_tag_double, - 7: self._read_tag_byte_array, - 8: self._read_tag_string, - 9: self._read_tag_list, - 10:self._read_tag_compound, - } - - read_method = read_tagmap[tagid] - l = [] - for _ in xrange(length): - l.append(read_method()) - return l - - def _read_tag_compound(self): - # Build a dictionary of all the tag names mapping to their payloads - tags = {} - while True: - # Read a tag - tagtype = ord(self._file.read(1)) - - if tagtype == 0: - break - - name = self._read_tag_string() - read_tagmap = { - 0: self._read_tag_end, - 1: self._read_tag_byte, - 2: self._read_tag_short, - 3: self._read_tag_int, - 4: self._read_tag_long, - 5: self._read_tag_float, - 6: self._read_tag_double, - 7: self._read_tag_byte_array, - 8: self._read_tag_string, - 9: self._read_tag_list, - 10:self._read_tag_compound, - } - payload = read_tagmap[tagtype]() - - tags[name] = payload - - return tags - - - - def read_all(self): - """Reads the entire file and returns (name, payload) - name is the name of the root tag, and payload is a dictionary mapping - names to their payloads - - """ - # Read tag type - tagtype = ord(self._file.read(1)) - if tagtype != 10: - raise Exception("Expected a tag compound") - - # Read the tag name - name = self._read_tag_string() - - payload = self._read_tag_compound() - - return name, payload - - -# For reference, the MCR format is outlined at -# -class MCRFileReader(object): - """A class for reading chunk region files, as introduced in the - Beta 1.3 update. It provides functions for opening individual - chunks (as instances of NBTFileReader), getting chunk timestamps, - and for listing chunks contained in the file.""" - - def __init__(self, filename): - self._file = None - self._filename = filename - # cache used when the entire header tables are read in get_chunks() - self._locations = None - self._timestamps = None - self._chunks = None - - def _read_24bit_int(self): - """Read in a 24-bit, big-endian int, used in the chunk - location table.""" - - ret = 0 - bytes = self._file.read(3) - for i in xrange(3): - ret = ret << 8 - ret += struct.unpack("B", bytes[i])[0] - - return ret - - def _read_chunk_location(self, x=None, y=None): - """Read and return the (offset, length) of the given chunk - coordinate, or None if the requested chunk doesn't exist. x - and y must be between 0 and 31, or None. If they are None, - then there will be no file seek before doing the read.""" - - if x is not None and y is not None: - if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): - raise ValueError("Chunk location out of range.") - - # check for a cached value - if self._locations: - return self._locations[x + y * 32] - - # go to the correct entry in the chunk location table - self._file.seek(4 * (x + y * 32)) - - # 3-byte offset in 4KiB sectors - offset_sectors = self._read_24bit_int() - - # 1-byte length in 4KiB sectors, rounded up - byte = self._file.read(1) - length_sectors = struct.unpack("B", byte)[0] - - # check for empty chunks - if offset_sectors == 0 or length_sectors == 0: - return None - - return (offset_sectors * 4096, length_sectors * 4096) - - def _read_chunk_timestamp(self, x=None, y=None): - """Read and return the last modification time of the given - chunk coordinate. x and y must be between 0 and 31, or - None. If they are, None, then there will be no file seek - before doing the read.""" - - if x is not None and y is not None: - if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): - raise ValueError("Chunk location out of range.") - - # check for a cached value - if self._timestamps: - return self._timestamps[x + y * 32] - - # go to the correct entry in the chunk timestamp table - self._file.seek(4 * (x + y * 32) + 4096) - - bytes = self._file.read(4) - timestamp = struct.unpack(">I", bytes)[0] - - return timestamp - - def get_chunk_info(self,closeFile = True): - """Return a list of all chunks contained in this region file, - as a list of (x, y) coordinate tuples. To load these chunks, - provide these coordinates to load_chunk().""" - - if self._chunks: - return self._chunks - - if self._file is None: - self._file = open(self._filename,'rb'); - - self._chunks = [] - self._locations = [] - self._timestamps = [] - - # go to the beginning of the file - self._file.seek(0) - - # read chunk location table - for y in xrange(32): - for x in xrange(32): - location = self._read_chunk_location() - self._locations.append(location) - if location: - self._chunks.append((x, y)) - - # read chunk timestamp table - for y in xrange(32): - for x in xrange(32): - timestamp = self._read_chunk_timestamp() - self._timestamps.append(timestamp) - - if closeFile: - #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!) - self._file.close() - self._file = None - return self._chunks - - def get_chunk_timestamp(self, x, y): - """Return the given chunk's modification time. If the given - chunk doesn't exist, this number may be nonsense. Like - load_chunk(), this will wrap x and y into the range [0, 31]. - """ - x = x % 32 - y = y % 32 - if self._timestamps is None: - self.get_chunk_info() - return self._timestamps[x + y * 32] - - def chunkExists(self, x, y): - """Determines if a chunk exists without triggering loading of the backend data""" - x = x % 32 - y = y % 32 - if self._locations is None: - self.get_chunk_info() - location = self._locations[x + y * 32] - return location is not None - - def load_chunk(self, x, y): - """Return a NBTFileReader instance for the given chunk, or - None if the given chunk doesn't exist in this region file. If - you provide an x or y not between 0 and 31, it will be - modulo'd into this range (x % 32, etc.) This is so you can - provide chunk coordinates in global coordinates, and still - have the chunks load out of regions properly.""" - x = x % 32 - y = y % 32 - if self._locations is None: - self.get_chunk_info() - - location = self._locations[x + y * 32] - if location is None: - return None - - if self._file is None: - self._file = open(self._filename,'rb'); - # seek to the data - self._file.seek(location[0]) - - # read in the chunk data header - bytes = self._file.read(4) - data_length = struct.unpack(">I", bytes)[0] - bytes = self._file.read(1) - compression = struct.unpack("B", bytes)[0] - - # figure out the compression - is_gzip = True - if compression == 1: - # gzip -- not used by the official client, but trivial to support here so... - is_gzip = True - elif compression == 2: - # deflate -- pure zlib stream - is_gzip = False - else: - # unsupported! - raise Exception("Unsupported chunk compression type: %i" % (compression)) - # turn the rest of the data into a StringIO object - # (using data_length - 1, as we already read 1 byte for compression) - data = self._file.read(data_length - 1) - data = StringIO.StringIO(data) - - return NBTFileReader(data, is_gzip=is_gzip) +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import gzip, zlib +import struct +import StringIO +import os + +# decorator to handle filename or object as first parameter +def _file_loader(func): + def wrapper(fileobj, *args): + if isinstance(fileobj, basestring): + if not os.path.isfile(fileobj): + return None + + # Is actually a filename + fileobj = open(fileobj, 'rb',4096) + return func(fileobj, *args) + return wrapper + +@_file_loader +def load(fileobj): + return NBTFileReader(fileobj).read_all() + +def load_from_region(filename, x, y): + nbt = load_region(filename).load_chunk(x, y) + if nbt is None: + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + return nbt.read_all() + +def load_region(filename): + return MCRFileReader(filename) + + +# compile the unpacker's into a classes +_byte = struct.Struct("b") +_short = struct.Struct(">h") +_int = struct.Struct(">i") +_long = struct.Struct(">q") +_float = struct.Struct(">f") +_double = struct.Struct(">d") + +_24bit_int = struct.Struct("B B B") +_unsigned_byte = struct.Struct("B") +_unsigned_int = struct.Struct(">I") +_chunk_header = struct.Struct(">I B") + +class NBTFileReader(object): + def __init__(self, fileobj, is_gzip=True): + if is_gzip: + self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') + else: + # pure zlib stream -- maybe later replace this with + # a custom zlib file object? + data = zlib.decompress(fileobj.read()) + self._file = StringIO.StringIO(data) + + # These private methods read the payload only of the following types + def _read_tag_end(self): + # Nothing to read + return 0 + + def _read_tag_byte(self): + byte = self._file.read(1) + return _byte.unpack(byte)[0] + + def _read_tag_short(self): + bytes = self._file.read(2) + global _short + return _short.unpack(bytes)[0] + + def _read_tag_int(self): + bytes = self._file.read(4) + global _int + return _int.unpack(bytes)[0] + + def _read_tag_long(self): + bytes = self._file.read(8) + global _long + return _long.unpack(bytes)[0] + + def _read_tag_float(self): + bytes = self._file.read(4) + global _float + return _float.unpack(bytes)[0] + + def _read_tag_double(self): + bytes = self._file.read(8) + global _double + return _double.unpack(bytes)[0] + + def _read_tag_byte_array(self): + length = self._read_tag_int() + bytes = self._file.read(length) + return bytes + + def _read_tag_string(self): + length = self._read_tag_short() + + # Read the string + string = self._file.read(length) + + # decode it and return + return string.decode("UTF-8") + + def _read_tag_list(self): + tagid = self._read_tag_byte() + length = self._read_tag_int() + + read_tagmap = { + 0: self._read_tag_end, + 1: self._read_tag_byte, + 2: self._read_tag_short, + 3: self._read_tag_int, + 4: self._read_tag_long, + 5: self._read_tag_float, + 6: self._read_tag_double, + 7: self._read_tag_byte_array, + 8: self._read_tag_string, + 9: self._read_tag_list, + 10:self._read_tag_compound, + } + + read_method = read_tagmap[tagid] + l = [] + for _ in xrange(length): + l.append(read_method()) + return l + + def _read_tag_compound(self): + # Build a dictionary of all the tag names mapping to their payloads + tags = {} + while True: + # Read a tag + tagtype = ord(self._file.read(1)) + + if tagtype == 0: + break + + name = self._read_tag_string() + read_tagmap = { + 0: self._read_tag_end, + 1: self._read_tag_byte, + 2: self._read_tag_short, + 3: self._read_tag_int, + 4: self._read_tag_long, + 5: self._read_tag_float, + 6: self._read_tag_double, + 7: self._read_tag_byte_array, + 8: self._read_tag_string, + 9: self._read_tag_list, + 10:self._read_tag_compound, + } + payload = read_tagmap[tagtype]() + + tags[name] = payload + + return tags + + + + def read_all(self): + """Reads the entire file and returns (name, payload) + name is the name of the root tag, and payload is a dictionary mapping + names to their payloads + + """ + # Read tag type + tagtype = ord(self._file.read(1)) + if tagtype != 10: + raise Exception("Expected a tag compound") + + # Read the tag name + name = self._read_tag_string() + + payload = self._read_tag_compound() + + return name, payload + + +# For reference, the MCR format is outlined at +# +class MCRFileReader(object): + """A class for reading chunk region files, as introduced in the + Beta 1.3 update. It provides functions for opening individual + chunks (as instances of NBTFileReader), getting chunk timestamps, + and for listing chunks contained in the file.""" + + def __init__(self, filename): + self._file = None + self._filename = filename + # cache used when the entire header tables are read in get_chunks() + self._locations = None + self._timestamps = None + self._chunks = None + + def _read_24bit_int(self): + """Read in a 24-bit, big-endian int, used in the chunk + location table.""" + + ret = 0 + bytes = self._file.read(3) + global _24bit_int + bytes = _24bit_int.unpack(bytes) + for i in xrange(3): + ret = ret << 8 + ret += bytes[i] + + return ret + + def _read_chunk_location(self, x=None, y=None): + """Read and return the (offset, length) of the given chunk + coordinate, or None if the requested chunk doesn't exist. x + and y must be between 0 and 31, or None. If they are None, + then there will be no file seek before doing the read.""" + + if x is not None and y is not None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._locations: + return self._locations[x + y * 32] + + # go to the correct entry in the chunk location table + self._file.seek(4 * (x + y * 32)) + + + # 3-byte offset in 4KiB sectors + offset_sectors = self._read_24bit_int() + global _unsigned_byte + # 1-byte length in 4KiB sectors, rounded up + byte = self._file.read(1) + length_sectors = _unsigned_byte.unpack(byte)[0] + + # check for empty chunks + if offset_sectors == 0 or length_sectors == 0: + return None + + return (offset_sectors * 4096, length_sectors * 4096) + + def _read_chunk_timestamp(self, x=None, y=None): + """Read and return the last modification time of the given + chunk coordinate. x and y must be between 0 and 31, or + None. If they are, None, then there will be no file seek + before doing the read.""" + + if x is not None and y is not None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._timestamps: + return self._timestamps[x + y * 32] + + # go to the correct entry in the chunk timestamp table + self._file.seek(4 * (x + y * 32) + 4096) + + bytes = self._file.read(4) + + global _unsigned_int + timestamp = _unsigned_int.unpack(bytes)[0] + + return timestamp + + def get_chunks(self): + """Return a list of all chunks contained in this region file, + as a list of (x, y) coordinate tuples. To load these chunks, + provide these coordinates to load_chunk().""" + + if self._chunks: + return self._chunks + if self._locations is None: + self.get_chunk_info() + self._chunks = filter(None,self._locations) + + return self._chunks + + def get_chunk_info(self,closeFile = True): + """Preloads region header information.""" + + if self._locations: + return + + if self._file is None: + self._file = open(self._filename,'rb'); + + self._chunks = None + self._locations = [] + self._timestamps = [] + + # go to the beginning of the file + self._file.seek(0) + + # read chunk location table + locations_append = self._locations.append + for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]: + locations_append(self._read_chunk_location()) + + # read chunk timestamp table + timestamp_append = self._timestamps.append + for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]: + timestamp_append(self._read_chunk_timestamp()) + + if closeFile: + #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!) + self._file.close() + self._file = None + return + + def get_chunk_timestamp(self, x, y): + """Return the given chunk's modification time. If the given + chunk doesn't exist, this number may be nonsense. Like + load_chunk(), this will wrap x and y into the range [0, 31]. + """ + x = x % 32 + y = y % 32 + if self._timestamps is None: + self.get_chunk_info() + return self._timestamps[x + y * 32] + + def chunkExists(self, x, y): + """Determines if a chunk exists without triggering loading of the backend data""" + x = x % 32 + y = y % 32 + if self._locations is None: + self.get_chunk_info() + location = self._locations[x + y * 32] + return location is not None + + def load_chunk(self, x, y): + """Return a NBTFileReader instance for the given chunk, or + None if the given chunk doesn't exist in this region file. If + you provide an x or y not between 0 and 31, it will be + modulo'd into this range (x % 32, etc.) This is so you can + provide chunk coordinates in global coordinates, and still + have the chunks load out of regions properly.""" + x = x % 32 + y = y % 32 + if self._locations is None: + self.get_chunk_info() + + location = self._locations[x + y * 32] + if location is None: + return None + + if self._file is None: + self._file = open(self._filename,'rb'); + # seek to the data + self._file.seek(location[0]) + + # read in the chunk data header + bytes = self._file.read(5) + data_length,compression = _chunk_header.unpack(bytes) + + # figure out the compression + is_gzip = True + if compression == 1: + # gzip -- not used by the official client, but trivial to support here so... + is_gzip = True + elif compression == 2: + # deflate -- pure zlib stream + is_gzip = False + else: + # unsupported! + raise Exception("Unsupported chunk compression type: %i" % (compression)) + # turn the rest of the data into a StringIO object + # (using data_length - 1, as we already read 1 byte for compression) + data = self._file.read(data_length - 1) + data = StringIO.StringIO(data) + + return NBTFileReader(data, is_gzip=is_gzip) From 8cfa50087aeb4e266fef7d67f145f5106fe63101 Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 17:22:57 +0800 Subject: [PATCH 04/11] Removed filting chunks from render_worldtile since _get_chunks_in_range can do it trivially before constructing the list --- quadtree.py | 19 ++++++------------- world.py | 6 +++--- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/quadtree.py b/quadtree.py index ce2dab1..c32b757 100644 --- a/quadtree.py +++ b/quadtree.py @@ -85,6 +85,9 @@ class QuadtreeGen(object): self.imgformat = imgformat self.optimizeimg = optimizeimg + self.lighting = rendermode in ("lighting", "night", "spawn") + self.night = rendermode in ("night", "spawn") + self.spawn = rendermode in ("spawn",) self.rendermode = rendermode # Make the destination dir @@ -410,8 +413,8 @@ class QuadtreeGen(object): # return (col, row, chunkx, chunky, regionpath) chunkx, chunky = unconvert_coords(col, row) #c = get_region_path(chunkx, chunky) - _, _, c = get_region((chunkx//32, chunky//32),(None,None,None)); - if c is not None: + _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None)); + if c is not None and mcr.chunkExists(chunkx,chunky): chunklist.append((col, row, chunkx, chunky, c)) return chunklist @@ -561,16 +564,7 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path) # anyways just in case). "chunks" should include up to rowstart-16 imgpath = path + "." + quadtree.imgformat - world = quadtree.world - # first, remove chunks from `chunks` that don't actually exist in - # their region files - def chunk_exists(chunk): - _, _, chunkx, chunky, region = chunk - r = world.load_region(region) - return r.chunkExists(chunkx, chunky) - chunks = filter(chunk_exists, chunks) - #stat the file, we need to know if it exists or it's mtime try: tile_mtime = os.stat(imgpath)[stat.ST_MTIME]; @@ -602,12 +596,11 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path) needs_rerender = False for col, row, chunkx, chunky, regionfile in chunks: # check region file mtime first. - regionMtime = world.get_region_mtime(regionfile) + region,regionMtime = world.get_region_mtime(regionfile) if regionMtime <= tile_mtime: continue # checking chunk mtime - region = world.load_region(regionfile) if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime: needs_rerender = True break diff --git a/world.py b/world.py index bd4d9a6..077a17e 100644 --- a/world.py +++ b/world.py @@ -80,7 +80,7 @@ class World(object): mcr = nbt.MCRFileReader(regionfile) mcr.get_chunk_info() regions[regionfile] = (mcr,os.path.getmtime(regionfile)) - regionfiles[(x,y)] = (x,y,regionfile) + regionfiles[(x,y)] = (x,y,regionfile,mcr) self.regionfiles = regionfiles self.regions = regions logging.debug("Done scanning regions") @@ -116,7 +116,7 @@ class World(object): def get_region_path(self, chunkX, chunkY): """Returns the path to the region that contains chunk (chunkX, chunkY) """ - _, _, regionfile = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None)); + _, _, regionfile,_ = self.regionfiles.get((chunkX//32, chunkY//32),(None,None,None,None)); return regionfile def load_from_region(self,filename, x, y): @@ -135,7 +135,7 @@ class World(object): return self.regions[filename][0] def get_region_mtime(self,filename): - return self.regions[filename][1] + return self.regions[filename] def convert_coords(self, chunkx, chunky): """Takes a coordinate (chunkx, chunky) where chunkx and chunky are From c700afb012b9bfb810f721c631110a409c78d309 Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 18:49:26 +0800 Subject: [PATCH 05/11] Fixed get_chunks, simplified get_chunk_info --- nbt.py | 773 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 388 insertions(+), 385 deletions(-) diff --git a/nbt.py b/nbt.py index 5258ffb..8a96a62 100644 --- a/nbt.py +++ b/nbt.py @@ -1,385 +1,388 @@ -# This file is part of the Minecraft Overviewer. -# -# Minecraft Overviewer is free software: you can redistribute it and/or -# modify it under the terms of the GNU General Public License as published -# by the Free Software Foundation, either version 3 of the License, or (at -# your option) any later version. -# -# Minecraft Overviewer is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with the Overviewer. If not, see . - -import gzip, zlib -import struct -import StringIO -import os - -# decorator to handle filename or object as first parameter -def _file_loader(func): - def wrapper(fileobj, *args): - if isinstance(fileobj, basestring): - if not os.path.isfile(fileobj): - return None - - # Is actually a filename - fileobj = open(fileobj, 'rb',4096) - return func(fileobj, *args) - return wrapper - -@_file_loader -def load(fileobj): - return NBTFileReader(fileobj).read_all() - -def load_from_region(filename, x, y): - nbt = load_region(filename).load_chunk(x, y) - if nbt is None: - return None ## return none. I think this is who we should indicate missing chunks - #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) - return nbt.read_all() - -def load_region(filename): - return MCRFileReader(filename) - - -# compile the unpacker's into a classes -_byte = struct.Struct("b") -_short = struct.Struct(">h") -_int = struct.Struct(">i") -_long = struct.Struct(">q") -_float = struct.Struct(">f") -_double = struct.Struct(">d") - -_24bit_int = struct.Struct("B B B") -_unsigned_byte = struct.Struct("B") -_unsigned_int = struct.Struct(">I") -_chunk_header = struct.Struct(">I B") - -class NBTFileReader(object): - def __init__(self, fileobj, is_gzip=True): - if is_gzip: - self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') - else: - # pure zlib stream -- maybe later replace this with - # a custom zlib file object? - data = zlib.decompress(fileobj.read()) - self._file = StringIO.StringIO(data) - - # These private methods read the payload only of the following types - def _read_tag_end(self): - # Nothing to read - return 0 - - def _read_tag_byte(self): - byte = self._file.read(1) - return _byte.unpack(byte)[0] - - def _read_tag_short(self): - bytes = self._file.read(2) - global _short - return _short.unpack(bytes)[0] - - def _read_tag_int(self): - bytes = self._file.read(4) - global _int - return _int.unpack(bytes)[0] - - def _read_tag_long(self): - bytes = self._file.read(8) - global _long - return _long.unpack(bytes)[0] - - def _read_tag_float(self): - bytes = self._file.read(4) - global _float - return _float.unpack(bytes)[0] - - def _read_tag_double(self): - bytes = self._file.read(8) - global _double - return _double.unpack(bytes)[0] - - def _read_tag_byte_array(self): - length = self._read_tag_int() - bytes = self._file.read(length) - return bytes - - def _read_tag_string(self): - length = self._read_tag_short() - - # Read the string - string = self._file.read(length) - - # decode it and return - return string.decode("UTF-8") - - def _read_tag_list(self): - tagid = self._read_tag_byte() - length = self._read_tag_int() - - read_tagmap = { - 0: self._read_tag_end, - 1: self._read_tag_byte, - 2: self._read_tag_short, - 3: self._read_tag_int, - 4: self._read_tag_long, - 5: self._read_tag_float, - 6: self._read_tag_double, - 7: self._read_tag_byte_array, - 8: self._read_tag_string, - 9: self._read_tag_list, - 10:self._read_tag_compound, - } - - read_method = read_tagmap[tagid] - l = [] - for _ in xrange(length): - l.append(read_method()) - return l - - def _read_tag_compound(self): - # Build a dictionary of all the tag names mapping to their payloads - tags = {} - while True: - # Read a tag - tagtype = ord(self._file.read(1)) - - if tagtype == 0: - break - - name = self._read_tag_string() - read_tagmap = { - 0: self._read_tag_end, - 1: self._read_tag_byte, - 2: self._read_tag_short, - 3: self._read_tag_int, - 4: self._read_tag_long, - 5: self._read_tag_float, - 6: self._read_tag_double, - 7: self._read_tag_byte_array, - 8: self._read_tag_string, - 9: self._read_tag_list, - 10:self._read_tag_compound, - } - payload = read_tagmap[tagtype]() - - tags[name] = payload - - return tags - - - - def read_all(self): - """Reads the entire file and returns (name, payload) - name is the name of the root tag, and payload is a dictionary mapping - names to their payloads - - """ - # Read tag type - tagtype = ord(self._file.read(1)) - if tagtype != 10: - raise Exception("Expected a tag compound") - - # Read the tag name - name = self._read_tag_string() - - payload = self._read_tag_compound() - - return name, payload - - -# For reference, the MCR format is outlined at -# -class MCRFileReader(object): - """A class for reading chunk region files, as introduced in the - Beta 1.3 update. It provides functions for opening individual - chunks (as instances of NBTFileReader), getting chunk timestamps, - and for listing chunks contained in the file.""" - - def __init__(self, filename): - self._file = None - self._filename = filename - # cache used when the entire header tables are read in get_chunks() - self._locations = None - self._timestamps = None - self._chunks = None - - def _read_24bit_int(self): - """Read in a 24-bit, big-endian int, used in the chunk - location table.""" - - ret = 0 - bytes = self._file.read(3) - global _24bit_int - bytes = _24bit_int.unpack(bytes) - for i in xrange(3): - ret = ret << 8 - ret += bytes[i] - - return ret - - def _read_chunk_location(self, x=None, y=None): - """Read and return the (offset, length) of the given chunk - coordinate, or None if the requested chunk doesn't exist. x - and y must be between 0 and 31, or None. If they are None, - then there will be no file seek before doing the read.""" - - if x is not None and y is not None: - if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): - raise ValueError("Chunk location out of range.") - - # check for a cached value - if self._locations: - return self._locations[x + y * 32] - - # go to the correct entry in the chunk location table - self._file.seek(4 * (x + y * 32)) - - - # 3-byte offset in 4KiB sectors - offset_sectors = self._read_24bit_int() - global _unsigned_byte - # 1-byte length in 4KiB sectors, rounded up - byte = self._file.read(1) - length_sectors = _unsigned_byte.unpack(byte)[0] - - # check for empty chunks - if offset_sectors == 0 or length_sectors == 0: - return None - - return (offset_sectors * 4096, length_sectors * 4096) - - def _read_chunk_timestamp(self, x=None, y=None): - """Read and return the last modification time of the given - chunk coordinate. x and y must be between 0 and 31, or - None. If they are, None, then there will be no file seek - before doing the read.""" - - if x is not None and y is not None: - if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): - raise ValueError("Chunk location out of range.") - - # check for a cached value - if self._timestamps: - return self._timestamps[x + y * 32] - - # go to the correct entry in the chunk timestamp table - self._file.seek(4 * (x + y * 32) + 4096) - - bytes = self._file.read(4) - - global _unsigned_int - timestamp = _unsigned_int.unpack(bytes)[0] - - return timestamp - - def get_chunks(self): - """Return a list of all chunks contained in this region file, - as a list of (x, y) coordinate tuples. To load these chunks, - provide these coordinates to load_chunk().""" - - if self._chunks: - return self._chunks - if self._locations is None: - self.get_chunk_info() - self._chunks = filter(None,self._locations) - - return self._chunks - - def get_chunk_info(self,closeFile = True): - """Preloads region header information.""" - - if self._locations: - return - - if self._file is None: - self._file = open(self._filename,'rb'); - - self._chunks = None - self._locations = [] - self._timestamps = [] - - # go to the beginning of the file - self._file.seek(0) - - # read chunk location table - locations_append = self._locations.append - for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]: - locations_append(self._read_chunk_location()) - - # read chunk timestamp table - timestamp_append = self._timestamps.append - for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]: - timestamp_append(self._read_chunk_timestamp()) - - if closeFile: - #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!) - self._file.close() - self._file = None - return - - def get_chunk_timestamp(self, x, y): - """Return the given chunk's modification time. If the given - chunk doesn't exist, this number may be nonsense. Like - load_chunk(), this will wrap x and y into the range [0, 31]. - """ - x = x % 32 - y = y % 32 - if self._timestamps is None: - self.get_chunk_info() - return self._timestamps[x + y * 32] - - def chunkExists(self, x, y): - """Determines if a chunk exists without triggering loading of the backend data""" - x = x % 32 - y = y % 32 - if self._locations is None: - self.get_chunk_info() - location = self._locations[x + y * 32] - return location is not None - - def load_chunk(self, x, y): - """Return a NBTFileReader instance for the given chunk, or - None if the given chunk doesn't exist in this region file. If - you provide an x or y not between 0 and 31, it will be - modulo'd into this range (x % 32, etc.) This is so you can - provide chunk coordinates in global coordinates, and still - have the chunks load out of regions properly.""" - x = x % 32 - y = y % 32 - if self._locations is None: - self.get_chunk_info() - - location = self._locations[x + y * 32] - if location is None: - return None - - if self._file is None: - self._file = open(self._filename,'rb'); - # seek to the data - self._file.seek(location[0]) - - # read in the chunk data header - bytes = self._file.read(5) - data_length,compression = _chunk_header.unpack(bytes) - - # figure out the compression - is_gzip = True - if compression == 1: - # gzip -- not used by the official client, but trivial to support here so... - is_gzip = True - elif compression == 2: - # deflate -- pure zlib stream - is_gzip = False - else: - # unsupported! - raise Exception("Unsupported chunk compression type: %i" % (compression)) - # turn the rest of the data into a StringIO object - # (using data_length - 1, as we already read 1 byte for compression) - data = self._file.read(data_length - 1) - data = StringIO.StringIO(data) - - return NBTFileReader(data, is_gzip=is_gzip) +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import gzip, zlib +import struct +import StringIO +import os + +# decorator to handle filename or object as first parameter +def _file_loader(func): + def wrapper(fileobj, *args): + if isinstance(fileobj, basestring): + if not os.path.isfile(fileobj): + return None + + # Is actually a filename + fileobj = open(fileobj, 'rb',4096) + return func(fileobj, *args) + return wrapper + +@_file_loader +def load(fileobj): + return NBTFileReader(fileobj).read_all() + +def load_from_region(filename, x, y): + nbt = load_region(filename).load_chunk(x, y) + if nbt is None: + return None ## return none. I think this is who we should indicate missing chunks + #raise IOError("No such chunk in region: (%i, %i)" % (x, y)) + return nbt.read_all() + +def load_region(filename): + return MCRFileReader(filename) + + +# compile the unpacker's into a classes +_byte = struct.Struct("b") +_short = struct.Struct(">h") +_int = struct.Struct(">i") +_long = struct.Struct(">q") +_float = struct.Struct(">f") +_double = struct.Struct(">d") + +_24bit_int = struct.Struct("B B B") +_unsigned_byte = struct.Struct("B") +_unsigned_int = struct.Struct(">I") +_chunk_header = struct.Struct(">I B") + +class NBTFileReader(object): + def __init__(self, fileobj, is_gzip=True): + if is_gzip: + self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') + else: + # pure zlib stream -- maybe later replace this with + # a custom zlib file object? + data = zlib.decompress(fileobj.read()) + self._file = StringIO.StringIO(data) + + # These private methods read the payload only of the following types + def _read_tag_end(self): + # Nothing to read + return 0 + + def _read_tag_byte(self): + byte = self._file.read(1) + return _byte.unpack(byte)[0] + + def _read_tag_short(self): + bytes = self._file.read(2) + global _short + return _short.unpack(bytes)[0] + + def _read_tag_int(self): + bytes = self._file.read(4) + global _int + return _int.unpack(bytes)[0] + + def _read_tag_long(self): + bytes = self._file.read(8) + global _long + return _long.unpack(bytes)[0] + + def _read_tag_float(self): + bytes = self._file.read(4) + global _float + return _float.unpack(bytes)[0] + + def _read_tag_double(self): + bytes = self._file.read(8) + global _double + return _double.unpack(bytes)[0] + + def _read_tag_byte_array(self): + length = self._read_tag_int() + bytes = self._file.read(length) + return bytes + + def _read_tag_string(self): + length = self._read_tag_short() + + # Read the string + string = self._file.read(length) + + # decode it and return + return string.decode("UTF-8") + + def _read_tag_list(self): + tagid = self._read_tag_byte() + length = self._read_tag_int() + + read_tagmap = { + 0: self._read_tag_end, + 1: self._read_tag_byte, + 2: self._read_tag_short, + 3: self._read_tag_int, + 4: self._read_tag_long, + 5: self._read_tag_float, + 6: self._read_tag_double, + 7: self._read_tag_byte_array, + 8: self._read_tag_string, + 9: self._read_tag_list, + 10:self._read_tag_compound, + } + + read_method = read_tagmap[tagid] + l = [] + for _ in xrange(length): + l.append(read_method()) + return l + + def _read_tag_compound(self): + # Build a dictionary of all the tag names mapping to their payloads + tags = {} + while True: + # Read a tag + tagtype = ord(self._file.read(1)) + + if tagtype == 0: + break + + name = self._read_tag_string() + read_tagmap = { + 0: self._read_tag_end, + 1: self._read_tag_byte, + 2: self._read_tag_short, + 3: self._read_tag_int, + 4: self._read_tag_long, + 5: self._read_tag_float, + 6: self._read_tag_double, + 7: self._read_tag_byte_array, + 8: self._read_tag_string, + 9: self._read_tag_list, + 10:self._read_tag_compound, + } + payload = read_tagmap[tagtype]() + + tags[name] = payload + + return tags + + + + def read_all(self): + """Reads the entire file and returns (name, payload) + name is the name of the root tag, and payload is a dictionary mapping + names to their payloads + + """ + # Read tag type + tagtype = ord(self._file.read(1)) + if tagtype != 10: + raise Exception("Expected a tag compound") + + # Read the tag name + name = self._read_tag_string() + + payload = self._read_tag_compound() + + return name, payload + + +# For reference, the MCR format is outlined at +# +class MCRFileReader(object): + """A class for reading chunk region files, as introduced in the + Beta 1.3 update. It provides functions for opening individual + chunks (as instances of NBTFileReader), getting chunk timestamps, + and for listing chunks contained in the file.""" + + def __init__(self, filename): + self._file = None + self._filename = filename + # cache used when the entire header tables are read in get_chunks() + self._locations = None + self._timestamps = None + self._chunks = None + + def _read_24bit_int(self): + """Read in a 24-bit, big-endian int, used in the chunk + location table.""" + + ret = 0 + bytes = self._file.read(3) + global _24bit_int + bytes = _24bit_int.unpack(bytes) + for i in xrange(3): + ret = ret << 8 + ret += bytes[i] + + return ret + + def _read_chunk_location(self, x=None, y=None): + """Read and return the (offset, length) of the given chunk + coordinate, or None if the requested chunk doesn't exist. x + and y must be between 0 and 31, or None. If they are None, + then there will be no file seek before doing the read.""" + + if x is not None and y is not None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._locations: + return self._locations[x + y * 32] + + # go to the correct entry in the chunk location table + self._file.seek(4 * (x + y * 32)) + + + # 3-byte offset in 4KiB sectors + offset_sectors = self._read_24bit_int() + global _unsigned_byte + # 1-byte length in 4KiB sectors, rounded up + byte = self._file.read(1) + length_sectors = _unsigned_byte.unpack(byte)[0] + + # check for empty chunks + if offset_sectors == 0 or length_sectors == 0: + return None + + return (offset_sectors * 4096, length_sectors * 4096) + + def _read_chunk_timestamp(self, x=None, y=None): + """Read and return the last modification time of the given + chunk coordinate. x and y must be between 0 and 31, or + None. If they are, None, then there will be no file seek + before doing the read.""" + + if x is not None and y is not None: + if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): + raise ValueError("Chunk location out of range.") + + # check for a cached value + if self._timestamps: + return self._timestamps[x + y * 32] + + # go to the correct entry in the chunk timestamp table + self._file.seek(4 * (x + y * 32) + 4096) + + bytes = self._file.read(4) + + global _unsigned_int + timestamp = _unsigned_int.unpack(bytes)[0] + + return timestamp + + def get_chunks(self): + """Return a list of all chunks contained in this region file, + as a list of (x, y) coordinate tuples. To load these chunks, + provide these coordinates to load_chunk().""" + + if self._chunks is not None: + return self._chunks + if self._locations is None: + self.get_chunk_info() + self._chunks = [] + for x in xrange(32): + for y in xrange(32): + if self._locations[x + y * 32] is not None: + self._chunks.append((x,y)) + return self._chunks + + def get_chunk_info(self,closeFile = True): + """Preloads region header information.""" + + if self._locations: + return + + if self._file is None: + self._file = open(self._filename,'rb') + + self._chunks = None + self._locations = [] + self._timestamps = [] + + # go to the beginning of the file + self._file.seek(0) + + # read chunk location table + locations_append = self._locations.append + for _ in xrange(32*32): + locations_append(self._read_chunk_location()) + + # read chunk timestamp table + timestamp_append = self._timestamps.append + for _ in xrange(32*32): + timestamp_append(self._read_chunk_timestamp()) + + if closeFile: + #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!) + self._file.close() + self._file = None + return + + def get_chunk_timestamp(self, x, y): + """Return the given chunk's modification time. If the given + chunk doesn't exist, this number may be nonsense. Like + load_chunk(), this will wrap x and y into the range [0, 31]. + """ + x = x % 32 + y = y % 32 + if self._timestamps is None: + self.get_chunk_info() + return self._timestamps[x + y * 32] + + def chunkExists(self, x, y): + """Determines if a chunk exists without triggering loading of the backend data""" + x = x % 32 + y = y % 32 + if self._locations is None: + self.get_chunk_info() + location = self._locations[x + y * 32] + return location is not None + + def load_chunk(self, x, y): + """Return a NBTFileReader instance for the given chunk, or + None if the given chunk doesn't exist in this region file. If + you provide an x or y not between 0 and 31, it will be + modulo'd into this range (x % 32, etc.) This is so you can + provide chunk coordinates in global coordinates, and still + have the chunks load out of regions properly.""" + x = x % 32 + y = y % 32 + if self._locations is None: + self.get_chunk_info() + + location = self._locations[x + y * 32] + if location is None: + return None + + if self._file is None: + self._file = open(self._filename,'rb'); + # seek to the data + self._file.seek(location[0]) + + # read in the chunk data header + bytes = self._file.read(5) + data_length,compression = _chunk_header.unpack(bytes) + + # figure out the compression + is_gzip = True + if compression == 1: + # gzip -- not used by the official client, but trivial to support here so... + is_gzip = True + elif compression == 2: + # deflate -- pure zlib stream + is_gzip = False + else: + # unsupported! + raise Exception("Unsupported chunk compression type: %i" % (compression)) + # turn the rest of the data into a StringIO object + # (using data_length - 1, as we already read 1 byte for compression) + data = self._file.read(data_length - 1) + data = StringIO.StringIO(data) + + return NBTFileReader(data, is_gzip=is_gzip) From ca36c9864194b902e39b3042ef77163dc9d1c2c7 Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 21:42:13 +0800 Subject: [PATCH 06/11] Initial commit for multi-layer rendering. Hardwired to use all 4 render modes at once. Todo: add config file/commandline argument support. --- googlemap.py | 3 +- overviewer.py | 17 +- quadtree.py | 641 ++++++++++++++++---------------------------------- rendernode.py | 346 +++++++++++++++++++++++++++ 4 files changed, 569 insertions(+), 438 deletions(-) create mode 100644 rendernode.py diff --git a/googlemap.py b/googlemap.py index fbecc03..f53aae0 100644 --- a/googlemap.py +++ b/googlemap.py @@ -53,7 +53,8 @@ class MapGen(object): def __init__(self, quadtrees, skipjs=False, web_assets_hook=None): """Generates a Google Maps interface for the given list of quadtrees. All of the quadtrees must have the same destdir, - image format, and world.""" + image format, and world. + Note:tiledir for each quadtree should be unique. By default the tiledir is determined by the rendermode""" self.skipjs = skipjs self.web_assets_hook = web_assets_hook diff --git a/overviewer.py b/overviewer.py index ffc3882..cb1b0a9 100755 --- a/overviewer.py +++ b/overviewer.py @@ -45,6 +45,7 @@ import composite import world import quadtree import googlemap +import rendernode helptext = """ %prog [OPTIONS] @@ -171,21 +172,29 @@ def main(): useBiomeData = os.path.exists(os.path.join(worlddir, 'biomes')) if not useBiomeData: logging.info("Notice: Not using biome data for tinting") - + # First do world-level preprocessing w = world.World(worlddir, useBiomeData=useBiomeData) w.go(options.procs) # create the quadtrees # TODO chunklist - q = quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode=options.rendermode) + q = [] + q.append(quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode='normal', tiledir='tiles')) + q.append(quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode='lighting')) + q.append(quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode='night')) + q.append(quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode='spawn')) + + #create the distributed render + r = rendernode.RenderNode(w,q) # write out the map and web assets - m = googlemap.MapGen([q,], skipjs=options.skipjs, web_assets_hook=options.web_assets_hook) + m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook) m.go(options.procs) # render the tiles! - q.go(options.procs) + r.go(options.procs) + def delete_all(worlddir, tiledir): # TODO should we delete tiledir here too? diff --git a/quadtree.py b/quadtree.py index c32b757..60c097d 100644 --- a/quadtree.py +++ b/quadtree.py @@ -46,32 +46,9 @@ This module has routines related to generating a quadtree of tiles def iterate_base4(d): """Iterates over a base 4 number with d digits""" return itertools.product(xrange(4), repeat=d) - -def catch_keyboardinterrupt(func): - """Decorator that catches a keyboardinterrupt and raises a real exception - so that multiprocessing will propagate it properly""" - @functools.wraps(func) - def newfunc(*args, **kwargs): - try: - return func(*args, **kwargs) - except KeyboardInterrupt: - logging.error("Ctrl-C caught!") - raise Exception("Exiting") - except: - import traceback - traceback.print_exc() - raise - return newfunc - -child_quadtree = None -def pool_initializer(quadtree): - logging.debug("Child process {0}".format(os.getpid())) - #stash the quadtree object in a global variable after fork() for windows compat. - global child_quadtree - child_quadtree = quadtree - + class QuadtreeGen(object): - def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, rendermode="normal"): + def __init__(self, worldobj, destdir, depth=None, tiledir=None, imgformat=None, optimizeimg=None, rendermode="normal"): """Generates a quadtree from the world given into the given dest directory @@ -93,6 +70,8 @@ class QuadtreeGen(object): # Make the destination dir if not os.path.exists(destdir): os.mkdir(destdir) + if tiledir is None: + tiledir = rendermode self.tiledir = tiledir if depth is None: @@ -128,23 +107,6 @@ class QuadtreeGen(object): self.destdir = destdir self.full_tiledir = os.path.join(destdir, tiledir) - - - def print_statusline(self, complete, total, level, unconditional=False): - if unconditional: - pass - elif complete < 100: - if not complete % 25 == 0: - return - elif complete < 1000: - if not complete % 100 == 0: - return - else: - if not complete % 1000 == 0: - return - logging.info("{0}/{1} tiles complete on level {2}/{3}".format( - complete, total, level, self.p)) - def _get_cur_depth(self): """How deep is the quadtree currently in the destdir? This glances in config.js to see what maxZoom is set to. @@ -219,64 +181,9 @@ class QuadtreeGen(object): os.rename(getpath("3", "0"), getpath("new3")) shutil.rmtree(getpath("3")) os.rename(getpath("new3"), getpath("3")) - - def _apply_render_worldtiles(self, pool,batch_size): - """Returns an iterator over result objects. Each time a new result is - requested, a new task is added to the pool and a result returned. - """ - - batch = [] - tiles = 0 - for path in iterate_base4(self.p): - # Get the range for this tile - colstart, rowstart = self._get_range_by_path(path) - colend = colstart + 2 - rowend = rowstart + 4 - - # This image is rendered at(relative to the worker's destdir): - tilepath = [str(x) for x in path] - tilepath = os.sep.join(tilepath) - #logging.debug("this is rendered at %s", dest) - - # Put this in the batch to be submited to the pool - batch.append((colstart, colend, rowstart, rowend, tilepath)) - tiles += 1 - if tiles >= batch_size: - tiles = 0 - yield pool.apply_async(func=render_worldtile_batch, args= [batch]) - batch = [] - - if tiles > 0: - yield pool.apply_async(func=render_worldtile_batch, args= (batch,)) - - - def _apply_render_inntertile(self, pool, zoom,batch_size): - """Same as _apply_render_worltiles but for the inntertile routine. - Returns an iterator that yields result objects from tasks that have - been applied to the pool. - """ - batch = [] - tiles = 0 - for path in iterate_base4(zoom): - # This image is rendered at(relative to the worker's destdir): - tilepath = [str(x) for x in path[:-1]] - tilepath = os.sep.join(tilepath) - name = str(path[-1]) - - - self.full_tiledir - batch.append((tilepath, name, self.imgformat, self.optimizeimg)) - tiles += 1 - if tiles >= batch_size: - tiles = 0 - yield pool.apply_async(func=render_innertile_batch, args= [batch]) - batch = [] - - if tiles > 0: - yield pool.apply_async(func=render_innertile_batch, args= [batch]) def go(self, procs): - """Renders all tiles""" + """Processing before tile rendering""" curdepth = self._get_cur_depth() if curdepth != -1: @@ -289,95 +196,8 @@ class QuadtreeGen(object): logging.warning("Your map seems to have shrunk. Re-arranging tiles, just a sec...") for _ in xrange(curdepth - self.p): self._decrease_depth() - - logging.debug("Parent process {0}".format(os.getpid())) - # Create a pool - if procs == 1: - pool = FakePool() - global child_quadtree - child_quadtree = self - else: - pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) - #warm up the pool so it reports all the worker id's - pool.map(bool,xrange(multiprocessing.cpu_count()),1) - - # Render the highest level of tiles from the chunks - results = collections.deque() - complete = 0 - total = 4**self.p - logging.info("Rendering highest zoom level of tiles now.") - logging.info("There are {0} tiles to render".format(total)) - logging.info("There are {0} total levels to render".format(self.p)) - logging.info("Don't worry, each level has only 25% as many tiles as the last.") - logging.info("The others will go faster") - count = 0 - batch_size = 8 - timestamp = time.time() - for result in self._apply_render_worldtiles(pool,batch_size): - results.append(result) - # every second drain some of the queue - timestamp2 = time.time() - if timestamp2 >= timestamp + 1: - timestamp = timestamp2 - count_to_remove = (1000//batch_size) - if count_to_remove < len(results): - while count_to_remove > 0: - count_to_remove -= 1 - complete += results.popleft().get() - self.print_statusline(complete, total, 1) - if len(results) > (10000//batch_size): - # Empty the queue before adding any more, so that memory - # required has an upper bound - while len(results) > (500//batch_size): - complete += results.popleft().get() - self.print_statusline(complete, total, 1) - - # Wait for the rest of the results - while len(results) > 0: - complete += results.popleft().get() - self.print_statusline(complete, total, 1) - - self.print_statusline(complete, total, 1, True) - - # Now do the other layers - for zoom in xrange(self.p-1, 0, -1): - level = self.p - zoom + 1 - assert len(results) == 0 - complete = 0 - total = 4**zoom - logging.info("Starting level {0}".format(level)) - timestamp = time.time() - for result in self._apply_render_inntertile(pool, zoom,batch_size): - results.append(result) - # every second drain some of the queue - timestamp2 = time.time() - if timestamp2 >= timestamp + 1: - timestamp = timestamp2 - count_to_remove = (1000//batch_size) - if count_to_remove < len(results): - while count_to_remove > 0: - count_to_remove -= 1 - complete += results.popleft().get() - self.print_statusline(complete, total, 1) - if len(results) > (10000/batch_size): - while len(results) > (500/batch_size): - complete += results.popleft().get() - self.print_statusline(complete, total, level) - # Empty the queue - while len(results) > 0: - complete += results.popleft().get() - self.print_statusline(complete, total, level) - - self.print_statusline(complete, total, level, True) - - logging.info("Done") - - pool.close() - pool.join() - - # Do the final one right here: - render_innertile(os.path.join(self.destdir, self.tiledir), "base", self.imgformat, self.optimizeimg) - + + def _get_range_by_path(self, path): """Returns the x, y chunk coordinates of this tile""" x, y = self.mincol, self.minrow @@ -394,259 +214,214 @@ class QuadtreeGen(object): ysize //= 2 return x, y + + def get_worldtiles(self): + """Returns an iterator over the tiles of the most detailed layer + """ + for path in iterate_base4(self.p): + # Get the range for this tile + colstart, rowstart = self._get_range_by_path(path) + colend = colstart + 2 + rowend = rowstart + 4 + + # This image is rendered at(relative to the worker's destdir): + tilepath = [str(x) for x in path] + tilepath = os.sep.join(tilepath) + #logging.debug("this is rendered at %s", dest) + + # Put this in the batch to be submited to the pool + yield [self,colstart, colend, rowstart, rowend, tilepath] + + def get_innertiles(self,zoom): + """Same as get_worldtiles but for the inntertile routine. + """ + for path in iterate_base4(zoom): + # This image is rendered at(relative to the worker's destdir): + tilepath = [str(x) for x in path[:-1]] + tilepath = os.sep.join(tilepath) + name = str(path[-1]) + + yield [self,tilepath, name] + + def render_innertile(self, dest, name): + """ + Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from + os.path.join(dest, name, "{0,1,2,3}.png") + """ + imgformat = self.imgformat + imgpath = os.path.join(dest, name) + "." + imgformat - def _get_chunks_in_range(self, colstart, colend, rowstart, rowend): - """Get chunks that are relevant to the tile rendering function that's - rendering that range""" - chunklist = [] - unconvert_coords = self.world.unconvert_coords - #get_region_path = self.world.get_region_path - get_region = self.world.regionfiles.get - for row in xrange(rowstart-16, rowend+1): - for col in xrange(colstart, colend+1): - # due to how chunks are arranged, we can only allow - # even row, even column or odd row, odd column - # otherwise, you end up with duplicates! - if row % 2 != col % 2: + if name == "base": + quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]] + else: + quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]] + + #stat the tile, we need to know if it exists or it's mtime + try: + tile_mtime = os.stat(imgpath)[stat.ST_MTIME]; + except OSError, e: + if e.errno != errno.ENOENT: + raise + tile_mtime = None + + #check mtimes on each part of the quad, this also checks if they exist + needs_rerender = tile_mtime is None + quadPath_filtered = [] + for path in quadPath: + try: + quad_mtime = os.stat(path[1])[stat.ST_MTIME]; + quadPath_filtered.append(path) + if quad_mtime > tile_mtime: + needs_rerender = True + except OSError: + # We need to stat all the quad files, so keep looping + pass + # do they all not exist? + if quadPath_filtered == []: + if tile_mtime is not None: + os.unlink(imgpath) + return + # quit now if we don't need rerender + if not needs_rerender: + return + #logging.debug("writing out innertile {0}".format(imgpath)) + + # Create the actual image now + img = Image.new("RGBA", (384, 384), (38,92,255,0)) + + # we'll use paste (NOT alpha_over) for quadtree generation because + # this is just straight image stitching, not alpha blending + + for path in quadPath_filtered: + try: + quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS) + img.paste(quad, path[0]) + except Exception, e: + logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e) + + # Save it + if self.imgformat == 'jpg': + img.save(imgpath, quality=95, subsampling=0) + else: # png + img.save(imgpath) + + if self.optimizeimg: + optimize_image(imgpath, self.imgformat, self.optimizeimg) + + + + def render_worldtile(self, chunks, colstart, colend, rowstart, rowend, path): + """Renders just the specified chunks into a tile and save it. Unlike usual + python conventions, rowend and colend are inclusive. Additionally, the + chunks around the edges are half-way cut off (so that neighboring tiles + will render the other half) + + chunks is a list of (col, row, chunkx, chunky, filename) of chunk + images that are relevant to this call (with their associated regions) + + The image is saved to path+"."+self.imgformat + + If there are no chunks, this tile is not saved (if it already exists, it is + deleted) + + Standard tile size has colend-colstart=2 and rowend-rowstart=4 + + There is no return value + """ + + # width of one chunk is 384. Each column is half a chunk wide. The total + # width is (384 + 192*(numcols-1)) since the first column contributes full + # width, and each additional one contributes half since they're staggered. + # However, since we want to cut off half a chunk at each end (384 less + # pixels) and since (colend - colstart + 1) is the number of columns + # inclusive, the equation simplifies to: + width = 192 * (colend - colstart) + # Same deal with height + height = 96 * (rowend - rowstart) + + # The standard tile size is 3 columns by 5 rows, which works out to 384x384 + # pixels for 8 total chunks. (Since the chunks are staggered but the grid + # is not, some grid coordinates do not address chunks) The two chunks on + # the middle column are shown in full, the two chunks in the middle row are + # half cut off, and the four remaining chunks are one quarter shown. + # The above example with cols 0-3 and rows 0-4 has the chunks arranged like this: + # 0,0 2,0 + # 1,1 + # 0,2 2,2 + # 1,3 + # 0,4 2,4 + + # Due to how the tiles fit together, we may need to render chunks way above + # this (since very few chunks actually touch the top of the sky, some tiles + # way above this one are possibly visible in this tile). Render them + # anyways just in case). "chunks" should include up to rowstart-16 + + imgpath = path + "." + self.imgformat + world = self.world + #stat the file, we need to know if it exists or it's mtime + try: + tile_mtime = os.stat(imgpath)[stat.ST_MTIME]; + except OSError, e: + if e.errno != errno.ENOENT: + raise + tile_mtime = None + + if not chunks: + # No chunks were found in this tile + if tile_mtime is not None: + os.unlink(imgpath) + return None + + # Create the directory if not exists + dirdest = os.path.dirname(path) + if not os.path.exists(dirdest): + try: + os.makedirs(dirdest) + except OSError, e: + # Ignore errno EEXIST: file exists. Since this is multithreaded, + # two processes could conceivably try and create the same directory + # at the same time. + if e.errno != errno.EEXIST: + raise + + # check chunk mtimes to see if they are newer + try: + needs_rerender = False + for col, row, chunkx, chunky, regionfile in chunks: + # check region file mtime first. + region,regionMtime = world.get_region_mtime(regionfile) + if regionMtime <= tile_mtime: continue - # return (col, row, chunkx, chunky, regionpath) - chunkx, chunky = unconvert_coords(col, row) - #c = get_region_path(chunkx, chunky) - _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None)); - if c is not None and mcr.chunkExists(chunkx,chunky): - chunklist.append((col, row, chunkx, chunky, c)) - return chunklist - -@catch_keyboardinterrupt -def render_innertile_batch(batch): - global child_quadtree - quadtree = child_quadtree - count = 0 - #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) - for job in batch: - count += 1 - dest = quadtree.full_tiledir+os.sep+job[0] - render_innertile(dest,job[1],job[2],job[3]) - return count - -def render_innertile(dest, name, imgformat, optimizeimg): - """ - Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from - os.path.join(dest, name, "{0,1,2,3}.png") - """ - imgpath = os.path.join(dest, name) + "." + imgformat - - if name == "base": - quadPath = [[(0,0),os.path.join(dest, "0." + imgformat)],[(192,0),os.path.join(dest, "1." + imgformat)], [(0, 192),os.path.join(dest, "2." + imgformat)],[(192,192),os.path.join(dest, "3." + imgformat)]] - else: - quadPath = [[(0,0),os.path.join(dest, name, "0." + imgformat)],[(192,0),os.path.join(dest, name, "1." + imgformat)],[(0, 192),os.path.join(dest, name, "2." + imgformat)],[(192,192),os.path.join(dest, name, "3." + imgformat)]] - - #stat the tile, we need to know if it exists or it's mtime - try: - tile_mtime = os.stat(imgpath)[stat.ST_MTIME]; - except OSError, e: - if e.errno != errno.ENOENT: - raise - tile_mtime = None - - #check mtimes on each part of the quad, this also checks if they exist - needs_rerender = tile_mtime is None - quadPath_filtered = [] - for path in quadPath: - try: - quad_mtime = os.stat(path[1])[stat.ST_MTIME]; - quadPath_filtered.append(path) - if quad_mtime > tile_mtime: - needs_rerender = True + # checking chunk mtime + if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime: + needs_rerender = True + break + + # if after all that, we don't need a rerender, return + if not needs_rerender: + return None except OSError: - # We need to stat all the quad files, so keep looping - pass - # do they all not exist? - if quadPath_filtered == []: - if tile_mtime is not None: - os.unlink(imgpath) - return - # quit now if we don't need rerender - if not needs_rerender: - return - #logging.debug("writing out innertile {0}".format(imgpath)) - - # Create the actual image now - img = Image.new("RGBA", (384, 384), (38,92,255,0)) - - # we'll use paste (NOT alpha_over) for quadtree generation because - # this is just straight image stitching, not alpha blending - - for path in quadPath_filtered: - try: - quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS) - img.paste(quad, path[0]) - except Exception, e: - logging.warning("Couldn't open %s. It may be corrupt, you may need to delete it. %s", path[1], e) - - # Save it - if imgformat == 'jpg': - img.save(imgpath, quality=95, subsampling=0) - else: # png - img.save(imgpath) - if optimizeimg: - optimize_image(imgpath, imgformat, optimizeimg) - -@catch_keyboardinterrupt -def render_worldtile_batch(batch): - global child_quadtree - quadtree = child_quadtree - count = 0 - _get_chunks_in_range = quadtree._get_chunks_in_range - #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) - for job in batch: - count += 1 - colstart = job[0] - colend = job[1] - rowstart = job[2] - rowend = job[3] - path = job[4] - path = quadtree.full_tiledir+os.sep+path - # (even if tilechunks is empty, render_worldtile will delete - # existing images if appropriate) - # And uses these chunks - tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) - #logging.debug(" tilechunks: %r", tilechunks) - - render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path) - return count - -def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path): - """Renders just the specified chunks into a tile and save it. Unlike usual - python conventions, rowend and colend are inclusive. Additionally, the - chunks around the edges are half-way cut off (so that neighboring tiles - will render the other half) - - chunks is a list of (col, row, chunkx, chunky, filename) of chunk - images that are relevant to this call (with their associated regions) - - The image is saved to path+"."+quadtree.imgformat - - If there are no chunks, this tile is not saved (if it already exists, it is - deleted) - - Standard tile size has colend-colstart=2 and rowend-rowstart=4 - - There is no return value - """ - - # width of one chunk is 384. Each column is half a chunk wide. The total - # width is (384 + 192*(numcols-1)) since the first column contributes full - # width, and each additional one contributes half since they're staggered. - # However, since we want to cut off half a chunk at each end (384 less - # pixels) and since (colend - colstart + 1) is the number of columns - # inclusive, the equation simplifies to: - width = 192 * (colend - colstart) - # Same deal with height - height = 96 * (rowend - rowstart) - - # The standard tile size is 3 columns by 5 rows, which works out to 384x384 - # pixels for 8 total chunks. (Since the chunks are staggered but the grid - # is not, some grid coordinates do not address chunks) The two chunks on - # the middle column are shown in full, the two chunks in the middle row are - # half cut off, and the four remaining chunks are one quarter shown. - # The above example with cols 0-3 and rows 0-4 has the chunks arranged like this: - # 0,0 2,0 - # 1,1 - # 0,2 2,2 - # 1,3 - # 0,4 2,4 - - # Due to how the tiles fit together, we may need to render chunks way above - # this (since very few chunks actually touch the top of the sky, some tiles - # way above this one are possibly visible in this tile). Render them - # anyways just in case). "chunks" should include up to rowstart-16 - - imgpath = path + "." + quadtree.imgformat - world = quadtree.world - #stat the file, we need to know if it exists or it's mtime - try: - tile_mtime = os.stat(imgpath)[stat.ST_MTIME]; - except OSError, e: - if e.errno != errno.ENOENT: - raise - tile_mtime = None + # couldn't get tile mtime, skip check + pass - if not chunks: - # No chunks were found in this tile - if tile_mtime is not None: - os.unlink(imgpath) - return None + #logging.debug("writing out worldtile {0}".format(imgpath)) - # Create the directory if not exists - dirdest = os.path.dirname(path) - if not os.path.exists(dirdest): - try: - os.makedirs(dirdest) - except OSError, e: - # Ignore errno EEXIST: file exists. Since this is multithreaded, - # two processes could conceivably try and create the same directory - # at the same time. - if e.errno != errno.EEXIST: - raise - - # check chunk mtimes to see if they are newer - try: - needs_rerender = False + # Compile this image + tileimg = Image.new("RGBA", (width, height), (38,92,255,0)) + + # col colstart will get drawn on the image starting at x coordinates -(384/2) + # row rowstart will get drawn on the image starting at y coordinates -(192/2) for col, row, chunkx, chunky, regionfile in chunks: - # check region file mtime first. - region,regionMtime = world.get_region_mtime(regionfile) - if regionMtime <= tile_mtime: - continue - - # checking chunk mtime - if region.get_chunk_timestamp(chunkx, chunky) > tile_mtime: - needs_rerender = True - break - - # if after all that, we don't need a rerender, return - if not needs_rerender: - return None - except OSError: - # couldn't get tile mtime, skip check - pass - - #logging.debug("writing out worldtile {0}".format(imgpath)) + xpos = -192 + (col-colstart)*192 + ypos = -96 + (row-rowstart)*96 - # Compile this image - tileimg = Image.new("RGBA", (width, height), (38,92,255,0)) + # draw the chunk! + # TODO POI queue + chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), self, False, None) - # col colstart will get drawn on the image starting at x coordinates -(384/2) - # row rowstart will get drawn on the image starting at y coordinates -(192/2) - for col, row, chunkx, chunky, regionfile in chunks: - xpos = -192 + (col-colstart)*192 - ypos = -96 + (row-rowstart)*96 + # Save them + tileimg.save(imgpath) - # draw the chunk! - # TODO POI queue - chunk.render_to_image((chunkx, chunky), tileimg, (xpos, ypos), quadtree, False, None) - - # Save them - tileimg.save(imgpath) - - if quadtree.optimizeimg: - optimize_image(imgpath, quadtree.imgformat, quadtree.optimizeimg) - -class FakeResult(object): - def __init__(self, res): - self.res = res - def get(self): - return self.res -class FakePool(object): - """A fake pool used to render things in sync. Implements a subset of - multiprocessing.Pool""" - def apply_async(self, func, args=(), kwargs=None): - if not kwargs: - kwargs = {} - result = func(*args, **kwargs) - return FakeResult(result) - def close(self): - pass - def join(self): - pass + if self.optimizeimg: + optimize_image(imgpath, self.imgformat, self.optimizeimg) diff --git a/rendernode.py b/rendernode.py new file mode 100644 index 0000000..5680beb --- /dev/null +++ b/rendernode.py @@ -0,0 +1,346 @@ +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import multiprocessing +import itertools +from itertools import cycle, islice +import os +import os.path +import functools +import re +import shutil +import collections +import json +import logging +import util +import cPickle +import stat +import errno +import time +from time import gmtime, strftime, sleep + + +""" +This module has routines related to distributing the render job to multipule nodes + +""" + +def catch_keyboardinterrupt(func): + """Decorator that catches a keyboardinterrupt and raises a real exception + so that multiprocessing will propagate it properly""" + @functools.wraps(func) + def newfunc(*args, **kwargs): + try: + return func(*args, **kwargs) + except KeyboardInterrupt: + logging.error("Ctrl-C caught!") + raise Exception("Exiting") + except: + import traceback + traceback.print_exc() + raise + return newfunc + +child_rendernode = None +def pool_initializer(rendernode): + logging.debug("Child process {0}".format(os.getpid())) + #stash the quadtree objects in a global variable after fork() for windows compat. + global child_rendernode + child_rendernode = rendernode + +#http://docs.python.org/library/itertools.html +def roundrobin(iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + # Recipe credited to George Sakkis + pending = len(iterables) + nexts = cycle(iter(it).next for it in iterables) + while pending: + try: + for next in nexts: + yield next() + except StopIteration: + pending -= 1 + nexts = cycle(islice(nexts, pending)) + + +class RenderNode(object): + def __init__(self, world, quadtrees): + """Distributes the rendering of a list of quadtrees. All of the quadtrees must have the same world.""" + + if not len(quadtrees) > 0: + raise ValueError("there must be at least one quadtree to work on") + + self.world = world + self.quadtrees = quadtrees + #bind an index value to the quadtree so we can find it again + i = 0 + for q in quadtrees: + q._render_index = i + i += 1 + + def print_statusline(self, complete, total, level, unconditional=False): + if unconditional: + pass + elif complete < 100: + if not complete % 25 == 0: + return + elif complete < 1000: + if not complete % 100 == 0: + return + else: + if not complete % 1000 == 0: + return + logging.info("{0}/{1} tiles complete on level {2}/{3}".format( + complete, total, level, self.max_p)) + + def go(self, procs): + """Renders all tiles""" + + logging.debug("Parent process {0}".format(os.getpid())) + # Create a pool + if procs == 1: + pool = FakePool() + global child_rendernode + child_rendernode = self + else: + pool = multiprocessing.Pool(processes=procs,initializer=pool_initializer,initargs=(self,)) + #warm up the pool so it reports all the worker id's + pool.map(bool,xrange(multiprocessing.cpu_count()),1) + + quadtrees = self.quadtrees + + # do per-quadtree init. + max_p = 0 + total = 0 + for q in quadtrees: + total += 4**q.p + if q.p > max_p: + max_p = q.p + q.go(procs) + self.max_p = max_p + # Render the highest level of tiles from the chunks + results = collections.deque() + complete = 0 + logging.info("Rendering highest zoom level of tiles now.") + logging.info("Rendering {0} layer{1}".format(len(quadtrees),'s' if len(quadtrees) > 1 else '' )) + logging.info("There are {0} tiles to render".format(total)) + logging.info("There are {0} total levels to render".format(self.max_p)) + logging.info("Don't worry, each level has only 25% as many tiles as the last.") + logging.info("The others will go faster") + count = 0 + batch_size = 4*len(quadtrees) + while batch_size < 10: + batch_size *= 2 + timestamp = time.time() + for result in self._apply_render_worldtiles(pool,batch_size): + results.append(result) + # every second drain some of the queue + timestamp2 = time.time() + if timestamp2 >= timestamp + 1: + timestamp = timestamp2 + count_to_remove = (1000//batch_size) + if count_to_remove < len(results): + while count_to_remove > 0: + count_to_remove -= 1 + complete += results.popleft().get() + self.print_statusline(complete, total, 1) + if len(results) > (10000//batch_size): + # Empty the queue before adding any more, so that memory + # required has an upper bound + while len(results) > (500//batch_size): + complete += results.popleft().get() + self.print_statusline(complete, total, 1) + + # Wait for the rest of the results + while len(results) > 0: + complete += results.popleft().get() + self.print_statusline(complete, total, 1) + + self.print_statusline(complete, total, 1, True) + + # Now do the other layers + for zoom in xrange(self.max_p-1, 0, -1): + level = self.max_p - zoom + 1 + assert len(results) == 0 + complete = 0 + total = 0 + for q in quadtrees: + if zoom <= q.p: + total += 4**zoom + logging.info("Starting level {0}".format(level)) + timestamp = time.time() + for result in self._apply_render_inntertile(pool, zoom,batch_size): + results.append(result) + # every second drain some of the queue + timestamp2 = time.time() + if timestamp2 >= timestamp + 1: + timestamp = timestamp2 + count_to_remove = (1000//batch_size) + if count_to_remove < len(results): + while count_to_remove > 0: + count_to_remove -= 1 + complete += results.popleft().get() + self.print_statusline(complete, total, 1) + if len(results) > (10000/batch_size): + while len(results) > (500/batch_size): + complete += results.popleft().get() + self.print_statusline(complete, total, level) + # Empty the queue + while len(results) > 0: + complete += results.popleft().get() + self.print_statusline(complete, total, level) + + self.print_statusline(complete, total, level, True) + + logging.info("Done") + + pool.close() + pool.join() + + # Do the final one right here: + for q in quadtrees: + q.render_innertile(os.path.join(q.destdir, q.tiledir), "base") + + + def _get_chunks_in_range(self, colstart, colend, rowstart, rowend): + """Get chunks that are relevant to the tile rendering function that's + rendering that range""" + chunklist = [] + unconvert_coords = self.world.unconvert_coords + #get_region_path = self.world.get_region_path + get_region = self.world.regionfiles.get + for row in xrange(rowstart-16, rowend+1): + for col in xrange(colstart, colend+1): + # due to how chunks are arranged, we can only allow + # even row, even column or odd row, odd column + # otherwise, you end up with duplicates! + if row % 2 != col % 2: + continue + + # return (col, row, chunkx, chunky, regionpath) + chunkx, chunky = unconvert_coords(col, row) + #c = get_region_path(chunkx, chunky) + _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None)); + if c is not None and mcr.chunkExists(chunkx,chunky): + chunklist.append((col, row, chunkx, chunky, c)) + return chunklist + + + + def _apply_render_worldtiles(self, pool,batch_size): + """Returns an iterator over result objects. Each time a new result is + requested, a new task is added to the pool and a result returned. + """ + if batch_size < len(self.quadtrees): + batch_size = len(self.quadtrees) + batch = [] + jobcount = 0 + # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks) + iterables = [q.get_worldtiles() for q in self.quadtrees] + for job in roundrobin(iterables): + # fixup so the worker knows which quadtree this is + job[0] = job[0]._render_index + # Put this in the batch to be submited to the pool + batch.append(job) + jobcount += 1 + if jobcount >= batch_size: + jobcount = 0 + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) + batch = [] + if jobcount > 0: + yield pool.apply_async(func=render_worldtile_batch, args= [batch]) + + def _apply_render_inntertile(self, pool, zoom,batch_size): + """Same as _apply_render_worltiles but for the inntertile routine. + Returns an iterator that yields result objects from tasks that have + been applied to the pool. + """ + + if batch_size < len(self.quadtrees): + batch_size = len(self.quadtrees) + batch = [] + jobcount = 0 + # roundrobin add tiles to a batch job (thus they should all roughly work on similar chunks) + iterables = [q.get_innertiles(zoom) for q in self.quadtrees if zoom <= q.p] + for job in roundrobin(iterables): + # fixup so the worker knows which quadtree this is + job[0] = job[0]._render_index + # Put this in the batch to be submited to the pool + batch.append(job) + jobcount += 1 + if jobcount >= batch_size: + jobcount = 0 + yield pool.apply_async(func=render_innertile_batch, args= [batch]) + batch = [] + + if jobcount > 0: + yield pool.apply_async(func=render_innertile_batch, args= [batch]) + +@catch_keyboardinterrupt +def render_worldtile_batch(batch): + global child_rendernode + rendernode = child_rendernode + count = 0 + _get_chunks_in_range = rendernode._get_chunks_in_range + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + quadtree = rendernode.quadtrees[job[0]] + colstart = job[1] + colend = job[2] + rowstart = job[3] + rowend = job[4] + path = job[5] + path = quadtree.full_tiledir+os.sep+path + # (even if tilechunks is empty, render_worldtile will delete + # existing images if appropriate) + # And uses these chunks + tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) + #logging.debug(" tilechunks: %r", tilechunks) + + quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path) + return count + +@catch_keyboardinterrupt +def render_innertile_batch(batch): + global child_rendernode + rendernode = child_rendernode + count = 0 + #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) + for job in batch: + count += 1 + quadtree = rendernode.quadtrees[job[0]] + dest = quadtree.full_tiledir+os.sep+job[1] + quadtree.render_innertile(dest=dest,name=job[2]) + return count + +class FakeResult(object): + def __init__(self, res): + self.res = res + def get(self): + return self.res +class FakePool(object): + """A fake pool used to render things in sync. Implements a subset of + multiprocessing.Pool""" + def apply_async(self, func, args=(), kwargs=None): + if not kwargs: + kwargs = {} + result = func(*args, **kwargs) + return FakeResult(result) + def close(self): + pass + def join(self): + pass + \ No newline at end of file From c7920ce61e726777a77d54b0512967806d1a6285 Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 23:23:42 +0800 Subject: [PATCH 07/11] Fixed multi-world support in multi-layer renderer --- quadtree.py | 23 +++++++++++++++++++++++ rendernode.py | 34 +++------------------------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/quadtree.py b/quadtree.py index 60c097d..2b532a1 100644 --- a/quadtree.py +++ b/quadtree.py @@ -215,6 +215,29 @@ class QuadtreeGen(object): return x, y + def get_chunks_in_range(self, colstart, colend, rowstart, rowend): + """Get chunks that are relevant to the tile rendering function that's + rendering that range""" + chunklist = [] + unconvert_coords = self.world.unconvert_coords + #get_region_path = self.world.get_region_path + get_region = self.world.regionfiles.get + for row in xrange(rowstart-16, rowend+1): + for col in xrange(colstart, colend+1): + # due to how chunks are arranged, we can only allow + # even row, even column or odd row, odd column + # otherwise, you end up with duplicates! + if row % 2 != col % 2: + continue + + # return (col, row, chunkx, chunky, regionpath) + chunkx, chunky = unconvert_coords(col, row) + #c = get_region_path(chunkx, chunky) + _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None)); + if c is not None and mcr.chunkExists(chunkx,chunky): + chunklist.append((col, row, chunkx, chunky, c)) + return chunklist + def get_worldtiles(self): """Returns an iterator over the tiles of the most detailed layer """ diff --git a/rendernode.py b/rendernode.py index 5680beb..6cfae8a 100644 --- a/rendernode.py +++ b/rendernode.py @@ -76,13 +76,12 @@ def roundrobin(iterables): class RenderNode(object): - def __init__(self, world, quadtrees): - """Distributes the rendering of a list of quadtrees. All of the quadtrees must have the same world.""" + def __init__(self, quadtrees): + """Distributes the rendering of a list of quadtrees.""" if not len(quadtrees) > 0: raise ValueError("there must be at least one quadtree to work on") - self.world = world self.quadtrees = quadtrees #bind an index value to the quadtree so we can find it again i = 0 @@ -213,32 +212,6 @@ class RenderNode(object): for q in quadtrees: q.render_innertile(os.path.join(q.destdir, q.tiledir), "base") - - def _get_chunks_in_range(self, colstart, colend, rowstart, rowend): - """Get chunks that are relevant to the tile rendering function that's - rendering that range""" - chunklist = [] - unconvert_coords = self.world.unconvert_coords - #get_region_path = self.world.get_region_path - get_region = self.world.regionfiles.get - for row in xrange(rowstart-16, rowend+1): - for col in xrange(colstart, colend+1): - # due to how chunks are arranged, we can only allow - # even row, even column or odd row, odd column - # otherwise, you end up with duplicates! - if row % 2 != col % 2: - continue - - # return (col, row, chunkx, chunky, regionpath) - chunkx, chunky = unconvert_coords(col, row) - #c = get_region_path(chunkx, chunky) - _, _, c, mcr = get_region((chunkx//32, chunky//32),(None,None,None,None)); - if c is not None and mcr.chunkExists(chunkx,chunky): - chunklist.append((col, row, chunkx, chunky, c)) - return chunklist - - - def _apply_render_worldtiles(self, pool,batch_size): """Returns an iterator over result objects. Each time a new result is requested, a new task is added to the pool and a result returned. @@ -293,7 +266,6 @@ def render_worldtile_batch(batch): global child_rendernode rendernode = child_rendernode count = 0 - _get_chunks_in_range = rendernode._get_chunks_in_range #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch))) for job in batch: count += 1 @@ -307,7 +279,7 @@ def render_worldtile_batch(batch): # (even if tilechunks is empty, render_worldtile will delete # existing images if appropriate) # And uses these chunks - tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend) + tilechunks = quadtree.get_chunks_in_range(colstart, colend, rowstart,rowend) #logging.debug(" tilechunks: %r", tilechunks) quadtree.render_worldtile(tilechunks,colstart, colend, rowstart, rowend, path) From e55b7045ea0a546ce405a0d90c428a2a6c37276f Mon Sep 17 00:00:00 2001 From: Xon Date: Wed, 23 Mar 2011 23:40:48 +0800 Subject: [PATCH 08/11] Updated findSigns.py & rerenderBlocks.py to work against region format --- chunk.py | 8 ++++---- contrib/findSigns.py | 38 +++++++++++++++++++++----------------- contrib/rerenderBlocks.py | 21 +++++++++++++-------- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/chunk.py b/chunk.py index 63de064..b9bc817 100644 --- a/chunk.py +++ b/chunk.py @@ -73,10 +73,10 @@ def get_blockarray(level): Block array, which just contains all the block ids""" return numpy.frombuffer(level['Blocks'], dtype=numpy.uint8).reshape((16,16,128)) -def get_blockarray_fromfile(world,filename): - """Same as get_blockarray except takes a filename and uses get_lvldata to - open it. This is a shortcut""" - level = get_lvldata(world,filename) +def get_blockarray_fromfile(filename): + """Same as get_blockarray except takes a filename. This is a shortcut""" + d = nbt.load_from_region(filename, x, y) + level = return d[1]['Level'] return get_blockarray(level) def get_skylight_array(level): diff --git a/contrib/findSigns.py b/contrib/findSigns.py index c390654..9e3b621 100644 --- a/contrib/findSigns.py +++ b/contrib/findSigns.py @@ -33,30 +33,34 @@ if os.path.exists(worlddir): else: sys.exit("Bad WorldDir") -matcher = re.compile(r"^c\..*\.dat$") +matcher = re.compile(r"^r\..*\.mcr$") POI = [] for dirpath, dirnames, filenames in os.walk(worlddir): for f in filenames: if matcher.match(f): + print f full = os.path.join(dirpath, f) - #print "inspecting %s" % full - data = nbt.load(full)[1]['Level']['TileEntities'] - for entity in data: - if entity['id'] == 'Sign': - msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']]) - #print "checking -->%s<--" % msg.strip() - if msg.strip(): - newPOI = dict(type="sign", - x= entity['x'], - y= entity['y'], - z= entity['z'], - msg=msg, - chunk= (entity['x']/16, entity['z']/16), - ) - POI.append(newPOI) - print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg']) + r = nbt.load_region(full) + chunks = r.get_chunks() + for x,y in chunks: + chunk = r.load_chunk(x,y).read_all() + data = chunk[1]['Level']['TileEntities'] + for entity in data: + if entity['id'] == 'Sign': + msg=' \n'.join([entity['Text1'], entity['Text2'], entity['Text3'], entity['Text4']]) + #print "checking -->%s<--" % msg.strip() + if msg.strip(): + newPOI = dict(type="sign", + x= entity['x'], + y= entity['y'], + z= entity['z'], + msg=msg, + chunk= (entity['x']/16, entity['z']/16), + ) + POI.append(newPOI) + print "Found sign at (%d, %d, %d): %r" % (newPOI['x'], newPOI['y'], newPOI['z'], newPOI['msg']) diff --git a/contrib/rerenderBlocks.py b/contrib/rerenderBlocks.py index 51940df..d0fa0df 100644 --- a/contrib/rerenderBlocks.py +++ b/contrib/rerenderBlocks.py @@ -7,8 +7,8 @@ blockID. The output is a chunklist file that is suitable to use with the Example: -python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> chunklist.txt - python overviewer.py --chunklist=chunklist.txt world/ output_dir/ +python contrib/rerenderBlocks.py --ids=46,79,91 --world=world/> regionlist.txt + python overviewer.py --regionlist=regionlist.txt world/ output_dir/ This will rerender any chunks that contain either TNT (46), Ice (79), or a Jack-O-Lantern (91) @@ -42,15 +42,20 @@ ids = map(lambda x: int(x),options.ids.split(",")) sys.stderr.write("Searching for these blocks: %r...\n" % ids) -matcher = re.compile(r"^c\..*\.dat$") +matcher = re.compile(r"^r\..*\.mcr$") for dirpath, dirnames, filenames in os.walk(options.world): for f in filenames: if matcher.match(f): full = os.path.join(dirpath, f) - blocks = get_blockarray_fromfile(full) - for i in ids: - if i in blocks: - print full - break + r = nbt.load_region(full) + chunks = r.get_chunks() + for x,y in chunks: + chunk = r.load_chunk(x,y).read_all() + blocks = get_blockarray(chunk[1]['Level']) + for i in ids: + if i in blocks: + print full + break + From 01790913b309a8298772a27fdd7f1290e2025af2 Mon Sep 17 00:00:00 2001 From: Xon Date: Fri, 25 Mar 2011 21:34:06 +0800 Subject: [PATCH 09/11] Check PyImport_ImportModule return result --- overviewer.py | 2 ++ src/iterate.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/overviewer.py b/overviewer.py index cb1b0a9..2c626c2 100755 --- a/overviewer.py +++ b/overviewer.py @@ -32,6 +32,8 @@ import util logging.basicConfig(level=logging.INFO,format="%(asctime)s [%(levelname)s] %(message)s") +#import this before to ensure it doesn't have an errors or c_overviewer will eat them +import chunk # make sure the c_overviewer extension is available try: import c_overviewer diff --git a/src/iterate.c b/src/iterate.c index 8b93cd8..1dcce1a 100644 --- a/src/iterate.c +++ b/src/iterate.c @@ -34,6 +34,12 @@ int init_chunk_render(void) { textures = PyImport_ImportModule("textures"); chunk_mod = PyImport_ImportModule("chunk"); + /* ensure none of these pointers are NULL */ + if ((!textures) || (!chunk_mod)) { + fprintf(stderr, "\ninit_chunk_render failed\n"); + return 1; + } + blockmap = PyObject_GetAttrString(textures, "blockmap"); special_blocks = PyObject_GetAttrString(textures, "special_blocks"); specialblockmap = PyObject_GetAttrString(textures, "specialblockmap"); From 21d04cd3fbeddd90b097128672fb0c28218d7212 Mon Sep 17 00:00:00 2001 From: Xon Date: Fri, 25 Mar 2011 21:42:18 +0800 Subject: [PATCH 10/11] Better error messages, removed import which triggered a failure --- overviewer.py | 2 -- src/iterate.c | 13 +++++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/overviewer.py b/overviewer.py index 2c626c2..cb1b0a9 100755 --- a/overviewer.py +++ b/overviewer.py @@ -32,8 +32,6 @@ import util logging.basicConfig(level=logging.INFO,format="%(asctime)s [%(levelname)s] %(message)s") -#import this before to ensure it doesn't have an errors or c_overviewer will eat them -import chunk # make sure the c_overviewer extension is available try: import c_overviewer diff --git a/src/iterate.c b/src/iterate.c index 1dcce1a..f535f39 100644 --- a/src/iterate.c +++ b/src/iterate.c @@ -32,11 +32,16 @@ int init_chunk_render(void) { if (blockmap) return 1; textures = PyImport_ImportModule("textures"); - chunk_mod = PyImport_ImportModule("chunk"); - /* ensure none of these pointers are NULL */ - if ((!textures) || (!chunk_mod)) { - fprintf(stderr, "\ninit_chunk_render failed\n"); + if ((!textures)) { + fprintf(stderr, "\ninit_chunk_render failed to load; textures\n"); + return 1; + } + + chunk_mod = PyImport_ImportModule("chunk"); + /* ensure none of these pointers are NULL */ + if ((!chunk_mod)) { + fprintf(stderr, "\ninit_chunk_render failed to load; chunk\n"); return 1; } From af3278e3ccccaf768de192a2673d6b5acb8c279c Mon Sep 17 00:00:00 2001 From: Xon Date: Fri, 25 Mar 2011 21:52:34 +0800 Subject: [PATCH 11/11] Merge fix ups --- chunk.py | 2 +- overviewer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chunk.py b/chunk.py index b9bc817..761ea86 100644 --- a/chunk.py +++ b/chunk.py @@ -76,7 +76,7 @@ def get_blockarray(level): def get_blockarray_fromfile(filename): """Same as get_blockarray except takes a filename. This is a shortcut""" d = nbt.load_from_region(filename, x, y) - level = return d[1]['Level'] + level = d[1]['Level'] return get_blockarray(level) def get_skylight_array(level): diff --git a/overviewer.py b/overviewer.py index cb1b0a9..3e37dbc 100755 --- a/overviewer.py +++ b/overviewer.py @@ -186,7 +186,7 @@ def main(): q.append(quadtree.QuadtreeGen(w, destdir, depth=options.zoom, imgformat=imgformat, optimizeimg=optimizeimg, rendermode='spawn')) #create the distributed render - r = rendernode.RenderNode(w,q) + r = rendernode.RenderNode(q) # write out the map and web assets m = googlemap.MapGen(q, skipjs=options.skipjs, web_assets_hook=options.web_assets_hook)