LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Fixed get_chunks, simplified get_chunk_info

Browse Source
This commit is contained in:
Xon
2011-03-23 18:49:26 +08:00
parent 8cfa50087a
commit c700afb012
1 changed files with 388 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

773
nbt.py
View File

@@ -1,385 +1,388 @@
# This file is part of the Minecraft Overviewer. # This file is part of the Minecraft Overviewer.
# #
# Minecraft Overviewer is free software: you can redistribute it and/or # Minecraft Overviewer is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published # modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or (at # by the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version. # your option) any later version.
# #
# Minecraft Overviewer is distributed in the hope that it will be useful, # Minecraft Overviewer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details. # Public License for more details.
# #
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with the Overviewer. If not, see <http://www.gnu.org/licenses/>. # with the Overviewer. If not, see <http://www.gnu.org/licenses/>.
import gzip, zlib import gzip, zlib
import struct import struct
import StringIO import StringIO
import os import os
# decorator to handle filename or object as first parameter # decorator to handle filename or object as first parameter
def _file_loader(func): def _file_loader(func):
def wrapper(fileobj, *args): def wrapper(fileobj, *args):
if isinstance(fileobj, basestring): if isinstance(fileobj, basestring):
if not os.path.isfile(fileobj): if not os.path.isfile(fileobj):
return None return None
# Is actually a filename # Is actually a filename
fileobj = open(fileobj, 'rb',4096) fileobj = open(fileobj, 'rb',4096)
return func(fileobj, *args) return func(fileobj, *args)
return wrapper return wrapper
@_file_loader @_file_loader
def load(fileobj): def load(fileobj):
return NBTFileReader(fileobj).read_all() return NBTFileReader(fileobj).read_all()
def load_from_region(filename, x, y): def load_from_region(filename, x, y):
nbt = load_region(filename).load_chunk(x, y) nbt = load_region(filename).load_chunk(x, y)
if nbt is None: if nbt is None:
return None ## return none. I think this is who we should indicate missing chunks return None ## return none. I think this is who we should indicate missing chunks
#raise IOError("No such chunk in region: (%i, %i)" % (x, y)) #raise IOError("No such chunk in region: (%i, %i)" % (x, y))
return nbt.read_all() return nbt.read_all()
def load_region(filename): def load_region(filename):
return MCRFileReader(filename) return MCRFileReader(filename)
# compile the unpacker's into a classes # compile the unpacker's into a classes
_byte = struct.Struct("b") _byte = struct.Struct("b")
_short = struct.Struct(">h") _short = struct.Struct(">h")
_int = struct.Struct(">i") _int = struct.Struct(">i")
_long = struct.Struct(">q") _long = struct.Struct(">q")
_float = struct.Struct(">f") _float = struct.Struct(">f")
_double = struct.Struct(">d") _double = struct.Struct(">d")
_24bit_int = struct.Struct("B B B") _24bit_int = struct.Struct("B B B")
_unsigned_byte = struct.Struct("B") _unsigned_byte = struct.Struct("B")
_unsigned_int = struct.Struct(">I") _unsigned_int = struct.Struct(">I")
_chunk_header = struct.Struct(">I B") _chunk_header = struct.Struct(">I B")
class NBTFileReader(object): class NBTFileReader(object):
def __init__(self, fileobj, is_gzip=True): def __init__(self, fileobj, is_gzip=True):
if is_gzip: if is_gzip:
self._file = gzip.GzipFile(fileobj=fileobj, mode='rb') self._file = gzip.GzipFile(fileobj=fileobj, mode='rb')
else: else:
# pure zlib stream -- maybe later replace this with # pure zlib stream -- maybe later replace this with
# a custom zlib file object? # a custom zlib file object?
data = zlib.decompress(fileobj.read()) data = zlib.decompress(fileobj.read())
self._file = StringIO.StringIO(data) self._file = StringIO.StringIO(data)
# These private methods read the payload only of the following types # These private methods read the payload only of the following types
def _read_tag_end(self): def _read_tag_end(self):
# Nothing to read # Nothing to read
return 0 return 0
def _read_tag_byte(self): def _read_tag_byte(self):
byte = self._file.read(1) byte = self._file.read(1)
return _byte.unpack(byte)[0] return _byte.unpack(byte)[0]
def _read_tag_short(self): def _read_tag_short(self):
bytes = self._file.read(2) bytes = self._file.read(2)
global _short global _short
return _short.unpack(bytes)[0] return _short.unpack(bytes)[0]
def _read_tag_int(self): def _read_tag_int(self):
bytes = self._file.read(4) bytes = self._file.read(4)
global _int global _int
return _int.unpack(bytes)[0] return _int.unpack(bytes)[0]
def _read_tag_long(self): def _read_tag_long(self):
bytes = self._file.read(8) bytes = self._file.read(8)
global _long global _long
return _long.unpack(bytes)[0] return _long.unpack(bytes)[0]
def _read_tag_float(self): def _read_tag_float(self):
bytes = self._file.read(4) bytes = self._file.read(4)
global _float global _float
return _float.unpack(bytes)[0] return _float.unpack(bytes)[0]
def _read_tag_double(self): def _read_tag_double(self):
bytes = self._file.read(8) bytes = self._file.read(8)
global _double global _double
return _double.unpack(bytes)[0] return _double.unpack(bytes)[0]
def _read_tag_byte_array(self): def _read_tag_byte_array(self):
length = self._read_tag_int() length = self._read_tag_int()
bytes = self._file.read(length) bytes = self._file.read(length)
return bytes return bytes
def _read_tag_string(self): def _read_tag_string(self):
length = self._read_tag_short() length = self._read_tag_short()
# Read the string # Read the string
string = self._file.read(length) string = self._file.read(length)
# decode it and return # decode it and return
return string.decode("UTF-8") return string.decode("UTF-8")
def _read_tag_list(self): def _read_tag_list(self):
tagid = self._read_tag_byte() tagid = self._read_tag_byte()
length = self._read_tag_int() length = self._read_tag_int()
read_tagmap = { read_tagmap = {
0: self._read_tag_end, 0: self._read_tag_end,
1: self._read_tag_byte, 1: self._read_tag_byte,
2: self._read_tag_short, 2: self._read_tag_short,
3: self._read_tag_int, 3: self._read_tag_int,
4: self._read_tag_long, 4: self._read_tag_long,
5: self._read_tag_float, 5: self._read_tag_float,
6: self._read_tag_double, 6: self._read_tag_double,
7: self._read_tag_byte_array, 7: self._read_tag_byte_array,
8: self._read_tag_string, 8: self._read_tag_string,
9: self._read_tag_list, 9: self._read_tag_list,
10:self._read_tag_compound, 10:self._read_tag_compound,
} }
read_method = read_tagmap[tagid] read_method = read_tagmap[tagid]
l = [] l = []
for _ in xrange(length): for _ in xrange(length):
l.append(read_method()) l.append(read_method())
return l return l
def _read_tag_compound(self): def _read_tag_compound(self):
# Build a dictionary of all the tag names mapping to their payloads # Build a dictionary of all the tag names mapping to their payloads
tags = {} tags = {}
while True: while True:
# Read a tag # Read a tag
tagtype = ord(self._file.read(1)) tagtype = ord(self._file.read(1))
if tagtype == 0: if tagtype == 0:
break break
name = self._read_tag_string() name = self._read_tag_string()
read_tagmap = { read_tagmap = {
0: self._read_tag_end, 0: self._read_tag_end,
1: self._read_tag_byte, 1: self._read_tag_byte,
2: self._read_tag_short, 2: self._read_tag_short,
3: self._read_tag_int, 3: self._read_tag_int,
4: self._read_tag_long, 4: self._read_tag_long,
5: self._read_tag_float, 5: self._read_tag_float,
6: self._read_tag_double, 6: self._read_tag_double,
7: self._read_tag_byte_array, 7: self._read_tag_byte_array,
8: self._read_tag_string, 8: self._read_tag_string,
9: self._read_tag_list, 9: self._read_tag_list,
10:self._read_tag_compound, 10:self._read_tag_compound,
} }
payload = read_tagmap[tagtype]() payload = read_tagmap[tagtype]()
tags[name] = payload tags[name] = payload
return tags return tags
def read_all(self): def read_all(self):
"""Reads the entire file and returns (name, payload) """Reads the entire file and returns (name, payload)
name is the name of the root tag, and payload is a dictionary mapping name is the name of the root tag, and payload is a dictionary mapping
names to their payloads names to their payloads
""" """
# Read tag type # Read tag type
tagtype = ord(self._file.read(1)) tagtype = ord(self._file.read(1))
if tagtype != 10: if tagtype != 10:
raise Exception("Expected a tag compound") raise Exception("Expected a tag compound")
# Read the tag name # Read the tag name
name = self._read_tag_string() name = self._read_tag_string()
payload = self._read_tag_compound() payload = self._read_tag_compound()
return name, payload return name, payload
# For reference, the MCR format is outlined at # For reference, the MCR format is outlined at
# <http://www.minecraftwiki.net/wiki/Beta_Level_Format> # <http://www.minecraftwiki.net/wiki/Beta_Level_Format>
class MCRFileReader(object): class MCRFileReader(object):
"""A class for reading chunk region files, as introduced in the """A class for reading chunk region files, as introduced in the
Beta 1.3 update. It provides functions for opening individual Beta 1.3 update. It provides functions for opening individual
chunks (as instances of NBTFileReader), getting chunk timestamps, chunks (as instances of NBTFileReader), getting chunk timestamps,
and for listing chunks contained in the file.""" and for listing chunks contained in the file."""
def __init__(self, filename): def __init__(self, filename):
self._file = None self._file = None
self._filename = filename self._filename = filename
# cache used when the entire header tables are read in get_chunks() # cache used when the entire header tables are read in get_chunks()
self._locations = None self._locations = None
self._timestamps = None self._timestamps = None
self._chunks = None self._chunks = None
def _read_24bit_int(self): def _read_24bit_int(self):
"""Read in a 24-bit, big-endian int, used in the chunk """Read in a 24-bit, big-endian int, used in the chunk
location table.""" location table."""
ret = 0 ret = 0
bytes = self._file.read(3) bytes = self._file.read(3)
global _24bit_int global _24bit_int
bytes = _24bit_int.unpack(bytes) bytes = _24bit_int.unpack(bytes)
for i in xrange(3): for i in xrange(3):
ret = ret << 8 ret = ret << 8
ret += bytes[i] ret += bytes[i]
return ret return ret
def _read_chunk_location(self, x=None, y=None): def _read_chunk_location(self, x=None, y=None):
"""Read and return the (offset, length) of the given chunk """Read and return the (offset, length) of the given chunk
coordinate, or None if the requested chunk doesn't exist. x coordinate, or None if the requested chunk doesn't exist. x
and y must be between 0 and 31, or None. If they are None, and y must be between 0 and 31, or None. If they are None,
then there will be no file seek before doing the read.""" then there will be no file seek before doing the read."""
if x is not None and y is not None: if x is not None and y is not None:
if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32):
raise ValueError("Chunk location out of range.") raise ValueError("Chunk location out of range.")
# check for a cached value # check for a cached value
if self._locations: if self._locations:
return self._locations[x + y * 32] return self._locations[x + y * 32]
# go to the correct entry in the chunk location table # go to the correct entry in the chunk location table
self._file.seek(4 * (x + y * 32)) self._file.seek(4 * (x + y * 32))
# 3-byte offset in 4KiB sectors # 3-byte offset in 4KiB sectors
offset_sectors = self._read_24bit_int() offset_sectors = self._read_24bit_int()
global _unsigned_byte global _unsigned_byte
# 1-byte length in 4KiB sectors, rounded up # 1-byte length in 4KiB sectors, rounded up
byte = self._file.read(1) byte = self._file.read(1)
length_sectors = _unsigned_byte.unpack(byte)[0] length_sectors = _unsigned_byte.unpack(byte)[0]
# check for empty chunks # check for empty chunks
if offset_sectors == 0 or length_sectors == 0: if offset_sectors == 0 or length_sectors == 0:
return None return None
return (offset_sectors * 4096, length_sectors * 4096) return (offset_sectors * 4096, length_sectors * 4096)
def _read_chunk_timestamp(self, x=None, y=None): def _read_chunk_timestamp(self, x=None, y=None):
"""Read and return the last modification time of the given """Read and return the last modification time of the given
chunk coordinate. x and y must be between 0 and 31, or chunk coordinate. x and y must be between 0 and 31, or
None. If they are, None, then there will be no file seek None. If they are, None, then there will be no file seek
before doing the read.""" before doing the read."""
if x is not None and y is not None: if x is not None and y is not None:
if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32): if (not x >= 0) or (not x < 32) or (not y >= 0) or (not y < 32):
raise ValueError("Chunk location out of range.") raise ValueError("Chunk location out of range.")
# check for a cached value # check for a cached value
if self._timestamps: if self._timestamps:
return self._timestamps[x + y * 32] return self._timestamps[x + y * 32]
# go to the correct entry in the chunk timestamp table # go to the correct entry in the chunk timestamp table
self._file.seek(4 * (x + y * 32) + 4096) self._file.seek(4 * (x + y * 32) + 4096)
bytes = self._file.read(4) bytes = self._file.read(4)
global _unsigned_int global _unsigned_int
timestamp = _unsigned_int.unpack(bytes)[0] timestamp = _unsigned_int.unpack(bytes)[0]
return timestamp return timestamp
def get_chunks(self): def get_chunks(self):
"""Return a list of all chunks contained in this region file, """Return a list of all chunks contained in this region file,
as a list of (x, y) coordinate tuples. To load these chunks, as a list of (x, y) coordinate tuples. To load these chunks,
provide these coordinates to load_chunk().""" provide these coordinates to load_chunk()."""
if self._chunks: if self._chunks is not None:
return self._chunks return self._chunks
if self._locations is None: if self._locations is None:
self.get_chunk_info() self.get_chunk_info()
self._chunks = filter(None,self._locations) self._chunks = []
for x in xrange(32):
return self._chunks for y in xrange(32):
if self._locations[x + y * 32] is not None:
def get_chunk_info(self,closeFile = True): self._chunks.append((x,y))
"""Preloads region header information.""" return self._chunks
if self._locations: def get_chunk_info(self,closeFile = True):
return """Preloads region header information."""
if self._file is None: if self._locations:
self._file = open(self._filename,'rb'); return
self._chunks = None if self._file is None:
self._locations = [] self._file = open(self._filename,'rb')
self._timestamps = []
self._chunks = None
# go to the beginning of the file self._locations = []
self._file.seek(0) self._timestamps = []
# read chunk location table # go to the beginning of the file
locations_append = self._locations.append self._file.seek(0)
for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]:
locations_append(self._read_chunk_location()) # read chunk location table
locations_append = self._locations.append
# read chunk timestamp table for _ in xrange(32*32):
timestamp_append = self._timestamps.append locations_append(self._read_chunk_location())
for x, y in [(x,y) for x in xrange(32) for y in xrange(32)]:
timestamp_append(self._read_chunk_timestamp()) # read chunk timestamp table
timestamp_append = self._timestamps.append
if closeFile: for _ in xrange(32*32):
#free the file object since it isn't safe to be reused in child processes (seek point goes wonky!) timestamp_append(self._read_chunk_timestamp())
self._file.close()
self._file = None if closeFile:
return #free the file object since it isn't safe to be reused in child processes (seek point goes wonky!)
self._file.close()
def get_chunk_timestamp(self, x, y): self._file = None
"""Return the given chunk's modification time. If the given return
chunk doesn't exist, this number may be nonsense. Like
load_chunk(), this will wrap x and y into the range [0, 31]. def get_chunk_timestamp(self, x, y):
""" """Return the given chunk's modification time. If the given
x = x % 32 chunk doesn't exist, this number may be nonsense. Like
y = y % 32 load_chunk(), this will wrap x and y into the range [0, 31].
if self._timestamps is None: """
self.get_chunk_info() x = x % 32
return self._timestamps[x + y * 32] y = y % 32
if self._timestamps is None:
def chunkExists(self, x, y): self.get_chunk_info()
"""Determines if a chunk exists without triggering loading of the backend data""" return self._timestamps[x + y * 32]
x = x % 32
y = y % 32 def chunkExists(self, x, y):
if self._locations is None: """Determines if a chunk exists without triggering loading of the backend data"""
self.get_chunk_info() x = x % 32
location = self._locations[x + y * 32] y = y % 32
return location is not None if self._locations is None:
self.get_chunk_info()
def load_chunk(self, x, y): location = self._locations[x + y * 32]
"""Return a NBTFileReader instance for the given chunk, or return location is not None
None if the given chunk doesn't exist in this region file. If
you provide an x or y not between 0 and 31, it will be def load_chunk(self, x, y):
modulo'd into this range (x % 32, etc.) This is so you can """Return a NBTFileReader instance for the given chunk, or
provide chunk coordinates in global coordinates, and still None if the given chunk doesn't exist in this region file. If
have the chunks load out of regions properly.""" you provide an x or y not between 0 and 31, it will be
x = x % 32 modulo'd into this range (x % 32, etc.) This is so you can
y = y % 32 provide chunk coordinates in global coordinates, and still
if self._locations is None: have the chunks load out of regions properly."""
self.get_chunk_info() x = x % 32
y = y % 32
location = self._locations[x + y * 32] if self._locations is None:
if location is None: self.get_chunk_info()
return None
location = self._locations[x + y * 32]
if self._file is None: if location is None:
self._file = open(self._filename,'rb'); return None
# seek to the data
self._file.seek(location[0]) if self._file is None:
self._file = open(self._filename,'rb');
# read in the chunk data header # seek to the data
bytes = self._file.read(5) self._file.seek(location[0])
data_length,compression = _chunk_header.unpack(bytes)
# read in the chunk data header
# figure out the compression bytes = self._file.read(5)
is_gzip = True data_length,compression = _chunk_header.unpack(bytes)
if compression == 1:
# gzip -- not used by the official client, but trivial to support here so... # figure out the compression
is_gzip = True is_gzip = True
elif compression == 2: if compression == 1:
# deflate -- pure zlib stream # gzip -- not used by the official client, but trivial to support here so...
is_gzip = False is_gzip = True
else: elif compression == 2:
# unsupported! # deflate -- pure zlib stream
raise Exception("Unsupported chunk compression type: %i" % (compression)) is_gzip = False
# turn the rest of the data into a StringIO object else:
# (using data_length - 1, as we already read 1 byte for compression) # unsupported!
data = self._file.read(data_length - 1) raise Exception("Unsupported chunk compression type: %i" % (compression))
data = StringIO.StringIO(data) # turn the rest of the data into a StringIO object
# (using data_length - 1, as we already read 1 byte for compression)
return NBTFileReader(data, is_gzip=is_gzip) data = self._file.read(data_length - 1)
data = StringIO.StringIO(data)
return NBTFileReader(data, is_gzip=is_gzip)