diff --git a/overviewer.py b/overviewer.py index e90f386..f739bdd 100755 --- a/overviewer.py +++ b/overviewer.py @@ -35,6 +35,7 @@ import logging from optparse import OptionParser from overviewer_core import util +from overviewer_core import logger from overviewer_core import textures from overviewer_core import optimizeimages, world from overviewer_core import configParser, tileset, assetmanager, dispatcher @@ -44,52 +45,36 @@ helptext = """ %prog [--rendermodes=...] [options] %prog --config= [options]""" -def configure_logger(loglevel=logging.INFO, verbose=False): - """Configures the root logger to our liking - - For a non-standard loglevel, pass in the level with which to configure the handler. - - For a more verbose options line, pass in verbose=True - - This function may be called more than once. - +def is_bare_console(): + """Returns true if Overviewer is running in a bare console in + Windows, that is, if overviewer wasn't started in a cmd.exe + session. """ - - logger = logging.getLogger() - - outstream = sys.stderr - if platform.system() == 'Windows': - # Our custom output stream processor knows how to deal with select ANSI - # color escape sequences - outstream = util.WindowsOutputStream() - formatter = util.ANSIColorFormatter(verbose) + try: + import ctypes + GetConsoleProcessList = ctypes.windll.kernel32.GetConsoleProcessList + num = GetConsoleProcessList(ctypes.byref(ctypes.c_int(0)), ctypes.c_int(1)) + if (num == 1): + return True + + except Exception: + pass + return False - elif sys.stderr.isatty(): - # terminal logging with ANSI color - formatter = util.ANSIColorFormatter(verbose) - - else: - # Let's not assume anything. Just text. - formatter = util.DumbFormatter(verbose) - - if hasattr(logger, 'overviewerHandler'): - # we have already set up logging so just replace the formatter - # this time with the new values - logger.overviewerHandler.setFormatter(formatter) - logger.setLevel(loglevel) - - else: - # Save our handler here so we can tell which handler was ours if the - # function is called again - logger.overviewerHandler = logging.StreamHandler(outstream) - logger.overviewerHandler.setFormatter(formatter) - logger.addHandler(logger.overviewerHandler) - logger.setLevel(loglevel) +def nice_exit(ret=0): + """Drop-in replacement for sys.exit that will automatically detect + bare consoles and wait for user input before closing. + """ + if ret and is_bare_console(): + print + print "Press [Enter] to close this window." + raw_input() + sys.exit(ret) def main(): # bootstrap the logger with defaults - configure_logger() + logger.configure() try: cpus = multiprocessing.cpu_count() @@ -132,8 +117,8 @@ def main(): options, args = parser.parse_args() # re-configure the logger now that we've processed the command line options - configure_logger(logging.INFO + 10*options.quiet - 10*options.verbose, - options.verbose > 0) + logger.configure(logging.INFO + 10*options.quiet - 10*options.verbose, + options.verbose > 0) ########################################################################## # This section of main() runs in response to any one-time options we have, @@ -171,7 +156,7 @@ def main(): if len(args) == 0 and not options.config: # first provide an appropriate error for bare-console users # that don't provide any options - if util.is_bare_console(): + if is_bare_console(): print "\n" print "The Overviewer is a console program. Please open a Windows command prompt" print "first and run Overviewer from there. Further documentation is available at" @@ -236,7 +221,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces. rendermodes = options.rendermodes.replace("-","_").split(",") # Now for some good defaults - renders = {} + renders = util.OrderedDict() for rm in rendermodes: renders["world-" + rm] = { "world": "world", @@ -350,6 +335,8 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # Set up the cache objects to use caches = [] caches.append(cache.LRUCache(size=100)) + if config.get("memcached_host", False): + caches.append(cache.Memcached(config['memcached_host'])) # TODO: optionally more caching layers here renders = config['renders'] @@ -483,10 +470,10 @@ if __name__ == "__main__": multiprocessing.freeze_support() try: ret = main() - util.exit(ret) + nice_exit(ret) except Exception, e: logging.exception("""An error has occurred. This may be a bug. Please let us know! See http://docs.overviewer.org/en/latest/index.html#help This is the error that occurred:""") - util.exit(1) + nice_exit(1) diff --git a/overviewer_core/assetmanager.py b/overviewer_core/assetmanager.py index 2b540cf..131d986 100644 --- a/overviewer_core/assetmanager.py +++ b/overviewer_core/assetmanager.py @@ -24,7 +24,7 @@ from PIL import Image import world import util -import overviewer_version +from files import FileReplacer, mirror_dir class AssetManager(object): """\ @@ -150,13 +150,13 @@ directory. global_assets = os.path.join(util.get_program_path(), "overviewer_core", "data", "web_assets") if not os.path.isdir(global_assets): global_assets = os.path.join(util.get_program_path(), "web_assets") - util.mirror_dir(global_assets, self.outputdir) + mirror_dir(global_assets, self.outputdir) # create overviewer.js from the source js files js_src = os.path.join(util.get_program_path(), "overviewer_core", "data", "js_src") if not os.path.isdir(js_src): js_src = os.path.join(util.get_program_path(), "js_src") - with util.FileReplacer(os.path.join(self.outputdir, "overviewer.js")) as tmpfile: + with FileReplacer(os.path.join(self.outputdir, "overviewer.js")) as tmpfile: with open(tmpfile, "w") as fout: # first copy in js_src/overviewer.js with open(os.path.join(js_src, "overviewer.js"), 'r') as f: @@ -169,7 +169,7 @@ directory. # write out config jsondump = json.dumps(dump, indent=4) - with util.FileReplacer(os.path.join(self.outputdir, "overviewerConfig.js")) as tmpfile: + with FileReplacer(os.path.join(self.outputdir, "overviewerConfig.js")) as tmpfile: with codecs.open(tmpfile, 'w', encoding='UTF-8') as f: f.write("var overviewerConfig = " + jsondump + ";\n") @@ -179,9 +179,9 @@ directory. index = codecs.open(indexpath, 'r', encoding='UTF-8').read() index = index.replace("{title}", "Minecraft Overviewer") index = index.replace("{time}", time.strftime("%a, %d %b %Y %H:%M:%S %Z", time.localtime()).decode(locale.getpreferredencoding())) - versionstr = "%s (%s)" % (overviewer_version.VERSION, overviewer_version.HASH[:7]) + versionstr = "%s (%s)" % (util.findGitVersion(), util.findGitHash()[:7]) index = index.replace("{version}", versionstr) - with util.FileReplacer(indexpath) as indexpath: + with FileReplacer(indexpath) as indexpath: with codecs.open(indexpath, 'w', encoding='UTF-8') as output: output.write(index) diff --git a/overviewer_core/cache.py b/overviewer_core/cache.py index b2f867d..fb6d8be 100644 --- a/overviewer_core/cache.py +++ b/overviewer_core/cache.py @@ -22,6 +22,7 @@ attribute. """ import functools import logging +import cPickle class LRUCache(object): """A simple, generic, in-memory LRU cache that implements the standard @@ -124,3 +125,30 @@ class LRUCache(object): cache[key] = link +# memcached is an option, but unless your IO costs are really high, it just +# ends up adding overhead and isn't worth it. +try: + import memcache +except ImportError: + class Memcached(object): + def __init__(*args): + raise ImportError("No module 'memcache' found. Please install python-memcached") +else: + class Memcached(object): + def __init__(self, conn='127.0.0.1:11211'): + self.conn = conn + self.mc = memcache.Client([conn], debug=0, pickler=cPickle.Pickler, unpickler=cPickle.Unpickler) + + def __getstate__(self): + return self.conn + def __setstate__(self, conn): + self.__init__(conn) + + def __getitem__(self, key): + v = self.mc.get(key) + if not v: + raise KeyError() + return v + + def __setitem__(self, key, value): + self.mc.set(key, value) diff --git a/overviewer_core/files.py b/overviewer_core/files.py new file mode 100644 index 0000000..1c72661 --- /dev/null +++ b/overviewer_core/files.py @@ -0,0 +1,122 @@ +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import os +import os.path +import tempfile +import shutil +import logging + +## useful recursive copy, that ignores common OS cruft +def mirror_dir(src, dst, entities=None): + '''copies all of the entities from src to dst''' + if not os.path.exists(dst): + os.mkdir(dst) + if entities and type(entities) != list: raise Exception("Expected a list, got a %r instead" % type(entities)) + + # files which are problematic and should not be copied + # usually, generated by the OS + skip_files = ['Thumbs.db', '.DS_Store'] + + for entry in os.listdir(src): + if entry in skip_files: + continue + if entities and entry not in entities: + continue + + if os.path.isdir(os.path.join(src,entry)): + mirror_dir(os.path.join(src, entry), os.path.join(dst, entry)) + elif os.path.isfile(os.path.join(src,entry)): + try: + shutil.copy(os.path.join(src, entry), os.path.join(dst, entry)) + except IOError as outer: + try: + # maybe permission problems? + src_stat = os.stat(os.path.join(src, entry)) + os.chmod(os.path.join(src, entry), src_stat.st_mode | stat.S_IRUSR) + dst_stat = os.stat(os.path.join(dst, entry)) + os.chmod(os.path.join(dst, entry), dst_stat.st_mode | stat.S_IWUSR) + except OSError: # we don't care if this fails + pass + shutil.copy(os.path.join(src, entry), os.path.join(dst, entry)) + # if this stills throws an error, let it propagate up + +# Define a context manager to handle atomic renaming or "just forget it write +# straight to the file" depending on whether os.rename provides atomic +# overwrites. +# Detect whether os.rename will overwrite files +with tempfile.NamedTemporaryFile() as f1: + with tempfile.NamedTemporaryFile() as f2: + try: + os.rename(f1.name,f2.name) + except OSError: + renameworks = False + else: + renameworks = True + # re-make this file so it can be deleted without error + open(f1.name, 'w').close() +del tempfile,f1,f2 +doc = """This class acts as a context manager for files that are to be written +out overwriting an existing file. + +The parameter is the destination filename. The value returned into the context +is the filename that should be used. On systems that support an atomic +os.rename(), the filename will actually be a temporary file, and it will be +atomically replaced over the destination file on exit. + +On systems that don't support an atomic rename, the filename returned is the +filename given. + +If an error is encountered, the file is attempted to be removed, and the error +is propagated. + +Example: + +with FileReplacer("config") as configname: + with open(configout, 'w') as configout: + configout.write(newconfig) +""" +if renameworks: + class FileReplacer(object): + __doc__ = doc + def __init__(self, destname): + self.destname = destname + self.tmpname = destname + ".tmp" + def __enter__(self): + # rename works here. Return a temporary filename + return self.tmpname + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type: + # error + try: + os.remove(self.tmpname) + except Exception, e: + logging.warning("An error was raised, so I was doing " + "some cleanup first, but I couldn't remove " + "'%s'!", self.tmpname) + else: + # atomic rename into place + os.rename(self.tmpname, self.destname) +else: + class FileReplacer(object): + __doc__ = doc + def __init__(self, destname): + self.destname = destname + def __enter__(self): + return self.destname + def __exit__(self, exc_type, exc_val, exc_tb): + return +del renameworks + diff --git a/overviewer_core/logger.py b/overviewer_core/logger.py new file mode 100644 index 0000000..96850cf --- /dev/null +++ b/overviewer_core/logger.py @@ -0,0 +1,298 @@ +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import sys +import os +import logging +import platform +import ctypes +from cStringIO import StringIO + +# Some cool code for colored logging: +# For background, add 40. For foreground, add 30 +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + +RESET_SEQ = "\033[0m" +COLOR_SEQ = "\033[1;%dm" +BOLD_SEQ = "\033[1m" + +# Windows colors, taken from WinCon.h +FOREGROUND_BLUE = 0x01 +FOREGROUND_GREEN = 0x02 +FOREGROUND_RED = 0x04 +FOREGROUND_BOLD = 0x08 +FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED + +BACKGROUND_BLACK = 0x00 +BACKGROUND_BLUE = 0x10 +BACKGROUND_GREEN = 0x20 +BACKGROUND_RED = 0x40 + +COLORIZE = { + #'INFO': WHITe, + 'DEBUG': CYAN, +} +HIGHLIGHT = { + 'CRITICAL': RED, + 'ERROR': RED, + 'WARNING': YELLOW, +} + + +class WindowsOutputStream(object): + """A file-like object that proxies sys.stderr and interprets simple ANSI + escape codes for color, translating them to the appropriate Windows calls. + + """ + def __init__(self, stream=None): + assert platform.system() == 'Windows' + self.stream = stream or sys.stderr + + # go go gadget ctypes + self.GetStdHandle = ctypes.windll.kernel32.GetStdHandle + self.SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute + self.STD_OUTPUT_HANDLE = ctypes.c_int(0xFFFFFFF5) + self.output_handle = self.GetStdHandle(self.STD_OUTPUT_HANDLE) + if self.output_handle == 0xFFFFFFFF: + raise Exception("Something failed in WindowsColorFormatter") + + + # default is white text on a black background + self.currentForeground = FOREGROUND_WHITE + self.currentBackground = BACKGROUND_BLACK + self.currentBold = 0 + + def updateWinColor(self, Fore=None, Back=None, Bold=False): + if Fore != None: self.currentForeground = Fore + if Back != None: self.currentBackground = Back + if Bold: + self.currentBold = FOREGROUND_BOLD + else: + self.currentBold = 0 + + self.SetConsoleTextAttribute(self.output_handle, + ctypes.c_int(self.currentForeground | self.currentBackground | self.currentBold)) + + def write(self, s): + + msg_strm = StringIO(s) + + while (True): + c = msg_strm.read(1) + if c == '': break + if c == '\033': + c1 = msg_strm.read(1) + if c1 != '[': # + sys.stream.write(c + c1) + continue + c2 = msg_strm.read(2) + if c2 == "0m": # RESET_SEQ + self.updateWinColor(Fore=FOREGROUND_WHITE, Back=BACKGROUND_BLACK) + + elif c2 == "1;": + color = "" + while(True): + nc = msg_strm.read(1) + if nc == 'm': break + color += nc + color = int(color) + if (color >= 40): # background + color = color - 40 + if color == BLACK: + self.updateWinColor(Back=BACKGROUND_BLACK) + if color == RED: + self.updateWinColor(Back=BACKGROUND_RED) + elif color == GREEN: + self.updateWinColor(Back=BACKGROUND_GREEN) + elif color == YELLOW: + self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_GREEN) + elif color == BLUE: + self.updateWinColor(Back=BACKGROUND_BLUE) + elif color == MAGENTA: + self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_BLUE) + elif color == CYAN: + self.updateWinColor(Back=BACKGROUND_GREEN | BACKGROUND_BLUE) + elif color == WHITE: + self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE) + elif (color >= 30): # foreground + color = color - 30 + if color == BLACK: + self.updateWinColor(Fore=FOREGROUND_BLACK) + if color == RED: + self.updateWinColor(Fore=FOREGROUND_RED) + elif color == GREEN: + self.updateWinColor(Fore=FOREGROUND_GREEN) + elif color == YELLOW: + self.updateWinColor(Fore=FOREGROUND_RED | FOREGROUND_GREEN) + elif color == BLUE: + self.updateWinColor(Fore=FOREGROUND_BLUE) + elif color == MAGENTA: + self.updateWinColor(Fore=FOREGROUND_RED | FOREGROUND_BLUE) + elif color == CYAN: + self.updateWinColor(Fore=FOREGROUND_GREEN | FOREGROUND_BLUE) + elif color == WHITE: + self.updateWinColor(Fore=FOREGROUND_WHITE) + + + + elif c2 == "1m": # BOLD_SEQ + pass + + else: + self.stream.write(c) + + + + def flush(self): + self.stream.flush() + +class HighlightingFormatter(logging.Formatter): + """Base class of our custom formatter + + """ + datefmt = "%Y-%m-%d %H:%M:%S" + funcName_len = 15 + + def __init__(self, verbose=False): + if verbose: + fmtstr = '%(fileandlineno)-18s %(pid)s %(asctime)s ' \ + '%(levelname)-8s %(message)s' + else: + fmtstr = '%(asctime)s ' '%(shortlevelname)-1s%(message)s' + + logging.Formatter.__init__(self, fmtstr, self.datefmt) + + def format(self, record): + """Add a few extra options to the record + + pid + The process ID + + fileandlineno + A combination filename:linenumber string, so it can be justified as + one entry in a format string. + + funcName + The function name truncated/padded to a fixed width characters + + shortlevelname + The level name truncated to 1 character + + """ + + record.shortlevelname = record.levelname[0] + ' ' + if record.levelname == 'INFO': record.shortlevelname = '' + + record.pid = os.getpid() + record.fileandlineno = "%s:%s" % (record.filename, record.lineno) + + # Set the max length for the funcName field, and left justify + l = self.funcName_len + record.funcName = ("%-" + str(l) + 's') % record.funcName[:l] + + return self.highlight(record) + + def highlight(self, record): + """This method applies any special effects such as colorization. It + should modify the records in the record object, and should return the + *formatted line*. This probably involves calling + logging.Formatter.format() + + Override this in subclasses + + """ + return logging.Formatter.format(self, record) + +class DumbFormatter(HighlightingFormatter): + """Formatter for dumb terminals that don't support color, or log files. + Prints a bunch of stars before a highlighted line. + + """ + def highlight(self, record): + if record.levelname in HIGHLIGHT: + line = logging.Formatter.format(self, record) + line = "*" * min(79,len(line)) + "\n" + line + return line + else: + return HighlightingFormatter.highlight(self, record) + + +class ANSIColorFormatter(HighlightingFormatter): + """Uses ANSI escape sequences to enable GLORIOUS EXTRA-COLOR! + + """ + def highlight(self, record): + if record.levelname in COLORIZE: + # Colorize just the levelname + # left justify again because the color sequence bumps the length up + # above 8 chars + levelname_color = COLOR_SEQ % (30 + COLORIZE[record.levelname]) + \ + "%-8s" % record.levelname + RESET_SEQ + record.levelname = levelname_color + return logging.Formatter.format(self, record) + + elif record.levelname in HIGHLIGHT: + # Colorize the entire line + line = logging.Formatter.format(self, record) + line = COLOR_SEQ % (40 + HIGHLIGHT[record.levelname]) + line + \ + RESET_SEQ + return line + + else: + # No coloring if it's not to be highlighted or colored + return logging.Formatter.format(self, record) + +def configure(loglevel=logging.INFO, verbose=False): + """Configures the root logger to our liking + + For a non-standard loglevel, pass in the level with which to configure the handler. + + For a more verbose options line, pass in verbose=True + + This function may be called more than once. + + """ + + logger = logging.getLogger() + + outstream = sys.stderr + + if platform.system() == 'Windows': + # Our custom output stream processor knows how to deal with select ANSI + # color escape sequences + outstream = WindowsOutputStream() + formatter = ANSIColorFormatter(verbose) + + elif sys.stderr.isatty(): + # terminal logging with ANSI color + formatter = ANSIColorFormatter(verbose) + + else: + # Let's not assume anything. Just text. + formatter = DumbFormatter(verbose) + + if hasattr(logger, 'overviewerHandler'): + # we have already set up logging so just replace the formatter + # this time with the new values + logger.overviewerHandler.setFormatter(formatter) + logger.setLevel(loglevel) + + else: + # Save our handler here so we can tell which handler was ours if the + # function is called again + logger.overviewerHandler = logging.StreamHandler(outstream) + logger.overviewerHandler.setFormatter(formatter) + logger.addHandler(logger.overviewerHandler) + logger.setLevel(loglevel) diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index fded558..e023312 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -44,6 +44,7 @@ # can be initialized and then appended/added to when the config file is parsed. from settingsValidators import * +import util # renders is a dictionary mapping strings to dicts. These dicts describe the # configuration for that render. Therefore, the validator for 'renders' is set @@ -55,7 +56,7 @@ from settingsValidators import * # objects with their respective validators. # config file. -renders = Setting(required=True, default={}, +renders = Setting(required=True, default=util.OrderedDict(), validator=make_dictValidator(validateStr, make_configDictValidator( { "world": Setting(required=True, validator=validateStr, default=None), @@ -81,8 +82,12 @@ renders = Setting(required=True, default={}, ))) # The worlds dict, mapping world names to world paths -worlds = Setting(required=True, validator=make_dictValidator(validateStr, validateWorldPath), default={}) +worlds = Setting(required=True, validator=make_dictValidator(validateStr, validateWorldPath), default=util.OrderedDict()) outputdir = Setting(required=True, validator=validateOutputDir, default=None) processes = Setting(required=True, validator=int, default=-1) + +# memcached is an option, but unless your IO costs are really high, it just +# ends up adding overhead and isn't worth it. +memcached_host = Setting(required=False, validator=str, default=None) diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 84d2d2a..f6fdd77 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -4,6 +4,7 @@ import os.path from collections import namedtuple import rendermodes +import util from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT class ValidationException(Exception): @@ -183,7 +184,7 @@ def make_dictValidator(keyvalidator, valuevalidator): """ def v(d): - newd = {} + newd = util.OrderedDict() for key, value in d.iteritems(): newd[keyvalidator(key)] = valuevalidator(value) return newd @@ -211,7 +212,7 @@ def make_configDictValidator(config, ignore_undefined=False): """ def configDictValidator(d): - newdict = {} + newdict = util.OrderedDict() # values are config keys that the user specified that don't match any # valid key diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index a20c22e..5c409d3 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -28,8 +28,8 @@ from itertools import product, izip from PIL import Image -from .util import iterate_base4, convert_coords, unconvert_coords, roundrobin -from .util import FileReplacer +from .util import roundrobin +from .files import FileReplacer from .optimizeimages import optimize_image import c_overviewer @@ -84,6 +84,11 @@ do_work(workobj) """ +# small but useful +def iterate_base4(d): + """Iterates over a base 4 number with d digits""" + return product(xrange(4), repeat=d) + # A named tuple class storing the row and column bounds for the to-be-rendered # world Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow")) @@ -1002,6 +1007,27 @@ class TileSet(object): # Nope. yield path, max_child_mtime, False +## +## Functions for converting (x, z) to (col, row) and back +## + +def convert_coords(chunkx, chunkz): + """Takes a coordinate (chunkx, chunkz) where chunkx and chunkz are + in the chunk coordinate system, and figures out the row and column + in the image each one should be. Returns (col, row).""" + + # columns are determined by the sum of the chunk coords, rows are the + # difference + # change this function, and you MUST change unconvert_coords + return (chunkx + chunkz, chunkz - chunkx) + +def unconvert_coords(col, row): + """Undoes what convert_coords does. Returns (chunkx, chunkz).""" + + # col + row = chunkz + chunkz => (col + row)/2 = chunkz + # col - row = chunkx + chunkx => (col - row)/2 = chunkx + return ((col - row) / 2, (col + row) / 2) + ###################### # The following two functions define the mapping from chunks to tiles and back. # The mapping from chunks to tiles (get_tiles_by_chunk()) is used during the diff --git a/overviewer_core/util.py b/overviewer_core/util.py index 4d5b133..b00faa8 100644 --- a/overviewer_core/util.py +++ b/overviewer_core/util.py @@ -18,17 +18,10 @@ Misc utility routines used by multiple files that don't belong anywhere else """ import imp -import os import os.path import sys -import platform from subprocess import Popen, PIPE -import logging -from cStringIO import StringIO -import ctypes -import platform from itertools import cycle, islice, product -import shutil def get_program_path(): if hasattr(sys, "frozen") or imp.is_frozen("__main__"): @@ -89,33 +82,6 @@ def findGitVersion(): except Exception: return "unknown" -def is_bare_console(): - """Returns true if Overviewer is running in a bare console in - Windows, that is, if overviewer wasn't started in a cmd.exe - session. - """ - if platform.system() == 'Windows': - try: - import ctypes - GetConsoleProcessList = ctypes.windll.kernel32.GetConsoleProcessList - num = GetConsoleProcessList(ctypes.byref(ctypes.c_int(0)), ctypes.c_int(1)) - if (num == 1): - return True - - except Exception: - pass - return False - -def exit(ret=0): - """Drop-in replacement for sys.exit that will automatically detect - bare consoles and wait for user input before closing. - """ - if ret and is_bare_console(): - print - print "Press [Enter] to close this window." - raw_input() - sys.exit(ret) - # http://docs.python.org/library/itertools.html def roundrobin(iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" @@ -130,367 +96,6 @@ def roundrobin(iterables): pending -= 1 nexts = cycle(islice(nexts, pending)) -def iterate_base4(d): - """Iterates over a base 4 number with d digits""" - return product(xrange(4), repeat=d) - - -def convert_coords(chunkx, chunkz): - """Takes a coordinate (chunkx, chunkz) where chunkx and chunkz are - in the chunk coordinate system, and figures out the row and column - in the image each one should be. Returns (col, row).""" - - # columns are determined by the sum of the chunk coords, rows are the - # difference - # change this function, and you MUST change unconvert_coords - return (chunkx + chunkz, chunkz - chunkx) - -def unconvert_coords(col, row): - """Undoes what convert_coords does. Returns (chunkx, chunkz).""" - - # col + row = chunkz + chunkz => (col + row)/2 = chunkz - # col - row = chunkx + chunkx => (col - row)/2 = chunkx - return ((col - row) / 2, (col + row) / 2) - -# Define a context manager to handle atomic renaming or "just forget it write -# straight to the file" depending on whether os.rename provides atomic -# overwrites. -# Detect whether os.rename will overwrite files -import tempfile -with tempfile.NamedTemporaryFile() as f1: - with tempfile.NamedTemporaryFile() as f2: - try: - os.rename(f1.name,f2.name) - except OSError: - renameworks = False - else: - renameworks = True - # re-make this file so it can be deleted without error - open(f1.name, 'w').close() -del tempfile,f1,f2 -doc = """This class acts as a context manager for files that are to be written -out overwriting an existing file. - -The parameter is the destination filename. The value returned into the context -is the filename that should be used. On systems that support an atomic -os.rename(), the filename will actually be a temporary file, and it will be -atomically replaced over the destination file on exit. - -On systems that don't support an atomic rename, the filename returned is the -filename given. - -If an error is encountered, the file is attempted to be removed, and the error -is propagated. - -Example: - -with FileReplacer("config") as configname: - with open(configout, 'w') as configout: - configout.write(newconfig) -""" -if renameworks: - class FileReplacer(object): - __doc__ = doc - def __init__(self, destname): - self.destname = destname - self.tmpname = destname + ".tmp" - def __enter__(self): - # rename works here. Return a temporary filename - return self.tmpname - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type: - # error - try: - os.remove(self.tmpname) - except Exception, e: - logging.warning("An error was raised, so I was doing " - "some cleanup first, but I couldn't remove " - "'%s'!", self.tmpname) - else: - # atomic rename into place - os.rename(self.tmpname, self.destname) -else: - class FileReplacer(object): - __doc__ = doc - def __init__(self, destname): - self.destname = destname - def __enter__(self): - return self.destname - def __exit__(self, exc_type, exc_val, exc_tb): - return -del renameworks - -# Logging related classes are below - -# Some cool code for colored logging: -# For background, add 40. For foreground, add 30 -BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) - -RESET_SEQ = "\033[0m" -COLOR_SEQ = "\033[1;%dm" -BOLD_SEQ = "\033[1m" - -# Windows colors, taken from WinCon.h -FOREGROUND_BLUE = 0x01 -FOREGROUND_GREEN = 0x02 -FOREGROUND_RED = 0x04 -FOREGROUND_BOLD = 0x08 -FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED - -BACKGROUND_BLACK = 0x00 -BACKGROUND_BLUE = 0x10 -BACKGROUND_GREEN = 0x20 -BACKGROUND_RED = 0x40 - -COLORIZE = { - #'INFO': WHITe, - 'DEBUG': CYAN, -} -HIGHLIGHT = { - 'CRITICAL': RED, - 'ERROR': RED, - 'WARNING': YELLOW, -} - - -class WindowsOutputStream(object): - """A file-like object that proxies sys.stderr and interprets simple ANSI - escape codes for color, translating them to the appropriate Windows calls. - - """ - def __init__(self, stream=None): - assert platform.system() == 'Windows' - self.stream = stream or sys.stderr - - # go go gadget ctypes - self.GetStdHandle = ctypes.windll.kernel32.GetStdHandle - self.SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute - self.STD_OUTPUT_HANDLE = ctypes.c_int(0xFFFFFFF5) - self.output_handle = self.GetStdHandle(self.STD_OUTPUT_HANDLE) - if self.output_handle == 0xFFFFFFFF: - raise Exception("Something failed in WindowsColorFormatter") - - - # default is white text on a black background - self.currentForeground = FOREGROUND_WHITE - self.currentBackground = BACKGROUND_BLACK - self.currentBold = 0 - - def updateWinColor(self, Fore=None, Back=None, Bold=False): - if Fore != None: self.currentForeground = Fore - if Back != None: self.currentBackground = Back - if Bold: - self.currentBold = FOREGROUND_BOLD - else: - self.currentBold = 0 - - self.SetConsoleTextAttribute(self.output_handle, - ctypes.c_int(self.currentForeground | self.currentBackground | self.currentBold)) - - def write(self, s): - - msg_strm = StringIO(s) - - while (True): - c = msg_strm.read(1) - if c == '': break - if c == '\033': - c1 = msg_strm.read(1) - if c1 != '[': # - sys.stream.write(c + c1) - continue - c2 = msg_strm.read(2) - if c2 == "0m": # RESET_SEQ - self.updateWinColor(Fore=FOREGROUND_WHITE, Back=BACKGROUND_BLACK) - - elif c2 == "1;": - color = "" - while(True): - nc = msg_strm.read(1) - if nc == 'm': break - color += nc - color = int(color) - if (color >= 40): # background - color = color - 40 - if color == BLACK: - self.updateWinColor(Back=BACKGROUND_BLACK) - if color == RED: - self.updateWinColor(Back=BACKGROUND_RED) - elif color == GREEN: - self.updateWinColor(Back=BACKGROUND_GREEN) - elif color == YELLOW: - self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_GREEN) - elif color == BLUE: - self.updateWinColor(Back=BACKGROUND_BLUE) - elif color == MAGENTA: - self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_BLUE) - elif color == CYAN: - self.updateWinColor(Back=BACKGROUND_GREEN | BACKGROUND_BLUE) - elif color == WHITE: - self.updateWinColor(Back=BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE) - elif (color >= 30): # foreground - color = color - 30 - if color == BLACK: - self.updateWinColor(Fore=FOREGROUND_BLACK) - if color == RED: - self.updateWinColor(Fore=FOREGROUND_RED) - elif color == GREEN: - self.updateWinColor(Fore=FOREGROUND_GREEN) - elif color == YELLOW: - self.updateWinColor(Fore=FOREGROUND_RED | FOREGROUND_GREEN) - elif color == BLUE: - self.updateWinColor(Fore=FOREGROUND_BLUE) - elif color == MAGENTA: - self.updateWinColor(Fore=FOREGROUND_RED | FOREGROUND_BLUE) - elif color == CYAN: - self.updateWinColor(Fore=FOREGROUND_GREEN | FOREGROUND_BLUE) - elif color == WHITE: - self.updateWinColor(Fore=FOREGROUND_WHITE) - - - - elif c2 == "1m": # BOLD_SEQ - pass - - else: - self.stream.write(c) - - - - def flush(self): - self.stream.flush() - -class HighlightingFormatter(logging.Formatter): - """Base class of our custom formatter - - """ - datefmt = "%Y-%m-%d %H:%M:%S" - funcName_len = 15 - - def __init__(self, verbose=False): - if verbose: - fmtstr = '%(fileandlineno)-18s %(pid)s %(asctime)s ' \ - '%(levelname)-8s %(message)s' - else: - fmtstr = '%(asctime)s ' '%(shortlevelname)-1s%(message)s' - - logging.Formatter.__init__(self, fmtstr, self.datefmt) - - def format(self, record): - """Add a few extra options to the record - - pid - The process ID - - fileandlineno - A combination filename:linenumber string, so it can be justified as - one entry in a format string. - - funcName - The function name truncated/padded to a fixed width characters - - shortlevelname - The level name truncated to 1 character - - """ - - record.shortlevelname = record.levelname[0] + ' ' - if record.levelname == 'INFO': record.shortlevelname = '' - - record.pid = os.getpid() - record.fileandlineno = "%s:%s" % (record.filename, record.lineno) - - # Set the max length for the funcName field, and left justify - l = self.funcName_len - record.funcName = ("%-" + str(l) + 's') % record.funcName[:l] - - return self.highlight(record) - - def highlight(self, record): - """This method applies any special effects such as colorization. It - should modify the records in the record object, and should return the - *formatted line*. This probably involves calling - logging.Formatter.format() - - Override this in subclasses - - """ - return logging.Formatter.format(self, record) - -class DumbFormatter(HighlightingFormatter): - """Formatter for dumb terminals that don't support color, or log files. - Prints a bunch of stars before a highlighted line. - - """ - def highlight(self, record): - if record.levelname in HIGHLIGHT: - line = logging.Formatter.format(self, record) - line = "*" * min(79,len(line)) + "\n" + line - return line - else: - return HighlightingFormatter.highlight(self, record) - - -class ANSIColorFormatter(HighlightingFormatter): - """Uses ANSI escape sequences to enable GLORIOUS EXTRA-COLOR! - - """ - def highlight(self, record): - if record.levelname in COLORIZE: - # Colorize just the levelname - # left justify again because the color sequence bumps the length up - # above 8 chars - levelname_color = COLOR_SEQ % (30 + COLORIZE[record.levelname]) + \ - "%-8s" % record.levelname + RESET_SEQ - record.levelname = levelname_color - return logging.Formatter.format(self, record) - - elif record.levelname in HIGHLIGHT: - # Colorize the entire line - line = logging.Formatter.format(self, record) - line = COLOR_SEQ % (40 + HIGHLIGHT[record.levelname]) + line + \ - RESET_SEQ - return line - - else: - # No coloring if it's not to be highlighted or colored - return logging.Formatter.format(self, record) - - -def mirror_dir(src, dst, entities=None): - '''copies all of the entities from src to dst''' - if not os.path.exists(dst): - os.mkdir(dst) - if entities and type(entities) != list: raise Exception("Expected a list, got a %r instead" % type(entities)) - - # files which are problematic and should not be copied - # usually, generated by the OS - skip_files = ['Thumbs.db', '.DS_Store'] - - for entry in os.listdir(src): - if entry in skip_files: - continue - if entities and entry not in entities: - continue - - if os.path.isdir(os.path.join(src,entry)): - mirror_dir(os.path.join(src, entry), os.path.join(dst, entry)) - elif os.path.isfile(os.path.join(src,entry)): - try: - shutil.copy(os.path.join(src, entry), os.path.join(dst, entry)) - except IOError as outer: - try: - # maybe permission problems? - src_stat = os.stat(os.path.join(src, entry)) - os.chmod(os.path.join(src, entry), src_stat.st_mode | stat.S_IRUSR) - dst_stat = os.stat(os.path.join(dst, entry)) - os.chmod(os.path.join(dst, entry), dst_stat.st_mode | stat.S_IWUSR) - except OSError: # we don't care if this fails - pass - shutil.copy(os.path.join(src, entry), os.path.join(dst, entry)) - # if this stills throws an error, let it propagate up - - def dict_subset(d, keys): "Return a new dictionary that is built from copying select keys from d" n = dict() @@ -499,4 +104,268 @@ def dict_subset(d, keys): n[key] = d[key] return n - +## (from http://code.activestate.com/recipes/576693/ [r9]) +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. + +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) + +# now replace all that with the official version, if available +try: + import collections + OrderedDict = collections.OrderedDict +except (ImportError, AttributeError): + pass diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 43b11d8..12a5d0a 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -18,6 +18,7 @@ import os import os.path from glob import glob import logging +import hashlib import numpy @@ -581,14 +582,13 @@ class CachedRegionSet(RegionSetWrapper): s += obj.regiondir logging.debug("Initializing a cache with key '%s'", s) - if len(s) > 32: - import hashlib - s = hashlib.md5(s).hexdigest() + + s = hashlib.md5(s).hexdigest() self.key = s def get_chunk(self, x, z): - key = (self.key, x, z) + key = hashlib.md5(repr((self.key, x, z))).hexdigest() for i, cache in enumerate(self.caches): try: retval = cache[key] diff --git a/test/test_all.py b/test/test_all.py index 5ef49b6..21ae591 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -26,9 +26,8 @@ if 0: self.lock = None root.addHandler(NullHandler()) else: - import overviewer - import logging - overviewer.configure_logger(logging.DEBUG, True) + from overviewer_core import logger + logger.configure(logging.DEBUG, True) if __name__ == "__main__": diff --git a/test/test_rendertileset.py b/test/test_rendertileset.py index fe00e3f..ea2e017 100644 --- a/test/test_rendertileset.py +++ b/test/test_rendertileset.py @@ -1,7 +1,6 @@ import unittest -from overviewer_core.tileset import RendertileSet -from overviewer_core.util import iterate_base4 +from overviewer_core.tileset import iterate_base4, RendertileSet class RendertileSetTest(unittest.TestCase): # If you change this definition, you must also change the hard-coded diff --git a/test/test_tileobj.py b/test/test_tileobj.py index 303cec9..d8cc913 100644 --- a/test/test_tileobj.py +++ b/test/test_tileobj.py @@ -1,7 +1,6 @@ import unittest -from overviewer_core.util import iterate_base4 -from overviewer_core.tileset import RenderTile +from overviewer_core.tileset import iterate_base4, RenderTile items = [ ((-4,-8), (0,0)), diff --git a/test/test_tileset.py b/test/test_tileset.py index 63be0a3..db23d80 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -6,7 +6,7 @@ import os import os.path import random -from overviewer_core import tileset, util +from overviewer_core import tileset # Supporing data # chunks list: chunkx, chunkz mapping to chunkmtime @@ -120,7 +120,7 @@ def get_tile_set(chunks): tile_set = defaultdict(int) for (chunkx, chunkz), chunkmtime in chunks.iteritems(): - col, row = util.convert_coords(chunkx, chunkz) + col, row = tileset.convert_coords(chunkx, chunkz) for tilec, tiler in tileset.get_tiles_by_chunk(col, row): tile = tileset.RenderTile.compute_path(tilec, tiler, 3)