From 4579998663d163b795cc2ccdcd5d0c43ad443a1a Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 15:34:36 -0500 Subject: [PATCH 01/45] Prevent opening and parsing files which haven't been modified since the last time that the render was done --- overviewer_core/tileset.py | 2 +- overviewer_core/world.py | 42 +++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 0f0e1c3..1660348 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -757,7 +757,7 @@ class TileSet(object): # Compare the last modified time of the chunk and tile. If the # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks(): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 9c2bc86..95effe5 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -272,7 +272,7 @@ class RegionSet(object): for x, y, regionfile in self._iterate_regionfiles(): # regionfile is a pathname - self.regionfiles[(x,y)] = regionfile + self.regionfiles[(x,y)] = (regionfile, os.path.getmtime(regionfile)) self.empty_chunk = [None,None] logging.debug("Done scanning regions") @@ -458,7 +458,7 @@ class RegionSet(object): """ - for (regionx, regiony), regionfile in self.regionfiles.iteritems(): + for (regionx, regiony), (regionfile, filemtime) in self.regionfiles.iteritems(): try: mcr = self._get_regionobj(regionfile) except nbt.CorruptRegionError: @@ -467,6 +467,27 @@ class RegionSet(object): for chunkx, chunky in mcr.get_chunks(): yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) + def iterate_newer_chunks(self, mtime): + """Returns an iterator over all chunk metadata in this world. Iterates + over tuples of integers (x,z,mtime) for each chunk. Other chunk data + is not returned here. + + """ + + for (regionx, regiony), (regionfile, filemtime) in self.regionfiles.iteritems(): + """ SKIP LOADING A REGION WHICH HAS NOT BEEN MODIFIED! """ + if (filemtime < mtime): + continue + + try: + mcr = self._get_regionobj(regionfile) + except nbt.CorruptRegionError: + logging.warning("Found a corrupt region file at %s,%s. Skipping it.", regionx, regiony) + continue + + for chunkx, chunky in mcr.get_chunks(): + yield chunkx+32*regionx, chunky+32*regiony, mcr.get_chunk_timestamp(chunkx, chunky) + def get_chunk_mtime(self, x, z): """Returns a chunk's mtime, or False if the chunk does not exist. This is therefore a dual purpose method. It corrects for the given north @@ -492,7 +513,7 @@ class RegionSet(object): Coords can be either be global chunk coords, or local to a region """ - regionfile = self.regionfiles.get((chunkX//32, chunkY//32),None) + (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),None) return regionfile def _iterate_regionfiles(self): @@ -536,6 +557,8 @@ class RegionSetWrapper(object): return self._r.get_chunk(x,z) def iterate_chunks(self): return self._r.iterate_chunks() + def iterate_newer_chunks(self,filemtime): + return self._r.iterate_newer_chunks(filemtime) def get_chunk_mtime(self, x, z): return self._r.get_chunk_mtime(x,z) @@ -622,6 +645,11 @@ class RotatedRegionSet(RegionSetWrapper): x,z = self.rotate(x,z) yield x,z,mtime + def iterate_newer_chunks(self, filemtime): + for x,z,mtime in super(RotatedRegionSet, self).iterate_newer_chunks(filemtime): + x,z = self.rotate(x,z) + yield x,z,mtime + class CroppedRegionSet(RegionSetWrapper): def __init__(self, rsetobj, xmin, zmin, xmax, zmax): super(CroppedRegionSet, self).__init__(rsetobj) @@ -645,6 +673,14 @@ class CroppedRegionSet(RegionSetWrapper): self.xmin <= x <= self.xmax and self.zmin <= z <= self.zmax ) + + def iterate_newer_chunks(self, filemtime): + return ((x,z,mtime) for (x,z,mtime) in super(CroppedRegionSet,self).iterate_newer_chunks(filemtime) + if + self.xmin <= x <= self.xmax and + self.zmin <= z <= self.zmax + ) + def get_chunk_mtime(self,x,z): if ( self.xmin <= x <= self.xmax and From 8458451044321d8857bcfe9adeea514da3fafe1c Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 15:55:51 -0500 Subject: [PATCH 02/45] Respect the markall parameter --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 1660348..6046ea4 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -757,8 +757,8 @@ class TileSet(object): # Compare the last modified time of the chunk and tile. If the # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if markall else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 2b2d929659bad4429353fd8ea2b608e3c3ffffef Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:04:11 -0500 Subject: [PATCH 03/45] Add the new function to the synthetic test --- test/test_tileset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_tileset.py b/test/test_tileset.py index f555eac..864538f 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -1,3 +1,4 @@ +M import unittest import tempfile import shutil @@ -53,6 +54,10 @@ class FakeRegionset(object): for (x,z),mtime in self.chunks.iteritems(): yield x,z,mtime + def iterate_newer_chunks(self, filemtime): + for (x,z),mtime in self.chunks.iteritems(): + yield x,z,mtime + def get_chunk_mtime(self, x, z): try: return self.chunks[x,z] From a0640e8bdb0ef81173f83962dafe77f8fc39b81e Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:10:48 -0500 Subject: [PATCH 04/45] extra character ? --- test/test_tileset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_tileset.py b/test/test_tileset.py index 864538f..5658d00 100644 --- a/test/test_tileset.py +++ b/test/test_tileset.py @@ -1,4 +1,3 @@ -M import unittest import tempfile import shutil From c438a37b295a14dbaa08ddfc01a05eb5ec8c97cd Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Mon, 20 Jan 2014 16:30:08 -0500 Subject: [PATCH 05/45] Default value is compatible with the expecteed result in fetching the region for a path --- overviewer_core/world.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/world.py b/overviewer_core/world.py index 95effe5..f20e317 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -513,7 +513,7 @@ class RegionSet(object): Coords can be either be global chunk coords, or local to a region """ - (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),None) + (regionfile,filemtime) = self.regionfiles.get((chunkX//32, chunkY//32),(None, None)) return regionfile def _iterate_regionfiles(self): From 6ee3eba550c4be1fc52a04095b2508a171d2449d Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:08:19 -0500 Subject: [PATCH 06/45] Do not use the newer than check on windows since apparently minecraft will not force a timestamp update on the file when they are written until it's closed. --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 6046ea4..d14bde2 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -758,7 +758,7 @@ class TileSet(object): # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if markall else self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall || platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 8023b52fdcc20a951349e151a473ff88d991e83e Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:52:53 -0500 Subject: [PATCH 07/45] Pythonic or --- overviewer_core/tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index d14bde2..41a8756 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -758,7 +758,7 @@ class TileSet(object): # tile is older, mark it in a RendertileSet object as dirty. - for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall || platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): + for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall or platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime): chunkcount += 1 if chunkmtime > max_chunk_mtime: From 18beae66249833164a151c71bf905cd3eaec0bc4 Mon Sep 17 00:00:00 2001 From: Patrick-Emmanuel Boulanger-Nadeau Date: Tue, 21 Jan 2014 10:55:13 -0500 Subject: [PATCH 08/45] Import the platform --- overviewer_core/tileset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 41a8756..8899993 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -24,6 +24,7 @@ import functools import time import errno import stat +import platform from collections import namedtuple from itertools import product, izip, chain From 09477ed8a0a8b06764404c60add0c8b2744ce1ad Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 4 Mar 2014 00:39:59 +0100 Subject: [PATCH 09/45] Rewrote image optimisation stuff. The old one was broken anyway. --- docs/config.rst | 88 +++++++++++++++++++-- overviewer_core/optimizeimages.py | 110 ++++++++++++++++++++------ overviewer_core/settingsDefinition.py | 3 +- overviewer_core/settingsValidators.py | 10 ++- overviewer_core/tileset.py | 12 +-- 5 files changed, 182 insertions(+), 41 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 5d63d21..d1bc878 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -545,18 +545,92 @@ values. The valid configuration keys are listed below. **Default:** ``95`` ``optimizeimg`` + + .. warning:: + Using image optimizers will increase render times significantly. + This option specifies which additional tools overviewer should use to optimize the filesize of png tiles. The tools used must be placed somewhere, where overviewer can find them, for example the "PATH" environment variable or a directory like /usr/bin. - This should be an integer between 0 and 3. - * ``1 - Use pngcrush`` - * ``2 - Use advdef`` - * ``3 - Use pngcrush and advdef (Not recommended)`` - Using this option may significantly increase render time, but will make - the resulting tiles smaller, with lossless image quality. - **Default:** ``0`` + The option is a list of Optimizer objects, which are then executed in + the order in which they're specified:: + + from optimizeimages import pngnq, optipng + worlds["world"] = "/path/to/world" + + renders["daytime"] = { + "world":"world", + "title":"day", + "rendermode":smooth_lighting, + "optimizeimg":[pngnq(sampling=1), optipng(olevel=3)], + } + + Here is a list of supported image optimization programs: + + ``pngnq`` + pngnq quantizes 32-bit RGBA images into 8-bit RGBA palette PNGs. This is + lossy, but reduces filesize significantly. Available settings: + + ``sampling`` + An integer between ``1`` and ``10``, ``1`` samples all pixels, is slow and yields + the best quality. Higher values sample less of the image, which makes + the process faster, but less accurate. + + **Default:** ``3`` + + ``dither`` + Either the string ``"n"`` for no dithering, or ``"f"`` for Floyd + Steinberg dithering. Dithering helps eliminate colorbanding, sometimes + increasing visual quality. + + .. warning:: + With pngnq version 1.0 (which is what Ubuntu 12.04 ships), the + dithering option is broken. Only the default, no dithering, + can be specified on those systems. + + **Default:** ``"n"`` + + .. warning:: + Because of several PIL bugs, only the most zoomed in level has transparency + when using pngnq. The other zoom levels have all transparency replaced by + black. This is *not* pngnq's fault, as pngnq supports multiple levels of + transparency just fine, it's PIL's fault for not even reading indexed + PNGs correctly. + + ``optipng`` + optipng tunes the deflate algorithm and removes unneeded channels from the PNG, + producing a smaller, lossless output image. It was inspired by pngcrush. + Available settings: + + ``olevel`` + An integer between ``0`` (few optimizations) and ``7`` (many optimizations). + The default should be satisfactory for everyone, higher levels than the default + see almost no benefit. + + **Default:** ``2`` + + ``pngcrush`` + pngcrush is very slow and not very good, you should use optipng in probably all cases. + However, Overviewer still allows you to use it because we're nice people like that. + Available settings: + + ``brute`` + Either ``True`` or ``False``. Cycles through all compression methods, and is very slow. + + .. note:: + There is practically no reason to ever use this. optipng will beat pngcrush, and + throwing more CPU time at pngcrush most likely won't help. If you think you need + this option, then you are most likely wrong. + + **Default:** ``False`` + + .. note:: + Don't forget to import the optimizers you use in your settings file, as shown in the + example above. + + **Default:** ``[]`` ``bgcolor`` This is the background color to be displayed behind the map. Its value diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index 4422feb..f31b6a1 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -17,36 +17,96 @@ import os import subprocess import shlex -pngcrush = "pngcrush" -optipng = "optipng" -advdef = "advdef" +class Optimizer: + binaryname = "" -def check_programs(level): + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + + def optimize(self, img): + raise NotImplementedError("I can't let you do that, Dave.") + + def fire_and_forget(self, args): + subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + +class NonAtomicOptimizer(Optimizer): + def cleanup(self, img): + os.rename(img + ".tmp", img) + + def fire_and_forget(self, args, img): + subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + self.cleanup(img) + +class PNGOptimizer: + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + +class JPEGOptimizer: + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + +class pngnq(NonAtomicOptimizer, PNGOptimizer): + binaryname = "pngnq" + + def __init__(self, sampling=3, dither="n"): + if sampling < 1 or sampling > 10: + raise Exception("Invalid sampling value '%d' for pngnq!" % sampling) + + if dither not in ["n", "f"]: + raise Exception("Invalid dither method '%s' for pngnq!" % dither) + + self.sampling = sampling + self.dither = dither + + def optimize(self, img): + if img.endswith(".tmp"): + extension = ".tmp" + else: + extension = ".png.tmp" + + NonAtomicOptimizer.fire_and_forget(self, [self.binaryname, "-s", str(self.sampling), + "-Q", self.dither, "-e", extension, img], img) + +class pngcrush(NonAtomicOptimizer, PNGOptimizer): + binaryname = "pngcrush" + # really can't be bothered to add some interface for all + # the pngcrush options, it sucks anyway + def __init__(self, brute=False): + self.brute = brute + + def optimize(self, img): + args = [self.binaryname, img, img + ".tmp"] + if self.brute == True: # Was the user an idiot? + args.insert(1, "-brute") + + NonAtomicOptimizer.fire_and_forget(self, args, img) + +class optipng(Optimizer, PNGOptimizer): + binaryname = "optipng" + + def __init__(self, olevel=2): + self.olevel = olevel + + def optimize(self, img): + Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img]) + + +def check_programs(optimizers): path = os.environ.get("PATH").split(os.pathsep) def exists_in_path(prog): result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) return len(result) != 0 - for prog,l in [(pngcrush,1), (advdef,2)]: - if l <= level: - if (not exists_in_path(prog)) and (not exists_in_path(prog + ".exe")): - raise Exception("Optimization prog %s for level %d not found!" % (prog, l)) - -def optimize_image(imgpath, imgformat, optimizeimg): - if imgformat == 'png': - if optimizeimg >= 1: - # we can't do an atomic replace here because windows is terrible - # so instead, we make temp files, delete the old ones, and rename - # the temp files. go windows! - subprocess.Popen([pngcrush, imgpath, imgpath + ".tmp"], - stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] - os.remove(imgpath) - os.rename(imgpath+".tmp", imgpath) - - if optimizeimg >= 2: - # the "-nc" it's needed to no broke the transparency of tiles - recompress_option = "-z2" if optimizeimg == 2 else "-z4" - subprocess.Popen([advdef, recompress_option,imgpath], stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).communicate()[0] + for opt in optimizers: + if (not exists_in_path(opt.binaryname)) and (not exists_in_path(opt.binaryname + ".exe")): + raise Exception("Optimization program '%s' was not found!" % opt.binaryname) +def optimize_image(imgpath, imgformat, optimizers): + for opt in optimizers: + if imgformat == 'png': + if isinstance(opt, PNGOptimizer): + opt.optimize(imgpath) + elif imgformat == 'jpg': + if isinstance(opt, JPEGOptimizer): + opt.optimize(imgpath) diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index 41b8970..c17a171 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -46,6 +46,7 @@ from settingsValidators import * import util from observer import ProgressBarObserver, LoggingObserver, JSObserver +from optimizeimages import pngnq, optipng, pngcrush import platform import sys @@ -72,7 +73,7 @@ renders = Setting(required=True, default=util.OrderedDict(), "imgquality": Setting(required=False, validator=validateImgQuality, default=95), "bgcolor": Setting(required=True, validator=validateBGColor, default="1a1a1a"), "defaultzoom": Setting(required=True, validator=validateDefaultZoom, default=1), - "optimizeimg": Setting(required=True, validator=validateOptImg, default=0), + "optimizeimg": Setting(required=True, validator=validateOptImg, default=[]), "nomarkers": Setting(required=False, validator=validateBool, default=None), "texturepath": Setting(required=False, validator=validateTexturePath, default=None), "renderchecks": Setting(required=False, validator=validateInt, default=None), diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 6b5afa9..0279ed7 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -5,6 +5,7 @@ from collections import namedtuple import rendermodes import util +from optimizeimages import Optimizer from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT class ValidationException(Exception): @@ -155,8 +156,13 @@ def validateBGColor(color): return color -def validateOptImg(opt): - return bool(opt) +def validateOptImg(optimizers): + if isinstance(optimizers, (int, long)): + raise ValidationException("You are using a deprecated method of specifying optimizeimg!") + for opt in optimizers: + if not isinstance(opt, Optimizer): + raise ValidationException("Invalid Optimizer!") + return optimizers def validateTexturePath(path): # Expand user dir in directories strings diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 3e13129..a2338a8 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -246,11 +246,7 @@ class TileSet(object): relevant in jpeg mode. optimizeimg - an integer indiating optimizations to perform on png outputs. 0 - indicates no optimizations. Only relevant in png mode. - 1 indicates pngcrush is run on all output images - 2 indicates pngcrush and advdef are run on all output images with advdef -z2 - 3 indicates pngcrush and advdef are run on all output images with advdef -z4 + A list of optimizer instances to use. rendermode Perhaps the most important/relevant option: a string indicating the @@ -892,7 +888,11 @@ class TileSet(object): try: #quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS) src = Image.open(path[1]) + # optimizeimg may have converted them to a palette image in the meantime + if src.mode != "RGB" and src.mode != "RGBA": + src = src.convert("RGBA") src.load() + quad = Image.new("RGBA", (192, 192), self.options['bgcolor']) resize_half(quad, src) img.paste(quad, path[0]) @@ -1017,7 +1017,7 @@ class TileSet(object): if self.options['optimizeimg']: optimize_image(tmppath, self.imgextension, self.options['optimizeimg']) - + os.utime(tmppath, (max_chunk_mtime, max_chunk_mtime)) def _iterate_and_check_tiles(self, path): From 1cf131a8fc410be95e3560c7f90fec970bdecde6 Mon Sep 17 00:00:00 2001 From: Luc Ritchie Date: Sat, 8 Mar 2014 23:01:40 -0500 Subject: [PATCH 10/45] Handle UUID player files semi-nicely in POIgen (14w10a+, 1.7.6+) --- overviewer_core/aux_files/genPOI.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 5fa4b00..367eb19 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -132,7 +132,11 @@ def handlePlayers(rset, render, worldpath): dimension = int(mystdim.group(1)) else: raise - playerdir = os.path.join(worldpath, "players") + # TODO: get player names from UUIDs once Mojang makes available an API to do it + playerdir = os.path.join(worldpath, "playerdata") + if not os.path.isdir(playerdir): + playerdir = os.path.join(worldpath, "players") + if os.path.isdir(playerdir): playerfiles = os.listdir(playerdir) playerfiles = [x for x in playerfiles if x.endswith(".dat")] From 0e1bd4369a0aaf0e1734fcc5974705dc9f0428be Mon Sep 17 00:00:00 2001 From: Luc Ritchie Date: Sun, 9 Mar 2014 16:22:00 -0400 Subject: [PATCH 11/45] Use Mojang's session API to get usernames from UUIDs --- overviewer_core/aux_files/genPOI.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 367eb19..72eae2f 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -19,6 +19,7 @@ import logging import json import sys import re +import urllib2 import Queue import multiprocessing @@ -30,6 +31,8 @@ from overviewer_core import logger from overviewer_core import nbt from overviewer_core import configParser, world +UUID_LOOKUP_URL = 'https://sessionserver.mojang.com/session/minecraft/profile/' + def replaceBads(s): "Replaces bad characters with good characters!" bads = [" ", "(", ")"] @@ -132,10 +135,11 @@ def handlePlayers(rset, render, worldpath): dimension = int(mystdim.group(1)) else: raise - # TODO: get player names from UUIDs once Mojang makes available an API to do it playerdir = os.path.join(worldpath, "playerdata") + useUUIDs = True if not os.path.isdir(playerdir): playerdir = os.path.join(worldpath, "players") + useUUIDs = False if os.path.isdir(playerdir): playerfiles = os.listdir(playerdir) @@ -156,6 +160,13 @@ def handlePlayers(rset, render, worldpath): logging.warning("Skipping bad player dat file %r", playerfile) continue playername = playerfile.split(".")[0] + if useUUIDs: + try: + profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + playername.replace('-','')).read()) + if 'name' in profile: + playername = profile['name'] + except ValueError: + logging.warning("Unable to get player name for UUID %s", playername) if isSinglePlayer: playername = 'Player' if data['Dimension'] == dimension: From a5b7c9617f908a9cb21544c23d578fb8ab0dcef7 Mon Sep 17 00:00:00 2001 From: kiyote Date: Sat, 29 Mar 2014 09:17:59 -0500 Subject: [PATCH 12/45] Added ['time'] attribute to Players POI It's nice that we can see where a player was at logout, but now we can see when that logout was. --- overviewer_core/aux_files/genPOI.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 5fa4b00..9ca0b92 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -15,6 +15,7 @@ markers.js holds a list of which markerSets are attached to each tileSet ''' import os +import time import logging import json import sys @@ -161,6 +162,8 @@ def handlePlayers(rset, render, worldpath): data['x'] = int(data['Pos'][0]) data['y'] = int(data['Pos'][1]) data['z'] = int(data['Pos'][2]) + # Time at last logout, calculated from last time the player's file was modified + data['time'] = time.ctime(os.path.getmtime(os.path.join(playerdir, playerfile)) rset._pois['Players'].append(data) if "SpawnX" in data and dimension == 0: # Spawn position (bed or main spawn) From 3f222796c573fd21e68a449df2e7c3577ae95b32 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 30 Mar 2014 18:50:46 -0400 Subject: [PATCH 13/45] Print a limit=1 traceback if failed to build --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index cc55dce..1b4b30d 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +import traceback # quick version check if not (sys.version_info[0] == 2 and sys.version_info[1] >= 6): @@ -272,6 +273,7 @@ class CustomBuild(build): build.run(self) print("\nBuild Complete") except Exception: + traceback.print_exc(limit=1) print("\nFailed to build Overviewer!") print("Please review the errors printed above and the build instructions") print("at . If you are") From a8fc3300b4e1432ec424cf279692851cf0be5a50 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 5 Apr 2014 20:29:58 -0400 Subject: [PATCH 14/45] general fixes for min/maxzoom, documentation cleared up hopeful fix for #1086 --- docs/config.rst | 20 +++++++++++++++----- overviewer_core/data/js_src/util.js | 21 +++++++++++++++++---- overviewer_core/tileset.py | 2 ++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 5d63d21..24067fe 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -566,13 +566,22 @@ values. The valid configuration keys are listed below. **Default:** ``#1a1a1a`` ``defaultzoom`` - This value specifies the default zoom level that the map will be opened - with. It has to be greater than 0. + This value specifies the default zoom level that the map will be + opened with. It has to be greater than 0, which corresponds to the + most zoomed-out level. If you use ``minzoom`` or ``maxzoom``, it + should be between those two. **Default:** ``1`` ``maxzoom`` - This specifies the maximum zoom allowed by the zoom control on the web page. + This specifies the maximum, closest in zoom allowed by the zoom + control on the web page. This is relative to 0, the farthest-out + image, so setting this to 8 will allow you to zoom in at most 8 + times. This is *not* relative to ``minzoom``, so setting + ``minzoom`` will shave off even more levels. If you wish to + specify how many zoom levels to leave off, instead of how many + total to use, use a negative number here. For example, setting + this to -2 will disable the two most zoomed-in levels. .. note:: @@ -583,8 +592,9 @@ values. The valid configuration keys are listed below. **Default:** Automatically set to most detailed zoom level ``minzoom`` - This specifies the minimum zoom allowed by the zoom control on the web page. For - example, setting this to 2 will disable the two most-zoomed out levels. + This specifies the minimum, farthest away zoom allowed by the zoom + control on the web page. For example, setting this to 2 will + disable the two most zoomed-out levels. .. note:: diff --git a/overviewer_core/data/js_src/util.js b/overviewer_core/data/js_src/util.js index 62bad82..6121cf5 100644 --- a/overviewer_core/data/js_src/util.js +++ b/overviewer_core/data/js_src/util.js @@ -119,7 +119,7 @@ overviewer.util = { zoom = overviewer.mapView.options.currentTileSet.get('minZoom'); } else { zoom = parseInt(zoom); - if (zoom < 0 && zoom + overviewer.mapView.options.currentTileSet.get('maxZoom') >= 0) { + if (zoom < 0) { // if zoom is negative, treat it as a "zoom out from max" zoom += overviewer.mapView.options.currentTileSet.get('maxZoom'); } else { @@ -127,6 +127,13 @@ overviewer.util = { zoom = overviewer.mapView.options.currentTileSet.get('defaultZoom'); } } + + // clip zoom + if (zoom > overviewer.mapView.options.currentTileSet.get('maxZoom')) + zoom = overviewer.mapView.options.currentTileSet.get('maxZoom'); + if (zoom < overviewer.mapView.options.currentTileSet.get('minZoom')) + zoom = overviewer.mapView.options.currentTileSet.get('minZoom'); + overviewer.map.setZoom(zoom); } @@ -512,9 +519,9 @@ overviewer.util = { } - if (zoom == currTileset.get('maxZoom')) { + if (zoom >= currTileset.get('maxZoom')) { zoom = 'max'; - } else if (zoom == currTileset.get('minZoom')) { + } else if (zoom <= currTileset.get('minZoom')) { zoom = 'min'; } else { // default to (map-update friendly) negative zooms @@ -556,7 +563,7 @@ overviewer.util = { zoom = tsetModel.get('minZoom'); } else { zoom = parseInt(zoom); - if (zoom < 0 && zoom + tsetModel.get('maxZoom') >= 0) { + if (zoom < 0) { // if zoom is negative, treat it as a "zoom out from max" zoom += tsetModel.get('maxZoom'); } else { @@ -565,6 +572,12 @@ overviewer.util = { } } + // clip zoom + if (zoom > tsetModel.get('maxZoom')) + zoom = tsetModel.get('maxZoom'); + if (zoom < tsetModel.get('minZoom')) + zoom = tsetModel.get('minZoom'); + overviewer.map.setCenter(latlngcoords); overviewer.map.setZoom(zoom); var locationmarker = new overviewer.views.LocationIconView(); diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index e665bff..032dcdb 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -552,7 +552,9 @@ class TileSet(object): poititle = self.options.get("poititle"), showlocationmarker = self.options.get("showlocationmarker") ) + d['maxZoom'] = min(self.treedepth, d['maxZoom']) d['minZoom'] = min(max(0, self.options.get("minzoom", 0)), d['maxZoom']) + d['defaultZoom'] = max(d['minZoom'], min(d['defaultZoom'], d['maxZoom'])) if isOverlay: d.update({"tilesets": self.options.get("overlay")}) From 8e5944d5c3fc1c1aae8331daa1b72af0c29d8702 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Thu, 10 Apr 2014 19:52:50 -0400 Subject: [PATCH 15/45] Catch some additional UUID lookup errors from urllib2 --- overviewer_core/aux_files/genPOI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 72eae2f..f0f5492 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -165,7 +165,7 @@ def handlePlayers(rset, render, worldpath): profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + playername.replace('-','')).read()) if 'name' in profile: playername = profile['name'] - except ValueError: + except (ValueError, urllib2.URLError): logging.warning("Unable to get player name for UUID %s", playername) if isSinglePlayer: playername = 'Player' From 9a3305932e5e1e72e8f78f29cb70e9ffa0c45343 Mon Sep 17 00:00:00 2001 From: rymate1234 Date: Fri, 11 Apr 2014 16:10:56 +0200 Subject: [PATCH 16/45] Add a small message stating where the render is and how to open it --- overviewer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/overviewer.py b/overviewer.py index f5b8969..a9a82ae 100755 --- a/overviewer.py +++ b/overviewer.py @@ -514,6 +514,8 @@ dir but you forgot to put quotes around the directory, since it contains spaces. if options.pid: os.remove(options.pid) + logging.info("Your render has been written to '%s', open index.html to view it" % destdir) + return 0 def list_worlds(): From 7d87d2565828da7e75ee497e2299f5a262a01bdc Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Tue, 15 Apr 2014 11:22:41 -0400 Subject: [PATCH 17/45] Fix ServerAnnounceObserver typo --- overviewer_core/observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index e9a5370..32a472c 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -345,7 +345,7 @@ class ServerAnnounceObserver(Observer): def update(self, current_value): super(ServerAnnounceObserver, self).update(current_value) - if self._need_update(current_value): + if self._need_update(): self._send_output('Rendered %d of %d tiles, %d%% complete' % (self.get_current_value(), self.get_max_value(), self.get_percentage())) From 7c23d6e86ac85f7e1bec66aab71a541347a979b4 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Thu, 1 May 2014 16:48:28 -0400 Subject: [PATCH 18/45] added rendercheck mode 3, the identity function of rendercheck modes! --- docs/config.rst | 8 ++++++++ overviewer.py | 17 +++++++++++------ overviewer_core/tileset.py | 36 +++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 24067fe..2b6cfe2 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -704,6 +704,14 @@ values. The valid configuration keys are listed below. 'forcerender': True, } +``renderchecks`` + This is an integer, and functions as a more complex form of + ``forcerender``. Setting it to 1 enables :option:`--check-tiles` + mode, setting it to 2 enables :option:`--forcerender`, and 3 tells + Overviewer to keep this particular render in the output, but + otherwise don't update it. It defaults to 0, which is the usual + update checking mode. + ``changelist`` This is a string. It names a file where it will write out, one per line, the path to tiles that have been updated. You can specify the same file for diff --git a/overviewer.py b/overviewer.py index f5b8969..71e1c09 100755 --- a/overviewer.py +++ b/overviewer.py @@ -318,19 +318,24 @@ dir but you forgot to put quotes around the directory, since it contains spaces. "--check-tiles, and --no-tile-checks. These options conflict.") parser.print_help() return 1 + + def set_renderchecks(checkname, num): + for name, render in config['renders'].iteritems(): + if render.get('renderchecks', 0) == 3: + logging.warning(checkname + " ignoring render " + repr(name) + " since it's marked as \"don't render\".") + else: + render['renderchecks'] = num + if options.forcerender: logging.info("Forcerender mode activated. ALL tiles will be rendered") - for render in config['renders'].itervalues(): - render['renderchecks'] = 2 + set_renderchecks("forcerender", 2) elif options.checktiles: logging.info("Checking all tiles for updates manually.") - for render in config['renders'].itervalues(): - render['renderchecks'] = 1 + set_renderchecks("checktiles", 1) elif options.notilechecks: logging.info("Disabling all tile mtime checks. Only rendering tiles "+ "that need updating since last render") - for render in config['renders'].itervalues(): - render['renderchecks'] = 0 + set_renderchecks("notilechecks", 0) if not config['renders']: logging.error("You must specify at least one render in your config file. See the docs if you're having trouble") diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 032dcdb..8d3e8d6 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -130,6 +130,14 @@ Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow")) # slowest, but SHOULD be specified if this is the first render because # the scan will forgo tile stat calls. It's also useful for changing # texture packs or other options that effect the output. + +# 3 +# A very special mode. Using this will not actually render +# anything, but will leave this tileset in the resulting +# map. Useful for renders that you want to keep, but not +# update. Since this mode is so simple, it's left out of the +# rest of this discussion. + # # For 0 our caller has explicitly requested not to check mtimes on disk to # speed things up. So the mode 0 chunk scan only looks at chunk mtimes and the @@ -238,6 +246,13 @@ class TileSet(object): useful for changing texture packs or other options that effect the output. + 3 + A very special mode. Using this will not actually render + anything, but will leave this tileset in the resulting + map. Useful for renders that you want to keep, but not + update. Since this mode is so simple, it's left out of the + rest of this discussion. + imgformat A string indicating the output format. Must be one of 'png' or 'jpeg' @@ -390,6 +405,11 @@ class TileSet(object): attribute for later use in iterate_work_items() """ + + # skip if we're told to + if self.options['renderchecks'] == 3: + return + # REMEMBER THAT ATTRIBUTES ASSIGNED IN THIS METHOD ARE NOT AVAILABLE IN # THE do_work() METHOD (because this is only called in the main process # not the workers) @@ -416,15 +436,16 @@ class TileSet(object): return 1 def get_phase_length(self, phase): - """Returns the number of work items in a given phase, or None if there - is no good estimate. + """Returns the number of work items in a given phase. """ # Yeah functional programming! + # and by functional we mean a bastardized python switch statement return { 0: lambda: self.dirtytree.count_all(), #there is no good way to guess this so just give total count 1: lambda: (4**(self.treedepth+1)-1)/3, 2: lambda: self.dirtytree.count_all(), + 3: lambda: 0, }[self.options['renderchecks']]() def iterate_work_items(self, phase): @@ -434,6 +455,10 @@ class TileSet(object): This method returns an iterator over (obj, [dependencies, ...]) """ + # skip if asked to + if self.options['renderchecks'] == 3: + return + # The following block of code implementes the changelist functionality. fd = self.options.get("changelist", None) if fd: @@ -536,6 +561,11 @@ class TileSet(object): def bgcolorformat(color): return "#%02x%02x%02x" % color[0:3] isOverlay = self.options.get("overlay") or (not any(isinstance(x, rendermodes.Base) for x in self.options.get("rendermode"))) + + # don't update last render time if we're leaving this alone + last_rendertime = self.last_rendertime + if self.options['renderchecks'] != 3: + last_rendertime = self.max_chunk_mtime d = dict(name = self.options.get('title'), zoomLevels = self.treedepth, @@ -546,7 +576,7 @@ class TileSet(object): bgcolor = bgcolorformat(self.options.get('bgcolor')), world = self.options.get('worldname_orig') + (" - " + self.options.get('dimension')[0] if self.options.get('dimension')[1] != 0 else ''), - last_rendertime = self.max_chunk_mtime, + last_rendertime = last_rendertime, imgextension = self.imgextension, isOverlay = isOverlay, poititle = self.options.get("poititle"), From 9487d6f5bdb45d50d8e0388126f35d310e887a0c Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sat, 3 May 2014 00:23:10 -0400 Subject: [PATCH 19/45] Added a --check-version option --- overviewer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/overviewer.py b/overviewer.py index 71e1c09..4d22e3a 100755 --- a/overviewer.py +++ b/overviewer.py @@ -84,6 +84,8 @@ def main(): help="Tries to locate the texture files. Useful for debugging texture problems.") parser.add_option("-V", "--version", dest="version", help="Displays version information and then exits", action="store_true") + parser.add_option("--check-version", dest="checkversion", + help="Fetchs information about the latest version of Overviewer", action="store_true") parser.add_option("--update-web-assets", dest='update_web_assets', action="store_true", help="Update web assets. Will *not* render tiles or update overviewerConfig.js") @@ -141,8 +143,28 @@ def main(): if options.verbose > 0: print("Python executable: %r" % sys.executable) print(sys.version) + if not options.checkversion: + return 0 + if options.checkversion: + print("Currently running Minecraft Overviewer %s" % util.findGitVersion()), + print("(%s)" % util.findGitHash()[:7]) + try: + import urllib + import json + latest_ver = json.loads(urllib.urlopen("http://overviewer.org/download.json").read())['src'] + print("Latest version of Minecraft Overviewer %s (%s)" % (latest_ver['version'], latest_ver['commit'][:7])) + print("See http://overviewer.org/downloads for more information") + except Exception: + print("Failed to fetch latest version info.") + if options.verbose > 0: + import traceback + traceback.print_exc() + else: + print("Re-run with --verbose for more details") + return 1 return 0 + if options.pid: if os.path.exists(options.pid): try: From 866c2fe064ed7045a1536b479b5739950ae3e44e Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 19:54:43 +0200 Subject: [PATCH 20/45] Work around and warn if old optimizeimg definition Before someone says this is incorrect because it only ever uses pngcrush: The old code always used pngcrush and nothing else anyway. This is absolutely correct and the old behaviour. I also added a check to make sure it's a list, as some people might forget the whole list thing. --- overviewer_core/settingsValidators.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 0279ed7..6b53bee 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -158,7 +158,12 @@ def validateBGColor(color): def validateOptImg(optimizers): if isinstance(optimizers, (int, long)): - raise ValidationException("You are using a deprecated method of specifying optimizeimg!") + from optimizeimages import pngcrush + import logging + logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.") + optimizers = [pngcrush()] + if not isinstance(optimizers, list): + raise ValidationException("optimizeimg is not a list. Make sure you specify them like [foo()], with square brackets.") for opt in optimizers: if not isinstance(opt, Optimizer): raise ValidationException("Invalid Optimizer!") From 8817972b36145f5531f25bb1da40b9cd064ecbc4 Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 20:10:42 +0200 Subject: [PATCH 21/45] Move the check for program availability into class We also actually execute it now. Go us! --- overviewer_core/optimizeimages.py | 23 ++++++++++++----------- overviewer_core/settingsValidators.py | 5 ++++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index f31b6a1..c877e75 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -16,6 +16,7 @@ import os import subprocess import shlex +import logging class Optimizer: binaryname = "" @@ -29,6 +30,17 @@ class Optimizer: def fire_and_forget(self, args): subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + def check_availability(self): + path = os.environ.get("PATH").split(os.pathsep) + + def exists_in_path(prog): + result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) + return len(result) != 0 + + if (not exists_in_path(self.binaryname)) and (not exists_in_path(self.binaryname + ".exe")): + raise Exception("Optimization program '%s' was not found!" % self.binaryname) + + class NonAtomicOptimizer(Optimizer): def cleanup(self, img): os.rename(img + ".tmp", img) @@ -91,17 +103,6 @@ class optipng(Optimizer, PNGOptimizer): Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img]) -def check_programs(optimizers): - path = os.environ.get("PATH").split(os.pathsep) - - def exists_in_path(prog): - result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) - return len(result) != 0 - - for opt in optimizers: - if (not exists_in_path(opt.binaryname)) and (not exists_in_path(opt.binaryname + ".exe")): - raise Exception("Optimization program '%s' was not found!" % opt.binaryname) - def optimize_image(imgpath, imgformat, optimizers): for opt in optimizers: if imgformat == 'png': diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 6b53bee..e61a1ab 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -7,6 +7,7 @@ import rendermodes import util from optimizeimages import Optimizer from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT +import logging class ValidationException(Exception): pass @@ -159,7 +160,6 @@ def validateBGColor(color): def validateOptImg(optimizers): if isinstance(optimizers, (int, long)): from optimizeimages import pngcrush - import logging logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.") optimizers = [pngcrush()] if not isinstance(optimizers, list): @@ -167,6 +167,9 @@ def validateOptImg(optimizers): for opt in optimizers: if not isinstance(opt, Optimizer): raise ValidationException("Invalid Optimizer!") + + opt.check_availability() + return optimizers def validateTexturePath(path): From 03561dccfad75faec34906c08677c00cb68dfdc6 Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 22:31:23 +0200 Subject: [PATCH 22/45] Use check_call(), add workaround for broken pngnq Also -f pngnq to write files, in case something didn't work last render. --- overviewer_core/optimizeimages.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index c877e75..da8d306 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -28,7 +28,7 @@ class Optimizer: raise NotImplementedError("I can't let you do that, Dave.") def fire_and_forget(self, args): - subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + subprocess.check_call(args) def check_availability(self): path = os.environ.get("PATH").split(os.pathsep) @@ -46,7 +46,7 @@ class NonAtomicOptimizer(Optimizer): os.rename(img + ".tmp", img) def fire_and_forget(self, args, img): - subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + subprocess.check_call(args) self.cleanup(img) class PNGOptimizer: @@ -76,8 +76,13 @@ class pngnq(NonAtomicOptimizer, PNGOptimizer): else: extension = ".png.tmp" - NonAtomicOptimizer.fire_and_forget(self, [self.binaryname, "-s", str(self.sampling), - "-Q", self.dither, "-e", extension, img], img) + args = [self.binaryname, "-s", str(self.sampling), "-f", "-e", extension, img] + # Workaround for poopbuntu 12.04 which ships an old broken pngnq + if self.dither != "n": + args.insert(1, "-Q") + args.insert(2, self.dither) + + NonAtomicOptimizer.fire_and_forget(self, args, img) class pngcrush(NonAtomicOptimizer, PNGOptimizer): binaryname = "pngcrush" From 6d28942626f063927c475e2794e7b3caa18f5d3b Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Thu, 8 May 2014 20:49:41 +0200 Subject: [PATCH 23/45] Clarify optimizeimg docs; add warnings The validator will now warn if it detects that a crushed output is fed into something that is not a crusher. The is_crusher method of an optimizer shall return True if the optimisation process is lossless, and does try to find optimal encoding parameters as opposed to only removing unneeded channels or reducing palettes. --- docs/config.rst | 15 +++++++++------ overviewer_core/optimizeimages.py | 13 +++++++++++++ overviewer_core/settingsValidators.py | 15 ++++++++++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index eb2bfa8..5db549c 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -557,7 +557,9 @@ values. The valid configuration keys are listed below. The option is a list of Optimizer objects, which are then executed in the order in which they're specified:: + # Import the optimizers we need from optimizeimages import pngnq, optipng + worlds["world"] = "/path/to/world" renders["daytime"] = { @@ -566,6 +568,10 @@ values. The valid configuration keys are listed below. "rendermode":smooth_lighting, "optimizeimg":[pngnq(sampling=1), optipng(olevel=3)], } + + .. note:: + Don't forget to import the optimizers you use in your config file, as shown in the + example above. Here is a list of supported image optimization programs: @@ -612,8 +618,9 @@ values. The valid configuration keys are listed below. **Default:** ``2`` ``pngcrush`` - pngcrush is very slow and not very good, you should use optipng in probably all cases. - However, Overviewer still allows you to use it because we're nice people like that. + pngcrush, like optipng, is a lossless PNG recompressor. If you are able to do so, it + is recommended to use optipng instead, as it generally yields better results in less + time. Available settings: ``brute`` @@ -626,10 +633,6 @@ values. The valid configuration keys are listed below. **Default:** ``False`` - .. note:: - Don't forget to import the optimizers you use in your settings file, as shown in the - example above. - **Default:** ``[]`` ``bgcolor`` diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index da8d306..1d79f71 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -39,6 +39,10 @@ class Optimizer: if (not exists_in_path(self.binaryname)) and (not exists_in_path(self.binaryname + ".exe")): raise Exception("Optimization program '%s' was not found!" % self.binaryname) + + def is_crusher(self): + """Should return True if the optimization is lossless, i.e. none of the actual image data will be changed.""" + raise NotImplementedError("I'm so abstract I can't even say whether I'm a crusher.") class NonAtomicOptimizer(Optimizer): @@ -84,6 +88,9 @@ class pngnq(NonAtomicOptimizer, PNGOptimizer): NonAtomicOptimizer.fire_and_forget(self, args, img) + def is_crusher(self): + return False + class pngcrush(NonAtomicOptimizer, PNGOptimizer): binaryname = "pngcrush" # really can't be bothered to add some interface for all @@ -98,6 +105,9 @@ class pngcrush(NonAtomicOptimizer, PNGOptimizer): NonAtomicOptimizer.fire_and_forget(self, args, img) + def is_crusher(self): + return True + class optipng(Optimizer, PNGOptimizer): binaryname = "optipng" @@ -106,6 +116,9 @@ class optipng(Optimizer, PNGOptimizer): def optimize(self, img): Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img]) + + def is_crusher(self): + return True def optimize_image(imgpath, imgformat, optimizers): diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index e61a1ab..fd9f757 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -160,16 +160,25 @@ def validateBGColor(color): def validateOptImg(optimizers): if isinstance(optimizers, (int, long)): from optimizeimages import pngcrush - logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.") + logging.warning("You're using a deprecated definition of optimizeimg. "\ + "We'll do what you say for now, but please fix this as soon as possible.") optimizers = [pngcrush()] if not isinstance(optimizers, list): - raise ValidationException("optimizeimg is not a list. Make sure you specify them like [foo()], with square brackets.") - for opt in optimizers: + raise ValidationException("What you passed to optimizeimg is not a list. "\ + "Make sure you specify them like [foo()], with square brackets.") + + for opt, next_opt in zip(optimizers, optimizers[1:]) + [(optimizers[-1], None)]: if not isinstance(opt, Optimizer): raise ValidationException("Invalid Optimizer!") opt.check_availability() + # Check whether the chaining is somewhat sane + if next_opt: + if opt.is_crusher() and not next_opt.is_crusher(): + logging.warning("You're feeding a crushed output into an optimizer that does not crush. "\ + "This is most likely pointless, and wastes time.") + return optimizers def validateTexturePath(path): From 6812cad59660d20a73a986577782c152a48432dd Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Fri, 9 May 2014 16:58:47 +0200 Subject: [PATCH 24/45] Fix validator for empty lists (the default value) Whoops. --- overviewer_core/settingsValidators.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index fd9f757..29553a7 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -167,17 +167,18 @@ def validateOptImg(optimizers): raise ValidationException("What you passed to optimizeimg is not a list. "\ "Make sure you specify them like [foo()], with square brackets.") - for opt, next_opt in zip(optimizers, optimizers[1:]) + [(optimizers[-1], None)]: - if not isinstance(opt, Optimizer): - raise ValidationException("Invalid Optimizer!") + if optimizers: + for opt, next_opt in zip(optimizers, optimizers[1:]) + [(optimizers[-1], None)]: + if not isinstance(opt, Optimizer): + raise ValidationException("Invalid Optimizer!") - opt.check_availability() + opt.check_availability() - # Check whether the chaining is somewhat sane - if next_opt: - if opt.is_crusher() and not next_opt.is_crusher(): - logging.warning("You're feeding a crushed output into an optimizer that does not crush. "\ - "This is most likely pointless, and wastes time.") + # Check whether the chaining is somewhat sane + if next_opt: + if opt.is_crusher() and not next_opt.is_crusher(): + logging.warning("You're feeding a crushed output into an optimizer that does not crush. "\ + "This is most likely pointless, and wastes time.") return optimizers From b6ac54a2b61d0b8f45adab36ac5179e83bcef710 Mon Sep 17 00:00:00 2001 From: matrixhacker Date: Fri, 9 May 2014 23:44:46 -0400 Subject: [PATCH 25/45] Added the ability to specify multiple crop zones. --- docs/config.rst | 12 ++++++++++-- overviewer.py | 17 +++++++++++------ overviewer_core/files.py | 19 +++++++++++++++++-- overviewer_core/settingsValidators.py | 26 +++++++++++++++++--------- overviewer_core/tileset.py | 9 ++++++++- 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 5db549c..e33e035 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -709,8 +709,8 @@ values. The valid configuration keys are listed below. .. _crop: ``crop`` - You can use this to render a small subset of your map, instead of the entire - thing. The format is (min x, min z, max x, max z). + You can use this to render one or more small subsets of your map, instead + of the entire thing. The format is [(min x, min z, max x, max z)]. The coordinates are block coordinates. The same you get with the debug menu in-game and the coordinates shown when you view a map. @@ -723,6 +723,14 @@ values. The valid configuration keys are listed below. 'crop': (-500, -500, 500, 500), } + Example that renders two 500 by 500 squares of land: + + renders['myrender'] = { + 'world': 'myworld', + 'title': "Multi cropped Example", + 'crop': [(-500, -500, 0, 0), (0, 0, 500, 500)] + } + This option performs a similar function to the old ``--regionlist`` option (which no longer exists). It is useful for example if someone has wandered really far off and made your map too large. You can set the crop for the diff --git a/overviewer.py b/overviewer.py index 43bd44f..83bb6de 100755 --- a/overviewer.py +++ b/overviewer.py @@ -488,16 +488,20 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # regionset cache pulls from the same underlying cache object. rset = world.CachedRegionSet(rset, caches) - # If a crop is requested, wrap the regionset here - if "crop" in render: - rset = world.CroppedRegionSet(rset, *render['crop']) - # If this is to be a rotated regionset, wrap it in a RotatedRegionSet # object if (render['northdirection'] > 0): rset = world.RotatedRegionSet(rset, render['northdirection']) logging.debug("Using RegionSet %r", rset) + # If a crop is requested, wrap the regionset here + if "crop" in render: + rsets = [] + for zone in render['crop']: + rsets.append(world.CroppedRegionSet(rset, *zone)) + else: + rsets = [rset] + ############################### # Do the final prep and create the TileSet object @@ -508,8 +512,9 @@ dir but you forgot to put quotes around the directory, since it contains spaces. render['name'] = render_name # perhaps a hack. This is stored here for the asset manager tileSetOpts = util.dict_subset(render, ["name", "imgformat", "renderchecks", "rerenderprob", "bgcolor", "defaultzoom", "imgquality", "optimizeimg", "rendermode", "worldname_orig", "title", "dimension", "changelist", "showspawn", "overlay", "base", "poititle", "maxzoom", "showlocationmarker", "minzoom"]) tileSetOpts.update({"spawn": w.find_true_spawn()}) # TODO find a better way to do this - tset = tileset.TileSet(w, rset, assetMrg, tex, tileSetOpts, tileset_dir) - tilesets.append(tset) + for rset in rsets: + tset = tileset.TileSet(w, rset, assetMrg, tex, tileSetOpts, tileset_dir) + tilesets.append(tset) # Do tileset preprocessing here, before we start dispatching jobs logging.info("Preprocessing...") diff --git a/overviewer_core/files.py b/overviewer_core/files.py index ca6d8a6..15647fe 100644 --- a/overviewer_core/files.py +++ b/overviewer_core/files.py @@ -19,6 +19,7 @@ import tempfile import shutil import logging import stat +import errno default_caps = {"chmod_works": True, "rename_works": True} @@ -150,6 +151,20 @@ class FileReplacer(object): else: # copy permission bits, if needed if self.caps.get("chmod_works") and os.path.exists(self.destname): - shutil.copymode(self.destname, self.tmpname) + try: + shutil.copymode(self.destname, self.tmpname) + except OSError, e: + # Ignore errno ENOENT: file does not exist. Due to a race + # condition, two processes could conceivably try and update + # the same temp file at the same time + if e.errno != errno.ENOENT: + raise # atomic rename into place - os.rename(self.tmpname, self.destname) + try: + os.rename(self.tmpname, self.destname) + except OSError, e: + # Ignore errno ENOENT: file does not exist. Due to a race + # condition, two processes could conceivably try and update + # the same temp file at the same time + if e.errno != errno.ENOENT: + raise diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 29553a7..0499815 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -225,15 +225,23 @@ def validateOutputDir(d): return expand_path(d) def validateCrop(value): - if len(value) != 4: - raise ValidationException("The value for the 'crop' setting must be a tuple of length 4") - a, b, c, d = tuple(int(x) for x in value) - - if a >= c: - a, c = c, a - if b >= d: - b, d = d, b - return (a, b, c, d) + if not isinstance(value, list): + value = [value] + + cropZones = [] + for zone in value: + if len(zone) != 4: + raise ValidationException("The value for the 'crop' setting must be an array of tuples of length 4") + a, b, c, d = tuple(int(x) for x in zone) + + if a >= c: + a, c = c, a + if b >= d: + b, d = d, b + + cropZones.append((a, b, c, d)) + + return cropZones def validateObserver(observer): if all(map(lambda m: hasattr(observer, m), ['start', 'add', 'update', 'finish'])): diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 0d90446..b744d6e 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -947,7 +947,14 @@ class TileSet(object): if self.options['optimizeimg']: optimize_image(tmppath, imgformat, self.options['optimizeimg']) - os.utime(tmppath, (max_mtime, max_mtime)) + try: + os.utime(tmppath, (max_mtime, max_mtime)) + except OSError, e: + # Ignore errno ENOENT: file does not exist. Due to a race + # condition, two processes could conceivably try and update + # the same temp file at the same time + if e.errno != errno.ENOENT: + raise def _render_rendertile(self, tile): """Renders the given render-tile. From 95de300276eacdf5766b9f4edcef29e2a8380631 Mon Sep 17 00:00:00 2001 From: Aaron Griffith Date: Sat, 10 May 2014 18:45:58 -0400 Subject: [PATCH 26/45] fix jar caching forcing default textures (caused by 183da128) --- overviewer_core/textures.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/overviewer_core/textures.py b/overviewer_core/textures.py index e01ce08..3d79a25 100644 --- a/overviewer_core/textures.py +++ b/overviewer_core/textures.py @@ -161,17 +161,6 @@ class Textures(object): return None if verbose: logging.info('search_zip_paths: ' + ', '.join(search_zip_paths)) - # we've sucessfully loaded something from here before, so let's quickly try - # this before searching again - if self.jar is not None: - for jarfilename in search_zip_paths: - try: - self.jar.getinfo(jarfilename) - if verbose: logging.info("Found (cached) %s in '%s'", jarfilename, self.jarpath) - return self.jar.open(jarfilename) - except (KeyError, IOError), e: - pass - # A texture path was given on the command line. Search this location # for the file first. if self.find_file_local_path: @@ -227,6 +216,17 @@ class Textures(object): if verbose: logging.info("Did not find the file in overviewer executable directory") if verbose: logging.info("Looking for installed minecraft jar files...") + # we've sucessfully loaded something from here before, so let's quickly try + # this before searching again + if self.jar is not None: + for jarfilename in search_zip_paths: + try: + self.jar.getinfo(jarfilename) + if verbose: logging.info("Found (cached) %s in '%s'", jarfilename, self.jarpath) + return self.jar.open(jarfilename) + except (KeyError, IOError), e: + pass + # Find an installed minecraft client jar and look in it for the texture # file we need. versiondir = "" From bb1c4a7b85e5ed3b81fc29118196613be3ef0fbd Mon Sep 17 00:00:00 2001 From: matrixhacker Date: Mon, 12 May 2014 14:47:45 -0400 Subject: [PATCH 27/45] Updated documentation and added an additional validation check for improperly formatted crop zones. --- docs/config.rst | 6 ++++-- overviewer_core/settingsValidators.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index e33e035..54ddfa1 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -709,8 +709,10 @@ values. The valid configuration keys are listed below. .. _crop: ``crop`` - You can use this to render one or more small subsets of your map, instead - of the entire thing. The format is [(min x, min z, max x, max z)]. + You can use this to render one or more small subsets of your map. The format + of an individual crop zone is (min x, min z, max x, max z); if you wish to + specify multiple crop zones, you may do so by specifying a list of crop zones, + i.e. [(min x1, min z1, max x1, max z1), (min x2, min z2, max x2, max z2)] The coordinates are block coordinates. The same you get with the debug menu in-game and the coordinates shown when you view a map. diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 0499815..2b3d28f 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -230,7 +230,7 @@ def validateCrop(value): cropZones = [] for zone in value: - if len(zone) != 4: + if not isinstance(zone, tuple) or len(zone) != 4: raise ValidationException("The value for the 'crop' setting must be an array of tuples of length 4") a, b, c, d = tuple(int(x) for x in zone) From b3f41c7928dad68969f8d9eeb29a7fb417f53ac3 Mon Sep 17 00:00:00 2001 From: Shadark Date: Wed, 14 May 2014 01:49:45 +0200 Subject: [PATCH 28/45] Fixed pngnq rename error in Windows Fixed error 183 (File already exists) in Windows when trying to use pngnq and trying to rename "file.png.tmp" to "file.png". --- overviewer_core/optimizeimages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index 1d79f71..0ccd4a2 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -47,6 +47,7 @@ class Optimizer: class NonAtomicOptimizer(Optimizer): def cleanup(self, img): + os.remove(img) os.rename(img + ".tmp", img) def fire_and_forget(self, args, img): From 5427b28ca2d75dafbc0064e14866ea949d834cff Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Mon, 26 May 2014 14:02:40 -0400 Subject: [PATCH 29/45] Use `is not None` instead of `!= None` --- overviewer_core/textures.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/overviewer_core/textures.py b/overviewer_core/textures.py index 3d79a25..73ecaf5 100644 --- a/overviewer_core/textures.py +++ b/overviewer_core/textures.py @@ -638,23 +638,23 @@ class Textures(object): increment = int(round((top[1] / 16.)*12.)) # range increment in the block height in pixels (half texture size) crop_height = increment top = top[0] - if side1 != None: + if side1 is not None: side1 = side1.copy() ImageDraw.Draw(side1).rectangle((0, 0,16,crop_height),outline=(0,0,0,0),fill=(0,0,0,0)) - if side2 != None: + if side2 is not None: side2 = side2.copy() ImageDraw.Draw(side2).rectangle((0, 0,16,crop_height),outline=(0,0,0,0),fill=(0,0,0,0)) - if side3 != None: + if side3 is not None: side3 = side3.copy() ImageDraw.Draw(side3).rectangle((0, 0,16,crop_height),outline=(0,0,0,0),fill=(0,0,0,0)) - if side4 != None: + if side4 is not None: side4 = side4.copy() ImageDraw.Draw(side4).rectangle((0, 0,16,crop_height),outline=(0,0,0,0),fill=(0,0,0,0)) img = Image.new("RGBA", (24,24), self.bgcolor) # first back sides - if side1 != None : + if side1 is not None : side1 = self.transform_image_side(side1) side1 = side1.transpose(Image.FLIP_LEFT_RIGHT) @@ -666,7 +666,7 @@ class Textures(object): alpha_over(img, side1, (0,0), side1) - if side2 != None : + if side2 is not None : side2 = self.transform_image_side(side2) # Darken this side. @@ -676,12 +676,12 @@ class Textures(object): alpha_over(img, side2, (12,0), side2) - if bottom != None : + if bottom is not None : bottom = self.transform_image_top(bottom) alpha_over(img, bottom, (0,12), bottom) # front sides - if side3 != None : + if side3 is not None : side3 = self.transform_image_side(side3) # Darken this side @@ -691,7 +691,7 @@ class Textures(object): alpha_over(img, side3, (0,6), side3) - if side4 != None : + if side4 is not None : side4 = self.transform_image_side(side4) side4 = side4.transpose(Image.FLIP_LEFT_RIGHT) @@ -702,7 +702,7 @@ class Textures(object): alpha_over(img, side4, (12,6), side4) - if top != None : + if top is not None : top = self.transform_image_top(top) alpha_over(img, top, (0, increment), top) From 8053eaca720dd13961830d28a22d84edca46d878 Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Fri, 30 May 2014 10:16:28 +0200 Subject: [PATCH 30/45] Lazily get names from UUIDs Instead of doing the UUID->name resolution for all players in every case, only do it when EntityId is accessed and the name hasn't been retrieved this run already. This makes genPOI usable for people who have many players on their servers but don't wish to use player POI while still using other genPOI features. To do this, a PlayerDict has been created, which contains a dirty hack to see if the requested item is EntityId and whether it hasn't been set already. --- overviewer_core/aux_files/genPOI.py | 47 ++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index f0f5492..2266c49 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -117,6 +117,27 @@ def handleEntities(rset, outputdir, render, rname, config): logging.info("Done.") +class PlayerDict(dict): + use_uuid = False + _name = '' + def __getitem__(self, item): + if item == "EntityId": + if not super(PlayerDict, self).has_key("EntityId"): + if self.use_uuid: + super(PlayerDict, self).__setitem__("EntityId", self.get_name_from_uuid()) + else: + super(PlayerDict, self).__setitem__("EntityId", self._name) + + return super(PlayerDict, self).__getitem__(item) + + def get_name_from_uuid(self): + try: + profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + self._name.replace('-','')).read()) + if 'name' in profile: + return profile['name'] + except (ValueError, urllib2.URLError): + logging.warning("Unable to get player name for UUID %s", playername) + def handlePlayers(rset, render, worldpath): if not hasattr(rset, "_pois"): rset._pois = dict(TileEntities=[], Entities=[]) @@ -153,37 +174,35 @@ def handlePlayers(rset, render, worldpath): rset._pois['Players'] = [] for playerfile in playerfiles: try: - data = nbt.load(os.path.join(playerdir, playerfile))[1] + data = PlayerDict(nbt.load(os.path.join(playerdir, playerfile))[1]) + data.use_uuid = useUUIDs if isSinglePlayer: data = data['Data']['Player'] except IOError: logging.warning("Skipping bad player dat file %r", playerfile) continue playername = playerfile.split(".")[0] - if useUUIDs: - try: - profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + playername.replace('-','')).read()) - if 'name' in profile: - playername = profile['name'] - except (ValueError, urllib2.URLError): - logging.warning("Unable to get player name for UUID %s", playername) + if isSinglePlayer: playername = 'Player' + + data._name = playername + if data['Dimension'] == dimension: # Position at last logout data['id'] = "Player" - data['EntityId'] = playername data['x'] = int(data['Pos'][0]) data['y'] = int(data['Pos'][1]) data['z'] = int(data['Pos'][2]) rset._pois['Players'].append(data) if "SpawnX" in data and dimension == 0: # Spawn position (bed or main spawn) - spawn = {"id": "PlayerSpawn", - "EntityId": playername, - "x": data['SpawnX'], - "y": data['SpawnY'], - "z": data['SpawnZ']} + spawn = PlayerDict() + spawn._name = playername + spawn["id"] = "PlayerSpawn" + spawn["x"] = data['SpawnX'] + spawn["y"] = data['SpawnY'] + spawn["z"] = data['SpawnZ'] rset._pois['Players'].append(spawn) def handleManual(rset, manualpois): From a7aab9d1b2de8eb9f74f52f832cd06c0a55ca307 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 1 Jun 2014 21:13:14 -0400 Subject: [PATCH 31/45] Try to prevent findGitHash from ever returning None --- overviewer_core/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/overviewer_core/util.py b/overviewer_core/util.py index 6b81d8b..9a74757 100644 --- a/overviewer_core/util.py +++ b/overviewer_core/util.py @@ -43,6 +43,7 @@ def findGitHash(): line = p.stdout.readlines()[0].strip() if line and len(line) == 40 and all(c in hexdigits for c in line): return line + return "unknown" except Exception: try: import overviewer_version From 381d66f36db884d2e17bd0bbcfe23089b85b182d Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Sun, 1 Jun 2014 21:13:55 -0400 Subject: [PATCH 32/45] Try to prevent findGitHash from ever returning None See #1093 --- overviewer_core/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overviewer_core/util.py b/overviewer_core/util.py index 9a74757..17ff8a3 100644 --- a/overviewer_core/util.py +++ b/overviewer_core/util.py @@ -43,13 +43,13 @@ def findGitHash(): line = p.stdout.readlines()[0].strip() if line and len(line) == 40 and all(c in hexdigits for c in line): return line - return "unknown" except Exception: try: import overviewer_version return overviewer_version.HASH except Exception: - return "unknown" + pass + return "unknown" def findGitVersion(): try: From c15b9383c8541e29860b6f097c0401e631151e60 Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 3 Jun 2014 18:08:27 +0200 Subject: [PATCH 33/45] Remove broken special case for ancient worlds And to whoever wrote that thing: "World10" is not 6 characters long, so the workaround was broken anyway. --- overviewer_core/world.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/overviewer_core/world.py b/overviewer_core/world.py index bbfc7cf..0597317 100644 --- a/overviewer_core/world.py +++ b/overviewer_core/world.py @@ -780,12 +780,7 @@ def get_worlds(): if not os.path.exists(world_dat): continue info = nbt.load(world_dat)[1] info['Data']['path'] = os.path.join(save_dir, dir).decode(loc) - if dir.startswith("World") and len(dir) == 6: - try: - world_n = int(dir[-1]) - ret[world_n] = info['Data'] - except ValueError: - pass + if 'LevelName' in info['Data'].keys(): ret[info['Data']['LevelName']] = info['Data'] From 192ff4c1a0474252e6478509c65f981a67afc6b1 Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Wed, 4 Jun 2014 11:44:38 +0200 Subject: [PATCH 34/45] Long overdue addition of information to docs * Add 2 FAQ entries, one concerning mod blocks, the other about the maps API. * Add note about The End and lighting strength. (Closes #1111) (Also, nice quads.) --- docs/config.rst | 10 ++++++++++ docs/faq.rst | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/docs/config.rst b/docs/config.rst index 5db549c..bac5182 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -442,6 +442,16 @@ values. The valid configuration keys are listed below. nether :ref:`rendermode`. Otherwise you'll just end up rendering the nether's ceiling. + .. note:: + + For the end, you will most likely want to turn down the strength of + the shadows, as you'd otherwise end up with a very dark result. + + e.g.:: + + end_lighting = [Base(), EdgeLines(), Lighting(strength=0.5)] + end_smooth_lighting = [Base(), EdgeLines(), SmoothLighting(strength=0.5)] + **Default:** ``"overworld"`` .. _option_rendermode: diff --git a/docs/faq.rst b/docs/faq.rst index 9e9b8a8..d425d38 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -8,6 +8,22 @@ Frequently Asked Questions General Questions ================= +Does the Overviewer work with mod blocks? +----------------------------------------- + +The Overviewer will render the world, but none of the blocks added by mods +will be visible. Currently, the blocks Overviewer supports are hardcoded, and +because there is no official Minecraft modding API as of the time of writing, +supporting mod blocks is not trivial. + +Can I view Overviewer maps without having an internet connection? +----------------------------------------------------------------- + +Not at the moment. The Overviewer relies on the Google maps API to display +maps, which your browser needs to load from Google. However, switching away +from Google Maps is something that will most likely be looked into in the +future. + When my map expands, I see remnants of another zoom level --------------------------------------------------------- From 57fd1e2ffb533e3ad470fd633816335be68daee3 Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Wed, 4 Jun 2014 12:17:29 +0200 Subject: [PATCH 35/45] config.rst restructuring with new headings This hopefully makes the page slightly less intimidating. --- docs/config.rst | 287 +++++++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 125 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index bac5182..3789990 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -19,8 +19,14 @@ Python, don't worry, it's pretty simple. Just follow the examples. Windows. This is required because the backslash ("\\") has special meaning in Python. +Examples +======== + +The following examples should give you an idea of what a configuration file looks +like, and also teach you some neat tricks. + A Simple Example -================ +---------------- :: @@ -60,7 +66,7 @@ The ``renders`` dictionary ``worlds["My world"]`` A more complicated example -========================== +-------------------------- :: worlds["survival"] = "/home/username/server/survivalworld" @@ -130,7 +136,7 @@ renders. example. A dynamic config file -===================== +--------------------- It might be handy to dynamically retrieve parameters. For instance, if you periodically render your last map backup which is located in a timestamped @@ -192,6 +198,9 @@ If the above doesn't make sense, just know that items in the config file take the form ``key = value``. Two items take a different form:, ``worlds`` and ``renders``, which are described below. +General +------- + ``worlds`` This is pre-defined as an empty dictionary. The config file is expected to add at least one item to it. @@ -259,6 +268,9 @@ the form ``key = value``. Two items take a different form:, ``worlds`` and processes = 2 +Observers +~~~~~~~~~ + .. _observer: ``observer = `` @@ -367,6 +379,8 @@ the form ``key = value``. Two items take a different form:, ``worlds`` and +Custom web assets +~~~~~~~~~~~~~~~~~ .. _customwebassets: @@ -411,6 +425,9 @@ values. The valid configuration keys are listed below. 'title': 'This render doesn't explicitly declare a world!', } +General +~~~~~~~ + ``world`` Specifies which world this render corresponds to. Its value should be a string from the appropriate key in the worlds dictionary. @@ -454,6 +471,9 @@ values. The valid configuration keys are listed below. **Default:** ``"overworld"`` +Rendering +~~~~~~~~~ + .. _option_rendermode: ``rendermode`` @@ -533,14 +553,107 @@ values. The valid configuration keys are listed below. **Default:** ``"upper-left"`` -.. _rerenderprob: +.. _option_overlay: -``rerenderprob`` - This is the probability that a tile will be rerendered even though there may - have been no changes to any blocks within that tile. Its value should be a - floating point number between 0.0 and 1.0. +``overlay`` + This specifies which renders that this render will be displayed on top of. + It should be a list of other renders. If this option is confusing, think + of this option's name as "overlay_on_to". - **Default:** ``0`` + If you leave this as an empty list, this overlay will be displayed on top + of all renders for the same world/dimension as this one. + + As an example, let's assume you have two renders, one called "day" and one + called "night". You want to create a Biome Overlay to be displayed on top + of the "day" render. Your config file might look like this: + + :: + + outputdir = "output_dir" + + + worlds["exmaple"] = "exmaple" + + renders['day'] = { + 'world': 'exmaple', + 'rendermode': 'smooth_lighting', + 'title': "Daytime Render", + } + renders['night'] = { + 'world': 'exmaple', + 'rendermode': 'night', + 'title': "Night Render", + } + + renders['biomeover'] = { + 'world': 'exmaple', + 'rendermode': [ClearBase(), BiomeOverlay()], + 'title': "Biome Coloring Overlay", + 'overlay': ['day'] + } + + **Default:** ``[]`` (an empty list) + +.. _option_texturepath: + +``texturepath`` + This is a where a specific texture or resource pack can be found to use + during this render. It can be a path to either a folder or a zip/jar file + containing the texture resources. If specifying a folder, this option should + point to a directory that *contains* the assets/ directory (it should not + point to the assets directory directly or any one particular texture image). + + Its value should be a string: the path on the filesystem to the resource + pack. + +.. _crop: + +``crop`` + You can use this to render a small subset of your map, instead of the entire + thing. The format is (min x, min z, max x, max z). + + The coordinates are block coordinates. The same you get with the debug menu + in-game and the coordinates shown when you view a map. + + Example that only renders a 1000 by 1000 square of land about the origin:: + + renders['myrender'] = { + 'world': 'myworld', + 'title': "Cropped Example", + 'crop': (-500, -500, 500, 500), + } + + This option performs a similar function to the old ``--regionlist`` option + (which no longer exists). It is useful for example if someone has wandered + really far off and made your map too large. You can set the crop for the + largest map you want to render (perhaps ``(-10000,-10000,10000,10000)``). It + could also be used to define a really small render showing off one + particular feature, perhaps from multiple angles. + + .. warning:: + + If you decide to change the bounds on a render, you may find it produces + unexpected results. It is recommended to not change the crop settings + once it has been rendered once. + + For an expansion to the bounds, because chunks in the new bounds have + the same mtime as the old, tiles will not automatically be updated, + leaving strange artifacts along the old border. You may need to use + :option:`--forcerender` to force those tiles to update. (You can use + the ``forcerender`` option on just one render by adding ``'forcerender': + True`` to that render's configuration) + + For reductions to the bounds, you will need to render your map at least + once with the :option:`--check-tiles` mode activated, and then once with + the :option:`--forcerender` option. The first run will go and delete tiles that + should no longer exist, while the second will render the tiles around + the edge properly. Also see :ref:`this faq entry`. + + Sorry there's no better way to handle these cases at the moment. It's a + tricky problem and nobody has devoted the effort to solve it yet. + +Image options +~~~~~~~~~~~~~ ``imgformat`` This is which image format to render the tiles into. Its value should be a @@ -645,12 +758,10 @@ values. The valid configuration keys are listed below. **Default:** ``[]`` -``bgcolor`` - This is the background color to be displayed behind the map. Its value - should be either a string in the standard HTML color syntax or a 4-tuple in - the format of (r,b,g,a). The alpha entry should be set to 0. +Zoom +~~~~ - **Default:** ``#1a1a1a`` +These options control the zooming behavior in the JavaScript output. ``defaultzoom`` This value specifies the default zoom level that the map will be @@ -691,6 +802,9 @@ values. The valid configuration keys are listed below. **Default:** 0 (zero, which does not disable any zoom levels) +Other HTML/JS output options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``showlocationmarker`` Allows you to specify whether to show the location marker when accessing a URL with coordinates specified. @@ -704,63 +818,50 @@ values. The valid configuration keys are listed below. tiles folder itself. For example, if the tile images start at http://domain.com/map/world_day/ you want to set this to http://domain.com/map/ -.. _option_texturepath: +.. _option_markers: -``texturepath`` - This is a where a specific texture or resource pack can be found to use - during this render. It can be a path to either a folder or a zip/jar file - containing the texture resources. If specifying a folder, this option should - point to a directory that *contains* the assets/ directory (it should not - point to the assets directory directly or any one particular texture image). +``markers`` + This controls the display of markers, signs, and other points of interest + in the output HTML. It should be a list of dictionaries. - Its value should be a string: the path on the filesystem to the resource - pack. + .. note:: -.. _crop: + Setting this configuration option alone does nothing. In order to get + markers and signs on our map, you must also run the genPO script. See + the :doc:`Signs and markers` section for more details and documenation. -``crop`` - You can use this to render a small subset of your map, instead of the entire - thing. The format is (min x, min z, max x, max z). - The coordinates are block coordinates. The same you get with the debug menu - in-game and the coordinates shown when you view a map. + **Default:** ``[]`` (an empty list) - Example that only renders a 1000 by 1000 square of land about the origin:: - renders['myrender'] = { - 'world': 'myworld', - 'title': "Cropped Example", - 'crop': (-500, -500, 500, 500), - } +``poititle`` + This controls the display name of the POI/marker dropdown control. - This option performs a similar function to the old ``--regionlist`` option - (which no longer exists). It is useful for example if someone has wandered - really far off and made your map too large. You can set the crop for the - largest map you want to render (perhaps ``(-10000,-10000,10000,10000)``). It - could also be used to define a really small render showing off one - particular feature, perhaps from multiple angles. + **Default:** "Signs" - .. warning:: +``showspawn`` + This is a boolean, and defaults to ``True``. If set to ``False``, then the spawn + icon will not be displayed on the rendered map. - If you decide to change the bounds on a render, you may find it produces - unexpected results. It is recommended to not change the crop settings - once it has been rendered once. +``bgcolor`` + This is the background color to be displayed behind the map. Its value + should be either a string in the standard HTML color syntax or a 4-tuple in + the format of (r,b,g,a). The alpha entry should be set to 0. - For an expansion to the bounds, because chunks in the new bounds have - the same mtime as the old, tiles will not automatically be updated, - leaving strange artifacts along the old border. You may need to use - :option:`--forcerender` to force those tiles to update. (You can use - the ``forcerender`` option on just one render by adding ``'forcerender': - True`` to that render's configuration) + **Default:** ``#1a1a1a`` - For reductions to the bounds, you will need to render your map at least - once with the :option:`--check-tiles` mode activated, and then once with - the :option:`--forcerender` option. The first run will go and delete tiles that - should no longer exist, while the second will render the tiles around - the edge properly. Also see :ref:`this faq entry`. +Map update behavior +~~~~~~~~~~~~~~~~~~~ + +.. _rerenderprob: + +``rerenderprob`` + This is the probability that a tile will be rerendered even though there may + have been no changes to any blocks within that tile. Its value should be a + floating point number between 0.0 and 1.0. + + **Default:** ``0`` - Sorry there's no better way to handle these cases at the moment. It's a - tricky problem and nobody has devoted the effort to solve it yet. ``forcerender`` This is a boolean. If set to ``True`` (or any non-false value) then this @@ -816,72 +917,6 @@ values. The valid configuration keys are listed below. removed some tiles, you may need to do some manual deletion on the remote side. -.. _option_markers: - -``markers`` - This controls the display of markers, signs, and other points of interest - in the output HTML. It should be a list of dictionaries. - - .. note:: - - Setting this configuration option alone does nothing. In order to get - markers and signs on our map, you must also run the genPO script. See - the :doc:`Signs and markers` section for more details and documenation. - - - **Default:** ``[]`` (an empty list) - - -``poititle`` - This controls the display name of the POI/marker dropdown control. - - **Default:** "Signs" - -.. _option_overlay: - -``overlay`` - This specifies which renders that this render will be displayed on top of. - It should be a list of other renders. If this option is confusing, think - of this option's name as "overlay_on_to". - - If you leave this as an empty list, this overlay will be displayed on top - of all renders for the same world/dimension as this one. - - As an example, let's assume you have two renders, one called "day" and one - called "night". You want to create a Biome Overlay to be displayed on top - of the "day" render. Your config file might look like this: - - :: - - outputdir = "output_dir" - - - worlds["exmaple"] = "exmaple" - - renders['day'] = { - 'world': 'exmaple', - 'rendermode': 'smooth_lighting', - 'title': "Daytime Render", - } - renders['night'] = { - 'world': 'exmaple', - 'rendermode': 'night', - 'title': "Night Render", - } - - renders['biomeover'] = { - 'world': 'exmaple', - 'rendermode': [ClearBase(), BiomeOverlay()], - 'title': "Biome Coloring Overlay", - 'overlay': ['day'] - } - - **Default:** ``[]`` (an empty list) - -``showspawn`` - This is a boolean, and defaults to ``True``. If set to ``False``, then the spawn - icon will not be displayed on the rendered map. - .. _customrendermodes: Custom Rendermodes and Rendermode Primitives @@ -1072,6 +1107,7 @@ BiomeOverlay Defining Custom Rendermodes --------------------------- + Each rendermode primitive listed above is a Python *class* that is automatically imported in the context of the config file (They come from overviewer_core.rendermodes). To define your own rendermode, simply define a @@ -1099,7 +1135,8 @@ are referencing the previously defined list, not one of the built-in rendermodes. Built-in Rendermodes -==================== +-------------------- + The built-in rendermodes are nothing but pre-defined lists of rendermode primitives for your convenience. Here are their definitions:: From 3c33080e3d86cc2df331f9fd6f7de57bb23ecf83 Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Wed, 4 Jun 2014 12:23:43 +0200 Subject: [PATCH 36/45] Update front page of the docs lol --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 066dac9..7ba351f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,7 +18,7 @@ The Minecraft Overviewer is a command-line tool for rendering high-resolution maps of Minecraft worlds. It generates a set of static html and image files and uses the Google Maps API to display a nice interactive map. -The Overviewer has been in active development for over a year and has many +The Overviewer has been in active development for several years and has many features, including day and night lighting, cave rendering, mineral overlays, and many plugins for even more features! It is written mostly in Python with critical sections in C as an extension module. @@ -114,7 +114,7 @@ Windows, Mac, and Linux as long as you have these software packages installed: * Python 2.6 or 2.7 (we are *not* yet compatible with Python 3.x) -* PIL (Python Imaging Library) +* PIL (Python Imaging Library) or Pillow * Numpy From 948d2fa7411cb02a447a680390b694225ee3f082 Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Wed, 4 Jun 2014 13:23:30 +0200 Subject: [PATCH 37/45] Minor fixes to documentation * Fix the copypasted section in the README * Update version and dates --- README.rst | 2 +- docs/conf.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 6bcfeba..f9adc1f 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ The Minecraft Overviewer is a command-line tool for rendering high-resolution maps of Minecraft worlds. It generates a set of static html and image files and uses the Google Maps API to display a nice interactive map. -The Overviewer has been in active development for over a year and has many +The Overviewer has been in active development for several years and has many features, including day and night lighting, cave rendering, mineral overlays, and many plugins for even more features! It is written mostly in Python with critical sections in C as an extension module. diff --git a/docs/conf.py b/docs/conf.py index 995cf10..607a1cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ master_doc = 'index' # General information about the project. project = u'Overviewer' -copyright = u'2010-2012 The Overviewer Team' +copyright = u'2010-2014 The Overviewer Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = "0.10" +version = "0.11" # The full version, including alpha/beta/rc tags. -release = "0.10" +release = "0.11" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From a77f4aa5f482288fb809838103d7b2bb293e6fc3 Mon Sep 17 00:00:00 2001 From: "Nicolas F." Date: Fri, 20 Jun 2014 19:53:34 +0200 Subject: [PATCH 38/45] Fix warning for unresolvable UUID Exception while catching an exception. Try to make an inception joke of that. --- overviewer_core/aux_files/genPOI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 2266c49..d71abaa 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -136,7 +136,7 @@ class PlayerDict(dict): if 'name' in profile: return profile['name'] except (ValueError, urllib2.URLError): - logging.warning("Unable to get player name for UUID %s", playername) + logging.warning("Unable to get player name for UUID %s", self._name) def handlePlayers(rset, render, worldpath): if not hasattr(rset, "_pois"): From 51e75a7d069a75c3c0ae83c143a5e43eec145a74 Mon Sep 17 00:00:00 2001 From: Aaron1011 Date: Wed, 9 Jul 2014 10:52:12 -0400 Subject: [PATCH 39/45] Fix typo --- docs/design/designdoc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/designdoc.rst b/docs/design/designdoc.rst index 31971ae..8b9a943 100644 --- a/docs/design/designdoc.rst +++ b/docs/design/designdoc.rst @@ -34,7 +34,7 @@ hereby called "blocks", where each block in the world's grid has a type that determines what it is (grass, stone, ...). This makes worlds relatively uncomplicated to render, the Overviewer simply determines what blocks to draw and where. Since everything in Minecraft is aligned to a strict grid, placement -and rendering decisions are completely deterministic and can be performed in an +and rendering decisions are completely deterministic and can be performed iteratively. The coordinate system for Minecraft has three axes. The X and Z axes are the From 792b049dd52ab575be87fbb97929589a0722109a Mon Sep 17 00:00:00 2001 From: Nicolas F Date: Fri, 18 Jul 2014 20:08:34 +0200 Subject: [PATCH 40/45] FIx genPOI dimension parsing Fixes Issue #1130 --- overviewer_core/aux_files/genPOI.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index d71abaa..305ff82 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -145,17 +145,12 @@ def handlePlayers(rset, render, worldpath): # only handle this region set once if 'Players' in rset._pois: return - dimension = None - try: - dimension = {None: 0, - 'DIM-1': -1, - 'DIM1': 1}[rset.get_type()] - except KeyError, e: - mystdim = re.match(r"^DIM_MYST(\d+)$", e.message) # Dirty hack. Woo! - if mystdim: - dimension = int(mystdim.group(1)) - else: - raise + + if rset.get_type(): + dimension = int(re.match(r"^DIM(_MYST)?(-?\d+)$", rset.get_type()).group(2)) + else: + dimension = 0 + playerdir = os.path.join(worldpath, "playerdata") useUUIDs = True if not os.path.isdir(playerdir): From 6cca3ed004ec6d547a886389eaeeda9c55dbf131 Mon Sep 17 00:00:00 2001 From: Brooks Date: Wed, 30 Jul 2014 22:24:23 -0400 Subject: [PATCH 41/45] Latest version of minecraft jar Updated the version variable in the short script to install the latest minecraft jar for textures. --- docs/running.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/running.rst b/docs/running.rst index 6ccfb96..2da1557 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -261,13 +261,13 @@ If you want or need to provide your own textures, you have several options: :: - VERSION=1.7.2 + VERSION=1.7.10 wget https://s3.amazonaws.com/Minecraft.Download/versions/${VERSION}/${VERSION}.jar -P ~/.minecraft/versions/${VERSION}/ If that's too confusing for you, then just take this single line and paste it into - a terminal to get 1.7.2 textures:: + a terminal to get 1.7.10 textures:: - wget https://s3.amazonaws.com/Minecraft.Download/versions/1.7.2/1.7.2.jar -P ~/.minecraft/versions/1.7.2/ + wget https://s3.amazonaws.com/Minecraft.Download/versions/1.7.10/1.7.10.jar -P ~/.minecraft/versions/1.7.10/ * You can also just run the launcher to install the client. From 55bbe26916b8558844a419973873986d663d1064 Mon Sep 17 00:00:00 2001 From: Franz Dietrich Date: Fri, 1 Aug 2014 00:25:39 +0200 Subject: [PATCH 42/45] Fix error in the config.rst file --- docs/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.rst b/docs/config.rst index fe90c6a..c617f52 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -625,7 +625,7 @@ Rendering 'crop': (-500, -500, 500, 500), } - Example that renders two 500 by 500 squares of land: + Example that renders two 500 by 500 squares of land:: renders['myrender'] = { 'world': 'myworld', From 59d277a13142f52fa02eeb5dfd9e7e3df13d5d04 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Tue, 19 Aug 2014 21:28:25 -0400 Subject: [PATCH 43/45] Change ctime to localtime. See discussion in #1082 --- overviewer_core/aux_files/genPOI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index 8c9910a..a6c6f83 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -191,7 +191,7 @@ def handlePlayers(rset, render, worldpath): data['y'] = int(data['Pos'][1]) data['z'] = int(data['Pos'][2]) # Time at last logout, calculated from last time the player's file was modified - data['time'] = time.ctime(os.path.getmtime(os.path.join(playerdir, playerfile)) + data['time'] = time.localtime(os.path.getmtime(os.path.join(playerdir, playerfile))) rset._pois['Players'].append(data) if "SpawnX" in data and dimension == 0: # Spawn position (bed or main spawn) From 322922b8e65bc7f4b4664cf2bcb3883e386fdff2 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Tue, 19 Aug 2014 22:08:15 -0400 Subject: [PATCH 44/45] Implement a UUID lookup cache, to avoid hitting the mojang server so much The cache is a gzip'd JSON file. Soon we will have a small script to help manage the cache See #1090 and #1117 --- overviewer_core/aux_files/genPOI.py | 44 +++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/overviewer_core/aux_files/genPOI.py b/overviewer_core/aux_files/genPOI.py index a6c6f83..199cfd6 100755 --- a/overviewer_core/aux_files/genPOI.py +++ b/overviewer_core/aux_files/genPOI.py @@ -23,6 +23,7 @@ import re import urllib2 import Queue import multiprocessing +import gzip from multiprocessing import Process from multiprocessing import Pool @@ -121,6 +122,30 @@ def handleEntities(rset, outputdir, render, rname, config): class PlayerDict(dict): use_uuid = False _name = '' + uuid_cache = None # A cache the UUID->profile lookups + + @classmethod + def load_cache(cls, outputdir): + cache_file = os.path.join(outputdir, "uuidcache.dat") + pid = multiprocessing.current_process().pid + if os.path.exists(cache_file): + gz = gzip.GzipFile(cache_file) + cls.uuid_cache = json.load(gz) + logging.info("Loaded UUID cache from %r with %d entries", cache_file, len(cls.uuid_cache.keys())) + else: + cls.uuid_cache = {} + logging.info("Initialized an empty UUID cache") + cls.save_cache(outputdir) + + + @classmethod + def save_cache(cls, outputdir): + cache_file = os.path.join(outputdir, "uuidcache.dat") + gz = gzip.GzipFile(cache_file, "wb") + json.dump(cls.uuid_cache, gz) + logging.info("Wrote UUID cache with %d entries", len(cls.uuid_cache.keys())) + + def __getitem__(self, item): if item == "EntityId": if not super(PlayerDict, self).has_key("EntityId"): @@ -132,14 +157,22 @@ class PlayerDict(dict): return super(PlayerDict, self).__getitem__(item) def get_name_from_uuid(self): + sname = self._name.replace('-','') try: - profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + self._name.replace('-','')).read()) + profile = PlayerDict.uuid_cache[sname] + return profile['name'] + except (KeyError,): + pass + + try: + profile = json.loads(urllib2.urlopen(UUID_LOOKUP_URL + sname).read()) if 'name' in profile: + PlayerDict.uuid_cache[sname] = profile return profile['name'] except (ValueError, urllib2.URLError): logging.warning("Unable to get player name for UUID %s", self._name) -def handlePlayers(rset, render, worldpath): +def handlePlayers(rset, render, worldpath, outputdir): if not hasattr(rset, "_pois"): rset._pois = dict(TileEntities=[], Entities=[]) @@ -168,6 +201,7 @@ def handlePlayers(rset, render, worldpath): isSinglePlayer = True rset._pois['Players'] = [] + for playerfile in playerfiles: try: data = PlayerDict(nbt.load(os.path.join(playerdir, playerfile))[1]) @@ -252,6 +286,8 @@ def main(): markersets = set() markers = dict() + PlayerDict.load_cache(destdir) + for rname, render in config['renders'].iteritems(): try: worldpath = config['worlds'][render['world']] @@ -291,7 +327,7 @@ def main(): if not options.skipscan: handleEntities(rset, os.path.join(destdir, rname), render, rname, config) - handlePlayers(rset, render, worldpath) + handlePlayers(rset, render, worldpath, destdir) handleManual(rset, render['manualpois']) logging.info("Done handling POIs") @@ -402,6 +438,8 @@ def main(): markerSetDict[name]['raw'].append(d) #print markerSetDict + PlayerDict.save_cache(destdir) + with open(os.path.join(destdir, "markersDB.js"), "w") as output: output.write("var markersDB=") json.dump(markerSetDict, output, indent=2) From 924d3967584d33b3f46b006b109950d87384db87 Mon Sep 17 00:00:00 2001 From: Andrew Chin Date: Tue, 19 Aug 2014 22:49:33 -0400 Subject: [PATCH 45/45] Show an ETA when using the plain text logger Closes #1088 --- overviewer_core/observer.py | 41 +++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index 32a472c..e17c8f5 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -92,17 +92,50 @@ class LoggingObserver(Observer): #this is an easy way to make the first update() call print a line self.last_update = -101 + # a fake ProgressBar, for the sake of ETA + class FakePBar(object): + def __init__(self): + self.maxval = None + self.currval = 0 + self.finished = False + self.start_time = None + self.seconds_elapsed = 0 + def finish(self): + self.update(self.maxval) + def update(self, value): + assert 0 <= value <= self.maxval + self.currval = value + if self.finished: + return False + if not self.start_time: + self.start_time = time.time() + self.seconds_elapsed = time.time() - self.start_time + + if value == self.maxval: + self.finished = True + + self.fake = FakePBar(); + self.eta = progressbar.ETA() + + def start(self, max_value): + self.fake.maxval = max_value + super(LoggingObserver, self).start(max_value) + + def finish(self): - logging.info("Rendered %d of %d. %d%% complete", self.get_max_value(), - self.get_max_value(), 100.0) + self.fake.finish() + logging.info("Rendered %d of %d. %d%% complete. %s", self.get_max_value(), + self.get_max_value(), 100.0, self.eta.update(self.fake)) super(LoggingObserver, self).finish() def update(self, current_value): super(LoggingObserver, self).update(current_value) + self.fake.update(current_value) + if self._need_update(): - logging.info("Rendered %d of %d. %d%% complete", + logging.info("Rendered %d of %d. %d%% complete. %s", self.get_current_value(), self.get_max_value(), - self.get_percentage()) + self.get_percentage(), self.eta.update(self.fake)) self.last_update = current_value return True return False