From 09477ed8a0a8b06764404c60add0c8b2744ce1ad Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 4 Mar 2014 00:39:59 +0100 Subject: [PATCH 1/4] Rewrote image optimisation stuff. The old one was broken anyway. --- docs/config.rst | 88 +++++++++++++++++++-- overviewer_core/optimizeimages.py | 110 ++++++++++++++++++++------ overviewer_core/settingsDefinition.py | 3 +- overviewer_core/settingsValidators.py | 10 ++- overviewer_core/tileset.py | 12 +-- 5 files changed, 182 insertions(+), 41 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 5d63d21..d1bc878 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -545,18 +545,92 @@ values. The valid configuration keys are listed below. **Default:** ``95`` ``optimizeimg`` + + .. warning:: + Using image optimizers will increase render times significantly. + This option specifies which additional tools overviewer should use to optimize the filesize of png tiles. The tools used must be placed somewhere, where overviewer can find them, for example the "PATH" environment variable or a directory like /usr/bin. - This should be an integer between 0 and 3. - * ``1 - Use pngcrush`` - * ``2 - Use advdef`` - * ``3 - Use pngcrush and advdef (Not recommended)`` - Using this option may significantly increase render time, but will make - the resulting tiles smaller, with lossless image quality. - **Default:** ``0`` + The option is a list of Optimizer objects, which are then executed in + the order in which they're specified:: + + from optimizeimages import pngnq, optipng + worlds["world"] = "/path/to/world" + + renders["daytime"] = { + "world":"world", + "title":"day", + "rendermode":smooth_lighting, + "optimizeimg":[pngnq(sampling=1), optipng(olevel=3)], + } + + Here is a list of supported image optimization programs: + + ``pngnq`` + pngnq quantizes 32-bit RGBA images into 8-bit RGBA palette PNGs. This is + lossy, but reduces filesize significantly. Available settings: + + ``sampling`` + An integer between ``1`` and ``10``, ``1`` samples all pixels, is slow and yields + the best quality. Higher values sample less of the image, which makes + the process faster, but less accurate. + + **Default:** ``3`` + + ``dither`` + Either the string ``"n"`` for no dithering, or ``"f"`` for Floyd + Steinberg dithering. Dithering helps eliminate colorbanding, sometimes + increasing visual quality. + + .. warning:: + With pngnq version 1.0 (which is what Ubuntu 12.04 ships), the + dithering option is broken. Only the default, no dithering, + can be specified on those systems. + + **Default:** ``"n"`` + + .. warning:: + Because of several PIL bugs, only the most zoomed in level has transparency + when using pngnq. The other zoom levels have all transparency replaced by + black. This is *not* pngnq's fault, as pngnq supports multiple levels of + transparency just fine, it's PIL's fault for not even reading indexed + PNGs correctly. + + ``optipng`` + optipng tunes the deflate algorithm and removes unneeded channels from the PNG, + producing a smaller, lossless output image. It was inspired by pngcrush. + Available settings: + + ``olevel`` + An integer between ``0`` (few optimizations) and ``7`` (many optimizations). + The default should be satisfactory for everyone, higher levels than the default + see almost no benefit. + + **Default:** ``2`` + + ``pngcrush`` + pngcrush is very slow and not very good, you should use optipng in probably all cases. + However, Overviewer still allows you to use it because we're nice people like that. + Available settings: + + ``brute`` + Either ``True`` or ``False``. Cycles through all compression methods, and is very slow. + + .. note:: + There is practically no reason to ever use this. optipng will beat pngcrush, and + throwing more CPU time at pngcrush most likely won't help. If you think you need + this option, then you are most likely wrong. + + **Default:** ``False`` + + .. note:: + Don't forget to import the optimizers you use in your settings file, as shown in the + example above. + + **Default:** ``[]`` ``bgcolor`` This is the background color to be displayed behind the map. Its value diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index 4422feb..f31b6a1 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -17,36 +17,96 @@ import os import subprocess import shlex -pngcrush = "pngcrush" -optipng = "optipng" -advdef = "advdef" +class Optimizer: + binaryname = "" -def check_programs(level): + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + + def optimize(self, img): + raise NotImplementedError("I can't let you do that, Dave.") + + def fire_and_forget(self, args): + subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + +class NonAtomicOptimizer(Optimizer): + def cleanup(self, img): + os.rename(img + ".tmp", img) + + def fire_and_forget(self, args, img): + subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + self.cleanup(img) + +class PNGOptimizer: + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + +class JPEGOptimizer: + def __init__(self): + raise NotImplementedError("I can't let you do that, Dave.") + +class pngnq(NonAtomicOptimizer, PNGOptimizer): + binaryname = "pngnq" + + def __init__(self, sampling=3, dither="n"): + if sampling < 1 or sampling > 10: + raise Exception("Invalid sampling value '%d' for pngnq!" % sampling) + + if dither not in ["n", "f"]: + raise Exception("Invalid dither method '%s' for pngnq!" % dither) + + self.sampling = sampling + self.dither = dither + + def optimize(self, img): + if img.endswith(".tmp"): + extension = ".tmp" + else: + extension = ".png.tmp" + + NonAtomicOptimizer.fire_and_forget(self, [self.binaryname, "-s", str(self.sampling), + "-Q", self.dither, "-e", extension, img], img) + +class pngcrush(NonAtomicOptimizer, PNGOptimizer): + binaryname = "pngcrush" + # really can't be bothered to add some interface for all + # the pngcrush options, it sucks anyway + def __init__(self, brute=False): + self.brute = brute + + def optimize(self, img): + args = [self.binaryname, img, img + ".tmp"] + if self.brute == True: # Was the user an idiot? + args.insert(1, "-brute") + + NonAtomicOptimizer.fire_and_forget(self, args, img) + +class optipng(Optimizer, PNGOptimizer): + binaryname = "optipng" + + def __init__(self, olevel=2): + self.olevel = olevel + + def optimize(self, img): + Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img]) + + +def check_programs(optimizers): path = os.environ.get("PATH").split(os.pathsep) def exists_in_path(prog): result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) return len(result) != 0 - for prog,l in [(pngcrush,1), (advdef,2)]: - if l <= level: - if (not exists_in_path(prog)) and (not exists_in_path(prog + ".exe")): - raise Exception("Optimization prog %s for level %d not found!" % (prog, l)) - -def optimize_image(imgpath, imgformat, optimizeimg): - if imgformat == 'png': - if optimizeimg >= 1: - # we can't do an atomic replace here because windows is terrible - # so instead, we make temp files, delete the old ones, and rename - # the temp files. go windows! - subprocess.Popen([pngcrush, imgpath, imgpath + ".tmp"], - stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] - os.remove(imgpath) - os.rename(imgpath+".tmp", imgpath) - - if optimizeimg >= 2: - # the "-nc" it's needed to no broke the transparency of tiles - recompress_option = "-z2" if optimizeimg == 2 else "-z4" - subprocess.Popen([advdef, recompress_option,imgpath], stderr=subprocess.STDOUT, - stdout=subprocess.PIPE).communicate()[0] + for opt in optimizers: + if (not exists_in_path(opt.binaryname)) and (not exists_in_path(opt.binaryname + ".exe")): + raise Exception("Optimization program '%s' was not found!" % opt.binaryname) +def optimize_image(imgpath, imgformat, optimizers): + for opt in optimizers: + if imgformat == 'png': + if isinstance(opt, PNGOptimizer): + opt.optimize(imgpath) + elif imgformat == 'jpg': + if isinstance(opt, JPEGOptimizer): + opt.optimize(imgpath) diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index 41b8970..c17a171 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -46,6 +46,7 @@ from settingsValidators import * import util from observer import ProgressBarObserver, LoggingObserver, JSObserver +from optimizeimages import pngnq, optipng, pngcrush import platform import sys @@ -72,7 +73,7 @@ renders = Setting(required=True, default=util.OrderedDict(), "imgquality": Setting(required=False, validator=validateImgQuality, default=95), "bgcolor": Setting(required=True, validator=validateBGColor, default="1a1a1a"), "defaultzoom": Setting(required=True, validator=validateDefaultZoom, default=1), - "optimizeimg": Setting(required=True, validator=validateOptImg, default=0), + "optimizeimg": Setting(required=True, validator=validateOptImg, default=[]), "nomarkers": Setting(required=False, validator=validateBool, default=None), "texturepath": Setting(required=False, validator=validateTexturePath, default=None), "renderchecks": Setting(required=False, validator=validateInt, default=None), diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 6b5afa9..0279ed7 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -5,6 +5,7 @@ from collections import namedtuple import rendermodes import util +from optimizeimages import Optimizer from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT class ValidationException(Exception): @@ -155,8 +156,13 @@ def validateBGColor(color): return color -def validateOptImg(opt): - return bool(opt) +def validateOptImg(optimizers): + if isinstance(optimizers, (int, long)): + raise ValidationException("You are using a deprecated method of specifying optimizeimg!") + for opt in optimizers: + if not isinstance(opt, Optimizer): + raise ValidationException("Invalid Optimizer!") + return optimizers def validateTexturePath(path): # Expand user dir in directories strings diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index 3e13129..a2338a8 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -246,11 +246,7 @@ class TileSet(object): relevant in jpeg mode. optimizeimg - an integer indiating optimizations to perform on png outputs. 0 - indicates no optimizations. Only relevant in png mode. - 1 indicates pngcrush is run on all output images - 2 indicates pngcrush and advdef are run on all output images with advdef -z2 - 3 indicates pngcrush and advdef are run on all output images with advdef -z4 + A list of optimizer instances to use. rendermode Perhaps the most important/relevant option: a string indicating the @@ -892,7 +888,11 @@ class TileSet(object): try: #quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS) src = Image.open(path[1]) + # optimizeimg may have converted them to a palette image in the meantime + if src.mode != "RGB" and src.mode != "RGBA": + src = src.convert("RGBA") src.load() + quad = Image.new("RGBA", (192, 192), self.options['bgcolor']) resize_half(quad, src) img.paste(quad, path[0]) @@ -1017,7 +1017,7 @@ class TileSet(object): if self.options['optimizeimg']: optimize_image(tmppath, self.imgextension, self.options['optimizeimg']) - + os.utime(tmppath, (max_chunk_mtime, max_chunk_mtime)) def _iterate_and_check_tiles(self, path): From 866c2fe064ed7045a1536b479b5739950ae3e44e Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 19:54:43 +0200 Subject: [PATCH 2/4] Work around and warn if old optimizeimg definition Before someone says this is incorrect because it only ever uses pngcrush: The old code always used pngcrush and nothing else anyway. This is absolutely correct and the old behaviour. I also added a check to make sure it's a list, as some people might forget the whole list thing. --- overviewer_core/settingsValidators.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 0279ed7..6b53bee 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -158,7 +158,12 @@ def validateBGColor(color): def validateOptImg(optimizers): if isinstance(optimizers, (int, long)): - raise ValidationException("You are using a deprecated method of specifying optimizeimg!") + from optimizeimages import pngcrush + import logging + logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.") + optimizers = [pngcrush()] + if not isinstance(optimizers, list): + raise ValidationException("optimizeimg is not a list. Make sure you specify them like [foo()], with square brackets.") for opt in optimizers: if not isinstance(opt, Optimizer): raise ValidationException("Invalid Optimizer!") From 8817972b36145f5531f25bb1da40b9cd064ecbc4 Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 20:10:42 +0200 Subject: [PATCH 3/4] Move the check for program availability into class We also actually execute it now. Go us! --- overviewer_core/optimizeimages.py | 23 ++++++++++++----------- overviewer_core/settingsValidators.py | 5 ++++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index f31b6a1..c877e75 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -16,6 +16,7 @@ import os import subprocess import shlex +import logging class Optimizer: binaryname = "" @@ -29,6 +30,17 @@ class Optimizer: def fire_and_forget(self, args): subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + def check_availability(self): + path = os.environ.get("PATH").split(os.pathsep) + + def exists_in_path(prog): + result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) + return len(result) != 0 + + if (not exists_in_path(self.binaryname)) and (not exists_in_path(self.binaryname + ".exe")): + raise Exception("Optimization program '%s' was not found!" % self.binaryname) + + class NonAtomicOptimizer(Optimizer): def cleanup(self, img): os.rename(img + ".tmp", img) @@ -91,17 +103,6 @@ class optipng(Optimizer, PNGOptimizer): Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img]) -def check_programs(optimizers): - path = os.environ.get("PATH").split(os.pathsep) - - def exists_in_path(prog): - result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path) - return len(result) != 0 - - for opt in optimizers: - if (not exists_in_path(opt.binaryname)) and (not exists_in_path(opt.binaryname + ".exe")): - raise Exception("Optimization program '%s' was not found!" % opt.binaryname) - def optimize_image(imgpath, imgformat, optimizers): for opt in optimizers: if imgformat == 'png': diff --git a/overviewer_core/settingsValidators.py b/overviewer_core/settingsValidators.py index 6b53bee..e61a1ab 100644 --- a/overviewer_core/settingsValidators.py +++ b/overviewer_core/settingsValidators.py @@ -7,6 +7,7 @@ import rendermodes import util from optimizeimages import Optimizer from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT +import logging class ValidationException(Exception): pass @@ -159,7 +160,6 @@ def validateBGColor(color): def validateOptImg(optimizers): if isinstance(optimizers, (int, long)): from optimizeimages import pngcrush - import logging logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.") optimizers = [pngcrush()] if not isinstance(optimizers, list): @@ -167,6 +167,9 @@ def validateOptImg(optimizers): for opt in optimizers: if not isinstance(opt, Optimizer): raise ValidationException("Invalid Optimizer!") + + opt.check_availability() + return optimizers def validateTexturePath(path): From 03561dccfad75faec34906c08677c00cb68dfdc6 Mon Sep 17 00:00:00 2001 From: CounterPillow Date: Tue, 6 May 2014 22:31:23 +0200 Subject: [PATCH 4/4] Use check_call(), add workaround for broken pngnq Also -f pngnq to write files, in case something didn't work last render. --- overviewer_core/optimizeimages.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/overviewer_core/optimizeimages.py b/overviewer_core/optimizeimages.py index c877e75..da8d306 100644 --- a/overviewer_core/optimizeimages.py +++ b/overviewer_core/optimizeimages.py @@ -28,7 +28,7 @@ class Optimizer: raise NotImplementedError("I can't let you do that, Dave.") def fire_and_forget(self, args): - subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + subprocess.check_call(args) def check_availability(self): path = os.environ.get("PATH").split(os.pathsep) @@ -46,7 +46,7 @@ class NonAtomicOptimizer(Optimizer): os.rename(img + ".tmp", img) def fire_and_forget(self, args, img): - subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] + subprocess.check_call(args) self.cleanup(img) class PNGOptimizer: @@ -76,8 +76,13 @@ class pngnq(NonAtomicOptimizer, PNGOptimizer): else: extension = ".png.tmp" - NonAtomicOptimizer.fire_and_forget(self, [self.binaryname, "-s", str(self.sampling), - "-Q", self.dither, "-e", extension, img], img) + args = [self.binaryname, "-s", str(self.sampling), "-f", "-e", extension, img] + # Workaround for poopbuntu 12.04 which ships an old broken pngnq + if self.dither != "n": + args.insert(1, "-Q") + args.insert(2, self.dither) + + NonAtomicOptimizer.fire_and_forget(self, args, img) class pngcrush(NonAtomicOptimizer, PNGOptimizer): binaryname = "pngcrush"