Merge pull request #1074 from CounterPillow/optimizerewrite

Rewrote image optimisation stuff.

docs/config.rst

@@ -545,18 +545,92 @@ values. The valid configuration keys are listed below.
     **Default:** ``95``
 
 ``optimizeimg``
+
+    .. warning::
+        Using image optimizers will increase render times significantly.
+
     This option specifies which additional tools overviewer should use to
     optimize the filesize of png tiles.
     The tools used must be placed somewhere, where overviewer can find them, for
     example the "PATH" environment variable or a directory like /usr/bin.
-    This should be an integer between 0 and 3.
-    * ``1 - Use pngcrush``
-    * ``2 - Use advdef``
-    * ``3 - Use pngcrush and advdef (Not recommended)``
-    Using this option may significantly increase render time, but will make
-    the resulting tiles smaller, with lossless image quality.
 
-    **Default:** ``0``
+    The option is a list of Optimizer objects, which are then executed in
+    the order in which they're specified::
+        
+        from optimizeimages import pngnq, optipng
+        worlds["world"] = "/path/to/world"
+
+        renders["daytime"] = {
+            "world":"world",
+            "title":"day",
+            "rendermode":smooth_lighting,
+            "optimizeimg":[pngnq(sampling=1), optipng(olevel=3)],
+        }
+    
+    Here is a list of supported image optimization programs:
+
+    ``pngnq``
+        pngnq quantizes 32-bit RGBA images into 8-bit RGBA palette PNGs. This is
+        lossy, but reduces filesize significantly. Available settings:
+        
+        ``sampling``
+            An integer between ``1`` and ``10``, ``1`` samples all pixels, is slow and yields
+            the best quality. Higher values sample less of the image, which makes
+            the process faster, but less accurate.
+
+            **Default:** ``3``
+
+        ``dither``
+            Either the string ``"n"`` for no dithering, or ``"f"`` for Floyd
+            Steinberg dithering. Dithering helps eliminate colorbanding, sometimes
+            increasing visual quality.
+
+            .. warning::
+                With pngnq version 1.0 (which is what Ubuntu 12.04 ships), the
+                dithering option is broken. Only the default, no dithering,
+                can be specified on those systems.
+
+            **Default:** ``"n"``
+
+        .. warning::
+            Because of several PIL bugs, only the most zoomed in level has transparency
+            when using pngnq. The other zoom levels have all transparency replaced by
+            black. This is *not* pngnq's fault, as pngnq supports multiple levels of
+            transparency just fine, it's PIL's fault for not even reading indexed
+            PNGs correctly.
+
+    ``optipng``
+        optipng tunes the deflate algorithm and removes unneeded channels from the PNG,
+        producing a smaller, lossless output image. It was inspired by pngcrush.
+        Available settings:
+
+        ``olevel``
+            An integer between ``0`` (few optimizations) and ``7`` (many optimizations).
+            The default should be satisfactory for everyone, higher levels than the default
+            see almost no benefit.
+
+            **Default:** ``2``
+
+    ``pngcrush``
+        pngcrush is very slow and not very good, you should use optipng in probably all cases.
+        However, Overviewer still allows you to use it because we're nice people like that.
+        Available settings:
+
+        ``brute``
+            Either ``True`` or ``False``. Cycles through all compression methods, and is very slow.
+
+            .. note::
+                There is practically no reason to ever use this. optipng will beat pngcrush, and
+                throwing more CPU time at pngcrush most likely won't help. If you think you need
+                this option, then you are most likely wrong.
+
+            **Default:** ``False``
+
+    .. note::
+        Don't forget to import the optimizers you use in your settings file, as shown in the
+        example above.
+
+    **Default:** ``[]``
 
 ``bgcolor``
     This is the background color to be displayed behind the map. Its value

overviewer_core/optimizeimages.py

@@ -16,37 +16,103 @@
 import os
 import subprocess
 import shlex
+import logging
 
-pngcrush = "pngcrush"
+class Optimizer:
-optipng = "optipng"
+    binaryname = ""
-advdef = "advdef"
 
-def check_programs(level):
+    def __init__(self):
-    path = os.environ.get("PATH").split(os.pathsep)
+        raise NotImplementedError("I can't let you do that, Dave.")
+
+    def optimize(self, img):
+        raise NotImplementedError("I can't let you do that, Dave.")
     
-    def exists_in_path(prog):
+    def fire_and_forget(self, args):
-        result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path)
+        subprocess.check_call(args)
-        return len(result) != 0
+
+    def check_availability(self):
+        path = os.environ.get("PATH").split(os.pathsep)
+        
+        def exists_in_path(prog):
+            result = filter(lambda x: os.path.exists(os.path.join(x, prog)), path)
+            return len(result) != 0
+
+        if (not exists_in_path(self.binaryname)) and (not exists_in_path(self.binaryname + ".exe")):
+            raise Exception("Optimization program '%s' was not found!" % self.binaryname)
+        
+
+class NonAtomicOptimizer(Optimizer):
+    def cleanup(self, img):
+        os.rename(img + ".tmp", img)
+
+    def fire_and_forget(self, args, img):
+        subprocess.check_call(args)
+        self.cleanup(img)
+
+class PNGOptimizer:
+    def __init__(self):
+        raise NotImplementedError("I can't let you do that, Dave.")
+
+class JPEGOptimizer:
+    def __init__(self):
+        raise NotImplementedError("I can't let you do that, Dave.")
+
+class pngnq(NonAtomicOptimizer, PNGOptimizer):
+    binaryname = "pngnq"
+
+    def __init__(self, sampling=3, dither="n"):
+        if sampling < 1 or sampling > 10:
+            raise Exception("Invalid sampling value '%d' for pngnq!" % sampling)
+
+        if dither not in ["n", "f"]:
+            raise Exception("Invalid dither method '%s' for pngnq!" % dither)
+
+        self.sampling = sampling
+        self.dither = dither
     
-    for prog,l in [(pngcrush,1), (advdef,2)]:
+    def optimize(self, img):
-        if l <= level:
+        if img.endswith(".tmp"):
-            if (not exists_in_path(prog)) and (not exists_in_path(prog + ".exe")):
+            extension = ".tmp"
-                raise Exception("Optimization prog %s for level %d not found!" % (prog, l))
+        else:
+            extension = ".png.tmp"
 
-def optimize_image(imgpath, imgformat, optimizeimg):
+        args = [self.binaryname, "-s", str(self.sampling), "-f", "-e", extension, img]
-    if imgformat == 'png':
+        # Workaround for poopbuntu 12.04 which ships an old broken pngnq
-        if optimizeimg >= 1:
+        if self.dither != "n":
-            # we can't do an atomic replace here because windows is terrible
+            args.insert(1, "-Q")
-            # so instead, we make temp files, delete the old ones, and rename
+            args.insert(2, self.dither)
-            # the temp files. go windows!
-            subprocess.Popen([pngcrush, imgpath, imgpath + ".tmp"],
-                stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
-            os.remove(imgpath)
-            os.rename(imgpath+".tmp", imgpath)
 
-        if optimizeimg >= 2:
+        NonAtomicOptimizer.fire_and_forget(self, args, img)
-            # the "-nc" it's needed to no broke the transparency of tiles
-            recompress_option = "-z2" if optimizeimg == 2 else "-z4"
-            subprocess.Popen([advdef, recompress_option,imgpath], stderr=subprocess.STDOUT,
-                stdout=subprocess.PIPE).communicate()[0]
 
+class pngcrush(NonAtomicOptimizer, PNGOptimizer):
+    binaryname = "pngcrush"
+    # really can't be bothered to add some interface for all
+    # the pngcrush options, it sucks anyway
+    def __init__(self, brute=False):
+        self.brute = brute
+        
+    def optimize(self, img):
+        args = [self.binaryname, img, img + ".tmp"]
+        if self.brute == True:  # Was the user an idiot?
+            args.insert(1, "-brute")
+
+        NonAtomicOptimizer.fire_and_forget(self, args, img)
+
+class optipng(Optimizer, PNGOptimizer):
+    binaryname = "optipng"
+
+    def __init__(self, olevel=2):
+        self.olevel = olevel
+    
+    def optimize(self, img):
+        Optimizer.fire_and_forget(self, [self.binaryname, "-o" + str(self.olevel), "-quiet", img])
+        
+
+def optimize_image(imgpath, imgformat, optimizers):
+        for opt in optimizers:
+            if imgformat == 'png':
+                if isinstance(opt, PNGOptimizer):
+                    opt.optimize(imgpath)
+            elif imgformat == 'jpg':
+                if isinstance(opt, JPEGOptimizer):
+                    opt.optimize(imgpath)

overviewer_core/settingsDefinition.py

@@ -46,6 +46,7 @@
 from settingsValidators import *
 import util
 from observer import ProgressBarObserver, LoggingObserver, JSObserver
+from optimizeimages import pngnq, optipng, pngcrush
 import platform
 import sys
 
@@ -72,7 +73,7 @@ renders = Setting(required=True, default=util.OrderedDict(),
             "imgquality": Setting(required=False, validator=validateImgQuality, default=95),
             "bgcolor": Setting(required=True, validator=validateBGColor, default="1a1a1a"),
             "defaultzoom": Setting(required=True, validator=validateDefaultZoom, default=1),
-            "optimizeimg": Setting(required=True, validator=validateOptImg, default=0),
+            "optimizeimg": Setting(required=True, validator=validateOptImg, default=[]),
             "nomarkers": Setting(required=False, validator=validateBool, default=None),
             "texturepath": Setting(required=False, validator=validateTexturePath, default=None),
             "renderchecks": Setting(required=False, validator=validateInt, default=None),

overviewer_core/settingsValidators.py

@@ -5,7 +5,9 @@ from collections import namedtuple
 
 import rendermodes
 import util
+from optimizeimages import Optimizer
 from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT
+import logging
 
 class ValidationException(Exception):
     pass
@@ -155,8 +157,20 @@ def validateBGColor(color):
         return color
 
 
-def validateOptImg(opt):
+def validateOptImg(optimizers):
-    return bool(opt)
+    if isinstance(optimizers, (int, long)):
+        from optimizeimages import pngcrush
+        logging.warning("You're using a deprecated definition of optimizeimg. We'll do what you say for now, but please fix this as soon as possible.")
+        optimizers = [pngcrush()]
+    if not isinstance(optimizers, list):
+        raise ValidationException("optimizeimg is not a list. Make sure you specify them like [foo()], with square brackets.")
+    for opt in optimizers:
+        if not isinstance(opt, Optimizer):
+            raise ValidationException("Invalid Optimizer!")
+
+        opt.check_availability()
+
+    return optimizers
 
 def validateTexturePath(path):
     # Expand user dir in directories strings

overviewer_core/tileset.py

@@ -262,11 +262,7 @@ class TileSet(object):
             relevant in jpeg mode.
 
         optimizeimg
-            an integer indiating optimizations to perform on png outputs. 0
+            A list of optimizer instances to use.
-            indicates no optimizations. Only relevant in png mode.
-            1 indicates pngcrush is run on all output images
-            2 indicates pngcrush and advdef are run on all output images with advdef -z2
-            3 indicates pngcrush and advdef are run on all output images with advdef -z4
 
         rendermode
             Perhaps the most important/relevant option: a string indicating the
@@ -925,7 +921,11 @@ class TileSet(object):
             try:
                 #quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
                 src = Image.open(path[1])
+                # optimizeimg may have converted them to a palette image in the meantime
+                if src.mode != "RGB" and src.mode != "RGBA":
+                    src = src.convert("RGBA")
                 src.load()
+
                 quad = Image.new("RGBA", (192, 192), self.options['bgcolor'])
                 resize_half(quad, src)
                 img.paste(quad, path[0])
@@ -1050,7 +1050,7 @@ class TileSet(object):
 
             if self.options['optimizeimg']:
                 optimize_image(tmppath, self.imgextension, self.options['optimizeimg'])
-
+            
             os.utime(tmppath, (max_chunk_mtime, max_chunk_mtime))
 
     def _iterate_and_check_tiles(self, path):