0

tileset: drastic code style fixes

We've also added a setup.cfg to specify what rules we want for
pycodestyle. We'll be ignoring some dumb ones that make no sense
and probably shouldn't exist, like both W503 and W504 being an
issue somehow???????

We're using a line length limit of 100. If you're on an 80 character
terminal and incredibly upset about this, then that's the nursing
home staff's problem, not ours.
This commit is contained in:
Nicolas F
2019-03-03 16:47:35 +01:00
parent 0499f8d168
commit 7f63dfe315
2 changed files with 240 additions and 210 deletions

View File

@@ -13,31 +13,32 @@
# You should have received a copy of the GNU General Public License along
# with the Overviewer. If not, see <http://www.gnu.org/licenses/>.
import errno
import functools
import itertools
import logging
import os
import os.path
import sys
import shutil
import random
import functools
import time
import errno
import stat
import platform
import random
import shutil
import stat
import sys
import time
from collections import namedtuple
from itertools import product, izip, chain
from itertools import chain, izip, product
from PIL import Image
from .util import roundrobin
from . import nbt
from . import world
import c_overviewer
import rendermodes
from c_overviewer import resize_half
from . import nbt, world
from .files import FileReplacer, get_fs_caps
from .optimizeimages import optimize_image
import rendermodes
import c_overviewer
from c_overviewer import resize_half
from .util import roundrobin
"""
@@ -90,15 +91,18 @@ do_work(workobj)
"""
# small but useful
def iterate_base4(d):
"""Iterates over a base 4 number with d digits"""
return product(xrange(4), repeat=d)
# A named tuple class storing the row and column bounds for the to-be-rendered
# world
Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow"))
# A note about the implementation of the different rendercheck modes:
#
# For reference, here's what the rendercheck modes are:
@@ -171,6 +175,8 @@ Bounds = namedtuple("Bounds", ("mincol", "maxcol", "minrow", "maxrow"))
# any additional checks.
__all__ = ["TileSet"]
class TileSet(object):
"""The TileSet object manages the work required to produce a set of tiles
on disk. It calculates the work that needs to be done and tells the
@@ -321,9 +327,8 @@ class TileSet(object):
# This is the typical code path for an initial render, make
# this a "forcerender"
self.options['renderchecks'] = 2
logging.debug("This is the first time rendering %s. Doing" +
" a full-render",
self.options['name'])
logging.debug("This is the first time rendering %s. Doing "
"a full-render", self.options['name'])
elif not os.path.exists(self.outputdir):
# Somehow the outputdir got erased but the metadata file is
# still there. That's strange!
@@ -337,27 +342,26 @@ class TileSet(object):
# The last render must have been interrupted. The default should be
# a check-tiles render then
logging.warning(
"The last render for '%s' didn't finish. I'll be " +
"scanning all the tiles to make sure everything's up "+
"The last render for '%s' didn't finish. I'll be "
"scanning all the tiles to make sure everything's up "
"to date.",
self.options['name'],
)
logging.warning("The total tile count will be (possibly "+
"wildly) inaccurate, because I don't know how many "+
"tiles need rendering. I'll be checking them as I go")
self.options['name'])
logging.warning(
"The total tile count will be (possibly "
"wildly) inaccurate, because I don't know how many "
"tiles need rendering. I'll be checking them as I go.")
if self.forcerendertime != 0:
logging.info(
"The unfinished render was a --forcerender. " +
"Rerendering any tiles older than %s",
time.strftime("%x %X", time.localtime(self.forcerendertime)),
)
"The unfinished render was a --forcerender. "
"Rerendering any tiles older than %s.",
time.strftime("%x %X", time.localtime(self.forcerendertime)))
self.options['renderchecks'] = 1
else:
logging.debug("No rendercheck mode specified for %s. "+
"Rendering tile whose chunks have changed since %s",
logging.debug(
"No rendercheck mode specified for %s. "
"Rendering tile whose chunks have changed since %s.",
self.options['name'],
time.strftime("%x %X", time.localtime(self.last_rendertime)),
)
time.strftime("%x %X", time.localtime(self.last_rendertime)))
self.options['renderchecks'] = 0
if not os.path.exists(self.outputdir):
@@ -366,7 +370,7 @@ class TileSet(object):
"The tile directory didn't exist, but you have specified "
"explicitly not to do a --fullrender (which is the default for "
"this situation). I'm overriding your decision and setting "
"--fullrender for just this run")
"--fullrender for just this run.")
self.options['renderchecks'] = 2
os.mkdir(self.outputdir)
@@ -393,6 +397,7 @@ class TileSet(object):
# do_preprocessing step
def __getstate__(self):
return self.world, self.regionset, self.am, self.textures, self.options, self.outputdir
def __setstate__(self, state):
self.__init__(*state)
@@ -413,8 +418,9 @@ class TileSet(object):
# This warning goes here so it's only shown once
if self.treedepth >= 15:
logging.warning("Just letting you know, your map requires %s zoom levels. This is REALLY big!",
self.treedepth)
logging.warning(
"Just letting you know, your map requires %s "
"zoom levels. This is REALLY big!", self.treedepth)
# Do any tile re-arranging if necessary. Skip if there was no config
# from the asset-manager, which typically indicates this is a new
@@ -460,6 +466,7 @@ class TileSet(object):
fd = self.options.get("changelist", None)
if fd:
logging.debug("Changelist activated for %s (fileno %s)", self, fd)
# This re-implements some of the logic from do_work()
def write_out(tilepath):
if len(tilepath) == self.treedepth:
@@ -479,7 +486,6 @@ class TileSet(object):
# file object may be garbage collected, closing the file.
os.write(fd, imgpath + "\n")
# See note at the top of this file about the rendercheck modes for an
# explanation of what this method does in different situations.
#
@@ -557,28 +563,35 @@ class TileSet(object):
"""
def bgcolorformat(color):
return "#%02x%02x%02x" % color[0:3]
isOverlay = self.options.get("overlay") or (not any(isinstance(x, rendermodes.Base) for x in self.options.get("rendermode")))
isOverlay = self.options.get("overlay") or \
(not any(isinstance(x, rendermodes.Base) for x in self.options.get("rendermode")))
# don't update last render time if we're leaving this alone
last_rendertime = self.last_rendertime
if self.options['renderchecks'] != 3:
last_rendertime = self.max_chunk_mtime
d = dict(name = self.options.get('title'),
d = dict(
name=self.options.get('title'),
zoomLevels=self.treedepth,
defaultZoom=self.options.get('defaultzoom'),
maxZoom = self.options.get('maxzoom', self.treedepth) if self.options.get('maxzoom', self.treedepth) >= 0 else self.treedepth+self.options.get('maxzoom'),
maxZoom=None,
path=self.options.get('name'),
base=self.options.get('base'),
bgcolor=bgcolorformat(self.options.get('bgcolor')),
world = self.options.get('worldname_orig') +
(" - " + self.options.get('dimension')[0] if self.options.get('dimension')[1] != 0 else ''),
world=None,
last_rendertime=last_rendertime,
imgextension=self.imgextension,
isOverlay=isOverlay,
poititle=self.options.get("poititle"),
showlocationmarker=self.options.get("showlocationmarker")
)
d['maxZoom'] = self.options.get('maxzoom', self.treedepth)
if d['maxZoom'] < 0:
d['maxZoom'] = self.treedepth + self.options.get('maxzoom')
d['world'] = self.options.get('worldname_orig')
if self.options.get('dimension')[1] != 0:
d['world'] += " - " + self.options.get('dimension')[0]
d['maxZoom'] = min(self.treedepth, d['maxZoom'])
d['minZoom'] = min(max(0, self.options.get("minzoom", 0)), d['maxZoom'])
d['defaultZoom'] = max(d['minZoom'], min(d['defaultZoom'], d['maxZoom']))
@@ -587,7 +600,7 @@ class TileSet(object):
d.update({"tilesets": self.options.get("overlay")})
# None means overworld
if (self.regionset.get_type() == None and self.options.get("showspawn", True)):
if (self.regionset.get_type() is None and self.options.get("showspawn", True)):
d.update({"spawn": self.options.get("spawn")})
else:
d.update({"spawn": False})
@@ -661,9 +674,9 @@ class TileSet(object):
# Skip a depth 1 tree. A depth 1 tree pretty much can't happen, so
# when we detect this it usually means the tree is actually empty
return
logging.debug("Current tree depth for %s is reportedly %s. Target tree depth is %s",
self.options['name'],
curdepth, self.treedepth)
logging.debug(
"Current tree depth for %s is reportedly %s. Target tree depth is %s.",
self.options['name'], curdepth, self.treedepth)
if self.treedepth != curdepth:
if self.treedepth > curdepth:
logging.warning("Your map seems to have expanded beyond its previous bounds.")
@@ -671,11 +684,13 @@ class TileSet(object):
for _ in xrange(self.treedepth - curdepth):
self._increase_depth()
elif self.treedepth < curdepth:
logging.warning("Your map seems to have shrunk. Did you delete some chunks? No problem. Re-arranging tiles, just a sec...")
logging.warning(
"Your map seems to have shrunk. Did you delete some "
"chunks? No problem. Re-arranging tiles, just a sec...")
for _ in xrange(curdepth - self.treedepth):
self._decrease_depth()
logging.info(
"There done. I'm switching to --check-tiles mode for "
"There, done. I'm switching to --check-tiles mode for "
"this one render. This will make sure any old tiles that "
"should no longer exist are deleted.")
self.options['renderchecks'] = 1
@@ -726,14 +741,16 @@ class TileSet(object):
p = getpath(f)
if os.path.exists(p):
os.rename(p, getpath(newdir, newf))
except:
# We're catching BaseException here since we'll also want to do this on
# exit.
except BaseException:
rollback_filerename(dirnum)
raise
except:
except BaseException:
rollback_mkdir(dirnum)
raise
os.rename(newdirpath, getpath(str(dirnum)))
except:
except BaseException:
logging.warning("Overviewer was interrupted during tile "
"re-arrangement.")
logging.warning("Rolling back changes...")
@@ -776,7 +793,8 @@ class TileSet(object):
os.rename(getpath("new3"), getpath("3"))
# Delete the files in the top directory to make sure they get re-created.
files = [str(num)+"."+self.imgextension for num in xrange(4)] + ["base." + self.imgextension]
files = [str(num) + "." + self.imgextension for num in xrange(4)] + \
["base." + self.imgextension]
for f in files:
try:
os.unlink(getpath(f))
@@ -822,14 +840,14 @@ class TileSet(object):
max_chunk_mtime = 0
# For each chunk, do this:
# For each tile that the chunk touches, do this:
# Compare the last modified time of the chunk and tile. If the
# tile is older, mark it in a RendertileSet object as dirty.
for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() if (markall or platform.system() == 'Windows') else self.regionset.iterate_newer_chunks(last_rendertime):
for chunkx, chunkz, chunkmtime in self.regionset.iterate_chunks() \
if (markall or platform.system() == 'Windows') \
else self.regionset.iterate_newer_chunks(last_rendertime):
chunkcount += 1
if chunkmtime > max_chunk_mtime:
@@ -848,8 +866,7 @@ class TileSet(object):
c < -xradius or
c >= xradius or
r < -yradius or
r >= yradius
):
r >= yradius):
continue
# Computes the path in the quadtree from the col,row coordinates
@@ -882,7 +899,8 @@ class TileSet(object):
dirty.add(tile.path)
t = int(time.time() - stime)
logging.debug("Finished chunk scan for %s. %s chunks scanned in %s second%s",
logging.debug(
"Finished chunk scan for %s. %s chunks scanned in %s second%s.",
self.options['name'], chunkcount, t,
"s" if t != 1 else "")
@@ -945,20 +963,17 @@ class TileSet(object):
# Ignore errors if it's "file doesn't exist"
if e.errno != errno.ENOENT:
raise
logging.warning("Tile %s was requested for render, but no children were found! This is probably a bug", imgpath)
logging.warning(
"Tile %s was requested for render, but no children were found! "
"This is probably a bug.", imgpath)
return
#logging.debug("writing out compositetile {0}".format(imgpath))
# Create the actual image now
img = Image.new("RGBA", (384, 384), self.options['bgcolor'])
# we'll use paste (NOT alpha_over) for quadtree generation because
# We'll use paste (NOT alpha_over) for quadtree generation because
# this is just straight image stitching, not alpha blending
for path in quadPath_filtered:
try:
#quad = Image.open(path[1]).resize((192,192), Image.ANTIALIAS)
src = Image.open(path[1])
# optimizeimg may have converted them to a palette image in the meantime
if src.mode != "RGB" and src.mode != "RGBA":
@@ -969,18 +984,23 @@ class TileSet(object):
resize_half(quad, src)
img.paste(quad, path[0])
except Exception, e:
logging.warning("Couldn't open %s. It may be corrupt. Error was '%s'", path[1], e)
logging.warning("I'm going to try and delete it. You will need to run the render again and with --check-tiles")
logging.warning("Couldn't open %s. It may be corrupt. Error was '%s'.", path[1], e)
logging.warning(
"I'm going to try and delete it. You will need to run "
"the render again and with --check-tiles.")
try:
os.unlink(path[1])
except Exception, e:
logging.error("While attempting to delete corrupt image %s, an error was encountered. You will need to delete it yourself. Error was '%s'", path[1], e)
logging.error(
"While attempting to delete corrupt image %s, an error was encountered. "
"You will need to delete it yourself. Error was '%s'", path[1], e)
# Save it
with FileReplacer(imgpath, capabilities=self.fs_caps) as tmppath:
if imgformat == 'jpg':
img.convert('RGB').save(tmppath, "jpeg", quality=self.options['imgquality'], subsampling=0)
else: # png
img.convert('RGB').save(tmppath, "jpeg", quality=self.options['imgquality'],
subsampling=0)
else: # PNG
img.save(tmppath, "png")
if self.options['optimizeimg']:
@@ -1017,7 +1037,8 @@ class TileSet(object):
if not chunks:
# No chunks were found in this tile
logging.warning("%s was requested for render, but no chunks found! This may be a bug", tile)
logging.warning("%s was requested for render, but no chunks found! "
"This may be a bug.", tile)
try:
os.unlink(imgpath)
except OSError, e:
@@ -1040,8 +1061,6 @@ class TileSet(object):
if e.errno != errno.EEXIST:
raise
#logging.debug("writing out worldtile {0}".format(imgpath))
# Compile this image
tileimg = Image.new("RGBA", (384, 384), self.options['bgcolor'])
@@ -1059,48 +1078,33 @@ class TileSet(object):
# draw the chunk!
try:
c_overviewer.render_loop(self.world, self.regionset, chunkx, chunky,
chunkz, tileimg, xpos, ypos,
c_overviewer.render_loop(
self.world, self.regionset, chunkx, chunky, chunkz, tileimg, xpos, ypos,
self.options['rendermode'], self.textures)
except nbt.CorruptionError:
# A warning and traceback was already printed by world.py's
# get_chunk()
logging.debug("Skipping the render of corrupt chunk at %s,%s and moving on.", chunkx, chunkz)
logging.debug("Skipping the render of corrupt chunk at %s,%s "
"and moving on.", chunkx, chunkz)
except world.ChunkDoesntExist:
# Some chunks are present on disk but not fully initialized.
# This is okay.
pass
except Exception, e:
logging.error("Could not render chunk %s,%s for some reason. This is likely a render primitive option error.", chunkx, chunkz)
logging.error("Could not render chunk %s,%s for some reason. "
"This is likely a render primitive option error.", chunkx, chunkz)
logging.error("Full error was:", exc_info=1)
sys.exit(1)
## Semi-handy routine for debugging the drawing routine
## Draw the outline of the top of the chunk
#import ImageDraw
#draw = ImageDraw.Draw(tileimg)
## Draw top outline
#draw.line([(192,0), (384,96)], fill='red')
#draw.line([(192,0), (0,96)], fill='red')
#draw.line([(0,96), (192,192)], fill='red')
#draw.line([(384,96), (192,192)], fill='red')
## Draw side outline
#draw.line([(0,96),(0,96+192)], fill='red')
#draw.line([(384,96),(384,96+192)], fill='red')
## Which chunk this is:
#draw.text((96,48), "C: %s,%s" % (chunkx, chunkz), fill='red')
#draw.text((96,96), "c,r: %s,%s" % (col, row), fill='red')
# Save them
with FileReplacer(imgpath, capabilities=self.fs_caps) as tmppath:
if self.imgextension == 'jpg':
tileimg.convert('RGB').save(tmppath, "jpeg", quality=self.options['imgquality'], subsampling=0)
else: # png
tileimg.convert('RGB').save(tmppath, "jpeg", quality=self.options['imgquality'],
subsampling=0)
else: # PNG
tileimg.save(tmppath, "png")
if self.options['optimizeimg']:
optimize_image(tmppath, self.imgextension, self.options['optimizeimg'])
os.utime(tmppath, (max_chunk_mtime, max_chunk_mtime))
def _iterate_and_check_tiles(self, path):
@@ -1164,7 +1168,7 @@ class TileSet(object):
"rendering and which don't, so it's important to "
"preserve the mtimes Overviewer sets. Please see our FAQ "
"page on docs.overviewer.org or ask us in IRC for more "
"information")
"information.")
logging.warning("Tile was: %s", imgpath)
if max_chunk_mtime > tile_mtime or tile_mtime < self.forcerendertime:
@@ -1261,10 +1265,10 @@ class TileSet(object):
logging.debug("Found a subtree that shouldn't exist. Deleting it: %s", dirpath)
shutil.rmtree(dirpath)
##
## Functions for converting (x, z) to (col, row) and back
##
#
# Functions for converting (x, z) to (col, row) and back
#
def convert_coords(chunkx, chunkz):
"""Takes a coordinate (chunkx, chunkz) where chunkx and chunkz are
in the chunk coordinate system, and figures out the row and column
@@ -1275,6 +1279,7 @@ def convert_coords(chunkx, chunkz):
# change this function, and you MUST change unconvert_coords
return (chunkx + chunkz, chunkz - chunkx)
def unconvert_coords(col, row):
"""Undoes what convert_coords does. Returns (chunkx, chunkz)."""
@@ -1282,6 +1287,7 @@ def unconvert_coords(col, row):
# col - row = chunkx + chunkx => (col - row)/2 = chunkx
return ((col - row) / 2, (col + row) / 2)
######################
# The following two functions define the mapping from chunks to tiles and back.
# The mapping from chunks to tiles (get_tiles_by_chunk()) is used during the
@@ -1314,6 +1320,7 @@ def get_tiles_by_chunk(chunkcol, chunkrow):
return product(colrange, rowrange)
def get_chunks_by_tile(tile, regionset):
"""Get chunk sections that are relevant to the given render-tile. Only
returns chunk sections that are in chunks that actually exist according to
@@ -1330,7 +1337,8 @@ def get_chunks_by_tile(tile, regionset):
# This is not a documented usage of this function and is used only for
# debugging
if regionset is None:
get_mtime = lambda x,y: True
def get_mtime(x, y):
return True
else:
get_mtime = regionset.get_chunk_mtime
@@ -1362,13 +1370,13 @@ def get_chunks_by_tile(tile, regionset):
odditer,
eveniter, eveniter,
odditer,
eveniter,eveniter,
)):
eveniter, eveniter,)):
chunkx, chunkz = unconvert_coords(col, row)
mtime = get_mtime(chunkx, chunkz)
if mtime:
yield (col, row, chunkx, y, chunkz, mtime)
class RendertileSet(object):
"""This object holds a set of render-tiles using a quadtree data structure.
It is typically used to hold tiles that need rendering. This implementation
@@ -1387,6 +1395,7 @@ class RendertileSet(object):
"""
__slots__ = ("depth", "children", "num_tiles", "num_tiles_all")
def __init__(self, depth):
"""Initialize a new tree with the specified depth. This actually
initializes a node, which is the root of a subtree, with `depth` levels
@@ -1443,10 +1452,14 @@ class RendertileSet(object):
if path:
# We are not at the leaf, recurse.
if children[childnum] == True:
# Don't try to make this "prettier" by getting rid of is True and replacing
# the elif with not children[childnum], Python 2 thinks that's semantically different
# and you suddenly have no tiles being rendered anymore.
# No, I don't know either.
if children[childnum] is True:
# The child is already in the tree.
return
elif children[childnum] == False:
elif children[childnum] is False:
# Expand all-false.
children[childnum] = [False] * 4
@@ -1494,7 +1507,9 @@ class RendertileSet(object):
raise ValueError("Level parameter must be between 1 and %s" % self.depth)
todepth = self.depth - level + 1
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, onlydepth=todepth, robin=robin, offset=offset))
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth,
onlydepth=todepth, robin=robin,
offset=offset))
def posttraversal(self, robin=False, offset=(0, 0)):
"""Returns an iterator over tile paths for every tile in the
@@ -1507,7 +1522,8 @@ class RendertileSet(object):
subtrees simultaneously in a round-robin manner.
"""
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth, robin=robin, offset=offset))
return (tuple(path) for path in self._iterate_helper([], self.children, self.depth,
robin=robin, offset=offset))
def _iterate_helper(self, path, children, depth, onlydepth=None, robin=False, offset=(0, 0)):
"""Returns an iterator over tile paths for every tile in the set."""
@@ -1527,7 +1543,9 @@ class RendertileSet(object):
for (childnum_, child), childoffset_ in distance_sort(enumerate(children_list), offset):
if child:
def go(childnum, childoffset):
for p in self._iterate_helper(path + [childnum], children_list[childnum], depth-1, onlydepth=onlydepth, offset=childoffset):
for p in self._iterate_helper(path + [childnum], children_list[childnum],
depth - 1, onlydepth=onlydepth,
offset=childoffset):
yield p
gens.append(go(childnum_, childoffset_))
@@ -1578,7 +1596,8 @@ class RendertileSet(object):
from itertools import imap
num = sum(imap(lambda _: 1, self.iterate()))
if num != self.num_tiles:
logging.error("Please report this to the developers: RendertileSet num_tiles=%r, count=%r, children=%r", self.num_tiles, num, self.children)
logging.error("Please report this to the developers: RendertileSet num_tiles=%r, "
"count=%r, children=%r", self.num_tiles, num, self.children)
return num
def count_all(self):
@@ -1592,9 +1611,11 @@ class RendertileSet(object):
from itertools import imap
num = sum(imap(lambda _: 1, self.posttraversal()))
if num != self.num_tiles_all:
logging.error("Please report this to the developers: RendertileSet num_tiles_all=%r, count_all=%r, children=%r", self.num_tiles, num, self.children)
logging.error("Please report this to the developers: RendertileSet num_tiles_all=%r, "
"count_all=%r, children=%r", self.num_tiles, num, self.children)
return num
def distance_sort(children, (off_x, off_y)):
order = []
for child, (dx, dy) in izip(children, [(-1, -1), (1, -1), (-1, 1), (1, 1)]):
@@ -1604,6 +1625,7 @@ def distance_sort(children, (off_x, off_y)):
return sorted(order, key=lambda (_, (x, y)): x * x + y * y)
class RenderTile(object):
"""A simple container class that represents a single render-tile.
@@ -1612,6 +1634,7 @@ class RenderTile(object):
"""
__slots__ = ("col", "row", "path")
def __init__(self, col, row, path):
"""Initialize the tile obj with the given parameters. It's probably
better to use one of the other constructors though
@@ -1625,7 +1648,8 @@ class RenderTile(object):
return "%s(%r,%r,%r)" % (self.__class__.__name__, self.col, self.row, self.path)
def __eq__(self, other):
return self.col == other.col and self.row == other.row and tuple(self.path) == tuple(other.path)
return(self.col == other.col and self.row == other.row
and tuple(self.path) == tuple(other.path))
def __ne__(self, other):
return not self == other
@@ -1633,6 +1657,7 @@ class RenderTile(object):
# To support pickling
def __getstate__(self):
return self.col, self.row, self.path
def __setstate__(self, state):
self.__init__(*state)
@@ -1650,7 +1675,6 @@ class RenderTile(object):
imgpath = ".".join((path, imgformat))
return imgpath
@classmethod
def from_path(cls, path):
"""Constructor that takes a path and computes the col,row address of

6
setup.cfg Normal file
View File

@@ -0,0 +1,6 @@
[pycodestyle]
max_line_length = 100
ignore = E221,E222,E741,W503,W504
statistics = True
[isort]
line_length = 100