0

overhaul to configParser. Parsing config works now.

This commit is contained in:
Andrew Brown
2012-02-04 21:23:44 -05:00
parent ceb98c4441
commit 6d95d80a73
8 changed files with 321 additions and 245 deletions

View File

@@ -86,44 +86,6 @@ def configure_logger(loglevel=logging.INFO, verbose=False):
logger.addHandler(logger.overviewerHandler)
logger.setLevel(loglevel)
def build_fake_settings(worldpath):
"""Builds and returns a renders dict as if it was parsed from a settings
file and returned with get_render_things()
This is used for the simple command line usage with no config file
"""
from overviewer_core import settingsDefinition, rendermodes
world = {}
# Seed this render with all the defaults
for defaultname, defaultinfo in settingsDefinition.render['values'].iteritems():
if 'default' in defaultinfo:
world[defaultname] = defaultinfo['default']
# Set required items for the render. If any new required items without
# defaults are added, this will need to be updated.
worlds = {'world': worldpath}
world['worldname'] = 'world'
world['title'] = "Overviewer Render"
world['rendermode'] = rendermodes.normal
renders = {worldpath: world}
# The following is mostly a copy/paste of the code in
# MultiWorldParser.validate(). Someone make MultiWorldParser more
# extensible to avoid this!
origs = dict()
for key in world:
definition = settingsDefinition.render['values'][key]
val = definition['validator'](world[key], world = worlds)
if definition.get('save_orig', False):
origs[key + "_orig"] = world[key]
world[key] = val
world['name'] = worldpath
world.update(origs)
renders[worldpath] = world
return renders
def main():
# bootstrap the logger with defaults
configure_logger()
@@ -249,42 +211,56 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
return 1
#########################################################################
# These two blocks of code unify config-file mode and command-line mode.
# When the blocks have exited, they are expected to have set the following
# vars:
# destdir - the output directory
# renders - the dict heirarchy
# These two halfs of this if statement unify config-file mode and
# command-line mode.
mw_parser = configParser.MultiWorldParser()
if not options.config:
# No config file mode.
worldpath, destdir = map(os.path.expanduser, args)
logging.debug("Using %r as the world directory", worldpath)
logging.debug("Using %r as the output directory", destdir)
renders = build_fake_settings(worldpath)
mw_parser.set_config_item("world", {'world': worldpath})
mw_parser.set_config_item("outputdir", destdir)
# Now for some good defaults
mw_parser.set_config_item("render", {'world': {
'worldname': 'world',
'title': 'Overviewer Render',
'rendermode': 'normal',
}})
else:
# Parse the config file
mw_parser = configParser.MultiWorldParser(options.config)
mw_parser.parse()
try:
mw_parser.validate()
except Exception:
logging.exception("Please investigate these errors in settings.py then try running Overviewer again")
return 1
mw_parser = configParser.MultiWorldParser()
mw_parser.parse(options.config)
try:
destdir = mw_parser.outputdir
except AttributeError:
# Will get caught by the error check just below
logging.debug("Attribute error while getting the outputdir from the config file. Will error in just a sec")
destdir = ""
else:
logging.debug("outputdir from parser: %r", destdir)
try:
config = mw_parser.get_validated_config()
except Exception:
logging.exception("An error was encountered with your configuration. See the info below.")
return 1
renders = mw_parser.get_render_things()
############################################################
# Final validation and creation of the destination directory
# Final validation steps and creation of the destination directory
if not config['render']:
logging.error("You must specify at least one render in your config file. See the docs if you're having trouble")
return 1
for rname, render in config['render'].iteritems():
# Convert render['worldname'] to the world path, and store the original
# in render['worldname_orig']
try:
worldpath = config['world'][render['worldname']]
except KeyError:
logging.error("Render %s's world is '%s', but I could not find a corresponding entry in the worlds dictionary.",
rname, render['worldname'])
return 1
render['worldname_orig'] = render['worldname']
render['worldname'] = worldpath
destdir = config['outputdir']
if not destdir:
logging.error("You must specify the output directory in your config file.")
logging.error("e.g. outputdir = '/path/to/outputdir'")
@@ -313,6 +289,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
# same for textures
texcache = {}
renders = config['render']
for render_name, render in renders.iteritems():
logging.debug("Found the following render thing: %r", render)
@@ -348,6 +325,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
os.mkdir(tileset_dir)
# only pass to the TileSet the options it really cares about
render['name'] = render_name # perhaps a hack. This is stored here for the asset manager
tileSetOpts = util.dict_subset(render, ["name", "imgformat", "renderchecks", "rerenderprob", "bgcolor", "imgquality", "optimizeimg", "rendermode", "worldname_orig", "title", "dimension"])
tset = tileset.TileSet(rset, assetMrg, tex, tileSetOpts, tileset_dir)
tilesets.append(tset)

View File

@@ -4,103 +4,72 @@ import os.path
import logging
import settingsDefinition
import settingsValidators
class MultiWorldParser(object):
"""A class that is used to parse a settings.py file. It should replace
ConfigOptionParser class above."""
"""A class that is used to parse a settings.py file.
def __init__(self, settings):
"""Settings is a path to a settings.py file"""
if not os.path.exists(settings) and not os.path.isfile(settings):
This class's job is to compile and validate the configuration settings for
a set of renders. It can read in configuration from the given file with the
parse() method, and one can set configuration options directly with the
set_config_item() method.
get_validated_config() validates and returns the validated config
"""
def __init__(self):
"""Initialize this parser object"""
# This maps config names to their values
self._config_state = {}
# Scan the settings definition and build the config state heirarchy.
# Also go ahead and set default values for non-required settings.
# This maps setting names to their values as given in
# settingsDefinition.py
self._settings = {}
for settingname in settingsDefinition.__all__:
setting = getattr(settingsDefinition, settingname)
assert isinstance(setting, settingsValidators.Setting)
self._settings[settingname] = setting
if not setting.required:
self._config_state[settingname] = setting.default
def set_config_item(self, itemname, itemvalue):
self._config_state[itemname] = itemvalue
def parse(self, settings_file):
"""Reads in the named file and parses it, storing the results in an
internal state awating to be validated and returned upon call to
get_render_settings()
"""
if not os.path.exists(settings_file) and not os.path.isfile(settings_file):
raise ValueError("bad settings file")
self.settings_file = settings
def parse(self):
# settingsDefinition.py defines what types of things you can put in a configfile
# anything in settingsDefinitino that is a dict with a "type" key is a definiton
defDicts = [x for x in dir(settingsDefinition) if type(getattr(settingsDefinition,x)) == dict and getattr(settingsDefinition,x).has_key("type") and x != "__builtins__"]
glob = dict()
for name in defDicts:
d = getattr(settingsDefinition, name)
glob[name] = d['type']()
# The global environment should be the rendermode module, so the config
# file has access to those resources.
import rendermodes
loc=dict()
for thing in dir(rendermodes):
thething = getattr(rendermodes, thing)
if isinstance(thething, type) and issubclass(thething, rendermodes.RenderPrimitive):
loc[thing] = thething
try:
execfile(self.settings_file, glob, loc)
# delete the builtins, we don't need it
del glob['__builtins__']
execfile(settings_file, rendermodes.__dict__, self._config_state)
except NameError, ex:
import traceback
traceback.print_exc()
logging.error("Error parsing %s. Please check the trackback above" % self.settings_file)
logging.exception("Error parsing %s. Please check the trackback for more info" % settings_file)
sys.exit(1)
except SyntaxError, ex:
import traceback
traceback.print_exc()
tb = sys.exc_info()[2]
#print tb.tb_frame.f_code.co_filename
logging.error("Error parsing %s. Please check the trackback above" % self.settings_file)
logging.exception("Error parsing %s. Please check the trackback for more info" % self.settings_file)
sys.exit(1)
for name in defDicts:
setattr(self, name, glob[name])
del glob[name]
# seed with the Overviewer defaults, then update with the user defaults
self.defaults = dict()
for name in defDicts:
d = getattr(settingsDefinition, name)
if d['type'] == dict and d['valuetype'] == dict:
for key in d['values']:
option = d['values'][key]
if option.has_key("default"):
self.defaults[key] = option["default"]
self.defaults.update(glob)
def validate(self):
origs = dict()
for worldname in self.render:
world = dict()
world.update(self.defaults)
world.update(self.render[worldname])
for key in world:
if key not in settingsDefinition.render['values']:
logging.warning("%r is not a known setting", key)
continue
definition = settingsDefinition.render['values'][key]
try:
val = definition['validator'](world[key], world = self.world)
if definition.get('save_orig', False):
origs[key + "_orig"] = world[key]
world[key] = val
except Exception as e:
logging.error("Error validating '%s' option in render definition for '%s':", key, worldname)
logging.error(e)
raise e
world['name'] = worldname
world.update(origs)
self.render[worldname] = world
def get_render_things(self):
return self.render
def get_validated_config(self):
"""Validate and return the configuration"""
# Okay, this is okay, isn't it? We're going to create the validation
# routine right here, right now. I hope this works!
validator = settingsValidators.make_configdictvalidator(self._settings)
# Woah. What just happened? No. WAIT, WHAT ARE YOU...
validated_config = validator(self._config_state)
# WHAT HAVE YOU DONE?
return validated_config
# WHAT HAVE YOU DOOOOOOOOOOONE????

View File

@@ -1,54 +1,73 @@
# This file defines all of the things than can
# appear in a settings.py file, as well as the
# function that can validate them
# This file describes the format of the config file. Each item defined in this
# module is expected to appear in the same format in a settings file. The only
# difference is, instead of actual values for the settings, one is to use a
# Setting object. Here is its signature:
# the validator should raise an exception if there
# is a problem parsing or validating the value.
# it should return the value to use (which gives
# the validator an oppertunity to cleanup/normalize/
# whaterver)
# Setting(required, validator, default)
# if a setting is not required, the validator is
# expected to return the default option
# required is a boolean indicating the user is required to provide this
# setting. In this case, default is unused and can be set to anything (None is
# a good choice).
# validator is a callable that takes the provided value and returns a
# cleaned/normalized value to use. It should raise an exception if there is a
# problem parsing or validating the value given.
# default is used instead of the user-provided value in the event that required
# is false. It is passed into the validator just the same. If default is None
# and required is False, then that value is skipped entirely and will not
# appear in the resulting parsed options.
# The signature for validators is validator(value_given). Remember that the
# default is passed in as value_given in the event that required is False and
# default is not None.
# This file doesn't specify the format or even the type of the setting values,
# that is up to the validators used.
from settingsValidators import *
# note that all defaults go thought the validator
render = {
"type": dict,
"valuetype": dict,
"values": {
"worldname": dict(required=True, validator=validateWorldPath, save_orig=True),
"dimension": dict(required=False, validator=validateDimension, default="default"),
"title": dict(required=True, validator=validateStr),
"rendermode": dict(required=False, validator=validateRenderMode),
"northdirection": dict(required=False, validator=validateNorthDirection, default=0),
"renderrange": dict(required=False, validator=validateRenderRange),
"forcerender": dict(required=False, validator=validateBool),
"stochasticrender": dict(required=False, validator=validateStochastic),
"imgformat": dict(required=False, validator=validateImgFormat, default="png"),
"imgquality": dict(required=False, validator=validateImgQuality),
"bgcolor": dict(required=False, validator=validateBGColor, default="1a1a1a"),
"optimizeimg": dict(required=False, validator=validateOptImg, default=0),
"nomarkers": dict(required=False, validator=validateBool),
"texturepath": dict(required=False, validator=validateTexturePath),
"renderchecks": dict(required=False, validator=validateInt, default=0),
"rerenderprob": dict(required=False, validator=validateFloat, default=0),
}
# This is the export list for this module. It defines which items defined in
# this module are recognized by the config parser. Don't forget to update this
# if you add new items!
__all__ = ['render', 'world', 'outputdir']
# render is a dictionary mapping names to dicts describing the configuration
# for that render. It is therefore set to a settings object with a dict
# validator configured to validate keys as strings and values as... values are
# set to validate as a "configdict", which is a dict mapping a set of strings
# to some value. the make_configdictvalidator function creates a validator to
# use here configured with the given set of keys and Setting objects with their
# respective validators.
# Perhaps unintuitively, this is set to required=False. Of course, if no
# renders are specified, this is an error. However, this is caught later on in
# the code, and it also lets an empty dict get defined beforehand for the
# config file.
render = Setting(required=False, default={},
validator=dictValidator(validateStr, make_configdictvalidator(
{
"worldname": Setting(required=True, validator=validateStr, default=None),
"dimension": Setting(required=False, validator=validateDimension, default="default"),
"title": Setting(required=True, validator=validateStr, default=None),
"rendermode": Setting(required=False, validator=validateRenderMode, default=None),
"northdirection": Setting(required=False, validator=validateNorthDirection, default=0),
"renderrange": Setting(required=False, validator=validateRenderRange, default=None),
"forcerender": Setting(required=False, validator=validateBool, default=None),
"stochasticrender": Setting(required=False, validator=validateStochastic, default=None),
"imgformat": Setting(required=False, validator=validateImgFormat, default="png"),
"imgquality": Setting(required=False, validator=validateImgQuality, default=None),
"bgcolor": Setting(required=False, validator=validateBGColor, default="1a1a1a"),
"optimizeimg": Setting(required=False, validator=validateOptImg, default=0),
"nomarkers": Setting(required=False, validator=validateBool, default=None),
"texturepath": Setting(required=False, validator=validateTexturePath, default=None),
"renderchecks": Setting(required=False, validator=validateInt, default=0),
"rerenderprob": Setting(required=False, validator=validateFloat, default=0),
}
)))
world = {
"type": dict,
"valuetype": str,
"value": dict(validator=validateStr)
}
outputdir = {
"type": str,
"value": dict(validator=validateOutputDir)
}
#defines the values for each member of the world dict
#world = dict(require
# The world dict, mapping world names to world paths
world = Setting(required=False, validator=dictValidator(validateStr, validateWorldPath), default={})
outputdir = Setting(required=True, validator=validateOutputDir, default=None)

View File

@@ -1,6 +1,7 @@
# see settingsDefinition.py
import os
import os.path
from collections import namedtuple
import rendermodes
from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT
@@ -8,20 +9,34 @@ from world import UPPER_LEFT, UPPER_RIGHT, LOWER_LEFT, LOWER_RIGHT
class ValidationException(Exception):
pass
def validateWorldPath(name, **kwargs):
world = kwargs.get('world', dict())
if name not in world.keys():
raise ValidationException("bad world name")
abs_path = os.path.abspath(world[name])
Setting = namedtuple("Setting", [
'required',
'validator',
'default',
])
def validateWorldPath(worldpath):
abs_path = os.path.abspath(worldpath)
if not os.path.exists(os.path.join(abs_path, "level.dat")):
raise ValidationException("No level.dat file in %r. Check the path for world %r" % (abs_path, name))
raise ValidationException("No level.dat file in %r. Are you sure you have the right path?" % (abs_path,))
return abs_path
def validateRenderMode(mode, **kwargs):
def validateRenderMode(mode):
# make sure that mode is a list of things that are all rendermode primative
if type(mode) != list:
if isinstance(mode, str):
# Try and find an item named "mode" in the rendermodes module
try:
mode = getattr(rendermodes, mode)
except AttributeError:
raise ValidationException("You must specify a valid rendermode, not '%s'. See the docs for valid rendermodes." % mode)
if isinstance(mode, rendermodes.RenderPrimitive):
mode = [mode]
if not isinstance(mode, list):
raise ValidationException("%r is not a valid list of rendermodes. It should be a list"% mode)
for m in mode:
if not isinstance(m, rendermodes.RenderPrimitive):
raise ValidationException("%r is not a valid rendermode primitive." % m)
@@ -29,7 +44,7 @@ def validateRenderMode(mode, **kwargs):
return mode
def validateNorthDirection(direction, **kwargs):
def validateNorthDirection(direction):
# normalize to integers
intdir = 0 #default
if type(direction) == int:
@@ -43,28 +58,28 @@ def validateNorthDirection(direction, **kwargs):
raise ValidationException("%r is not a valid north direction" % direction)
return intdir
def validateRenderRange(r, **kwargs):
def validateRenderRange(r):
raise NotImplementedError("render range")
def validateStochastic(s, **kwargs):
def validateStochastic(s):
val = float(s)
if val < 0 or val > 1:
raise ValidationException("%r is not a valid stochastic value. Should be between 0.0 and 1.0" % s)
return val
def validateImgFormat(fmt, **kwargs):
def validateImgFormat(fmt):
if fmt not in ("png", "jpg", "jpeg"):
raise ValidationException("%r is not a valid image format" % fmt)
if fmt == "jpeg": fmt = "jpg"
return fmt
def validateImgQuality(qual, **kwargs):
def validateImgQuality(qual):
intqual = int(qual)
if (intqual < 0 or intqual > 100):
raise ValidationException("%r is not a valid image quality" % intqual)
return intqual
def validateBGColor(color, **kwargs):
def validateBGColor(color):
"""BG color must be an HTML color, with an option leading # (hash symbol)
returns an (r,b,g) 3-tuple
"""
@@ -86,32 +101,80 @@ def validateBGColor(color, **kwargs):
return color
def validateOptImg(opt, **kwargs):
def validateOptImg(opt):
return bool(opt)
def validateTexturePath(path, **kwargs):
def validateTexturePath(path):
# Expand user dir in directories strings
path = os.path.expanduser(path)
# TODO assert this path exists?
return path
def validateBool(b, **kwargs):
def validateBool(b):
return bool(b)
def validateFloat(f, **kwargs):
def validateFloat(f):
return float(f)
def validateInt(i, **kwargs):
def validateInt(i):
return int(i)
def validateStr(s, **kwargs):
def validateStr(s):
return str(s)
def validateDimension(d, **kwargs):
def validateDimension(d):
if d in ["nether", "overworld", "end", "default"]:
return d
raise ValidationException("%r is not a valid dimension" % d)
def validateOutputDir(d, **kwargs):
return os.path.abs(d)
def validateOutputDir(d):
if not d.strip():
raise ValidationException("You must specify a valid output directory")
return os.path.abspath(d)
def dictValidator(keyvalidator, valuevalidator):
"""Compose a dict validator by validating each key/value combination with
the given key validator and value validator
"""
def v(d):
newd = {}
for key, value in d.iteritems():
newd[keyvalidator(key)] = valuevalidator(value)
return newd
return v
def make_configdictvalidator(config):
"""Okay, stay with me here, this may get confusing. This function returns a
validator that validates a "configdict". This is a term I just made up to
refer to a dict where keys are strings and values are something. The
argument to /this/ function is a dictionary mapping those key names to
Setting objects. When the validator validates, it calls all the appropriate
validators to validate each item in the configdict.
I hope that makes sense.
"""
def configDictValidator(d):
newdict = {}
for configkey, configsetting in config.iteritems():
if configkey in d:
# This key /was/ specified in the user's dict. Make sure it validates.
newdict[configkey] = configsetting.validator(d[configkey])
else:
# The user did not give us this key. If it's required, send up
# an error. Otherwise, just return the default.
if configsetting.required:
raise ValidationException("Required key '%s' was not specified" % configkey)
elif configsetting.default is not None:
newdict[configkey] = configsetting.validator(configsetting.default)
# Now that all the defined keys have been accounted for, check to make
# sure any unauthorized keys were not specified.
for key in d.iterkeys():
if key not in config:
raise ValidationException("'%s' is not a configuration item" % key)
return newdict
return configDictValidator

View File

@@ -1,14 +1,17 @@
world['test'] = "test/data/settings/test_world"
worldname = 'test'
rendermode = "normal"
render["world"] = {
"rendermode": "normal",
"northdirection": ["upper-left"],
render["myworld"] = {
"title": "myworld title",
"worldname": "test",
"rendermode": normal,
"northdirection": "upper-left",
}
render["otherworld"] = {
"rendermode": "foo",
"title": "otherworld title",
"worldname": "test",
"rendermode": normal,
"bgcolor": "#ffffff"
}
outputdir = "/tmp/fictional/outputdir"

View File

@@ -1,8 +0,0 @@
bgcolor="#000000"
world['test'] = "test/data/settings/test_world"
render["world"] = {
"worldname": "test",
"bgcolor":"ffff"
}

View File

@@ -2,6 +2,9 @@ world['test'] = "test/data/settings/test_world"
render["world"] = {
"worldname": "test",
"title": "myworld title",
"rendermode": "bad_rendermode",
"northdirection": ["upper-left"],
"northdirection": "upper-left",
}
outputdir = "/tmp/fictional/outputdir"

View File

@@ -3,32 +3,81 @@ import unittest
from overviewer_core import configParser
from overviewer_core.settingsValidators import ValidationException
from overviewer_core import world
from overviewer_core import rendermodes
class SettingsTest(unittest.TestCase):
def setUp(self):
self.s = configParser.MultiWorldParser()
def test_missing(self):
"Validates that a non-existant settings.py causes an exception"
self.assertRaises(ValueError, configParser.MultiWorldParser, "doesnotexist.py")
self.assertRaises(ValueError, self.s.parse, "doesnotexist.py")
def test_existing_file(self):
s = configParser.MultiWorldParser("test/data/settings/settings_test_1.py")
s.parse()
s.validate()
things = s.get_render_things()
# no exceptions so far. that's good
self.assertEquals(things['world']['bgcolor'], (26,26,26,0))
self.assertEquals(things['otherworld']['bgcolor'], (255,255,255,0))
self.s.parse("test/data/settings/settings_test_1.py")
things = self.s.get_validated_config()
# no exceptions so far. that's a good thing
# Test the default
self.assertEquals(things['render']['myworld']['bgcolor'], (26,26,26,0))
# Test a non-default
self.assertEquals(things['render']['otherworld']['bgcolor'], (255,255,255,0))
self.assertEquals(things['render']['myworld']['northdirection'],
world.UPPER_LEFT)
def test_rendermode_validation(self):
s = configParser.MultiWorldParser("test/data/settings/settings_test_rendermode.py")
s.parse()
self.s.parse("test/data/settings/settings_test_rendermode.py")
self.assertRaises(ValidationException,s.validate)
self.assertRaises(ValidationException,self.s.get_validated_config)
def test_bgcolor_validation(self):
s = configParser.MultiWorldParser("test/data/settings/settings_test_bgcolor.py")
s.parse()
def test_manual(self):
"""Tests that manually setting the config parser works, you don't have
to do it from a file
self.assertRaises(ValidationException, s.validate)
"""
fromfile = configParser.MultiWorldParser()
fromfile.parse("test/data/settings/settings_test_1.py")
self.s.set_config_item("world", {
'test': "test/data/settings/test_world",
})
self.s.set_config_item("render", {
"myworld": {
"title": "myworld title",
"worldname": "test",
"rendermode": rendermodes.normal,
"northdirection": "upper-left",
},
"otherworld": {
"title": "otherworld title",
"worldname": "test",
"rendermode": rendermodes.normal,
"bgcolor": "#ffffff"
},
})
self.s.set_config_item("outputdir", "/tmp/fictional/outputdir")
self.assertEquals(fromfile.get_validated_config(), self.s.get_validated_config())
def test_rendermode_string(self):
self.s.set_config_item("world", {
'test': "test/data/settings/test_world",
})
self.s.set_config_item("outputdir", "/tmp/fictional/outputdir")
self.s.set_config_item("render", {
"myworld": {
"title": "myworld title",
"worldname": "test",
"rendermode": "normal",
"northdirection": "upper-left",
},
})
p = self.s.get_validated_config()
self.assertEquals(p['render']['myworld']['rendermode'], rendermodes.normal)
if __name__ == "__main__":
unittest.main()