From 9319fee138f424fc8407ddb7e5369a94d0b9ddb5 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Wed, 8 Feb 2012 21:05:02 -0500 Subject: [PATCH] added processes option to command line and config file Also the config file parser now just looks for all Settings objects in settingsDefinitions.py, not the items in __all__ --- docs/config.rst | 9 +++++++++ docs/running.rst | 8 ++++++++ overviewer.py | 10 +++++++++- overviewer_core/configParser.py | 5 +++-- overviewer_core/dispatcher.py | 1 + overviewer_core/settingsDefinition.py | 6 +----- 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 4b66a86..bd6a670 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -233,6 +233,15 @@ These values are set directly in the config file. Example:: This is a string indicating the path to the texture pack to use for rendering. +.. _processes: + +``processes = num_procs`` + This specifies the number of worker processes to spawn on the local machine + to do work. It defaults to the number of CPU cores you have, if not + specified. + + This can also be specified with :option:`--processes <-p>` + TODO: More to come here .. _customrendermodes: diff --git a/docs/running.rst b/docs/running.rst index a65db98..a09ece2 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -146,6 +146,14 @@ only have to use once-in-a-while. or :option:`--check-tiles` is in effect. This option overrides :option:`--forcerender` and :option:`--check-tiles`. +.. cmdoption:: -p , --processes + + This specifies the number of worker processes to spawn on the local machine + to do work. It defaults to the number of CPU cores you have, if not + specified. + + This option can also be specified in the config file as :ref:`processes ` + .. _installing-textures: Installing the Textures diff --git a/overviewer.py b/overviewer.py index 724f3dd..915723a 100755 --- a/overviewer.py +++ b/overviewer.py @@ -101,6 +101,8 @@ def main(): # Parse for basic options parser = OptionParser(usage=helptext) parser.add_option("--config", dest="config", action="store", help="Specify the config file to use.") + parser.add_option("-p", "--processes", dest="procs", action="store", type="int", + help="The number of local worker processes to spawn. Defaults to the number of CPU cores your computer has") # Useful one-time render modifiers: parser.add_option("--forcerender", dest="forcerender", action="store_true", @@ -235,6 +237,12 @@ dir but you forgot to put quotes around the directory, since it contains spaces. mw_parser = configParser.MultiWorldParser() mw_parser.parse(options.config) + # Add in the command options here, perhaps overriding values specified in + # the config + if options.procs: + mw_parser.set_config_item("processes", options.procs) + + # Now parse and return the validated config try: config = mw_parser.get_validated_config() except Exception: @@ -332,7 +340,7 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # multiprocessing dispatcher - dispatch = dispatcher.MultiprocessingDispatcher() + dispatch = dispatcher.MultiprocessingDispatcher(local_procs=config['processes']) def print_status(*args): logging.info("Status callback: %r", args) dispatch.render_all(tilesets, print_status) diff --git a/overviewer_core/configParser.py b/overviewer_core/configParser.py index e804d54..ee47d01 100644 --- a/overviewer_core/configParser.py +++ b/overviewer_core/configParser.py @@ -27,9 +27,10 @@ class MultiWorldParser(object): # This maps setting names to their values as given in # settingsDefinition.py self._settings = {} - for settingname in settingsDefinition.__all__: + for settingname in dir(settingsDefinition): setting = getattr(settingsDefinition, settingname) - assert isinstance(setting, settingsValidators.Setting) + if not isinstance(setting, settingsValidators.Setting): + continue self._settings[settingname] = setting diff --git a/overviewer_core/dispatcher.py b/overviewer_core/dispatcher.py index 8f9b7f8..655fd04 100644 --- a/overviewer_core/dispatcher.py +++ b/overviewer_core/dispatcher.py @@ -19,6 +19,7 @@ import multiprocessing.managers import cPickle as pickle import Queue import time +import logging from signals import Signal diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index 1bbe2c3..c34073d 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -45,11 +45,6 @@ from settingsValidators import * -# This is the export list for this module. It defines which items defined in -# this module are recognized by the config parser. Don't forget to update this -# if you add new items! -__all__ = ['render', 'world', 'outputdir'] - # render is a dictionary mapping strings to dicts. These dicts describe the # configuration for that render. Therefore, the validator for 'render' is set # to a dict validator configured to validate keys as strings and values as... @@ -87,3 +82,4 @@ world = Setting(required=True, validator=make_dictValidator(validateStr, validat outputdir = Setting(required=True, validator=validateOutputDir, default=None) +processes = Setting(required=True, validator=int, default=-1)