From f3d23fb8f8cbb05117e68bbf75d7113b68b74353 Mon Sep 17 00:00:00 2001 From: aheadley Date: Sat, 17 Mar 2012 14:34:57 -0400 Subject: [PATCH] clean up observer code --- overviewer.py | 14 ++-- overviewer_core/dispatcher.py | 146 ++------------------------------ overviewer_core/observer.py | 147 +++++++++++++++++++++++++++++++++ overviewer_core/progressbar.py | 7 +- overviewer_core/tileset.py | 6 +- 5 files changed, 170 insertions(+), 150 deletions(-) create mode 100644 overviewer_core/observer.py diff --git a/overviewer.py b/overviewer.py index 798e597..c87d3ef 100755 --- a/overviewer.py +++ b/overviewer.py @@ -40,7 +40,7 @@ from overviewer_core import textures from overviewer_core import optimizeimages, world from overviewer_core import configParser, tileset, assetmanager, dispatcher from overviewer_core import cache -from overviewer_core import progressbar +from overviewer_core import observer helptext = """ %prog [--rendermodes=...] [options] @@ -393,11 +393,15 @@ dir but you forgot to put quotes around the directory, since it contains spaces. # multiprocessing dispatcher if config['processes'] == 1: - dispatch = dispatcher.Dispatcher(observer=dispatcher.ProgressBarObserver) + dispatch = dispatcher.Dispatcher() else: - dispatch = dispatcher.MultiprocessingDispatcher(local_procs=config['processes'], - observer=dispatcher.ProgressBarObserver) - dispatch.render_all(tilesets) + dispatch = dispatcher.MultiprocessingDispatcher( + local_procs=config['processes']) + if platform.system() == 'Windows' or not sys.stderr.isatty(): + obs = observer.LoggingObserver() + else: + obs = observer.ProgressBarObserver() + dispatch.render_all(tilesets, obs) dispatch.close() assetMrg.finalize(tilesets) diff --git a/overviewer_core/dispatcher.py b/overviewer_core/dispatcher.py index 5d35835..cfb3faa 100644 --- a/overviewer_core/dispatcher.py +++ b/overviewer_core/dispatcher.py @@ -19,10 +19,6 @@ import multiprocessing.managers import cPickle as pickle import Queue import time -import logging -import progressbar -import sys - from signals import Signal class Dispatcher(object): @@ -32,7 +28,7 @@ class Dispatcher(object): possible to create a Dispatcher that distributes this work to many worker processes. """ - def __init__(self, observer=None): + def __init__(self): super(Dispatcher, self).__init__() # list of (tileset, workitem) tuples @@ -42,9 +38,7 @@ class Dispatcher(object): # keeps track of jobs waiting to run after dependencies finish self._pending_jobs = [] - self.observer_type = observer or LoggingObserver - - def render_all(self, tilesetlist): + def render_all(self, tilesetlist, observer): """Render all of the tilesets in the given tilesetlist. status_callback is called periodically to update status. The callback should take the following arguments: @@ -78,54 +72,18 @@ class Dispatcher(object): break else: total_jobs += jobs_for_tileset - finished_jobs = 0 - - self.observer = self.observer_type(total_jobs) - - # do the first status update - #self._status_update(status_callback, phase, finished_jobs, total_jobs, force=True) - self.observer.start() - self._status_update(phase, finished_jobs, True) + observer.start(total_jobs) # go through these iterators round-robin style for tileset, (workitem, deps) in util.roundrobin(work_iterators): self._pending_jobs.append((tileset, workitem, deps)) - finished_jobs += self._dispatch_jobs() - self._status_update(phase, finished_jobs) + observer.add(self._dispatch_jobs()) # after each phase, wait for the work to finish while len(self._pending_jobs) > 0 or len(self._running_jobs) > 0: - finished_jobs += self._dispatch_jobs() - self._status_update(phase, finished_jobs) + observer.add(self._dispatch_jobs()) - self.observer.finish() - - def _status_update(self, phase, completed, force=False): - if force or completed - self.observer.get_current_value() > \ - self.observer.MIN_UPDATE_INTERVAL: - self.observer.update(completed) - - """ - def _status_update(self, callback, phase, completed, total, force=False): - # always called with force=True at the beginning, so that can - # be used to set up state. After that, it is called after - # every _dispatch_jobs() often; this function is used to - # decide how often the actual status callback should be - # called. - if force: - self._last_status_update = completed - if callback: - callback(phase, completed, total) - return - - if callback is None: - return - - update_interval = 100 # XXX arbitrary - if self._last_status_update < 0 or completed >= self._last_status_update + update_interval or completed < self._last_status_update: - self._last_status_update = completed - callback(phase, completed, total) - """ + observer.finish() def _dispatch_jobs(self): # helper function to dispatch pending jobs when their @@ -309,12 +267,12 @@ class MultiprocessingDispatcher(Dispatcher): """A subclass of Dispatcher that spawns worker processes and distributes jobs to them to speed up processing. """ - def __init__(self, local_procs=-1, address=None, authkey=None, observer=None): + def __init__(self, local_procs=-1, address=None, authkey=None): """Creates the dispatcher. local_procs should be the number of worker processes to spawn. If it's omitted (or negative) the number of available CPUs is used instead. """ - super(MultiprocessingDispatcher, self).__init__(observer=observer) + super(MultiprocessingDispatcher, self).__init__() # automatic local_procs handling if local_procs < 0: @@ -422,91 +380,3 @@ class MultiprocessingDispatcher(Dispatcher): m.connect() p = MultiprocessingDispatcherProcess(m) p.run() - -class Observer(object): - """ - """ - - MIN_UPDATE_INTERVAL = 100 - - def __init__(self, max_value): - self._current_value = None - self._max_value = max_value - self.start_time = None - self.end_time = None - - def start(self): - self.start_time = time.time() - self.update(0) - return self - - def is_started(self): - return self.start_time is not None - - def finish(self): - self.end_time = time.time() - - def is_finished(self): - return self.end_time is not None - - def is_running(self): - return self.is_started() and not self.is_finished() - - def add(self, amount): - self.update(self.get_current_value() + amount) - - def update(self, current_value): - """ - """ - self._current_value = current_value - - def get_percentage(self): - if self.get_max_value() is 0: - return 100.0 - else: - return self.get_current_value() * 100.0 / self.get_max_value() - - def get_current_value(self): - return self._current_value - - def get_max_value(self): - return self._max_value - -class LoggingObserver(Observer): - """ - """ - def update(self, current_value): - super(LoggingObserver, self).update(current_value) - if self.get_max_value() is None: - logging.info("Rendered %d tiles.", self.get_current_value()) - else: - logging.info("Rendered %d of %d. %d%% complete", - self.get_current_value(), self.get_max_value(), - self.get_percentage()) - -default_widgets = ['Rendering: ', progressbar.FractionWidget(), ' (', - progressbar.Percentage(), ') ', progressbar.Bar(left='[', right=']'), - ' ', progressbar.ETA()] -class ProgressBarObserver(progressbar.ProgressBar): - """ - """ - - MIN_UPDATE_INTERVAL = 100 - - def __init__(self, max_value, widgets=default_widgets, term_width=None, - fd=sys.stderr): - super(ProgressBarObserver, self).__init__(maxval=max_value, - widgets=widgets, term_width=term_width, fd=fd) - - def is_started(self): - return self.start_time is not None - - def finish(self): - self._end_time = time.time() - super(ProgressBarObserver, self).finish() - - def get_current_value(self): - return self.currval - - def get_max_value(self): - return self.maxval diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py new file mode 100644 index 0000000..2034323 --- /dev/null +++ b/overviewer_core/observer.py @@ -0,0 +1,147 @@ +# This file is part of the Minecraft Overviewer. +# +# Minecraft Overviewer is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# Minecraft Overviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the Overviewer. If not, see . + +import time +import logging +import progressbar +import sys + +class Observer(object): + """Base class that defines the observer interface. + """ + + def __init__(self): + self._current_value = None + self._max_value = None + self.start_time = None + self.end_time = None + + def start(self, max_value): + self._set_max_value(max_value) + self.start_time = time.time() + self.update(0) + return self + + def is_started(self): + return self.start_time is not None + + def finish(self): + self.end_time = time.time() + + def is_finished(self): + return self.end_time is not None + + def is_running(self): + return self.is_started() and not self.is_finished() + + def add(self, amount): + if amount: + self.update(self.get_current_value() + amount) + + def update(self, current_value): + """ + """ + self._current_value = current_value + return False + + def get_percentage(self): + if self.get_max_value() is 0: + return 100.0 + else: + return self.get_current_value() * 100.0 / self.get_max_value() + + def get_current_value(self): + return self._current_value + + def get_max_value(self): + return self._max_value + + def _set_max_value(self, max_value): + self._max_value = max_value + +class LoggingObserver(Observer): + """Simple observer that just outputs status through logging. + """ + def __init__(self): + super(Observer, self).__init__() + #this is an easy way to make the first update() call print a line + self.last_update = -101 + + def finish(self): + logging.info("Rendered %d of %d. %d%% complete", self.get_max_value(), + self.get_max_value(), 100.0) + super(LoggingObserver, self).finish() + + def update(self, current_value): + super(LoggingObserver, self).update(current_value) + if self._need_update(): + logging.info("Rendered %d of %d. %d%% complete", + self.get_current_value(), self.get_max_value(), + self.get_percentage()) + self.last_update = current_value + return True + return False + + def _need_update(self): + cur_val = self.get_current_value() + if cur_val < 100: + return cur_val - self.last_update > 10 + elif cur_val < 500: + return cur_val - self.last_update > 50 + else: + return cur_val - self.last_update > 100 + +default_widgets = ['Rendering: ', progressbar.FractionWidget(), ' (', + progressbar.Percentage(), ') ', progressbar.Bar(left='[', right=']'), + ' ', progressbar.ETA()] +class ProgressBarObserver(progressbar.ProgressBar, Observer): + """Display progress through a progressbar. + """ + + UPDATE_INTERVAL = 25 + + def __init__(self, widgets=default_widgets, term_width=None, fd=sys.stderr): + super(ProgressBarObserver, self).__init__(widgets=widgets, + term_width=term_width, fd=fd) + self.last_update = -101 + + def start(self, max_value): + self._set_max_value(max_value) + super(ProgressBarObserver, self).start() + + def is_started(self): + return self.start_time is not None + + def finish(self): + self._end_time = time.time() + super(ProgressBarObserver, self).finish() + + def update(self, current_value): + if super(ProgressBarObserver, self).update(current_value): + self.last_update = self.get_current_value() + + percentage = Observer.get_percentage + + def get_current_value(self): + return self.currval + + def get_max_value(self): + return self.maxval + + def _set_max_value(self, max_value): + self.maxval = max_value + + def _need_update(self): + return self.get_current_value() - self.last_update > self.UPDATE_INTERVAL diff --git a/overviewer_core/progressbar.py b/overviewer_core/progressbar.py index c4fc722..6acfe89 100644 --- a/overviewer_core/progressbar.py +++ b/overviewer_core/progressbar.py @@ -233,7 +233,6 @@ class ProgressBar(object): self.currval = 0 self.finished = False - self.prev_percentage = -1 self.start_time = None self.seconds_elapsed = 0 @@ -270,23 +269,23 @@ class ProgressBar(object): return ''.join(self._format_widgets()).ljust(self.term_width) def _need_update(self): - return int(self.percentage()) != int(self.prev_percentage) + return True def update(self, value): "Updates the progress bar to a new value." assert 0 <= value <= self.maxval self.currval = value if not self._need_update() or self.finished: - return + return False if not self.start_time: self.start_time = time.time() self.seconds_elapsed = time.time() - self.start_time - self.prev_percentage = self.percentage() if value != self.maxval: self.fd.write(self._format_line() + '\r') else: self.finished = True self.fd.write(self._format_line() + '\n') + return True def start(self): """Start measuring time, and prints the bar at 0%. diff --git a/overviewer_core/tileset.py b/overviewer_core/tileset.py index a22358e..6ee7cf7 100644 --- a/overviewer_core/tileset.py +++ b/overviewer_core/tileset.py @@ -318,9 +318,9 @@ class TileSet(object): "to date.", self.options['name'], ) - logging.warning("You won't get percentage progress for "+ - "this run only, because I don't know how many tiles "+ - "need rendering. I'll be checking them as I go") + logging.warning("The total tile count will be (possibly "+ + "wildly) inaccurate, because I don't know how many "+ + "tiles need rendering. I'll be checking them as I go") self.options['renderchecks'] = 1 else: logging.debug("No rendercheck mode specified for %s. "+