LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

clean up observer code

Browse Source
This commit is contained in:
aheadley
2012-03-17 14:34:57 -04:00
parent 46b04dd09a
commit f3d23fb8f8
5 changed files with 170 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
overviewer.py
View File

@@ -40,7 +40,7 @@ from overviewer_core import textures
from overviewer_core import optimizeimages, world
from overviewer_core import configParser, tileset, assetmanager, dispatcher
from overviewer_core import cache
from overviewer_core import progressbar
from overviewer_core import observer
helptext = """
%prog [--rendermodes=...] [options] <World> <Output Dir>
@@ -393,11 +393,15 @@ dir but you forgot to put quotes around the directory, since it contains spaces.
# multiprocessing dispatcher
if config['processes'] == 1:
dispatch = dispatcher.Dispatcher(observer=dispatcher.ProgressBarObserver)
dispatch = dispatcher.Dispatcher()
else:
dispatch = dispatcher.MultiprocessingDispatcher(local_procs=config['processes'],
observer=dispatcher.ProgressBarObserver)
dispatch.render_all(tilesets)
dispatch = dispatcher.MultiprocessingDispatcher(
local_procs=config['processes'])
if platform.system() == 'Windows' or not sys.stderr.isatty():
obs = observer.LoggingObserver()
else:
obs = observer.ProgressBarObserver()
dispatch.render_all(tilesets, obs)
dispatch.close()
assetMrg.finalize(tilesets)

146
overviewer_core/dispatcher.py
View File

@@ -19,10 +19,6 @@ import multiprocessing.managers
import cPickle as pickle
import Queue
import time
import logging
import progressbar
import sys
from signals import Signal
class Dispatcher(object):
@@ -32,7 +28,7 @@ class Dispatcher(object):
possible to create a Dispatcher that distributes this work to many
worker processes.
"""
def __init__(self, observer=None):
def __init__(self):
super(Dispatcher, self).__init__()
# list of (tileset, workitem) tuples
@@ -42,9 +38,7 @@ class Dispatcher(object):
# keeps track of jobs waiting to run after dependencies finish
self._pending_jobs = []
self.observer_type = observer or LoggingObserver
def render_all(self, tilesetlist):
def render_all(self, tilesetlist, observer):
"""Render all of the tilesets in the given
tilesetlist. status_callback is called periodically to update
status. The callback should take the following arguments:
@@ -78,54 +72,18 @@ class Dispatcher(object):
break
else:
total_jobs += jobs_for_tileset
finished_jobs = 0
self.observer = self.observer_type(total_jobs)
# do the first status update
#self._status_update(status_callback, phase, finished_jobs, total_jobs, force=True)
self.observer.start()
self._status_update(phase, finished_jobs, True)
observer.start(total_jobs)
# go through these iterators round-robin style
for tileset, (workitem, deps) in util.roundrobin(work_iterators):
self._pending_jobs.append((tileset, workitem, deps))
finished_jobs += self._dispatch_jobs()
self._status_update(phase, finished_jobs)
observer.add(self._dispatch_jobs())
# after each phase, wait for the work to finish
while len(self._pending_jobs) > 0 or len(self._running_jobs) > 0:
finished_jobs += self._dispatch_jobs()
self._status_update(phase, finished_jobs)
observer.add(self._dispatch_jobs())
self.observer.finish()
def _status_update(self, phase, completed, force=False):
if force or completed - self.observer.get_current_value() > \
self.observer.MIN_UPDATE_INTERVAL:
self.observer.update(completed)
"""
def _status_update(self, callback, phase, completed, total, force=False):
# always called with force=True at the beginning, so that can
# be used to set up state. After that, it is called after
# every _dispatch_jobs() often; this function is used to
# decide how often the actual status callback should be
# called.
if force:
self._last_status_update = completed
if callback:
callback(phase, completed, total)
return
if callback is None:
return
update_interval = 100 # XXX arbitrary
if self._last_status_update < 0 or completed >= self._last_status_update + update_interval or completed < self._last_status_update:
self._last_status_update = completed
callback(phase, completed, total)
"""
observer.finish()
def _dispatch_jobs(self):
# helper function to dispatch pending jobs when their
@@ -309,12 +267,12 @@ class MultiprocessingDispatcher(Dispatcher):
"""A subclass of Dispatcher that spawns worker processes and
distributes jobs to them to speed up processing.
"""
def __init__(self, local_procs=-1, address=None, authkey=None, observer=None):
def __init__(self, local_procs=-1, address=None, authkey=None):
"""Creates the dispatcher. local_procs should be the number of
worker processes to spawn. If it's omitted (or negative)
the number of available CPUs is used instead.
"""
super(MultiprocessingDispatcher, self).__init__(observer=observer)
super(MultiprocessingDispatcher, self).__init__()
# automatic local_procs handling
if local_procs < 0:
@@ -422,91 +380,3 @@ class MultiprocessingDispatcher(Dispatcher):
m.connect()
p = MultiprocessingDispatcherProcess(m)
p.run()
class Observer(object):
"""
"""
MIN_UPDATE_INTERVAL = 100
def __init__(self, max_value):
self._current_value = None
self._max_value = max_value
self.start_time = None
self.end_time = None
def start(self):
self.start_time = time.time()
self.update(0)
return self
def is_started(self):
return self.start_time is not None
def finish(self):
self.end_time = time.time()
def is_finished(self):
return self.end_time is not None
def is_running(self):
return self.is_started() and not self.is_finished()
def add(self, amount):
self.update(self.get_current_value() + amount)
def update(self, current_value):
"""
"""
self._current_value = current_value
def get_percentage(self):
if self.get_max_value() is 0:
return 100.0
else:
return self.get_current_value() * 100.0 / self.get_max_value()
def get_current_value(self):
return self._current_value
def get_max_value(self):
return self._max_value
class LoggingObserver(Observer):
"""
"""
def update(self, current_value):
super(LoggingObserver, self).update(current_value)
if self.get_max_value() is None:
logging.info("Rendered %d tiles.", self.get_current_value())
else:
logging.info("Rendered %d of %d. %d%% complete",
self.get_current_value(), self.get_max_value(),
self.get_percentage())
default_widgets = ['Rendering: ', progressbar.FractionWidget(), ' (',
progressbar.Percentage(), ') ', progressbar.Bar(left='[', right=']'),
' ', progressbar.ETA()]
class ProgressBarObserver(progressbar.ProgressBar):
"""
"""
MIN_UPDATE_INTERVAL = 100
def __init__(self, max_value, widgets=default_widgets, term_width=None,
fd=sys.stderr):
super(ProgressBarObserver, self).__init__(maxval=max_value,
widgets=widgets, term_width=term_width, fd=fd)
def is_started(self):
return self.start_time is not None
def finish(self):
self._end_time = time.time()
super(ProgressBarObserver, self).finish()
def get_current_value(self):
return self.currval
def get_max_value(self):
return self.maxval

147
overviewer_core/observer.py Normal file
View File

@@ -0,0 +1,147 @@
# This file is part of the Minecraft Overviewer.
#
# Minecraft Overviewer is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# Minecraft Overviewer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the Overviewer. If not, see <http://www.gnu.org/licenses/>.
import time
import logging
import progressbar
import sys
class Observer(object):
"""Base class that defines the observer interface.
"""
def __init__(self):
self._current_value = None
self._max_value = None
self.start_time = None
self.end_time = None
def start(self, max_value):
self._set_max_value(max_value)
self.start_time = time.time()
self.update(0)
return self
def is_started(self):
return self.start_time is not None
def finish(self):
self.end_time = time.time()
def is_finished(self):
return self.end_time is not None
def is_running(self):
return self.is_started() and not self.is_finished()
def add(self, amount):
if amount:
self.update(self.get_current_value() + amount)
def update(self, current_value):
"""
"""
self._current_value = current_value
return False
def get_percentage(self):
if self.get_max_value() is 0:
return 100.0
else:
return self.get_current_value() * 100.0 / self.get_max_value()
def get_current_value(self):
return self._current_value
def get_max_value(self):
return self._max_value
def _set_max_value(self, max_value):
self._max_value = max_value
class LoggingObserver(Observer):
"""Simple observer that just outputs status through logging.
"""
def __init__(self):
super(Observer, self).__init__()
#this is an easy way to make the first update() call print a line
self.last_update = -101
def finish(self):
logging.info("Rendered %d of %d. %d%% complete", self.get_max_value(),
self.get_max_value(), 100.0)
super(LoggingObserver, self).finish()
def update(self, current_value):
super(LoggingObserver, self).update(current_value)
if self._need_update():
logging.info("Rendered %d of %d. %d%% complete",
self.get_current_value(), self.get_max_value(),
self.get_percentage())
self.last_update = current_value
return True
return False
def _need_update(self):
cur_val = self.get_current_value()
if cur_val < 100:
return cur_val - self.last_update > 10
elif cur_val < 500:
return cur_val - self.last_update > 50
else:
return cur_val - self.last_update > 100
default_widgets = ['Rendering: ', progressbar.FractionWidget(), ' (',
progressbar.Percentage(), ') ', progressbar.Bar(left='[', right=']'),
' ', progressbar.ETA()]
class ProgressBarObserver(progressbar.ProgressBar, Observer):
"""Display progress through a progressbar.
"""
UPDATE_INTERVAL = 25
def __init__(self, widgets=default_widgets, term_width=None, fd=sys.stderr):
super(ProgressBarObserver, self).__init__(widgets=widgets,
term_width=term_width, fd=fd)
self.last_update = -101
def start(self, max_value):
self._set_max_value(max_value)
super(ProgressBarObserver, self).start()
def is_started(self):
return self.start_time is not None
def finish(self):
self._end_time = time.time()
super(ProgressBarObserver, self).finish()
def update(self, current_value):
if super(ProgressBarObserver, self).update(current_value):
self.last_update = self.get_current_value()
percentage = Observer.get_percentage
def get_current_value(self):
return self.currval
def get_max_value(self):
return self.maxval
def _set_max_value(self, max_value):
self.maxval = max_value
def _need_update(self):
return self.get_current_value() - self.last_update > self.UPDATE_INTERVAL

7
overviewer_core/progressbar.py
View File

@@ -233,7 +233,6 @@ class ProgressBar(object):
self.currval = 0
self.finished = False
self.prev_percentage = -1
self.start_time = None
self.seconds_elapsed = 0
@@ -270,23 +269,23 @@ class ProgressBar(object):
return ''.join(self._format_widgets()).ljust(self.term_width)
def _need_update(self):
return int(self.percentage()) != int(self.prev_percentage)
return True
def update(self, value):
"Updates the progress bar to a new value."
assert 0 <= value <= self.maxval
self.currval = value
if not self._need_update() or self.finished:
return
return False
if not self.start_time:
self.start_time = time.time()
self.seconds_elapsed = time.time() - self.start_time
self.prev_percentage = self.percentage()
if value != self.maxval:
self.fd.write(self._format_line() + '\r')
else:
self.finished = True
self.fd.write(self._format_line() + '\n')
return True
def start(self):
"""Start measuring time, and prints the bar at 0%.

6
overviewer_core/tileset.py
View File

@@ -318,9 +318,9 @@ class TileSet(object):
"to date.",
self.options['name'],
)
logging.warning("You won't get percentage progress for "+
"this run only, because I don't know how many tiles "+
"need rendering. I'll be checking them as I go")
logging.warning("The total tile count will be (possibly "+
"wildly) inaccurate, because I don't know how many "+
"tiles need rendering. I'll be checking them as I go")
self.options['renderchecks'] = 1
else:
logging.debug("No rendercheck mode specified for %s. "+