From 55df1bcf78d32be3150649de10d4334332c7303e Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Sun, 8 Apr 2012 23:03:26 +0100 Subject: [PATCH 1/6] Add new Javascript observer Output render progress onto the rendered map using Javascript A minimum period between updates can be specified --- overviewer_core/data/js_src/util.js | 4 + overviewer_core/data/js_src/views.js | 21 +++- .../data/web_assets/overviewer.css | 2 +- overviewer_core/observer.py | 96 +++++++++++++++++++ overviewer_core/settingsDefinition.py | 2 +- 5 files changed, 122 insertions(+), 3 deletions(-) diff --git a/overviewer_core/data/js_src/util.js b/overviewer_core/data/js_src/util.js index 0e738da..3d6c36e 100644 --- a/overviewer_core/data/js_src/util.js +++ b/overviewer_core/data/js_src/util.js @@ -58,6 +58,10 @@ overviewer.util = { var coordsdiv = new overviewer.views.CoordboxView({tagName: 'DIV'}); coordsdiv.render(); + var progressdiv = new overviewer.views.ProgressView({tagName: 'DIV'}); + progressdiv.render(); + progressdiv.updateProgress(); + if (overviewer.collections.haveSigns) { var signs = new overviewer.views.SignControlView(); signs.registerEvents(signs); diff --git a/overviewer_core/data/js_src/views.js b/overviewer_core/data/js_src/views.js index 323c1fc..cbc6453 100644 --- a/overviewer_core/data/js_src/views.js +++ b/overviewer_core/data/js_src/views.js @@ -108,7 +108,26 @@ overviewer.views.CoordboxView = Backbone.View.extend({ } }); - +overviewer.views.ProgressView = Backbone.View.extend({ + initialize: function() { + this.el.id = 'progressDiv'; + this.el.innerHTML = 'Current Render Progress'; + overviewer.map.controls[google.maps.ControlPosition.BOTTOM_RIGHT].push(this.el); + this.hidden = true; + }, + updateProgress: function() { + e = this; + $.getJSON('progress.js', null, function(d){ + e.el.hidden = false; + e.el.innerHTML = d['message']; + if (d.update > 0) { + setTimeout("e.updateProgress()", d.update); + } else { + e.el.hidden = true; + } + }); + } +}); /* GoogleMapView is responsible for dealing with the GoogleMaps API to create the */ diff --git a/overviewer_core/data/web_assets/overviewer.css b/overviewer_core/data/web_assets/overviewer.css index 0ae76e5..e1f1527 100644 --- a/overviewer_core/data/web_assets/overviewer.css +++ b/overviewer_core/data/web_assets/overviewer.css @@ -86,7 +86,7 @@ body { } -#link, #coordsDiv { +#link, #coordsDiv, #progressDiv { background-color: #fff; /* fallback */ background-color: rgba(255,255,255,0.55); border: 1px solid rgb(0, 0, 0); diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index c58a0d9..15cb6d7 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -17,6 +17,7 @@ import time import logging import progressbar import sys +import os class Observer(object): """Base class that defines the observer interface. @@ -164,3 +165,98 @@ class ProgressBarObserver(progressbar.ProgressBar, Observer): def _need_update(self): return self.get_current_value() - self.last_update > self.UPDATE_INTERVAL + +class JSObserver(Observer): + """Display progress on index.html using JavaScript + """ + + def __init__(self, outputdir, minrefresh=5): + """Initialise observer + outputdir must be set to the map output directory path + minrefresh specifies the minimum gap between requests, in seconds + """ + self.last_update = -11 + self.last_update_time = -1 + self._current_value = -1 + self.minrefresh = 1000*minrefresh + self.logfile = os.path.join(outputdir, "progress.js") + + f = open(self.logfile, "w", 0) + f.write('{"message": "Render in progress", "update": %d}' % self.minrefresh) + f.close() + + def start(self, max_value): + f = open(self.logfile, "w", 0) + f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh) + self.start_time=time.time() + self._set_max_value(max_value) + + def is_started(self): + return self.start_time is not None + + def finish(self): + """Signals the end of the processes, should be called after the + process is done. + """ + self.end_time = time.time() + duration = self.end_time - self.start_time + f = open(self.logfile, "w", 0) + f.write('{"message": "Render completed in %dm %ds", "update": "false"}' % (duration//60, duration - duration//60)) + f.close() + + def is_finished(self): + return self.end_time is not None + + def is_running(self): + return self.is_started() and not self.is_finished() + + def add(self, amount): + """Shortcut to update by increments instead of absolute values. Zero + amounts are ignored. + """ + if amount: + self.update(self.get_current_value() + amount) + + def update(self, current_value): + """Set the progress value. Should be between 0 and max_value. Returns + whether this update is actually displayed. + """ + self._current_value = current_value + if self._need_update(): + refresh = max(1500*(time.time() - self.last_update_time), self.minrefresh) + f = open(self.logfile, "w", 0) + f.write('{"message": "Rendered %d of %d tiles (%d%%)", "update": %d }' % (self.get_current_value(), self.get_max_value(), self.get_percentage(), refresh)) + f.close() + self.last_update_time = time.time() + self.last_update = current_value + return True + return False + + def get_percentage(self): + """Get the current progress percentage. Assumes 100% if max_value is 0 + """ + if self.get_max_value() is 0: + return 100.0 + else: + return self.get_current_value() * 100.0 / self.get_max_value() + + def get_current_value(self): + return self._current_value + + def get_max_value(self): + return self._max_value + + def _set_max_value(self, max_value): + self._max_value = max_value + + def _need_update(self): + cur_val = self.get_current_value() + if (time.time() - self.last_update_time) <= self.minrefresh//1000: + return False + if cur_val < 100: + return cur_val - self.last_update > 10 + elif cur_val < 500: + return cur_val - self.last_update > 50 + else: + return cur_val - self.last_update > 100 + diff --git a/overviewer_core/settingsDefinition.py b/overviewer_core/settingsDefinition.py index d2e357c..cfb6539 100644 --- a/overviewer_core/settingsDefinition.py +++ b/overviewer_core/settingsDefinition.py @@ -45,7 +45,7 @@ from settingsValidators import * import util -from observer import ProgressBarObserver, LoggingObserver +from observer import ProgressBarObserver, LoggingObserver, JSObserver import platform import sys From 8ec3518eb223b583330a043124cb740dc13b4e95 Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Sun, 8 Apr 2012 23:09:45 +0100 Subject: [PATCH 2/6] Fix missing bracket in observer.py --- overviewer_core/observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index 15cb6d7..e6fa14f 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -187,7 +187,7 @@ class JSObserver(Observer): def start(self, max_value): f = open(self.logfile, "w", 0) - f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh) + f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) self.start_time=time.time() self._set_max_value(max_value) From ed01a3cf07d981432192d814b548c037f0ed1c23 Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Mon, 9 Apr 2012 00:23:15 +0100 Subject: [PATCH 3/6] Minor tweaks for JSObserver --- overviewer_core/data/js_src/views.js | 1 + overviewer_core/observer.py | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/overviewer_core/data/js_src/views.js b/overviewer_core/data/js_src/views.js index cbc6453..e66267e 100644 --- a/overviewer_core/data/js_src/views.js +++ b/overviewer_core/data/js_src/views.js @@ -114,6 +114,7 @@ overviewer.views.ProgressView = Backbone.View.extend({ this.el.innerHTML = 'Current Render Progress'; overviewer.map.controls[google.maps.ControlPosition.BOTTOM_RIGHT].push(this.el); this.hidden = true; + $.ajaxSetup({cache: false}); }, updateProgress: function() { e = this; diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index e6fa14f..32053cd 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -181,10 +181,6 @@ class JSObserver(Observer): self.minrefresh = 1000*minrefresh self.logfile = os.path.join(outputdir, "progress.js") - f = open(self.logfile, "w", 0) - f.write('{"message": "Render in progress", "update": %d}' % self.minrefresh) - f.close() - def start(self, max_value): f = open(self.logfile, "w", 0) f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) @@ -251,8 +247,6 @@ class JSObserver(Observer): def _need_update(self): cur_val = self.get_current_value() - if (time.time() - self.last_update_time) <= self.minrefresh//1000: - return False if cur_val < 100: return cur_val - self.last_update > 10 elif cur_val < 500: From c7fd50bd2d8473fa284809c76036b30a916c15b3 Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Wed, 18 Apr 2012 20:48:49 +0100 Subject: [PATCH 4/6] Add documentation for JSObserver Adds documentation for JSObserver and updates documentation for the observer option in general --- docs/config.rst | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index dac2c12..2e4e70b 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -165,6 +165,8 @@ the form ``key = value``. Two items take a different form:, ``worlds`` and **You must specify at least one render** +.. _outputdir: + ``outputdir = ""`` This is the path to the output directory where the rendered tiles will be saved. @@ -199,11 +201,38 @@ the form ``key = value``. Two items take a different form:, ``worlds`` and observer to be used. The observer object is expected to have at least ``start``, ``add``, ``update``, and ``finish`` methods. - e.g.:: + If you want to specify an observer manually, try something like: + :: + from observer import ProgressBarObserver() observer = ProgressBarObserver() -.. _outputdir: + There are currently three observers available: ``LoggingObserver``, + ``ProgressBarObserver`` and ``JSObserver``. + + ``LoggingObserver`` + This gives the normal/older style output and is the default when output + is redirected to a file or when running on Windows + + ``ProgressBarObserver`` + This is used by default when the output is a terminal. Displays a text based + progress bar and some statistics. + + ``JSObserver`` + This will display render progress on the output map in the bottom right + corner of the screen. ``JSObserver`` must be invoked with two parameters. + + The first is output directory. For simplicity, specify this as ``outputdir`` + and place this line after setting ``outputdir = ""``. + + The second parameter is the minimum interval between progress updates in + seconds. Progress information won't be written to file or requested by + your web browser more frequently than this interval. + :: + + from observer import JSObserver + observer = JSObserver(outputdir, 10) + .. _renderdict: From 806e517e98d78d7ed3098cd8c64a53b7c89e666a Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Wed, 18 Apr 2012 21:39:21 +0100 Subject: [PATCH 5/6] Fix tab/spaces in observer.py --- overviewer_core/observer.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index 32053cd..f6c0bfb 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -172,20 +172,20 @@ class JSObserver(Observer): def __init__(self, outputdir, minrefresh=5): """Initialise observer - outputdir must be set to the map output directory path - minrefresh specifies the minimum gap between requests, in seconds - """ - self.last_update = -11 - self.last_update_time = -1 - self._current_value = -1 - self.minrefresh = 1000*minrefresh + outputdir must be set to the map output directory path + minrefresh specifies the minimum gap between requests, in seconds + """ + self.last_update = -11 + self.last_update_time = -1 + self._current_value = -1 + self.minrefresh = 1000*minrefresh self.logfile = os.path.join(outputdir, "progress.js") def start(self, max_value): f = open(self.logfile, "w", 0) - f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) + f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) self.start_time=time.time() - self._set_max_value(max_value) + self._set_max_value(max_value) def is_started(self): return self.start_time is not None @@ -195,9 +195,9 @@ class JSObserver(Observer): process is done. """ self.end_time = time.time() - duration = self.end_time - self.start_time + duration = self.end_time - self.start_time f = open(self.logfile, "w", 0) - f.write('{"message": "Render completed in %dm %ds", "update": "false"}' % (duration//60, duration - duration//60)) + f.write('{"message": "Render completed in %dm %ds", "update": "false"}' % (duration//60, duration - duration//60)) f.close() def is_finished(self): @@ -221,11 +221,11 @@ class JSObserver(Observer): if self._need_update(): refresh = max(1500*(time.time() - self.last_update_time), self.minrefresh) f = open(self.logfile, "w", 0) - f.write('{"message": "Rendered %d of %d tiles (%d%%)", "update": %d }' % (self.get_current_value(), self.get_max_value(), self.get_percentage(), refresh)) + f.write('{"message": "Rendered %d of %d tiles (%d%%)", "update": %d }' % (self.get_current_value(), self.get_max_value(), self.get_percentage(), refresh)) f.close() self.last_update_time = time.time() - self.last_update = current_value - return True + self.last_update = current_value + return True return False def get_percentage(self): From 62322b452aeafa28fca24fe48156e3769b0fcfae Mon Sep 17 00:00:00 2001 From: Thomas Lake Date: Wed, 18 Apr 2012 22:04:19 +0100 Subject: [PATCH 6/6] Keep file open and flush changes Suggested by aheadley in IRC --- overviewer_core/observer.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/overviewer_core/observer.py b/overviewer_core/observer.py index f6c0bfb..7db5061 100644 --- a/overviewer_core/observer.py +++ b/overviewer_core/observer.py @@ -179,11 +179,13 @@ class JSObserver(Observer): self.last_update_time = -1 self._current_value = -1 self.minrefresh = 1000*minrefresh - self.logfile = os.path.join(outputdir, "progress.js") + self.logfile = open(os.path.join(outputdir, "progress.js"), "w+", 0) def start(self, max_value): - f = open(self.logfile, "w", 0) - f.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) + self.logfile.seek(0) + self.logfile.write('{"message": "Rendering %d tiles", "update": %s}' % (max_value, self.minrefresh)) + self.logfile.truncate() + self.logfile.flush() self.start_time=time.time() self._set_max_value(max_value) @@ -196,9 +198,10 @@ class JSObserver(Observer): """ self.end_time = time.time() duration = self.end_time - self.start_time - f = open(self.logfile, "w", 0) - f.write('{"message": "Render completed in %dm %ds", "update": "false"}' % (duration//60, duration - duration//60)) - f.close() + self.logfile.seek(0) + self.logfile.write('{"message": "Render completed in %dm %ds", "update": "false"}' % (duration//60, duration - duration//60)) + self.logfile.truncate() + self.logfile.close() def is_finished(self): return self.end_time is not None @@ -220,9 +223,10 @@ class JSObserver(Observer): self._current_value = current_value if self._need_update(): refresh = max(1500*(time.time() - self.last_update_time), self.minrefresh) - f = open(self.logfile, "w", 0) - f.write('{"message": "Rendered %d of %d tiles (%d%%)", "update": %d }' % (self.get_current_value(), self.get_max_value(), self.get_percentage(), refresh)) - f.close() + self.logfile.seek(0) + self.logfile.write('{"message": "Rendered %d of %d tiles (%d%%)", "update": %d }' % (self.get_current_value(), self.get_max_value(), self.get_percentage(), refresh)) + self.logfile.truncate() + self.logfile.flush() self.last_update_time = time.time() self.last_update = current_value return True