dispatcher status callback

Right now it's called every 100 work items. This number is completely
arbitrary and should probably be tuned.

overviewer_core/dispatcher.py

@@ -38,11 +38,14 @@ class Dispatcher(object):
         # list of (tileset, workitem, dependencies) tuples
         # keeps track of jobs waiting to run after dependencies finish
         self._pending_jobs = []
-    
+        
     def render_all(self, tilesetlist, status_callback):
         """Render all of the tilesets in the given
         tilesetlist. status_callback is called periodically to update
-        status.  """
+        status. The callback should take the following arguments:
+        (phase, items_completed, total_items), where total_items may
+        be none if there is no useful estimate.
+        """
         # TODO use status callback
         
         # setup tilesetlist
@@ -63,14 +66,52 @@ class Dispatcher(object):
                         return ((tset, workitem) for workitem in tset.iterate_work_items(p))
                     work_iterators.append(make_work_iterator(tileset, phase))
             
+            # keep track of total jobs, and how many jobs are done
+            total_jobs = 0
+            for tileset, phases in zip(tilesetlist, num_phases):
+                if phase < phases:
+                    jobs_for_tileset = tileset.get_phase_length(phase)
+                    # if one is unknown, the total is unknown
+                    if jobs_for_tileset is None:
+                        total_jobs = None
+                        break
+                    else:
+                        total_jobs += jobs_for_tileset
+            finished_jobs = 0
+            
+            # do the first status update
+            self._status_update(status_callback, phase, finished_jobs, total_jobs, force=True)
+            
             # go through these iterators round-robin style
             for tileset, (workitem, deps) in util.roundrobin(work_iterators):
                 self._pending_jobs.append((tileset, workitem, deps))
-                self._dispatch_jobs()
+                finished_jobs += self._dispatch_jobs()
+                self._status_update(status_callback, phase, finished_jobs, total_jobs)
             
             # after each phase, wait for the work to finish
             while len(self._pending_jobs) > 0 or len(self._running_jobs) > 0:
-                self._dispatch_jobs()
+                finished_jobs += self._dispatch_jobs()
+                self._status_update(status_callback, phase, finished_jobs, total_jobs)
+    
+    def _status_update(self, callback, phase, completed, total, force=False):
+        # always called with force=True at the beginning, so that can
+        # be used to set up state. After that, it is called after
+        # every _dispatch_jobs() often; this function is used to
+        # decide how often the actual status callback should be
+        # called.
+        if force:
+            self._last_status_update = completed
+            if callback:
+                callback(phase, completed, total)
+            return
+        
+        if callback is None:
+            return
+        
+        update_interval = 100 # XXX arbitrary
+        if self._last_status_update < 0 or completed >= self._last_status_update + update_interval or completed < self._last_status_update:
+            self._last_status_update = completed
+            callback(phase, completed, total)
     
     def _dispatch_jobs(self):
         # helper function to dispatch pending jobs when their
@@ -104,6 +145,8 @@ class Dispatcher(object):
             self._running_jobs.remove(job)
         for job in dispatched_jobs:
             self._pending_jobs.remove(job)
+        
+        return len(finished_jobs)
     
     def close(self):
         """Close the Dispatcher. This should be called when you are

overviewer_core/tileset.py

@@ -44,6 +44,12 @@ get_num_phases()
     other phases... all work done by one phase is done before the next phase is
     started.
 
+get_phase_length(phase)
+    This method returns an integer indicating how many work items there are in
+    this phase. This number is used for purely informational purposes. It can
+    be exact, or an estimate. If there is no useful information on the size of
+    a phase, return None.
+
 iterate_work_items(phase)
     Takes a phase number (a non-negative integer). This method should return an
     iterator over work items and a list of dependencies i.e. (work_item, [d1,
@@ -227,6 +233,12 @@ class TileSet(object):
 
         """
         return 1
+    
+    def get_phase_length(self, phase):
+        """Returns the number of work items in a given phase, or None if there
+        is no good estimate.
+        """
+        return None
 
     def iterate_work_items(self, phase):
         """Iterates over the dirty tiles in the tree at level depth-phase. So