From 2be64f2aa7a041becea95c3d2014c8ce8b527025 Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sat, 19 Mar 2011 16:01:33 +0800
Subject: [PATCH 1/8] render_inntertile & render_inntertile now called in
 batches in the worker process, speeds up update scan with a lot of tiles to
 skip.

---
 quadtree.py | 84 ++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 24 deletions(-)

diff --git a/quadtree.py b/quadtree.py
index fbee945..67eb7d3 100644
--- a/quadtree.py
+++ b/quadtree.py
@@ -292,10 +292,13 @@ class QuadtreeGen(object):
             shutil.rmtree(getpath("3"))
             os.rename(getpath("new3"), getpath("3"))
 
-    def _apply_render_worldtiles(self, pool):
+    def _apply_render_worldtiles(self, pool,batch_size):
         """Returns an iterator over result objects. Each time a new result is
         requested, a new task is added to the pool and a result returned.
         """
+             
+        batch = []
+        tiles = 0
         for path in iterate_base4(self.p):
             # Get the range for this tile
             colstart, rowstart = self._get_range_by_path(path)
@@ -310,24 +313,42 @@ class QuadtreeGen(object):
             tilechunks = self._get_chunks_in_range(colstart, colend, rowstart,
                     rowend)
             #logging.debug(" tilechunks: %r", tilechunks)
-
-            # Put this in the pool
+            
+            # Put this in the batch to be submited to the pool
             # (even if tilechunks is empty, render_worldtile will delete
-            # existing images if appropriate)
-            yield pool.apply_async(func=render_worldtile, args= (self,
-                tilechunks, colstart, colend, rowstart, rowend, dest))
+            # existing images if appropriate)                           
+            batch.append((tilechunks, colstart, colend, rowstart, rowend, dest))
+            tiles += 1
+            if tiles >= batch_size:
+                tiles = 0
+                yield pool.apply_async(func=render_worldtile_batch, args= (self,batch))
+                batch = []
 
-    def _apply_render_inntertile(self, pool, zoom):
+        if tiles > 0:
+            yield pool.apply_async(func=render_worldtile_batch, args= (self,batch))
+
+
+    def _apply_render_inntertile(self, pool, zoom,batch_size):
         """Same as _apply_render_worltiles but for the inntertile routine.
         Returns an iterator that yields result objects from tasks that have
         been applied to the pool.
         """
+        batch = []
+        tiles = 0        
         for path in iterate_base4(zoom):
             # This image is rendered at:
             dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path[:-1]))
             name = str(path[-1])
-
-            yield pool.apply_async(func=render_innertile, args= (dest, name, self.imgformat, self.optimizeimg))
+            
+            batch.append((dest, name, self.imgformat, self.optimizeimg))
+            tiles += 1
+            if tiles >= batch_size:
+                tiles = 0
+                yield pool.apply_async(func=render_innertile_batch, args= (batch))
+                batch = []
+            
+        if tiles > 0:            
+            yield pool.apply_async(func=render_innertile_batch, args= (batch))
 
     def go(self, procs):
         """Renders all tiles"""
@@ -359,20 +380,21 @@ class QuadtreeGen(object):
         logging.info("There are {0} total levels to render".format(self.p))
         logging.info("Don't worry, each level has only 25% as many tiles as the last.")
         logging.info("The others will go faster")
-        for result in self._apply_render_worldtiles(pool):
+        count = 0
+        batch_size = 50
+        for result in self._apply_render_worldtiles(pool,batch_size):
             results.append(result)
-            if len(results) > 10000:
+            if len(results) > (10000/batch_size):
                 # Empty the queue before adding any more, so that memory
                 # required has an upper bound
-                while len(results) > 500:
-                    results.popleft().get()
-                    complete += 1
+                while len(results) > (500/batch_size):
+                    complete += results.popleft().get()
                     self.print_statusline(complete, total, 1)
 
         # Wait for the rest of the results
         while len(results) > 0:
-            results.popleft().get()
-            complete += 1
+
+            complete += results.popleft().get()
             self.print_statusline(complete, total, 1)
 
         self.print_statusline(complete, total, 1, True)
@@ -384,17 +406,15 @@ class QuadtreeGen(object):
             complete = 0
             total = 4**zoom
             logging.info("Starting level {0}".format(level))
-            for result in self._apply_render_inntertile(pool, zoom):
+            for result in self._apply_render_inntertile(pool, zoom,batch_size):
                 results.append(result)
-                if len(results) > 10000:
-                    while len(results) > 500:
-                        results.popleft().get()
-                        complete += 1
+                if len(results) > (10000/batch_size):
+                    while len(results) > (500/batch_size):
+                        complete += results.popleft().get()
                         self.print_statusline(complete, total, level)
             # Empty the queue
             while len(results) > 0:
-                results.popleft().get()
-                complete += 1
+                complete += results.popleft().get()
                 self.print_statusline(complete, total, level)
 
             self.print_statusline(complete, total, level, True)
@@ -448,6 +468,14 @@ class QuadtreeGen(object):
         return chunklist
 
 @catch_keyboardinterrupt
+def render_innertile_batch( batch):    
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1
+        render_worldtile(job[0],job[1],job[2],job[3])
+    return count
+    
 def render_innertile(dest, name, imgformat, optimizeimg):
     """
     Renders a tile at os.path.join(dest, name)+".ext" by taking tiles from
@@ -512,6 +540,14 @@ def render_innertile(dest, name, imgformat, optimizeimg):
             optimize_image(imgpath, imgformat, optimizeimg)
 
 @catch_keyboardinterrupt
+def render_worldtile_batch(quadtree, batch):    
+    count = 0
+    #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
+    for job in batch:
+        count += 1
+        render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5])
+    return count
+
 def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):
     """Renders just the specified chunks into a tile and save it. Unlike usual
     python conventions, rowend and colend are inclusive. Additionally, the
@@ -529,7 +565,7 @@ def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path)
     Standard tile size has colend-colstart=2 and rowend-rowstart=4
 
     There is no return value
-    """
+    """    
     
     # width of one chunk is 384. Each column is half a chunk wide. The total
     # width is (384 + 192*(numcols-1)) since the first column contributes full

From e113a24ae0279637abfb1d92a60cd14b14a7e135 Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sat, 19 Mar 2011 16:23:11 +0800
Subject: [PATCH 2/8] Fixed render_innertile_batch

---
 quadtree.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/quadtree.py b/quadtree.py
index 67eb7d3..d577325 100644
--- a/quadtree.py
+++ b/quadtree.py
@@ -344,11 +344,11 @@ class QuadtreeGen(object):
             tiles += 1
             if tiles >= batch_size:
                 tiles = 0
-                yield pool.apply_async(func=render_innertile_batch, args= (batch))
+                yield pool.apply_async(func=render_innertile_batch, args= (self,batch))
                 batch = []
             
         if tiles > 0:            
-            yield pool.apply_async(func=render_innertile_batch, args= (batch))
+            yield pool.apply_async(func=render_innertile_batch, args= (self,batch))
 
     def go(self, procs):
         """Renders all tiles"""
@@ -468,12 +468,12 @@ class QuadtreeGen(object):
         return chunklist
 
 @catch_keyboardinterrupt
-def render_innertile_batch( batch):    
+def render_innertile_batch(quadtree, batch):    
     count = 0
     #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
     for job in batch:
         count += 1
-        render_worldtile(job[0],job[1],job[2],job[3])
+        render_innertile(job[0],job[1],job[2],job[3])
     return count
     
 def render_innertile(dest, name, imgformat, optimizeimg):

From b9433173c92498684d8397717318055457b3a9a3 Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sat, 19 Mar 2011 23:36:26 +0800
Subject: [PATCH 3/8] Fixed worker processes being passed the full quadtree
 object. Caused massive performance regressions when caching stuff in
 quadtree.world Offloaded self._get_chunks_in_range into worker process.

---
 quadtree.py | 61 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/quadtree.py b/quadtree.py
index d577325..452b481 100644
--- a/quadtree.py
+++ b/quadtree.py
@@ -82,6 +82,8 @@ def catch_keyboardinterrupt(func):
             raise
     return newfunc
 
+child_quadtree = None
+
 class QuadtreeGen(object):
     def __init__(self, worldobj, destdir, depth=None, tiledir="tiles", imgformat=None, optimizeimg=None, lighting=False, night=False, spawn=False):
         """Generates a quadtree from the world given into the
@@ -308,24 +310,17 @@ class QuadtreeGen(object):
             # This image is rendered at:
             dest = os.path.join(self.destdir, self.tiledir, *(str(x) for x in path))
             #logging.debug("this is rendered at %s", dest)
-
-            # And uses these chunks
-            tilechunks = self._get_chunks_in_range(colstart, colend, rowstart,
-                    rowend)
-            #logging.debug(" tilechunks: %r", tilechunks)
             
-            # Put this in the batch to be submited to the pool
-            # (even if tilechunks is empty, render_worldtile will delete
-            # existing images if appropriate)                           
-            batch.append((tilechunks, colstart, colend, rowstart, rowend, dest))
+            # Put this in the batch to be submited to the pool                        
+            batch.append((colstart, colend, rowstart, rowend, dest))
             tiles += 1
             if tiles >= batch_size:
                 tiles = 0
-                yield pool.apply_async(func=render_worldtile_batch, args= (self,batch))
+                yield pool.apply_async(func=render_worldtile_batch, args= [batch])
                 batch = []
 
         if tiles > 0:
-            yield pool.apply_async(func=render_worldtile_batch, args= (self,batch))
+            yield pool.apply_async(func=render_worldtile_batch, args= (batch,))
 
 
     def _apply_render_inntertile(self, pool, zoom,batch_size):
@@ -344,12 +339,15 @@ class QuadtreeGen(object):
             tiles += 1
             if tiles >= batch_size:
                 tiles = 0
-                yield pool.apply_async(func=render_innertile_batch, args= (self,batch))
+                yield pool.apply_async(func=render_innertile_batch, args= [batch])
                 batch = []
             
-        if tiles > 0:            
-            yield pool.apply_async(func=render_innertile_batch, args= (self,batch))
+        if tiles > 0:       
+            yield pool.apply_async(func=render_innertile_batch, args= [batch])
 
+    def pool_initializer(args):
+        logging.debug("Child process {0}".format(os.getpid()))
+        
     def go(self, procs):
         """Renders all tiles"""
 
@@ -365,12 +363,18 @@ class QuadtreeGen(object):
                 for _ in xrange(curdepth - self.p):
                     self._decrease_depth()
 
+        logging.debug("Parent process {0}".format(os.getpid()))
+        #stash the quadtree object so child process's can 
+        global child_quadtree
+        child_quadtree = self
         # Create a pool
         if procs == 1:
             pool = FakePool()
         else:
-            pool = multiprocessing.Pool(processes=procs)
-
+            pool = multiprocessing.Pool(processes=procs,initializer=self.pool_initializer,initargs=())
+            #warm up the pool so it reports all the worker id's
+            pool.map(bool,xrange(multiprocessing.cpu_count()),1)
+        
         # Render the highest level of tiles from the chunks
         results = collections.deque()
         complete = 0
@@ -381,7 +385,7 @@ class QuadtreeGen(object):
         logging.info("Don't worry, each level has only 25% as many tiles as the last.")
         logging.info("The others will go faster")
         count = 0
-        batch_size = 50
+        batch_size = 10
         for result in self._apply_render_worldtiles(pool,batch_size):
             results.append(result)
             if len(results) > (10000/batch_size):
@@ -393,7 +397,6 @@ class QuadtreeGen(object):
 
         # Wait for the rest of the results
         while len(results) > 0:
-
             complete += results.popleft().get()
             self.print_statusline(complete, total, 1)
 
@@ -468,7 +471,7 @@ class QuadtreeGen(object):
         return chunklist
 
 @catch_keyboardinterrupt
-def render_innertile_batch(quadtree, batch):    
+def render_innertile_batch(batch):    
     count = 0
     #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
     for job in batch:
@@ -540,12 +543,28 @@ def render_innertile(dest, name, imgformat, optimizeimg):
             optimize_image(imgpath, imgformat, optimizeimg)
 
 @catch_keyboardinterrupt
-def render_worldtile_batch(quadtree, batch):    
+def render_worldtile_batch(batch):  
+    global child_quadtree
+    return render_worldtile_batch_(child_quadtree, batch)
+
+def render_worldtile_batch_(quadtree, batch):   
     count = 0
+    _get_chunks_in_range = quadtree._get_chunks_in_range
     #logging.debug("{0} working on batch of size {1}".format(os.getpid(),len(batch)))
     for job in batch:
         count += 1
-        render_worldtile(quadtree,job[0],job[1],job[2],job[3],job[4],job[5])
+        colstart = job[0]
+        colend = job[1]
+        rowstart = job[2]
+        rowend = job[3]
+        path = job[4]
+        # (even if tilechunks is empty, render_worldtile will delete
+        # existing images if appropriate)    
+        # And uses these chunks
+        tilechunks = _get_chunks_in_range(colstart, colend, rowstart,rowend)
+        #logging.debug(" tilechunks: %r", tilechunks)
+            
+        render_worldtile(quadtree,tilechunks,colstart, colend, rowstart, rowend, path)
     return count
 
 def render_worldtile(quadtree, chunks, colstart, colend, rowstart, rowend, path):

From 6f340dceee3fda5cb1a9a52b9b849196211fad57 Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sun, 20 Mar 2011 01:19:23 +0800
Subject: [PATCH 4/8] New conrtib script to validate a region file

---
 contrib/validateRegionFile.py | 76 +++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 contrib/validateRegionFile.py

diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py
new file mode 100644
index 0000000..5bd7e14
--- /dev/null
+++ b/contrib/validateRegionFile.py
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+usage = "python contrib/%prog [OPTIONS] <regionfilename>"
+
+description = """
+This script will delete files from the old chunk-based cache, a lot
+like the old `gmap.py -d World/` command. You should only use this if
+you're updating from an older version of Overviewer, and you want to
+clean up your world folder.
+"""
+
+from optparse import OptionParser
+import sys
+import re
+import os.path
+import logging
+
+# sys.path wrangling, so we can access Overviewer code
+overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
+sys.path.insert(0, overviewer_dir)
+
+import nbt
+import chunk
+
+def main():
+    parser = OptionParser(usage=usage, description=description)
+#    parser.add_option("-d", "--dry-run", dest="dry", action="store_true",
+#                      help="Don't actually delete anything. Best used with -v.")
+    
+    opt, args = parser.parse_args()
+    
+    if not len(args) == 1:
+        parser.print_help()
+        sys.exit(1)
+                
+    regionfile = args[0]
+      
+    if not os.path.exists(regionfile):
+        parser.print_help()
+        print "\nFile not found"
+        sys.exit(1)
+    chunk_pass = 0    
+    chunk_total = 0
+    print( "Loading region: %s" % ( regionfile))
+    try:
+        mcr = nbt.load_region(regionfile)
+    except IOError, e:
+        print("Error opening regionfile. It may be corrupt. %s"%( e))
+        pass
+    if mcr is not None:
+        try:
+            chunks = mcr.get_chunk_info(False)
+        except IOError, e:
+            print("Error opening regionfile. It may be corrupt. %s"%( e))
+            chunks = []
+            pass            
+        for x, y in chunks:
+            chunk_total += 1
+            #try:
+            chunk_data = mcr.load_chunk(x, y)
+            if chunk_data is None:
+               print("Chunk %s:%s is unexpectedly empty"%(x, y))
+            else:
+                try:
+                    processed = chunk_data.read_all()
+                    if processed == []:
+                        print("Chunk %s:%s is an unexpectedly empty set"%(x, y))
+                    else:
+                        chunk_pass += 1
+                except Exception, e:
+                    print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e))
+    else:                
+        print("Error opening regionfile.")
+    print("Done; Passed %s/%s"%(chunk_pass,chunk_total))
+if __name__ == "__main__":
+    main()

From cfabf161485d57879de91b963938b75504af850d Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sun, 20 Mar 2011 01:22:34 +0800
Subject: [PATCH 5/8] Clarify if opening the region file failed or if the
 headers are wonky

---
 contrib/validateRegionFile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py
index 5bd7e14..3846784 100644
--- a/contrib/validateRegionFile.py
+++ b/contrib/validateRegionFile.py
@@ -51,7 +51,7 @@ def main():
         try:
             chunks = mcr.get_chunk_info(False)
         except IOError, e:
-            print("Error opening regionfile. It may be corrupt. %s"%( e))
+            print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e))
             chunks = []
             pass            
         for x, y in chunks:

From 76f85d0d2c68d805c1d2e4190f6913ab7a9bc0b1 Mon Sep 17 00:00:00 2001
From: Xon <Xon@localhost>
Date: Sun, 20 Mar 2011 01:57:47 +0800
Subject: [PATCH 6/8] Added ctrl-c handling, output is a single line (verbose
 reports what error occured), added optparsing, supports multipule files or a
 dir

---
 contrib/validateRegionFile.py | 109 +++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 47 deletions(-)

diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py
index 3846784..1e5320a 100644
--- a/contrib/validateRegionFile.py
+++ b/contrib/validateRegionFile.py
@@ -1,12 +1,9 @@
 #!/usr/bin/python
 
-usage = "python contrib/%prog [OPTIONS] <regionfilename>"
+usage = "python contrib/%prog [OPTIONS] (<regionfilename>)*"
 
 description = """
-This script will delete files from the old chunk-based cache, a lot
-like the old `gmap.py -d World/` command. You should only use this if
-you're updating from an older version of Overviewer, and you want to
-clean up your world folder.
+This script will valide a minecraft region file for errors
 """
 
 from optparse import OptionParser
@@ -21,56 +18,74 @@ sys.path.insert(0, overviewer_dir)
 
 import nbt
 import chunk
+import quadtree
 
 def main():
     parser = OptionParser(usage=usage, description=description)
-#    parser.add_option("-d", "--dry-run", dest="dry", action="store_true",
-#                      help="Don't actually delete anything. Best used with -v.")
+    parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files")
+    parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed")
     
     opt, args = parser.parse_args()
     
-    if not len(args) == 1:
+    if opt.regiondir:
+        if os.path.exists(opt.regiondir):
+            for dirpath, dirnames, filenames in os.walk(opt.regiondir, 'region'):        
+                if not dirnames and filenames and "DIM-1" not in dirpath:
+                    for f in filenames:
+                        if f.startswith("r.") and f.endswith(".mcr"):
+                            p = f.split(".")
+                            args.append(os.path.join(dirpath, f))
+                        
+    if len(args) < 1:
+        print "You must list at least one region file"
         parser.print_help()
         sys.exit(1)
-                
-    regionfile = args[0]
-      
-    if not os.path.exists(regionfile):
-        parser.print_help()
-        print "\nFile not found"
-        sys.exit(1)
-    chunk_pass = 0    
-    chunk_total = 0
-    print( "Loading region: %s" % ( regionfile))
-    try:
-        mcr = nbt.load_region(regionfile)
-    except IOError, e:
-        print("Error opening regionfile. It may be corrupt. %s"%( e))
-        pass
-    if mcr is not None:
+   
+    for regionfile in args:
+        _,shortname = os.path.split(regionfile)
+        chunk_pass = 0    
+        chunk_total = 0          
+        if not os.path.exists(regionfile):
+            print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
+            continue          
         try:
-            chunks = mcr.get_chunk_info(False)
+            mcr = nbt.load_region(regionfile)
         except IOError, e:
-            print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e))
-            chunks = []
-            pass            
-        for x, y in chunks:
-            chunk_total += 1
-            #try:
-            chunk_data = mcr.load_chunk(x, y)
-            if chunk_data is None:
-               print("Chunk %s:%s is unexpectedly empty"%(x, y))
-            else:
-                try:
-                    processed = chunk_data.read_all()
-                    if processed == []:
-                        print("Chunk %s:%s is an unexpectedly empty set"%(x, y))
-                    else:
-                        chunk_pass += 1
-                except Exception, e:
-                    print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e))
-    else:                
-        print("Error opening regionfile.")
-    print("Done; Passed %s/%s"%(chunk_pass,chunk_total))
+            if options.verbose:
+                print("Error opening regionfile. It may be corrupt. %s"%( e))
+        if mcr is not None:
+            try:
+                chunks = mcr.get_chunk_info(False)
+            except IOError, e:
+                if options.verbose:
+                    print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e))
+                chunks = []
+            for x, y in chunks:
+                chunk_total += 1
+                #try:
+                chunk_data = mcr.load_chunk(x, y)
+                if chunk_data is None:
+                    if options.verbose:
+                        print("Chunk %s:%s is unexpectedly empty"%(x, y))
+                else:
+                    try:
+                        processed = chunk_data.read_all()
+                        if processed == []:
+                            if options.verbose:
+                                print("Chunk %s:%s is an unexpectedly empty set"%(x, y))
+                        else:
+                            chunk_pass += 1
+                    except Exception, e:
+                        if options.verbose:
+                            print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e))
+        else:           
+            if options.verbose:    
+                print("Error opening regionfile.")
+         
+        print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except KeyboardInterrupt:
+        print "Caught Ctrl-C"
+        

From fae67de9f0fb156b5a3d2408ab3431130aab9ccc Mon Sep 17 00:00:00 2001
From: Alex Headley <aheadley@waysaboutstuff.com>
Date: Sat, 19 Mar 2011 14:28:19 -0400
Subject: [PATCH 7/8] fixed some bugs in the validateRegion script

---
 contrib/validateRegionFile.py | 45 +++++++++++++++++++++++------------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py
index 1e5320a..34ec1fe 100644
--- a/contrib/validateRegionFile.py
+++ b/contrib/validateRegionFile.py
@@ -12,15 +12,15 @@ import re
 import os.path
 import logging
 
-# sys.path wrangling, so we can access Overviewer code
-overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
-sys.path.insert(0, overviewer_dir)
 
-import nbt
-import chunk
-import quadtree
 
 def main():
+    # sys.path wrangling, so we can access Overviewer code
+    overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
+    sys.path.insert(0, overviewer_dir)
+    import nbt
+    #import chunk
+    #import quadtree
     parser = OptionParser(usage=usage, description=description)
     parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files")
     parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed")
@@ -46,41 +46,56 @@ def main():
         chunk_pass = 0    
         chunk_total = 0          
         if not os.path.exists(regionfile):
-            print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
+            print("File not found: %s"%( regionfile))
+            #print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
             continue          
         try:
             mcr = nbt.load_region(regionfile)
         except IOError, e:
-            if options.verbose:
+            if opt.verbose:
                 print("Error opening regionfile. It may be corrupt. %s"%( e))
+            continue
         if mcr is not None:
             try:
                 chunks = mcr.get_chunk_info(False)
             except IOError, e:
-                if options.verbose:
+                if opt.verbose:
                     print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e))
                 chunks = []
+                continue
+            except Exception, e:
+                if opt.verbose:
+                    print("Error opening regionfile (%s): %s"%( regionfile,e))
+                continue
             for x, y in chunks:
                 chunk_total += 1
-                #try:
-                chunk_data = mcr.load_chunk(x, y)
+                try:
+                    chunk_data = mcr.load_chunk(x, y)
+                except Exception, e:
+                    if opt.verbose:
+                        print("Error reading chunk (%i,%i): %s"%(x,y,e))
+                    continue
                 if chunk_data is None:
-                    if options.verbose:
+                    if opt.verbose:
                         print("Chunk %s:%s is unexpectedly empty"%(x, y))
+                    continue
                 else:
                     try:
                         processed = chunk_data.read_all()
                         if processed == []:
-                            if options.verbose:
+                            if opt.verbose:
                                 print("Chunk %s:%s is an unexpectedly empty set"%(x, y))
+                            continue
                         else:
                             chunk_pass += 1
                     except Exception, e:
-                        if options.verbose:
+                        if opt.verbose:
                             print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e))
+                        continue
         else:           
-            if options.verbose:    
+            if opt.verbose:    
                 print("Error opening regionfile.")
+            continue
          
         print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
 if __name__ == "__main__":

From 5ae361824bf6ce00360b30dd9277a92a33005945 Mon Sep 17 00:00:00 2001
From: Alex Headley <aheadley@waysaboutstuff.com>
Date: Sat, 19 Mar 2011 17:45:20 -0400
Subject: [PATCH 8/8] cleaned up contrib/validateRegionFile script

---
 contrib/validateRegionFile.py | 201 +++++++++++++++++-----------------
 1 file changed, 100 insertions(+), 101 deletions(-)

diff --git a/contrib/validateRegionFile.py b/contrib/validateRegionFile.py
index 34ec1fe..7dd2763 100644
--- a/contrib/validateRegionFile.py
+++ b/contrib/validateRegionFile.py
@@ -1,106 +1,105 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
-usage = "python contrib/%prog [OPTIONS] (<regionfilename>)*"
-
-description = """
-This script will valide a minecraft region file for errors
-"""
-
-from optparse import OptionParser
-import sys
-import re
 import os.path
-import logging
+import sys
+overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
+sys.path.insert(0, overviewer_dir)
+import nbt
 
-
-
-def main():
-    # sys.path wrangling, so we can access Overviewer code
-    overviewer_dir = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
-    sys.path.insert(0, overviewer_dir)
-    import nbt
-    #import chunk
-    #import quadtree
-    parser = OptionParser(usage=usage, description=description)
-    parser.add_option("-r", "--regions", dest="regiondir", help="Use path to the regions instead of a list of files")
-    parser.add_option("-v", dest="verbose", action="store_true", help="Lists why a chunk in a region failed")
-    
-    opt, args = parser.parse_args()
-    
-    if opt.regiondir:
-        if os.path.exists(opt.regiondir):
-            for dirpath, dirnames, filenames in os.walk(opt.regiondir, 'region'):        
-                if not dirnames and filenames and "DIM-1" not in dirpath:
-                    for f in filenames:
-                        if f.startswith("r.") and f.endswith(".mcr"):
-                            p = f.split(".")
-                            args.append(os.path.join(dirpath, f))
-                        
-    if len(args) < 1:
-        print "You must list at least one region file"
-        parser.print_help()
-        sys.exit(1)
-   
-    for regionfile in args:
-        _,shortname = os.path.split(regionfile)
-        chunk_pass = 0    
-        chunk_total = 0          
-        if not os.path.exists(regionfile):
-            print("File not found: %s"%( regionfile))
-            #print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
-            continue          
-        try:
-            mcr = nbt.load_region(regionfile)
-        except IOError, e:
-            if opt.verbose:
-                print("Error opening regionfile. It may be corrupt. %s"%( e))
-            continue
-        if mcr is not None:
-            try:
-                chunks = mcr.get_chunk_info(False)
-            except IOError, e:
-                if opt.verbose:
-                    print("Error opening regionfile(bad header info). It may be corrupt. %s"%( e))
-                chunks = []
-                continue
-            except Exception, e:
-                if opt.verbose:
-                    print("Error opening regionfile (%s): %s"%( regionfile,e))
-                continue
-            for x, y in chunks:
-                chunk_total += 1
-                try:
-                    chunk_data = mcr.load_chunk(x, y)
-                except Exception, e:
-                    if opt.verbose:
-                        print("Error reading chunk (%i,%i): %s"%(x,y,e))
-                    continue
-                if chunk_data is None:
-                    if opt.verbose:
-                        print("Chunk %s:%s is unexpectedly empty"%(x, y))
-                    continue
-                else:
-                    try:
-                        processed = chunk_data.read_all()
-                        if processed == []:
-                            if opt.verbose:
-                                print("Chunk %s:%s is an unexpectedly empty set"%(x, y))
-                            continue
-                        else:
-                            chunk_pass += 1
-                    except Exception, e:
-                        if opt.verbose:
-                            print("Error opening chunk (%i, %i) It may be corrupt. %s"%( x, y, e))
-                        continue
-        else:           
-            if opt.verbose:    
-                print("Error opening regionfile.")
-            continue
-         
-        print("Region:%s Passed %s/%s"%(shortname,chunk_pass,chunk_total))
-if __name__ == "__main__":
+def check_region(region_filename):
+    chunk_errors = []
+    if not os.path.exists(region_filename):
+        raise Exception('Region file not found: %s' % region_filename)
     try:
-        main()
-    except KeyboardInterrupt:
-        print "Caught Ctrl-C"
+        region = nbt.load_region(region_filename)
+    except IOError, e:
+        raise Exception('Error loading region (%s): %s' % (region_filename, e))
+    try:
+        chunks = region.get_chunk_info(False)
+    except IOError, e:
+        raise Exception('Error reading region header (%s): %s' % (region_filename, e))
+    except Exception, e:
+        raise Exception('Error reading region (%s): %s' % (region_filename, e))
+    for x,y in chunks:
+        try:
+            check_chunk(region, x, y)
+        except Exception, e:
+            chunk_errors.append(e)
+    return (chunk_errors, len(chunks))
+    
+def check_chunk(region, x, y):
+    try:
+        data = region.load_chunk(x ,y)
+    except Exception, e:
+        raise Exception('Error reading chunk (%i, %i): %s' % (x, y, e))
+    if data is None:
+        raise Exception('Chunk (%i, %i) is unexpectedly empty' % (x, y))
+    else:
+        try:
+            processed_data = data.read_all()
+        except Exception, e:
+            raise Exception('Error reading chunk (%i, %i) data: %s' % (x, y, e))
+        if processed_data == []:
+            raise Exception('Chunk (%i, %i) is an unexpectedly empty set' % (x, y))
+
+if __name__ == '__main__':
+    try:
+        from optparse import OptionParser
+
+        parser = OptionParser(usage='python contrib/%prog [OPTIONS] <path/to/regions|path/to/regions/*.mcr|regionfile1.mcr regionfile2.mcr ...>',
+                              description='This script will valide a minecraft region file for errors.')
+        parser.add_option('-v', dest='verbose', action='store_true', help='Print additional information.')
+        opts, args = parser.parse_args()
         
+        region_files = []
+        for path in args:
+            if os.path.isdir(path):
+                for dirpath, dirnames, filenames in os.walk(path, True):
+                    for filename in filenames:
+                        if filename.startswith('r.') and filename.endswith('.mcr'):
+                            if filename not in region_files:
+                                region_files.append(os.path.join(dirpath, filename))
+                        elif opts.verbose:
+                            print('Ignoring non-region file: %s' % os.path.join(dirpath, filename))
+            elif os.path.isfile(path):
+                dirpath,filename = os.path.split(path)
+                if filename.startswith('r.') and filename.endswith('.mcr'):
+                    if path not in region_files:
+                        region_files.append(path)
+                else:
+                    print('Ignoring non-region file: %s' % path)
+            else:
+                if opts.verbose:
+                    print('Ignoring arg: %s' % path)
+        if len(region_files) < 1:
+            print 'You must list at least one region file.'
+            parser.print_help()
+            sys.exit(1)
+        else:
+            overall_chunk_total = 0
+            bad_chunk_total = 0
+            bad_region_total = 0
+            for region_file in region_files:
+                try:
+                    (chunk_errors, region_chunks) = check_region(region_file)
+                    bad_chunk_total += len(chunk_errors)
+                    overall_chunk_total += region_chunks
+                except Exception, e:
+                    bad_region_total += 1
+                    print('FAILED(%s): %s' % (region_file, e))
+                else:
+                    if len(chunk_errors) is not 0:
+                        print('WARNING(%s) Chunks: %i/%' % (region_file, region_chunks - len(chunk_errors), region_chunks))
+                        if opts.verbose:
+                            for error in chunk_errors:
+                                print(error)
+                    elif opts.verbose:
+                            print ('PASSED(%s) Chunks: %i/%i' % (region_file, region_chunks - len(chunk_errors), region_chunks))
+            if opts.verbose:
+                print 'REGIONS: %i/%i' % (len(region_files) - bad_region_total, len(region_files))
+                print 'CHUNKS: %i/%i' % (overall_chunk_total - bad_chunk_total, overall_chunk_total)
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except Exception, e:
+        print('ERROR: %s' % e)
+