LogalDeveloper/Minecraft-Overviewer
Archived
0
Fork 0
Code Activity

Merge pull request #1064 from overviewer/fscaps

Browse Source 
Try to track the capabilities of our outputdir filesystem.
This commit is contained in:
Andrew Chin
2014-02-16 22:38:41 -05:00
parent bfe06c1464 d11960ecfb
commit 2c35ab8cc7
3 changed files with 78 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
overviewer_core/assetmanager.py
View File

@@ -25,7 +25,7 @@ from PIL import Image
import world
import util
from files import FileReplacer, mirror_dir
from files import FileReplacer, mirror_dir, get_fs_caps
class AssetManager(object):
"""\
@@ -44,6 +44,8 @@ directory.
self.custom_assets_dir = custom_assets_dir
self.renders = dict()
self.fs_caps = get_fs_caps(self.outputdir)
# look for overviewerConfig in self.outputdir
try:
with open(os.path.join(self.outputdir, "overviewerConfig.js")) as c:
@@ -148,7 +150,7 @@ directory.
# write out config
jsondump = json.dumps(dump, indent=4)
with FileReplacer(os.path.join(self.outputdir, "overviewerConfig.js")) as tmpfile:
with FileReplacer(os.path.join(self.outputdir, "overviewerConfig.js"), capabilities=self.fs_caps) as tmpfile:
with codecs.open(tmpfile, 'w', encoding='UTF-8') as f:
f.write("var overviewerConfig = " + jsondump + ";\n")
@@ -162,12 +164,12 @@ directory.
global_assets = os.path.join(util.get_program_path(), "overviewer_core", "data", "web_assets")
if not os.path.isdir(global_assets):
global_assets = os.path.join(util.get_program_path(), "web_assets")
mirror_dir(global_assets, self.outputdir)
mirror_dir(global_assets, self.outputdir, capabilities=self.fs_caps)
if self.custom_assets_dir:
# Could have done something fancy here rather than just overwriting
# the global files, but apparently this what we used to do pre-rewrite.
mirror_dir(self.custom_assets_dir, self.outputdir)
mirror_dir(self.custom_assets_dir, self.outputdir, capabilities=self.fs_caps)
# write a dummy baseMarkers.js if none exists
if not os.path.exists(os.path.join(self.outputdir, "baseMarkers.js")):
@@ -179,7 +181,7 @@ directory.
js_src = os.path.join(util.get_program_path(), "overviewer_core", "data", "js_src")
if not os.path.isdir(js_src):
js_src = os.path.join(util.get_program_path(), "js_src")
with FileReplacer(os.path.join(self.outputdir, "overviewer.js")) as tmpfile:
with FileReplacer(os.path.join(self.outputdir, "overviewer.js"), capabilities=self.fs_caps) as tmpfile:
with open(tmpfile, "w") as fout:
# first copy in js_src/overviewer.js
with open(os.path.join(js_src, "overviewer.js"), 'r') as f:
@@ -199,6 +201,6 @@ directory.
versionstr = "%s (%s)" % (util.findGitVersion(), util.findGitHash()[:7])
index = index.replace("{version}", versionstr)
with FileReplacer(indexpath) as indexpath:
with FileReplacer(indexpath, capabilities=self.fs_caps) as indexpath:
with codecs.open(indexpath, 'w', encoding='UTF-8') as output:
output.write(index)

99
overviewer_core/files.py
View File

@@ -20,9 +20,50 @@ import shutil
import logging
import stat
default_caps = {"chmod_works": True, "rename_works": True}
def get_fs_caps(dir_to_test):
return {"chmod_works": does_chmod_work(dir_to_test),
"rename_works": does_rename_work(dir_to_test)
}
def does_chmod_work(dir_to_test):
"Detects if chmod works in a given directory"
# a CIFS mounted FS is the only thing known to reliably not provide chmod
if not os.path.isdir(dir_to_test):
return True
f1 = tempfile.NamedTemporaryFile(dir=dir_to_test)
try:
f1_stat = os.stat(f1.name)
os.chmod(f1.name, f1_stat.st_mode | stat.S_IRUSR)
chmod_works = True
logging.debug("Detected that chmods work in %r" % dir_to_test)
except OSError:
chmod_works = False
logging.debug("Detected that chmods do NOT work in %r" % dir_to_test)
return chmod_works
def does_rename_work(dir_to_test):
with tempfile.NamedTemporaryFile(dir=dir_to_test) as f1:
with tempfile.NamedTemporaryFile(dir=dir_to_test) as f2:
try:
os.rename(f1.name,f2.name)
except OSError:
renameworks = False
logging.debug("Detected that overwriting renames do NOT work in %r" % dir_to_test)
else:
renameworks = True
logging.debug("Detected that overwriting renames work in %r" % dir_to_test)
# re-make this file so it can be deleted without error
open(f1.name, 'w').close()
return renameworks
## useful recursive copy, that ignores common OS cruft
def mirror_dir(src, dst, entities=None):
def mirror_dir(src, dst, entities=None, capabilities=default_caps):
'''copies all of the entities from src to dst'''
chmod_works = capabilities.get("chmod_works")
if not os.path.exists(dst):
os.mkdir(dst)
if entities and type(entities) != list: raise Exception("Expected a list, got a %r instead" % type(entities))
@@ -38,10 +79,13 @@ def mirror_dir(src, dst, entities=None):
continue
if os.path.isdir(os.path.join(src,entry)):
mirror_dir(os.path.join(src, entry), os.path.join(dst, entry))
mirror_dir(os.path.join(src, entry), os.path.join(dst, entry), capabilities=capabilities)
elif os.path.isfile(os.path.join(src,entry)):
try:
shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
if chmod_works:
shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
else:
shutil.copyfile(os.path.join(src, entry), os.path.join(dst, entry))
except IOError as outer:
try:
# maybe permission problems?
@@ -51,24 +95,16 @@ def mirror_dir(src, dst, entities=None):
os.chmod(os.path.join(dst, entry), dst_stat.st_mode | stat.S_IWUSR)
except OSError: # we don't care if this fails
pass
shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
# if this stills throws an error, let it propagate up
# try again; if this stills throws an error, let it propagate up
if chmod_works:
shutil.copy(os.path.join(src, entry), os.path.join(dst, entry))
else:
shutil.copyfile(os.path.join(src, entry), os.path.join(dst, entry))
# Define a context manager to handle atomic renaming or "just forget it write
# straight to the file" depending on whether os.rename provides atomic
# overwrites.
# Detect whether os.rename will overwrite files
with tempfile.NamedTemporaryFile() as f1:
with tempfile.NamedTemporaryFile() as f2:
try:
os.rename(f1.name,f2.name)
except OSError:
renameworks = False
else:
renameworks = True
# re-make this file so it can be deleted without error
open(f1.name, 'w').close()
del tempfile,f1,f2
doc = """This class acts as a context manager for files that are to be written
out overwriting an existing file.
@@ -89,16 +125,20 @@ with FileReplacer("config") as configname:
with open(configout, 'w') as configout:
configout.write(newconfig)
"""
if renameworks:
class FileReplacer(object):
__doc__ = doc
def __init__(self, destname):
self.destname = destname
class FileReplacer(object):
__doc__ = doc
def __init__(self, destname, capabilities=default_caps):
self.caps = capabilities
self.destname = destname
if self.caps.get("rename_works"):
self.tmpname = destname + ".tmp"
def __enter__(self):
def __enter__(self):
if self.caps.get("rename_works"):
# rename works here. Return a temporary filename
return self.tmpname
def __exit__(self, exc_type, exc_val, exc_tb):
return self.destname
def __exit__(self, exc_type, exc_val, exc_tb):
if self.caps.get("rename_works"):
if exc_type:
# error
try:
@@ -109,18 +149,7 @@ if renameworks:
"'%s'!", self.tmpname)
else:
# copy permission bits, if needed
if os.path.exists(self.destname):
if self.caps.get("chmod_works") and os.path.exists(self.destname):
shutil.copymode(self.destname, self.tmpname)
# atomic rename into place
os.rename(self.tmpname, self.destname)
else:
class FileReplacer(object):
__doc__ = doc
def __init__(self, destname):
self.destname = destname
def __enter__(self):
return self.destname
def __exit__(self, exc_type, exc_val, exc_tb):
return
del renameworks

9
overviewer_core/tileset.py
View File

@@ -31,7 +31,7 @@ from PIL import Image
from .util import roundrobin
from . import nbt
from .files import FileReplacer
from .files import FileReplacer, get_fs_caps
from .optimizeimages import optimize_image
import rendermodes
import c_overviewer
@@ -357,6 +357,9 @@ class TileSet(object):
self.options['renderchecks'] = 2
os.mkdir(self.outputdir)
# must wait until outputdir exists
self.fs_caps = get_fs_caps(self.outputdir)
if self.options['renderchecks'] == 2:
# Set forcerendertime so that upon an interruption the next render
# will continue where we left off.
@@ -902,7 +905,7 @@ class TileSet(object):
logging.error("While attempting to delete corrupt image %s, an error was encountered. You will need to delete it yourself. Error was '%s'", path[1], e)
# Save it
with FileReplacer(imgpath) as tmppath:
with FileReplacer(imgpath, capabilities=self.fs_caps) as tmppath:
if imgformat == 'jpg':
img.save(tmppath, "jpeg", quality=self.options['imgquality'], subsampling=0)
else: # png
@@ -1006,7 +1009,7 @@ class TileSet(object):
#draw.text((96,96), "c,r: %s,%s" % (col, row), fill='red')
# Save them
with FileReplacer(imgpath) as tmppath:
with FileReplacer(imgpath, capabilities=self.fs_caps) as tmppath:
if self.imgextension == 'jpg':
tileimg.save(tmppath, "jpeg", quality=self.options['imgquality'], subsampling=0)
else: # png