Skip to content

P
pyfs
Commit e1a641f3 by willmcgugan
Options

Added thread syncronisation to filesystem objects

parent 53b69ae5
Showing with 562 additions and 344 deletions
fs/fs.py
...@@ -5,7 +5,10 @@ import os.path ...@@ -5,7 +5,10 @@ import os.path
import fnmatch import fnmatch
from itertools import chain from itertools import chain
import datetime import datetime
try:
import threading
except ImportError:
import dummy_threading as threadding
error_msgs = { error_msgs = {
...@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2): ...@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2):
print_dir(fs, path, 0) print_dir(fs, path, 0)
def _synchronize(func):
def acquire_lock(self, *args, **kwargs):
self._lock.acquire()
try:
return func(self, *args, **kwargs)
finally:
self._lock.release()
acquire_lock.__doc__ = func.__doc__
return acquire_lock
class FS(object): class FS(object):
"""The base class for Filesystem objects. An instance of a class derived from FS is an abstraction """The base class for Filesystem objects. An instance of a class derived from FS is an abstraction
...@@ -310,6 +325,13 @@ class FS(object): ...@@ -310,6 +325,13 @@ class FS(object):
""" """
def __init__(self, thread_syncronize=False):
if thread_syncronize:
self._lock = threading.RLock()
else:
self._lock = None
def _resolve(self, pathname): def _resolve(self, pathname):
resolved_path = resolvepath(pathname) resolved_path = resolvepath(pathname)
return resolved_path return resolved_path
......
fs/memoryfs.py
...@@ -2,7 +2,9 @@ ...@@ -2,7 +2,9 @@
import os import os
import datetime import datetime
from fs import FS, pathsplit, _iteratepath, FSError, print_fs from fs import _iteratepath
from fs import *
try: try:
from cStringIO import StringIO from cStringIO import StringIO
...@@ -29,27 +31,22 @@ class MemoryFile(object): ...@@ -29,27 +31,22 @@ class MemoryFile(object):
self.mem_file.write(value) self.mem_file.write(value)
elif _check_mode(mode, 'w'): elif _check_mode(mode, 'w'):
self.mem_file = StringIO() self.mem_file = StringIO()
elif _check_mode(mode, 'ra'): elif _check_mode(mode, 'ra'):
self.mem_file = StringIO() self.mem_file = StringIO()
self.mem_file.write(value) self.mem_file.write(value)
elif _check_mode(mode, 'r'): elif _check_mode(mode, 'r'):
self.mem_file = StringIO(value) self.mem_file = StringIO(value)
else: else:
if value is not None: if value is not None:
self.mem_file = StringIO(value) self.mem_file = StringIO(value)
else: else:
self.mem_file = StringIO() self.mem_file = StringIO()
assert self.mem_file is not None, "self.mem_file should have a value" assert self.mem_file is not None, "self.mem_file should have a value"
self.closed = False self.closed = False
def __del__(self): def __del__(self):
...@@ -109,11 +106,8 @@ class MemoryFS(FS): ...@@ -109,11 +106,8 @@ class MemoryFS(FS):
contents = {} contents = {}
self.contents = contents self.contents = contents
self.data = None self.data = None
self.locks = 0 self.locks = 0
self.created_time = datetime.datetime.now() self.created_time = datetime.datetime.now()
def lock(self): def lock(self):
...@@ -145,6 +139,7 @@ class MemoryFS(FS): ...@@ -145,6 +139,7 @@ class MemoryFS(FS):
return self.dir_entry_factory(*args, **kwargs) return self.dir_entry_factory(*args, **kwargs)
def __init__(self, file_factory=None): def __init__(self, file_factory=None):
FS.__init__(self, thread_syncronize=True)
self.dir_entry_factory = MemoryFS.DirEntry self.dir_entry_factory = MemoryFS.DirEntry
self.file_factory = file_factory or MemoryFile self.file_factory = file_factory or MemoryFile
...@@ -154,186 +149,239 @@ class MemoryFS(FS): ...@@ -154,186 +149,239 @@ class MemoryFS(FS):
return "<MemoryFS>" return "<MemoryFS>"
def _get_dir_entry(self, dirpath): def _get_dir_entry(self, dirpath):
current_dir = self.root self._lock.acquire()
for path_component in _iteratepath(dirpath): try:
dir_entry = current_dir.contents.get(path_component, None) current_dir = self.root
if dir_entry is None: for path_component in _iteratepath(dirpath):
return None dir_entry = current_dir.contents.get(path_component, None)
current_dir = dir_entry if dir_entry is None:
return None
current_dir = dir_entry
return current_dir return current_dir
finally:
self._lock.release()
def desc(self, path): def desc(self, path):
if self.isdir(path): self._lock.acquire()
return "Memory dir" try:
elif self.isfile(path): if self.isdir(path):
return "Memory file object" return "Memory dir"
else: elif self.isfile(path):
return "No description available" return "Memory file object"
else:
return "No description available"
finally:
self._lock.release()
def isdir(self, path): def isdir(self, path):
dir_item = self._get_dir_entry(self._resolve(path)) self._lock.acquire()
if dir_item is None: try:
return False dir_item = self._get_dir_entry(self._resolve(path))
return dir_item.isdir() if dir_item is None:
return False
return dir_item.isdir()
finally:
self._lock.release()
def isfile(self, path): def isfile(self, path):
dir_item = self._get_dir_entry(self._resolve(path)) self._lock.acquire()
if dir_item is None: try:
return False dir_item = self._get_dir_entry(self._resolve(path))
return dir_item.isfile() if dir_item is None:
return False
return dir_item.isfile()
finally:
self._lock.release()
def exists(self, path): def exists(self, path):
return self._get_dir_entry(path) is not None self._lock.acquire()
try:
return self._get_dir_entry(path) is not None
finally:
self._lock.release()
def makedir(self, dirname, mode=0777, recursive=False, allow_recreate=False): def makedir(self, dirname, mode=0777, recursive=False, allow_recreate=False):
fullpath = dirname self._lock.acquire()
dirpath, dirname = pathsplit(dirname) try:
fullpath = dirname
dirpath, dirname = pathsplit(dirname)
if recursive:
parent_dir = self._get_dir_entry(dirpath)
if parent_dir is not None:
if parent_dir.isfile():
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
else:
if not allow_recreate:
if dirname in parent_dir.contents:
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
current_dir = self.root
for path_component in _iteratepath(dirpath)[:-1]:
dir_item = current_dir.contents.get(path_component, None)
if dir_item is None:
break
if not dir_item.isdir():
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
current_dir = dir_item.contents
current_dir = self.root
for path_component in _iteratepath(dirpath):
dir_item = current_dir.contents.get(path_component, None)
if dir_item is None:
new_dir = self._make_dir_entry("dir", path_component)
current_dir.contents[path_component] = new_dir
current_dir = new_dir
else:
current_dir = dir_item
parent_dir = current_dir
if recursive: else:
parent_dir = self._get_dir_entry(dirpath) parent_dir = self._get_dir_entry(dirpath)
if parent_dir is not None: if parent_dir is None:
if parent_dir.isfile(): raise ResourceNotFoundError("NO_DIR", dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
else:
if not allow_recreate:
if dirname in parent_dir.contents:
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
current_dir = self.root
for path_component in _iteratepath(dirpath)[:-1]:
dir_item = current_dir.contents.get(path_component, None)
if dir_item is None:
break
if not dir_item.isdir():
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
current_dir = dir_item.contents
current_dir = self.root dir_item = parent_dir.contents.get(dirname, None)
for path_component in _iteratepath(dirpath): if dir_item is not None:
dir_item = current_dir.contents.get(path_component, None) if dir_item.isdir():
if dir_item is None: if not allow_recreate:
new_dir = self._make_dir_entry("dir", path_component) raise FSError("DIR_EXISTS", dirname)
current_dir.contents[path_component] = new_dir
current_dir = new_dir
else: else:
current_dir = dir_item raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
parent_dir = current_dir
else:
parent_dir = self._get_dir_entry(dirpath)
if parent_dir is None:
raise ResourceNotFoundError("NO_DIR", dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")
dir_item = parent_dir.contents.get(dirname, None)
if dir_item is not None:
if dir_item.isdir():
if not allow_recreate:
raise FSError("DIR_EXISTS", dirname)
else:
raise ResourceNotFoundError("NO_DIR", dirname, msg="Can not create a directory, because path references a file: %(path)s")
if dir_item is None: if dir_item is None:
parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname) parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
return self return self
finally:
self._lock.release()
def _lock_dir_entry(self, path): def _lock_dir_entry(self, path):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
dir_entry.lock() try:
dir_entry = self._get_dir_entry(path)
dir_entry.lock()
finally:
self._lock.release()
def _unlock_dir_entry(self, path): def _unlock_dir_entry(self, path):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
dir_entry.unlock() try:
dir_entry = self._get_dir_entry(path)
dir_entry.unlock()
finally:
self._lock.release()
def _is_dir_locked(self, path): def _is_dir_locked(self, path):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
return dir_entry.islocked() try:
dir_entry = self._get_dir_entry(path)
return dir_entry.islocked()
finally:
self._lock.release()
def open(self, path, mode="r", **kwargs): def open(self, path, mode="r", **kwargs):
filepath, filename = pathsplit(path) self._lock.acquire()
parent_dir_entry = self._get_dir_entry(filepath) try:
filepath, filename = pathsplit(path)
if parent_dir_entry is None or not parent_dir_entry.isdir(): parent_dir_entry = self._get_dir_entry(filepath)
raise ResourceNotFoundError("NO_FILE", path)
if 'r' in mode or 'a' in mode: if parent_dir_entry is None or not parent_dir_entry.isdir():
if filename not in parent_dir_entry.contents:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
file_dir_entry = parent_dir_entry.contents[filename] if 'r' in mode or 'a' in mode:
if filename not in parent_dir_entry.contents:
raise ResourceNotFoundError("NO_FILE", path)
if 'a' in mode and file_dir_entry.islocked(): file_dir_entry = parent_dir_entry.contents[filename]
raise ResourceLockedError("FILE_LOCKED", path)
self._lock_dir_entry(path) if 'a' in mode and file_dir_entry.islocked():
mem_file = self.file_factory(path, self, file_dir_entry.data, mode) raise ResourceLockedError("FILE_LOCKED", path)
return mem_file
elif 'w' in mode: self._lock_dir_entry(path)
if filename not in parent_dir_entry.contents: mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
file_dir_entry = self._make_dir_entry("file", filename) return mem_file
parent_dir_entry.contents[filename] = file_dir_entry
else:
file_dir_entry = parent_dir_entry.contents[filename]
if file_dir_entry.islocked(): elif 'w' in mode:
raise ResourceLockedError("FILE_LOCKED", path) if filename not in parent_dir_entry.contents:
file_dir_entry = self._make_dir_entry("file", filename)
parent_dir_entry.contents[filename] = file_dir_entry
else:
file_dir_entry = parent_dir_entry.contents[filename]
self._lock_dir_entry(path) if file_dir_entry.islocked():
raise ResourceLockedError("FILE_LOCKED", path)
mem_file = self.file_factory(path, self, None, mode) self._lock_dir_entry(path)
return mem_file
if parent_dir_entry is None: mem_file = self.file_factory(path, self, None, mode)
raise ResourceNotFoundError("NO_FILE", path) return mem_file
if parent_dir_entry is None:
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def remove(self, path): def remove(self, path):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
if dir_entry is None: if dir_entry is None:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
if dir_entry.islocked(): if dir_entry.islocked():
raise ResourceLockedError("FILE_LOCKED", path) raise ResourceLockedError("FILE_LOCKED", path)
pathname, dirname = pathsplit(path) pathname, dirname = pathsplit(path)
parent_dir = self._get_dir_entry(pathname) parent_dir = self._get_dir_entry(pathname)
del parent_dir.contents[dirname] del parent_dir.contents[dirname]
finally:
self._lock.release()
def _on_close_memory_file(self, path, value): def _on_close_memory_file(self, path, value):
filepath, filename = pathsplit(path) self._lock.acquire()
dir_entry = self._get_dir_entry(path) try:
dir_entry.data = value filepath, filename = pathsplit(path)
self._unlock_dir_entry(path) dir_entry = self._get_dir_entry(path)
dir_entry.data = value
self._unlock_dir_entry(path)
finally:
self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False): def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
if dir_entry is None: try:
raise ResourceNotFoundError("NO_DIR", path) dir_entry = self._get_dir_entry(path)
paths = dir_entry.contents.keys() if dir_entry is None:
raise ResourceNotFoundError("NO_DIR", path)
paths = dir_entry.contents.keys()
return self._listdir_helper(path, paths, wildcard, full, absolute, hidden, dirs_only, files_only) return self._listdir_helper(path, paths, wildcard, full, absolute, hidden, dirs_only, files_only)
finally:
self._lock.release()
def getinfo(self, path): def getinfo(self, path):
dir_entry = self._get_dir_entry(path) self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
if dir_entry is None: if dir_entry is None:
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError("NO_RESOURCE", path)
info = {}
info['created_time'] = dir_entry.created_time
if dir_entry.isfile(): info = {}
info['size'] = len(dir_entry.data) info['created_time'] = dir_entry.created_time
return info if dir_entry.isfile():
info['size'] = len(dir_entry.data)
def ishidden(self, pathname): return info
return False finally:
self._lock.release()
......
fs/mountfs.py
#!/usr/bin/env python #!/usr/bin/env python
from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative, _synchronize
from objecttree import ObjectTree from objecttree import ObjectTree
from memoryfs import MemoryFS from memoryfs import MemoryFS
...@@ -21,8 +21,8 @@ class MountFS(FS): ...@@ -21,8 +21,8 @@ class MountFS(FS):
return {} return {}
self.info_callable = info_callable or no_info_callable self.info_callable = info_callable or no_info_callable
def __init__(self): def __init__(self):
FS.__init__(self, thread_syncronize=True)
self.mount_tree = ObjectTree() self.mount_tree = ObjectTree()
def _delegate(self, path): def _delegate(self, path):
...@@ -39,188 +39,243 @@ class MountFS(FS): ...@@ -39,188 +39,243 @@ class MountFS(FS):
return self, head_path, tail_path return self, head_path, tail_path
def desc(self, path): def desc(self, path):
fs, mount_path, delegate_path = self._delegate(path) self._lock.acquire()
if fs is self: try:
if fs.isdir(path): fs, mount_path, delegate_path = self._delegate(path)
return "Mount dir" if fs is self:
else: if fs.isdir(path):
return "Mounted file" return "Mount dir"
return "Mounted dir, maps to path %s on %s" % (delegate_path, str(fs)) else:
return "Mounted file"
return "Mounted dir, maps to path %s on %s" % (delegate_path, str(fs))
finally:
self._lock.release()
def isdir(self, path): def isdir(self, path):
fs, mount_path, delegate_path = self._delegate(path) self._lock.acquire()
if fs is None: try:
raise ResourceNotFoundError("NO_RESOURCE", path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
if fs is self: raise ResourceNotFoundError("NO_RESOURCE", path)
object = self.mount_tree.get(path, None)
return isinstance(object, dict) if fs is self:
else: object = self.mount_tree.get(path, None)
return fs.isdir(delegate_path) return isinstance(object, dict)
else:
return fs.isdir(delegate_path)
finally:
self._lock.release()
def isfile(self, path): def isfile(self, path):
fs, mount_path, delegate_path = self._delegate(path) self._lock.acquire()
if fs is None: try:
return ResourceNotFoundError("NO_RESOURCE", path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
if fs is self: return ResourceNotFoundError("NO_RESOURCE", path)
object = self.mount_tree.get(path, None)
return type(object) is MountFS.FileMount
else:
return fs.isfile(delegate_path)
if fs is self:
object = self.mount_tree.get(path, None)
return type(object) is MountFS.FileMount
else:
return fs.isfile(delegate_path)
finally:
self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False): def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
raise ResourceNotFoundError("NO_DIR", path)
if fs is self:
if files_only:
return []
paths = self.mount_tree[path].keys()
return self._listdir_helper(path,
paths,
wildcard,
full,
absolute,
hidden,
dirs_only,
files_only)
else:
paths = fs.listdir(delegate_path,
wildcard=wildcard,
full=False,
absolute=False,
hidden=hidden,
dirs_only=dirs_only,
files_only=files_only)
if full or absolute:
if full:
path = makeabsolute(path)
else:
path = makerelative(path)
paths = [pathjoin(path, p) for p in paths]
return paths self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
raise ResourceNotFoundError("NO_DIR", path)
if fs is self:
if files_only:
return []
paths = self.mount_tree[path].keys()
return self._listdir_helper(path,
paths,
wildcard,
full,
absolute,
hidden,
dirs_only,
files_only)
else:
paths = fs.listdir(delegate_path,
wildcard=wildcard,
full=False,
absolute=False,
hidden=hidden,
dirs_only=dirs_only,
files_only=files_only)
if full or absolute:
if full:
path = makeabsolute(path)
else:
path = makerelative(path)
paths = [pathjoin(path, p) for p in paths]
return paths
finally:
self._lock.release()
def open(self, path, mode="r", **kwargs): def open(self, path, mode="r", **kwargs):
path = normpath(path) self._lock.acquire()
object = self.mount_tree.get(path, None) try:
if type(object) is MountFS.FileMount: path = normpath(path)
callable = object.open_callable object = self.mount_tree.get(path, None)
return callable(path, mode, **kwargs) if type(object) is MountFS.FileMount:
callable = object.open_callable
return callable(path, mode, **kwargs)
fs, mount_path, delegate_path = self._delegate(path)
fs, mount_path, delegate_path = self._delegate(path) if fs is None:
raise ResourceNotFoundError("NO_FILE", path)
if fs is None: return fs.open(delegate_path, mode, **kwargs)
raise ResourceNotFoundError("NO_FILE", path)
return fs.open(delegate_path, mode, **kwargs) finally:
self._lock.release()
def exists(self, path): def exists(self, path):
path = normpath(path) self._lock.acquire()
fs, mount_path, delegate_path = self._delegate(path) try:
if fs is None: path = normpath(path)
return False fs, mount_path, delegate_path = self._delegate(path)
if fs is self: if fs is None:
return path in self.mount_tree return False
return fs.exists(delegate_path) if fs is self:
return path in self.mount_tree
def removedir(self, path, recursive=False): return fs.exists(delegate_path)
path = normpath(path) finally:
fs, mount_path, delegate_path = self._delegate(path) self._lock.release()
if fs is not self: def removedir(self, path, recursive=False):
return fs.removedir(delegate_path, path)
object = self.mount_tree.get(path, None) self._lock.acquire()
if object is None or not isinstance(object, dict): try:
raise ResourceNotFound("NO_DIR", path)
if not recursive and len(object): path = normpath(path)
raise OperationFailedError("REMOVEDIR_FAILED", path) fs, mount_path, delegate_path = self._delegate(path)
del self.mount_tree[delegate_path] if fs is not self:
return fs.removedir(delegate_path, path)
def rename(self, src, dst): object = self.mount_tree.get(path, None)
if object is None or not isinstance(object, dict):
raise ResourceNotFound("NO_DIR", path)
if not recursive and len(object):
raise OperationFailedError("REMOVEDIR_FAILED", path)
fs1, mount_path1, delegate_path1 = self._delegate(path) del self.mount_tree[delegate_path]
fs2, mount_path2, delegate_path2 = self._delegate(path)
if fs1 is not fs2: finally:
raise OperationFailedError("RENAME_FAILED", src) self._lock.release()
if fs1 is not self: def rename(self, src, dst):
return fs1.rename(delegate_path1, delegate_path2)
path_src = normpath(src) self._lock.acquire()
path_dst = normpath(dst) try:
fs1, mount_path1, delegate_path1 = self._delegate(path)
fs2, mount_path2, delegate_path2 = self._delegate(path)
object = self.mount_tree.get(path_src, None) if fs1 is not fs2:
object2 = self.mount_tree.get(path_dst, None) raise OperationFailedError("RENAME_FAILED", src)
if object1 is None: if fs1 is not self:
raise NoResourceError("NO_RESOURCE", src) return fs1.rename(delegate_path1, delegate_path2)
# TODO! path_src = normpath(src)
path_dst = normpath(dst)
raise UnsupportedError("UNSUPPORTED", src) object = self.mount_tree.get(path_src, None)
object2 = self.mount_tree.get(path_dst, None)
if object1 is None:
raise NoResourceError("NO_RESOURCE", src)
# TODO!
raise UnsupportedError("UNSUPPORTED", src)
finally:
self._lock.release()
def mountdir(self, path, fs): def mountdir(self, path, fs):
path = normpath(path) """Mounts a directory on a given path.
self.mount_tree[path] = MountFS.DirMount(path, fs)
path -- A path within the MountFS
fs -- A filesystem object to mount
"""
self._lock.acquire()
try:
path = normpath(path)
self.mount_tree[path] = MountFS.DirMount(path, fs)
finally:
self._lock.release()
def mountfile(self, path, open_callable=None, info_callable=None): def mountfile(self, path, open_callable=None, info_callable=None):
path = normpath(path) self._lock.acquire()
self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable) try:
path = normpath(path)
self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable)
finally:
self._lock.release()
def getinfo(self, path): def getinfo(self, path):
path = normpath(path) self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError("NO_RESOURCE", path)
if fs is self: if fs is self:
if self.isfile(path): if self.isfile(path):
return self.mount_tree[path].info_callable(path) return self.mount_tree[path].info_callable(path)
return {} return {}
return fs.getinfo(delegate_path) return fs.getinfo(delegate_path)
finally:
self._lock.release()
def getsize(self, path): def getsize(self, path):
path = normpath(path) self._lock.acquire()
fs, mount_path, delegate_path = self._delegate(path) try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
if fs is self: if fs is self:
object = self.mount_tree.get(path, None) object = self.mount_tree.get(path, None)
if object is None or isinstance(object, dict): if object is None or isinstance(object, dict):
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
size = self.mount_tree[path].info_callable(path).get("size", None) size = self.mount_tree[path].info_callable(path).get("size", None)
return size return size
return fs.getinfo(delegate_path).getsize() return fs.getinfo(delegate_path).getsize()
except:
self._lock.release()
...@@ -337,6 +392,8 @@ class MountFS(FS): ...@@ -337,6 +392,8 @@ class MountFS(FS):
if __name__ == "__main__": if __name__ == "__main__":
help(MountFS)
fs1 = MemoryFS() fs1 = MemoryFS()
fs1.makedir("Memroot/B/C/D", recursive=True) fs1.makedir("Memroot/B/C/D", recursive=True)
fs1.open("test.txt", 'w').write("Hello, World!") fs1.open("test.txt", 'w').write("Hello, World!")
......
fs/multifs.py
...@@ -5,7 +5,7 @@ from fs import FS, FSError ...@@ -5,7 +5,7 @@ from fs import FS, FSError
class MultiFS(FS): class MultiFS(FS):
def __init__(self): def __init__(self):
FS.__init__(self) FS.__init__(self, thread_syncronize=True)
self.fs_sequence = [] self.fs_sequence = []
self.fs_lookup = {} self.fs_lookup = {}
...@@ -14,23 +14,39 @@ class MultiFS(FS): ...@@ -14,23 +14,39 @@ class MultiFS(FS):
return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence) return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence)
def addfs(self, name, fs): def addfs(self, name, fs):
if name in self.fs_lookup: self._lock.acquire()
raise ValueError("Name already exists.") try:
if name in self.fs_lookup:
raise ValueError("Name already exists.")
self.fs_sequence.append(fs) self.fs_sequence.append(fs)
self.fs_lookup[name] = fs self.fs_lookup[name] = fs
finally:
self._lock.release()
def removefs(self, name): def removefs(self, name):
fs = self.fs_lookup[name] self._lock.acquire()
self.fs_sequence.remove(fs) try:
del self.fs_lookup[name] fs = self.fs_lookup[name]
self.fs_sequence.remove(fs)
del self.fs_lookup[name]
finally:
self._lock.release()
def __getitem__(self, name): def __getitem__(self, name):
return self.fs_lookup[name] self._lock.acquire()
try:
return self.fs_lookup[name]
finally:
self._lock.release()
def __iter__(self): def __iter__(self):
return iter(self.fs_sequence) self._lock.acquire()
try:
return iter(self.fs_sequence[:])
finally:
self._lock.release()
def _delegate_search(self, path): def _delegate_search(self, path):
for fs in self: for fs in self:
...@@ -39,97 +55,148 @@ class MultiFS(FS): ...@@ -39,97 +55,148 @@ class MultiFS(FS):
return None return None
def which(self, path): def which(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
for fs_name, fs_object in self.fs_lookup.iteritems(): for fs in self:
if fs is fs_object: if fs.exists(path):
return fs_name, fs for fs_name, fs_object in self.fs_lookup.iteritems():
return None, None if fs is fs_object:
return fs_name, fs
return None, None
finally:
self._lock.release()
def getsyspath(self, path): def getsyspath(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.getsyspath(path) fs = self._delegate_search(path)
if fs is not None:
return fs.getsyspath(path)
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def desc(self, path): def desc(self, path):
if not self.exists(path): self._lock.acquire()
raise FSError("NO_RESOURCE", path) try:
if not self.exists(path):
name, fs = self.which(path) raise FSError("NO_RESOURCE", path)
if name is None:
return ""
return "%s, on %s (%s)" % (fs.desc(path), name, fs)
name, fs = self.which(path)
if name is None:
return ""
return "%s, on %s (%s)" % (fs.desc(path), name, fs)
finally:
self._lock.release()
def open(self, path, mode="r", buffering=-1, **kwargs): def open(self, path, mode="r", buffering=-1, **kwargs):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
fs_file = fs.open(path, mode, buffering, **kwargs) for fs in self:
return fs_file if fs.exists(path):
fs_file = fs.open(path, mode, buffering, **kwargs)
return fs_file
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def exists(self, path): def exists(self, path):
return self._delegate_search(path) is not None self._lock.acquire()
try:
return self._delegate_search(path) is not None
finally:
self._lock.release()
def isdir(self, path): def isdir(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isdir(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isdir(path)
return False
finally:
self._lock.release()
def isfile(self, path): def isfile(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isfile(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def ishidden(self, path): def ishidden(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isfile(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def listdir(self, path="./", *args, **kwargs): def listdir(self, path="./", *args, **kwargs):
paths = [] self._lock.acquire()
for fs in self: try:
try: paths = []
paths += fs.listdir(path, *args, **kwargs) for fs in self:
except FSError, e: try:
pass paths += fs.listdir(path, *args, **kwargs)
except FSError, e:
return list(set(paths)) pass
return list(set(paths))
finally:
self._lock.release()
def remove(self, path): def remove(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
fs.remove(path) for fs in self:
return if fs.exists(path):
raise ResourceNotFoundError("NO_FILE", path) fs.remove(path)
return
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def removedir(self, path, recursive=False): def removedir(self, path, recursive=False):
for fs in self: self._lock.acquire()
if fs.isdir(path): try:
fs.removedir(path, recursive) for fs in self:
return if fs.isdir(path):
raise ResourceNotFoundError("NO_DIR", path) fs.removedir(path, recursive)
return
raise ResourceNotFoundError("NO_DIR", path)
finally:
self._lock.release()
def rename(self, src, dst): def rename(self, src, dst):
for fs in self: self._lock.acquire()
if fs.exists(src): try:
fs.rename(src, dst) for fs in self:
return if fs.exists(src):
raise FSError("NO_RESOURCE", path) fs.rename(src, dst)
return
raise FSError("NO_RESOURCE", path)
finally:
self._lock.release()
def getinfo(self, path): def getinfo(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
return fs.getinfo(path) for fs in self:
if fs.exists(path):
raise ResourceNotFoundError("NO_FILE", path) return fs.getinfo(path)
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
if __name__ == "__main__": if __name__ == "__main__":
......
fs/osfs.py
...@@ -5,6 +5,7 @@ from fs import * ...@@ -5,6 +5,7 @@ from fs import *
class OSFS(FS): class OSFS(FS):
def __init__(self, root_path): def __init__(self, root_path):
FS.__init__(self)
expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path))) expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path)))
......
fs/path.py
#!/usr/bin/env python #!/usr/bin/env python
import fs
from os import path from os import path
class Path(unicode): class Path(unicode):
def __repr__(self):
return "Path(%s)" % unicode.__repr__(self)
def __add__(self, path):
return self.__class__(unicode.__add__(self, path))
def __radd__(self, path):
return self.__class__(path.__add__(self))
def join(self, *paths): def join(self, *paths):
return Path(path.join(self, *paths)) return self.__class__(fs.pathjoin(self, *paths))
def _get_ext(self): def _get_ext(self):
return path.splitext(self)[-1] return path.splitext(self)[-1]
...@@ -13,24 +22,38 @@ class Path(unicode): ...@@ -13,24 +22,38 @@ class Path(unicode):
def _get_head(self): def _get_head(self):
head, tail = path.split(self) head, tail = path.split(self)
return Path(head) return self.__class__(head)
head = property(_get_head, None, "Retrieve the head of the path") head = property(_get_head, None, "Retrieve the head of the path")
def _get_tail(self): def _get_tail(self):
head, tail = path.split(self) head, tail = path.split(self)
return Path(tail) return self.__class__(tail)
tail = property(_get_tail, None, "Retrieve the head of the path") tail = property(_get_tail, None, "Retrieve the head of the path")
def splitall(self):
return [p for p in self.split('/') if p]
def replace(self, s1, s2):
return self.__class__(unicode.replace(self, s1, s2))
def __getitem__(self, slice):
return self.__class__(unicode.__getitem__(self, slice))
def __div__(self, pth): def __div__(self, pth):
return self.join(pth) return self.join(pth)
__truediv__ = __div__
if __name__ == "__main__": if __name__ == "__main__":
p1 = Path("a/b") p1 = Path("a/b")
p2 = p1.join("c/d.txt") p2 = p1.join("c/d.txt")
print repr(p1.replace('a', 'HAI!'))
print repr(p1[0])
print repr(p1 + p2)
print p1 / "d/e/f" print p1 / "d/e/f"
print p2 print p2
print p2.ext print p2.ext
print p2.head print p2.head
print p2.tail print p2.tail
print p2
print p2.splitall()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment