Skip to content

P
pyfs
Commit e1a641f3 by willmcgugan
Options

Added thread syncronisation to filesystem objects

parent 53b69ae5
Showing with 245 additions and 27 deletions
fs/fs.py
......@@ -5,7 +5,10 @@ import os.path
import fnmatch
from itertools import chain
import datetime
try:
import threading
except ImportError:
import dummy_threading as threadding
error_msgs = {
......@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2):
print_dir(fs, path, 0)
def _synchronize(func):
def acquire_lock(self, *args, **kwargs):
self._lock.acquire()
try:
return func(self, *args, **kwargs)
finally:
self._lock.release()
acquire_lock.__doc__ = func.__doc__
return acquire_lock
class FS(object):
"""The base class for Filesystem objects. An instance of a class derived from FS is an abstraction
......@@ -310,6 +325,13 @@ class FS(object):
"""
def __init__(self, thread_syncronize=False):
if thread_syncronize:
self._lock = threading.RLock()
else:
self._lock = None
def _resolve(self, pathname):
resolved_path = resolvepath(pathname)
return resolved_path
......
fs/memoryfs.py
......@@ -2,7 +2,9 @@
import os
import datetime
from fs import FS, pathsplit, _iteratepath, FSError, print_fs
from fs import _iteratepath
from fs import *
try:
from cStringIO import StringIO
......@@ -29,27 +31,22 @@ class MemoryFile(object):
self.mem_file.write(value)
elif _check_mode(mode, 'w'):
self.mem_file = StringIO()
elif _check_mode(mode, 'ra'):
self.mem_file = StringIO()
self.mem_file.write(value)
elif _check_mode(mode, 'r'):
self.mem_file = StringIO(value)
else:
if value is not None:
self.mem_file = StringIO(value)
else:
self.mem_file = StringIO()
assert self.mem_file is not None, "self.mem_file should have a value"
self.closed = False
def __del__(self):
......@@ -109,11 +106,8 @@ class MemoryFS(FS):
contents = {}
self.contents = contents
self.data = None
self.locks = 0
self.created_time = datetime.datetime.now()
def lock(self):
......@@ -145,6 +139,7 @@ class MemoryFS(FS):
return self.dir_entry_factory(*args, **kwargs)
def __init__(self, file_factory=None):
FS.__init__(self, thread_syncronize=True)
self.dir_entry_factory = MemoryFS.DirEntry
self.file_factory = file_factory or MemoryFile
......@@ -154,6 +149,8 @@ class MemoryFS(FS):
return "<MemoryFS>"
def _get_dir_entry(self, dirpath):
self._lock.acquire()
try:
current_dir = self.root
for path_component in _iteratepath(dirpath):
dir_entry = current_dir.contents.get(path_component, None)
......@@ -162,31 +159,51 @@ class MemoryFS(FS):
current_dir = dir_entry
return current_dir
finally:
self._lock.release()
def desc(self, path):
self._lock.acquire()
try:
if self.isdir(path):
return "Memory dir"
elif self.isfile(path):
return "Memory file object"
else:
return "No description available"
finally:
self._lock.release()
def isdir(self, path):
self._lock.acquire()
try:
dir_item = self._get_dir_entry(self._resolve(path))
if dir_item is None:
return False
return dir_item.isdir()
finally:
self._lock.release()
def isfile(self, path):
self._lock.acquire()
try:
dir_item = self._get_dir_entry(self._resolve(path))
if dir_item is None:
return False
return dir_item.isfile()
finally:
self._lock.release()
def exists(self, path):
self._lock.acquire()
try:
return self._get_dir_entry(path) is not None
finally:
self._lock.release()
def makedir(self, dirname, mode=0777, recursive=False, allow_recreate=False):
self._lock.acquire()
try:
fullpath = dirname
dirpath, dirname = pathsplit(dirname)
......@@ -238,20 +255,36 @@ class MemoryFS(FS):
parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
return self
finally:
self._lock.release()
def _lock_dir_entry(self, path):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
dir_entry.lock()
finally:
self._lock.release()
def _unlock_dir_entry(self, path):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
dir_entry.unlock()
finally:
self._lock.release()
def _is_dir_locked(self, path):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
return dir_entry.islocked()
finally:
self._lock.release()
def open(self, path, mode="r", **kwargs):
self._lock.acquire()
try:
filepath, filename = pathsplit(path)
parent_dir_entry = self._get_dir_entry(filepath)
......@@ -288,8 +321,12 @@ class MemoryFS(FS):
if parent_dir_entry is None:
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def remove(self, path):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
if dir_entry is None:
......@@ -303,22 +340,34 @@ class MemoryFS(FS):
parent_dir = self._get_dir_entry(pathname)
del parent_dir.contents[dirname]
finally:
self._lock.release()
def _on_close_memory_file(self, path, value):
self._lock.acquire()
try:
filepath, filename = pathsplit(path)
dir_entry = self._get_dir_entry(path)
dir_entry.data = value
self._unlock_dir_entry(path)
finally:
self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
if dir_entry is None:
raise ResourceNotFoundError("NO_DIR", path)
paths = dir_entry.contents.keys()
return self._listdir_helper(path, paths, wildcard, full, absolute, hidden, dirs_only, files_only)
finally:
self._lock.release()
def getinfo(self, path):
self._lock.acquire()
try:
dir_entry = self._get_dir_entry(path)
if dir_entry is None:
......@@ -331,9 +380,8 @@ class MemoryFS(FS):
info['size'] = len(dir_entry.data)
return info
def ishidden(self, pathname):
return False
finally:
self._lock.release()
......
fs/mountfs.py
#!/usr/bin/env python
from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative
from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative, _synchronize
from objecttree import ObjectTree
from memoryfs import MemoryFS
......@@ -21,8 +21,8 @@ class MountFS(FS):
return {}
self.info_callable = info_callable or no_info_callable
def __init__(self):
FS.__init__(self, thread_syncronize=True)
self.mount_tree = ObjectTree()
def _delegate(self, path):
......@@ -39,6 +39,8 @@ class MountFS(FS):
return self, head_path, tail_path
def desc(self, path):
self._lock.acquire()
try:
fs, mount_path, delegate_path = self._delegate(path)
if fs is self:
if fs.isdir(path):
......@@ -46,8 +48,12 @@ class MountFS(FS):
else:
return "Mounted file"
return "Mounted dir, maps to path %s on %s" % (delegate_path, str(fs))
finally:
self._lock.release()
def isdir(self, path):
self._lock.acquire()
try:
fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
raise ResourceNotFoundError("NO_RESOURCE", path)
......@@ -57,9 +63,13 @@ class MountFS(FS):
return isinstance(object, dict)
else:
return fs.isdir(delegate_path)
finally:
self._lock.release()
def isfile(self, path):
self._lock.acquire()
try:
fs, mount_path, delegate_path = self._delegate(path)
if fs is None:
return ResourceNotFoundError("NO_RESOURCE", path)
......@@ -69,9 +79,13 @@ class MountFS(FS):
return type(object) is MountFS.FileMount
else:
return fs.isfile(delegate_path)
finally:
self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
......@@ -107,9 +121,13 @@ class MountFS(FS):
paths = [pathjoin(path, p) for p in paths]
return paths
finally:
self._lock.release()
def open(self, path, mode="r", **kwargs):
self._lock.acquire()
try:
path = normpath(path)
object = self.mount_tree.get(path, None)
if type(object) is MountFS.FileMount:
......@@ -123,8 +141,14 @@ class MountFS(FS):
return fs.open(delegate_path, mode, **kwargs)
finally:
self._lock.release()
def exists(self, path):
self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
......@@ -136,8 +160,14 @@ class MountFS(FS):
return fs.exists(delegate_path)
finally:
self._lock.release()
def removedir(self, path, recursive=False):
self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
......@@ -153,9 +183,13 @@ class MountFS(FS):
del self.mount_tree[delegate_path]
def rename(self, src, dst):
finally:
self._lock.release()
def rename(self, src, dst):
self._lock.acquire()
try:
fs1, mount_path1, delegate_path1 = self._delegate(path)
fs2, mount_path2, delegate_path2 = self._delegate(path)
......@@ -177,20 +211,35 @@ class MountFS(FS):
# TODO!
raise UnsupportedError("UNSUPPORTED", src)
finally:
self._lock.release()
def mountdir(self, path, fs):
"""Mounts a directory on a given path.
path -- A path within the MountFS
fs -- A filesystem object to mount
def mountdir(self, path, fs):
"""
self._lock.acquire()
try:
path = normpath(path)
self.mount_tree[path] = MountFS.DirMount(path, fs)
finally:
self._lock.release()
def mountfile(self, path, open_callable=None, info_callable=None):
self._lock.acquire()
try:
path = normpath(path)
self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable)
finally:
self._lock.release()
def getinfo(self, path):
self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
......@@ -203,8 +252,12 @@ class MountFS(FS):
return self.mount_tree[path].info_callable(path)
return {}
return fs.getinfo(delegate_path)
finally:
self._lock.release()
def getsize(self, path):
self._lock.acquire()
try:
path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path)
......@@ -221,6 +274,8 @@ class MountFS(FS):
return size
return fs.getinfo(delegate_path).getsize()
except:
self._lock.release()
......@@ -337,6 +392,8 @@ class MountFS(FS):
if __name__ == "__main__":
help(MountFS)
fs1 = MemoryFS()
fs1.makedir("Memroot/B/C/D", recursive=True)
fs1.open("test.txt", 'w').write("Hello, World!")
......
fs/multifs.py
......@@ -5,7 +5,7 @@ from fs import FS, FSError
class MultiFS(FS):
def __init__(self):
FS.__init__(self)
FS.__init__(self, thread_syncronize=True)
self.fs_sequence = []
self.fs_lookup = {}
......@@ -14,23 +14,39 @@ class MultiFS(FS):
return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence)
def addfs(self, name, fs):
self._lock.acquire()
try:
if name in self.fs_lookup:
raise ValueError("Name already exists.")
self.fs_sequence.append(fs)
self.fs_lookup[name] = fs
finally:
self._lock.release()
def removefs(self, name):
self._lock.acquire()
try:
fs = self.fs_lookup[name]
self.fs_sequence.remove(fs)
del self.fs_lookup[name]
finally:
self._lock.release()
def __getitem__(self, name):
self._lock.acquire()
try:
return self.fs_lookup[name]
finally:
self._lock.release()
def __iter__(self):
return iter(self.fs_sequence)
self._lock.acquire()
try:
return iter(self.fs_sequence[:])
finally:
self._lock.release()
def _delegate_search(self, path):
for fs in self:
......@@ -39,21 +55,31 @@ class MultiFS(FS):
return None
def which(self, path):
self._lock.acquire()
try:
for fs in self:
if fs.exists(path):
for fs_name, fs_object in self.fs_lookup.iteritems():
if fs is fs_object:
return fs_name, fs
return None, None
finally:
self._lock.release()
def getsyspath(self, path):
self._lock.acquire()
try:
fs = self._delegate_search(path)
if fs is not None:
return fs.getsyspath(path)
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def desc(self, path):
self._lock.acquire()
try:
if not self.exists(path):
raise FSError("NO_RESOURCE", path)
......@@ -61,39 +87,62 @@ class MultiFS(FS):
if name is None:
return ""
return "%s, on %s (%s)" % (fs.desc(path), name, fs)
finally:
self._lock.release()
def open(self, path, mode="r", buffering=-1, **kwargs):
self._lock.acquire()
try:
for fs in self:
if fs.exists(path):
fs_file = fs.open(path, mode, buffering, **kwargs)
return fs_file
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def exists(self, path):
self._lock.acquire()
try:
return self._delegate_search(path) is not None
finally:
self._lock.release()
def isdir(self, path):
self._lock.acquire()
try:
fs = self._delegate_search(path)
if fs is not None:
return fs.isdir(path)
return False
finally:
self._lock.release()
def isfile(self, path):
self._lock.acquire()
try:
fs = self._delegate_search(path)
if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def ishidden(self, path):
self._lock.acquire()
try:
fs = self._delegate_search(path)
if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def listdir(self, path="./", *args, **kwargs):
self._lock.acquire()
try:
paths = []
for fs in self:
try:
......@@ -102,34 +151,52 @@ class MultiFS(FS):
pass
return list(set(paths))
finally:
self._lock.release()
def remove(self, path):
self._lock.acquire()
try:
for fs in self:
if fs.exists(path):
fs.remove(path)
return
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def removedir(self, path, recursive=False):
self._lock.acquire()
try:
for fs in self:
if fs.isdir(path):
fs.removedir(path, recursive)
return
raise ResourceNotFoundError("NO_DIR", path)
finally:
self._lock.release()
def rename(self, src, dst):
self._lock.acquire()
try:
for fs in self:
if fs.exists(src):
fs.rename(src, dst)
return
raise FSError("NO_RESOURCE", path)
finally:
self._lock.release()
def getinfo(self, path):
self._lock.acquire()
try:
for fs in self:
if fs.exists(path):
return fs.getinfo(path)
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
if __name__ == "__main__":
......
fs/osfs.py
......@@ -5,6 +5,7 @@ from fs import *
class OSFS(FS):
def __init__(self, root_path):
FS.__init__(self)
expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path)))
......
fs/path.py
#!/usr/bin/env python
import fs
from os import path
class Path(unicode):
def __repr__(self):
return "Path(%s)" % unicode.__repr__(self)
def __add__(self, path):
return self.__class__(unicode.__add__(self, path))
def __radd__(self, path):
return self.__class__(path.__add__(self))
def join(self, *paths):
return Path(path.join(self, *paths))
return self.__class__(fs.pathjoin(self, *paths))
def _get_ext(self):
return path.splitext(self)[-1]
......@@ -13,24 +22,38 @@ class Path(unicode):
def _get_head(self):
head, tail = path.split(self)
return Path(head)
return self.__class__(head)
head = property(_get_head, None, "Retrieve the head of the path")
def _get_tail(self):
head, tail = path.split(self)
return Path(tail)
return self.__class__(tail)
tail = property(_get_tail, None, "Retrieve the head of the path")
def splitall(self):
return [p for p in self.split('/') if p]
def replace(self, s1, s2):
return self.__class__(unicode.replace(self, s1, s2))
def __getitem__(self, slice):
return self.__class__(unicode.__getitem__(self, slice))
def __div__(self, pth):
return self.join(pth)
__truediv__ = __div__
if __name__ == "__main__":
p1 = Path("a/b")
p2 = p1.join("c/d.txt")
print repr(p1.replace('a', 'HAI!'))
print repr(p1[0])
print repr(p1 + p2)
print p1 / "d/e/f"
print p2
print p2.ext
print p2.head
print p2.tail
print p2
print p2.splitall()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment