Added thread syncronisation to filesystem objects

fs/fs.py

@@ -5,7 +5,10 @@ import os.path
 import fnmatch
 from itertools import chain
 import datetime
-
+try:
+    import threading
+except ImportError:
+    import dummy_threading as threadding
 
 error_msgs = {
 
@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2):
     print_dir(fs, path, 0)
 
 
+def _synchronize(func):
+    def acquire_lock(self, *args, **kwargs):
+        self._lock.acquire()
+        try:
+            return func(self, *args, **kwargs)
+        finally:
+            self._lock.release()
+    acquire_lock.__doc__ = func.__doc__
+    return acquire_lock
+
+
+
 class FS(object):
 
     """The base class for Filesystem objects. An instance of a class derived from FS is an abstraction
@@ -310,6 +325,13 @@ class FS(object):
 
     """
 
+    def __init__(self, thread_syncronize=False):
+
+        if thread_syncronize:
+            self._lock = threading.RLock()
+        else:
+            self._lock = None
+
     def _resolve(self, pathname):
         resolved_path = resolvepath(pathname)
         return resolved_path

fs/memoryfs.py

@@ -2,7 +2,9 @@
 
 import os
 import datetime
-from fs import FS, pathsplit, _iteratepath, FSError, print_fs
+from fs import _iteratepath
+from fs import *
+
 
 try:
     from cStringIO import StringIO
@@ -29,27 +31,22 @@ class MemoryFile(object):
             self.mem_file.write(value)
 
         elif _check_mode(mode, 'w'):
-
             self.mem_file = StringIO()
 
         elif _check_mode(mode, 'ra'):
-
             self.mem_file = StringIO()
             self.mem_file.write(value)
 
         elif _check_mode(mode, 'r'):
-
             self.mem_file = StringIO(value)
 
         else:
-
             if value is not None:
                 self.mem_file = StringIO(value)
             else:
                 self.mem_file = StringIO()
 
         assert self.mem_file is not None, "self.mem_file should have a value"
-
         self.closed = False
 
     def __del__(self):
@@ -109,11 +106,8 @@ class MemoryFS(FS):
                 contents = {}
 
             self.contents = contents
-
             self.data = None
-
             self.locks = 0
-
             self.created_time = datetime.datetime.now()
 
         def lock(self):
@@ -145,6 +139,7 @@ class MemoryFS(FS):
         return self.dir_entry_factory(*args, **kwargs)
 
     def __init__(self, file_factory=None):
+        FS.__init__(self, thread_syncronize=True)
         self.dir_entry_factory = MemoryFS.DirEntry
         self.file_factory = file_factory or MemoryFile
 
@@ -154,6 +149,8 @@ class MemoryFS(FS):
         return "<MemoryFS>"
 
     def _get_dir_entry(self, dirpath):
+        self._lock.acquire()
+        try:
             current_dir = self.root
             for path_component in _iteratepath(dirpath):
                 dir_entry = current_dir.contents.get(path_component, None)
@@ -162,31 +159,51 @@ class MemoryFS(FS):
                 current_dir = dir_entry
 
             return current_dir
+        finally:
+            self._lock.release()
 
     def desc(self, path):
+        self._lock.acquire()
+        try:
             if self.isdir(path):
                 return "Memory dir"
             elif self.isfile(path):
                 return "Memory file object"
             else:
                 return "No description available"
+        finally:
+            self._lock.release()
 
     def isdir(self, path):
+        self._lock.acquire()
+        try:
             dir_item = self._get_dir_entry(self._resolve(path))
             if dir_item is None:
                 return False
             return dir_item.isdir()
+        finally:
+            self._lock.release()
 
     def isfile(self, path):
+        self._lock.acquire()
+        try:
             dir_item = self._get_dir_entry(self._resolve(path))
             if dir_item is None:
                 return False
             return dir_item.isfile()
+        finally:
+            self._lock.release()
 
     def exists(self, path):
+        self._lock.acquire()
+        try:
             return self._get_dir_entry(path) is not None
+        finally:
+            self._lock.release()
 
     def makedir(self, dirname, mode=0777, recursive=False, allow_recreate=False):
+        self._lock.acquire()
+        try:
             fullpath = dirname
             dirpath, dirname = pathsplit(dirname)
 
@@ -238,20 +255,36 @@ class MemoryFS(FS):
                 parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
 
             return self
+        finally:
+            self._lock.release()
 
     def _lock_dir_entry(self, path):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
             dir_entry.lock()
+        finally:
+            self._lock.release()
 
     def _unlock_dir_entry(self, path):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
             dir_entry.unlock()
+        finally:
+            self._lock.release()
 
     def _is_dir_locked(self, path):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
             return dir_entry.islocked()
+        finally:
+            self._lock.release()
 
     def open(self, path, mode="r", **kwargs):
+        self._lock.acquire()
+        try:
             filepath, filename = pathsplit(path)
             parent_dir_entry = self._get_dir_entry(filepath)
 
@@ -288,8 +321,12 @@ class MemoryFS(FS):
 
             if parent_dir_entry is None:
                 raise ResourceNotFoundError("NO_FILE", path)
+        finally:
+            self._lock.release()
 
     def remove(self, path):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
 
             if dir_entry is None:
@@ -303,22 +340,34 @@ class MemoryFS(FS):
             parent_dir = self._get_dir_entry(pathname)
 
             del parent_dir.contents[dirname]
+        finally:
+            self._lock.release()
 
     def _on_close_memory_file(self, path, value):
+        self._lock.acquire()
+        try:
             filepath, filename = pathsplit(path)
             dir_entry = self._get_dir_entry(path)
             dir_entry.data = value
             self._unlock_dir_entry(path)
+        finally:
+            self._lock.release()
 
     def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
             if dir_entry is None:
                 raise ResourceNotFoundError("NO_DIR", path)
             paths = dir_entry.contents.keys()
 
             return self._listdir_helper(path, paths, wildcard, full, absolute, hidden, dirs_only, files_only)
+        finally:
+            self._lock.release()
 
     def getinfo(self, path):
+        self._lock.acquire()
+        try:
             dir_entry = self._get_dir_entry(path)
 
             if dir_entry is None:
@@ -331,9 +380,8 @@ class MemoryFS(FS):
                 info['size'] = len(dir_entry.data)
 
             return info
-
-    def ishidden(self, pathname):
-        return False
+        finally:
+            self._lock.release()
 
 
 

fs/mountfs.py

 #!/usr/bin/env python
 
-from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative
+from fs import FS, FSError, pathjoin, pathsplit, print_fs, _iteratepath, normpath, makeabsolute, makerelative, _synchronize
 from objecttree import ObjectTree
 from memoryfs import MemoryFS
 
@@ -21,8 +21,8 @@ class MountFS(FS):
                 return {}
             self.info_callable = info_callable or no_info_callable
 
-
     def __init__(self):
+        FS.__init__(self, thread_syncronize=True)
         self.mount_tree = ObjectTree()
 
     def _delegate(self, path):
@@ -39,6 +39,8 @@ class MountFS(FS):
         return self, head_path, tail_path
 
     def desc(self, path):
+        self._lock.acquire()
+        try:
             fs, mount_path, delegate_path = self._delegate(path)
             if fs is self:
                 if fs.isdir(path):
@@ -46,8 +48,12 @@ class MountFS(FS):
                 else:
                     return "Mounted file"
             return "Mounted dir, maps to path %s on %s" % (delegate_path, str(fs))
+        finally:
+            self._lock.release()
 
     def isdir(self, path):
+        self._lock.acquire()
+        try:
             fs, mount_path, delegate_path = self._delegate(path)
             if fs is None:
                 raise ResourceNotFoundError("NO_RESOURCE", path)
@@ -57,9 +63,13 @@ class MountFS(FS):
                 return isinstance(object, dict)
             else:
                 return fs.isdir(delegate_path)
+        finally:
+            self._lock.release()
 
     def isfile(self, path):
 
+        self._lock.acquire()
+        try:
             fs, mount_path, delegate_path = self._delegate(path)
             if fs is None:
                 return ResourceNotFoundError("NO_RESOURCE", path)
@@ -69,9 +79,13 @@ class MountFS(FS):
                 return type(object) is MountFS.FileMount
             else:
                 return fs.isfile(delegate_path)
-
+        finally:
+            self._lock.release()
 
     def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=False, dirs_only=False, files_only=False):
+
+        self._lock.acquire()
+        try:
             path = normpath(path)
             fs, mount_path, delegate_path = self._delegate(path)
 
@@ -107,9 +121,13 @@ class MountFS(FS):
                     paths = [pathjoin(path, p) for p in paths]
 
                 return paths
+        finally:
+            self._lock.release()
 
     def open(self, path, mode="r", **kwargs):
 
+        self._lock.acquire()
+        try:
             path = normpath(path)
             object = self.mount_tree.get(path, None)
             if type(object) is MountFS.FileMount:
@@ -123,8 +141,14 @@ class MountFS(FS):
 
             return fs.open(delegate_path, mode, **kwargs)
 
+        finally:
+            self._lock.release()
+
     def exists(self, path):
 
+        self._lock.acquire()
+        try:
+
             path = normpath(path)
             fs, mount_path, delegate_path = self._delegate(path)
 
@@ -136,8 +160,14 @@ class MountFS(FS):
 
             return fs.exists(delegate_path)
 
+        finally:
+            self._lock.release()
+
     def removedir(self, path, recursive=False):
 
+        self._lock.acquire()
+        try:
+
             path = normpath(path)
             fs, mount_path, delegate_path = self._delegate(path)
 
@@ -153,9 +183,13 @@ class MountFS(FS):
 
             del self.mount_tree[delegate_path]
 
-    def rename(self, src, dst):
+        finally:
+            self._lock.release()
 
+    def rename(self, src, dst):
 
+        self._lock.acquire()
+        try:
             fs1, mount_path1, delegate_path1 = self._delegate(path)
             fs2, mount_path2, delegate_path2 = self._delegate(path)
 
@@ -177,20 +211,35 @@ class MountFS(FS):
             # TODO!
 
             raise UnsupportedError("UNSUPPORTED", src)
+        finally:
+            self._lock.release()
 
+    def mountdir(self, path, fs):
+        """Mounts a directory on a given path.
 
+        path -- A path within the MountFS
+        fs -- A filesystem object to mount
 
-
-    def mountdir(self, path, fs):
+        """
+        self._lock.acquire()
+        try:
             path = normpath(path)
             self.mount_tree[path] = MountFS.DirMount(path, fs)
+        finally:
+            self._lock.release()
 
     def mountfile(self, path, open_callable=None, info_callable=None):
+        self._lock.acquire()
+        try:
             path = normpath(path)
             self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable)
+        finally:
+            self._lock.release()
 
     def getinfo(self, path):
 
+        self._lock.acquire()
+        try:
             path = normpath(path)
 
             fs, mount_path, delegate_path = self._delegate(path)
@@ -203,8 +252,12 @@ class MountFS(FS):
                     return self.mount_tree[path].info_callable(path)
                 return {}
             return fs.getinfo(delegate_path)
+        finally:
+            self._lock.release()
 
     def getsize(self, path):
+        self._lock.acquire()
+        try:
             path = normpath(path)
             fs, mount_path, delegate_path = self._delegate(path)
 
@@ -221,6 +274,8 @@ class MountFS(FS):
                 return size
 
             return fs.getinfo(delegate_path).getsize()
+        except:
+            self._lock.release()
 
 
 
@@ -337,6 +392,8 @@ class MountFS(FS):
 
 if __name__ == "__main__":
 
+    help(MountFS)
+
     fs1 = MemoryFS()
     fs1.makedir("Memroot/B/C/D", recursive=True)
     fs1.open("test.txt", 'w').write("Hello, World!")

fs/multifs.py

@@ -5,7 +5,7 @@ from fs import FS, FSError
 class MultiFS(FS):
 
     def __init__(self):
-        FS.__init__(self)
+        FS.__init__(self, thread_syncronize=True)
 
         self.fs_sequence = []
         self.fs_lookup =  {}
@@ -14,23 +14,39 @@ class MultiFS(FS):
         return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence)
 
     def addfs(self, name, fs):
+        self._lock.acquire()
+        try:
             if name in self.fs_lookup:
                 raise ValueError("Name already exists.")
 
             self.fs_sequence.append(fs)
             self.fs_lookup[name] = fs
+        finally:
+            self._lock.release()
 
     def removefs(self, name):
+        self._lock.acquire()
+        try:
             fs = self.fs_lookup[name]
             self.fs_sequence.remove(fs)
             del self.fs_lookup[name]
-
+        finally:
+            self._lock.release()
 
     def __getitem__(self, name):
+        self._lock.acquire()
+        try:
             return self.fs_lookup[name]
+        finally:
+            self._lock.release()
 
     def __iter__(self):
-        return iter(self.fs_sequence)
+        self._lock.acquire()
+        try:
+            return iter(self.fs_sequence[:])
+        finally:
+            self._lock.release()
+
 
     def _delegate_search(self, path):
         for fs in self:
@@ -39,21 +55,31 @@ class MultiFS(FS):
         return None
 
     def which(self, path):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.exists(path):
                     for fs_name, fs_object in self.fs_lookup.iteritems():
                         if fs is fs_object:
                             return fs_name, fs
             return None, None
+        finally:
+            self._lock.release()
 
     def getsyspath(self, path):
+        self._lock.acquire()
+        try:
             fs = self._delegate_search(path)
             if fs is not None:
                 return fs.getsyspath(path)
 
             raise ResourceNotFoundError("NO_FILE", path)
+        finally:
+            self._lock.release()
 
     def desc(self, path):
+        self._lock.acquire()
+        try:
             if not self.exists(path):
                 raise FSError("NO_RESOURCE", path)
 
@@ -61,39 +87,62 @@ class MultiFS(FS):
             if name is None:
                 return ""
             return "%s, on %s (%s)" % (fs.desc(path), name, fs)
-
+        finally:
+            self._lock.release()
 
 
     def open(self, path, mode="r", buffering=-1, **kwargs):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.exists(path):
                     fs_file = fs.open(path, mode, buffering, **kwargs)
                     return fs_file
 
             raise ResourceNotFoundError("NO_FILE", path)
+        finally:
+            self._lock.release()
 
     def exists(self, path):
+        self._lock.acquire()
+        try:
             return self._delegate_search(path) is not None
+        finally:
+            self._lock.release()
 
     def isdir(self, path):
+        self._lock.acquire()
+        try:
             fs = self._delegate_search(path)
             if fs is not None:
                 return fs.isdir(path)
             return False
+        finally:
+            self._lock.release()
 
     def isfile(self, path):
+        self._lock.acquire()
+        try:
             fs = self._delegate_search(path)
             if fs is not None:
                 return fs.isfile(path)
             return False
+        finally:
+            self._lock.release()
 
     def ishidden(self, path):
+        self._lock.acquire()
+        try:
             fs = self._delegate_search(path)
             if fs is not None:
                 return fs.isfile(path)
             return False
+        finally:
+            self._lock.release()
 
     def listdir(self, path="./", *args, **kwargs):
+        self._lock.acquire()
+        try:
             paths = []
             for fs in self:
                 try:
@@ -102,34 +151,52 @@ class MultiFS(FS):
                     pass
 
             return list(set(paths))
+        finally:
+            self._lock.release()
 
     def remove(self, path):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.exists(path):
                     fs.remove(path)
                     return
             raise ResourceNotFoundError("NO_FILE", path)
+        finally:
+            self._lock.release()
 
     def removedir(self, path, recursive=False):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.isdir(path):
                     fs.removedir(path, recursive)
                     return
             raise ResourceNotFoundError("NO_DIR", path)
+        finally:
+            self._lock.release()
 
     def rename(self, src, dst):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.exists(src):
                     fs.rename(src, dst)
                     return
             raise FSError("NO_RESOURCE", path)
+        finally:
+            self._lock.release()
 
     def getinfo(self, path):
+        self._lock.acquire()
+        try:
             for fs in self:
                 if fs.exists(path):
                     return fs.getinfo(path)
 
             raise ResourceNotFoundError("NO_FILE", path)
+        finally:
+            self._lock.release()
 
 
 if __name__ == "__main__":

fs/osfs.py

@@ -5,6 +5,7 @@ from fs import *
 class OSFS(FS):
 
     def __init__(self, root_path):
+        FS.__init__(self)
 
         expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path)))
 

fs/path.py

 #!/usr/bin/env python
-
+import fs
 from os import path
 
 class Path(unicode):
 
+    def __repr__(self):
+        return "Path(%s)" % unicode.__repr__(self)
+
+    def __add__(self, path):
+        return self.__class__(unicode.__add__(self, path))
+
+    def __radd__(self, path):
+        return self.__class__(path.__add__(self))
+
     def join(self, *paths):
-        return Path(path.join(self, *paths))
+        return self.__class__(fs.pathjoin(self, *paths))
 
     def _get_ext(self):
         return path.splitext(self)[-1]
@@ -13,24 +22,38 @@ class Path(unicode):
 
     def _get_head(self):
         head, tail = path.split(self)
-        return Path(head)
+        return self.__class__(head)
     head = property(_get_head, None, "Retrieve the head of the path")
 
     def _get_tail(self):
         head, tail = path.split(self)
-        return Path(tail)
+        return self.__class__(tail)
     tail = property(_get_tail, None, "Retrieve the head of the path")
 
+    def splitall(self):
+        return [p for p in self.split('/') if p]
+
+    def replace(self, s1, s2):
+        return self.__class__(unicode.replace(self, s1, s2))
+
+    def __getitem__(self, slice):
+        return self.__class__(unicode.__getitem__(self, slice))
 
     def __div__(self, pth):
         return self.join(pth)
+    __truediv__ = __div__
 
 if __name__ == "__main__":
 
     p1 = Path("a/b")
     p2 = p1.join("c/d.txt")
+    print repr(p1.replace('a', 'HAI!'))
+    print repr(p1[0])
+    print repr(p1 + p2)
     print p1 / "d/e/f"
     print p2
     print p2.ext
     print p2.head
     print p2.tail
+    print p2
+    print p2.splitall()