Skip to content

P
pyfs
Commit e1a641f3 by willmcgugan
Options

Added thread syncronisation to filesystem objects

parent 53b69ae5
Showing with 193 additions and 80 deletions
fs/fs.py
...@@ -5,7 +5,10 @@ import os.path ...@@ -5,7 +5,10 @@ import os.path
import fnmatch import fnmatch
from itertools import chain from itertools import chain
import datetime import datetime
try:
import threading
except ImportError:
import dummy_threading as threadding
error_msgs = { error_msgs = {
...@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2): ...@@ -303,6 +306,18 @@ def print_fs(fs, path="/", max_levels=None, indent=' '*2):
print_dir(fs, path, 0) print_dir(fs, path, 0)
def _synchronize(func):
def acquire_lock(self, *args, **kwargs):
self._lock.acquire()
try:
return func(self, *args, **kwargs)
finally:
self._lock.release()
acquire_lock.__doc__ = func.__doc__
return acquire_lock
class FS(object): class FS(object):
"""The base class for Filesystem objects. An instance of a class derived from FS is an abstraction """The base class for Filesystem objects. An instance of a class derived from FS is an abstraction
...@@ -310,6 +325,13 @@ class FS(object): ...@@ -310,6 +325,13 @@ class FS(object):
""" """
def __init__(self, thread_syncronize=False):
if thread_syncronize:
self._lock = threading.RLock()
else:
self._lock = None
def _resolve(self, pathname): def _resolve(self, pathname):
resolved_path = resolvepath(pathname) resolved_path = resolvepath(pathname)
return resolved_path return resolved_path
......
fs/multifs.py
...@@ -5,7 +5,7 @@ from fs import FS, FSError ...@@ -5,7 +5,7 @@ from fs import FS, FSError
class MultiFS(FS): class MultiFS(FS):
def __init__(self): def __init__(self):
FS.__init__(self) FS.__init__(self, thread_syncronize=True)
self.fs_sequence = [] self.fs_sequence = []
self.fs_lookup = {} self.fs_lookup = {}
...@@ -14,23 +14,39 @@ class MultiFS(FS): ...@@ -14,23 +14,39 @@ class MultiFS(FS):
return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence) return "<MultiFS: %s>" % ", ".join(str(fs) for fs in self.fs_sequence)
def addfs(self, name, fs): def addfs(self, name, fs):
if name in self.fs_lookup: self._lock.acquire()
raise ValueError("Name already exists.") try:
if name in self.fs_lookup:
raise ValueError("Name already exists.")
self.fs_sequence.append(fs) self.fs_sequence.append(fs)
self.fs_lookup[name] = fs self.fs_lookup[name] = fs
finally:
self._lock.release()
def removefs(self, name): def removefs(self, name):
fs = self.fs_lookup[name] self._lock.acquire()
self.fs_sequence.remove(fs) try:
del self.fs_lookup[name] fs = self.fs_lookup[name]
self.fs_sequence.remove(fs)
del self.fs_lookup[name]
finally:
self._lock.release()
def __getitem__(self, name): def __getitem__(self, name):
return self.fs_lookup[name] self._lock.acquire()
try:
return self.fs_lookup[name]
finally:
self._lock.release()
def __iter__(self): def __iter__(self):
return iter(self.fs_sequence) self._lock.acquire()
try:
return iter(self.fs_sequence[:])
finally:
self._lock.release()
def _delegate_search(self, path): def _delegate_search(self, path):
for fs in self: for fs in self:
...@@ -39,97 +55,148 @@ class MultiFS(FS): ...@@ -39,97 +55,148 @@ class MultiFS(FS):
return None return None
def which(self, path): def which(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
for fs_name, fs_object in self.fs_lookup.iteritems(): for fs in self:
if fs is fs_object: if fs.exists(path):
return fs_name, fs for fs_name, fs_object in self.fs_lookup.iteritems():
return None, None if fs is fs_object:
return fs_name, fs
return None, None
finally:
self._lock.release()
def getsyspath(self, path): def getsyspath(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.getsyspath(path) fs = self._delegate_search(path)
if fs is not None:
return fs.getsyspath(path)
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def desc(self, path): def desc(self, path):
if not self.exists(path): self._lock.acquire()
raise FSError("NO_RESOURCE", path) try:
if not self.exists(path):
name, fs = self.which(path) raise FSError("NO_RESOURCE", path)
if name is None:
return ""
return "%s, on %s (%s)" % (fs.desc(path), name, fs)
name, fs = self.which(path)
if name is None:
return ""
return "%s, on %s (%s)" % (fs.desc(path), name, fs)
finally:
self._lock.release()
def open(self, path, mode="r", buffering=-1, **kwargs): def open(self, path, mode="r", buffering=-1, **kwargs):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
fs_file = fs.open(path, mode, buffering, **kwargs) for fs in self:
return fs_file if fs.exists(path):
fs_file = fs.open(path, mode, buffering, **kwargs)
return fs_file
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def exists(self, path): def exists(self, path):
return self._delegate_search(path) is not None self._lock.acquire()
try:
return self._delegate_search(path) is not None
finally:
self._lock.release()
def isdir(self, path): def isdir(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isdir(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isdir(path)
return False
finally:
self._lock.release()
def isfile(self, path): def isfile(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isfile(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def ishidden(self, path): def ishidden(self, path):
fs = self._delegate_search(path) self._lock.acquire()
if fs is not None: try:
return fs.isfile(path) fs = self._delegate_search(path)
return False if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def listdir(self, path="./", *args, **kwargs): def listdir(self, path="./", *args, **kwargs):
paths = [] self._lock.acquire()
for fs in self: try:
try: paths = []
paths += fs.listdir(path, *args, **kwargs) for fs in self:
except FSError, e: try:
pass paths += fs.listdir(path, *args, **kwargs)
except FSError, e:
return list(set(paths)) pass
return list(set(paths))
finally:
self._lock.release()
def remove(self, path): def remove(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
fs.remove(path) for fs in self:
return if fs.exists(path):
raise ResourceNotFoundError("NO_FILE", path) fs.remove(path)
return
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
def removedir(self, path, recursive=False): def removedir(self, path, recursive=False):
for fs in self: self._lock.acquire()
if fs.isdir(path): try:
fs.removedir(path, recursive) for fs in self:
return if fs.isdir(path):
raise ResourceNotFoundError("NO_DIR", path) fs.removedir(path, recursive)
return
raise ResourceNotFoundError("NO_DIR", path)
finally:
self._lock.release()
def rename(self, src, dst): def rename(self, src, dst):
for fs in self: self._lock.acquire()
if fs.exists(src): try:
fs.rename(src, dst) for fs in self:
return if fs.exists(src):
raise FSError("NO_RESOURCE", path) fs.rename(src, dst)
return
raise FSError("NO_RESOURCE", path)
finally:
self._lock.release()
def getinfo(self, path): def getinfo(self, path):
for fs in self: self._lock.acquire()
if fs.exists(path): try:
return fs.getinfo(path) for fs in self:
if fs.exists(path):
raise ResourceNotFoundError("NO_FILE", path) return fs.getinfo(path)
raise ResourceNotFoundError("NO_FILE", path)
finally:
self._lock.release()
if __name__ == "__main__": if __name__ == "__main__":
......
fs/osfs.py
...@@ -5,6 +5,7 @@ from fs import * ...@@ -5,6 +5,7 @@ from fs import *
class OSFS(FS): class OSFS(FS):
def __init__(self, root_path): def __init__(self, root_path):
FS.__init__(self)
expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path))) expanded_path = normpath(os.path.expanduser(os.path.expandvars(root_path)))
......
fs/path.py
#!/usr/bin/env python #!/usr/bin/env python
import fs
from os import path from os import path
class Path(unicode): class Path(unicode):
def __repr__(self):
return "Path(%s)" % unicode.__repr__(self)
def __add__(self, path):
return self.__class__(unicode.__add__(self, path))
def __radd__(self, path):
return self.__class__(path.__add__(self))
def join(self, *paths): def join(self, *paths):
return Path(path.join(self, *paths)) return self.__class__(fs.pathjoin(self, *paths))
def _get_ext(self): def _get_ext(self):
return path.splitext(self)[-1] return path.splitext(self)[-1]
...@@ -13,24 +22,38 @@ class Path(unicode): ...@@ -13,24 +22,38 @@ class Path(unicode):
def _get_head(self): def _get_head(self):
head, tail = path.split(self) head, tail = path.split(self)
return Path(head) return self.__class__(head)
head = property(_get_head, None, "Retrieve the head of the path") head = property(_get_head, None, "Retrieve the head of the path")
def _get_tail(self): def _get_tail(self):
head, tail = path.split(self) head, tail = path.split(self)
return Path(tail) return self.__class__(tail)
tail = property(_get_tail, None, "Retrieve the head of the path") tail = property(_get_tail, None, "Retrieve the head of the path")
def splitall(self):
return [p for p in self.split('/') if p]
def replace(self, s1, s2):
return self.__class__(unicode.replace(self, s1, s2))
def __getitem__(self, slice):
return self.__class__(unicode.__getitem__(self, slice))
def __div__(self, pth): def __div__(self, pth):
return self.join(pth) return self.join(pth)
__truediv__ = __div__
if __name__ == "__main__": if __name__ == "__main__":
p1 = Path("a/b") p1 = Path("a/b")
p2 = p1.join("c/d.txt") p2 = p1.join("c/d.txt")
print repr(p1.replace('a', 'HAI!'))
print repr(p1[0])
print repr(p1 + p2)
print p1 / "d/e/f" print p1 / "d/e/f"
print p2 print p2
print p2.ext print p2.ext
print p2.head print p2.head
print p2.tail print p2.tail
print p2
print p2.splitall()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment