split FS wrapper implementations into independent submodules

fs/__init__.py

@@ -15,7 +15,7 @@ implementations of this interface such as:
 
 """
 
-__version__ = "0.2.0a9"
+__version__ = "0.2.0a10"
 __author__ = "Will McGugan (will@willmcgugan.com)"
 
 #  'base' imports * from 'path' and 'errors', so their
@@ -44,4 +44,4 @@ def set_thread_synchronize_default(sync):
 import os
 SEEK_CUR = os.SEEK_CUR
 SEEK_END = os.SEEK_END
-SEEK_SET = os.SEEK_SET
\ No newline at end of file
+SEEK_SET = os.SEEK_SET

fs/remote.py

@@ -25,7 +25,8 @@ import time
 import copy
 
 from fs.base import FS, threading
-from fs.wrapfs import WrapFS, LazyFS, wrap_fs_methods
+from fs.wrapfs import WrapFS, wrap_fs_methods
+from fs.wrapfs.lazyfs import LazyFS
 from fs.path import *
 from fs.errors import *
 

fs/wrapfs.py → fs/wrapfs/__init__.py

@@ -9,9 +9,9 @@ another FS object and provide some transformation of its contents.  It could
 be very useful for implementing e.g. transparent encryption or compression
 services.
 
-As a simple example of how this class could be used, the 'HideDotFiles' class
-implements the standard unix shell functionality of hiding dot files in
-directory listings.
+For a simple example of how this class could be used, see the 'HideDotFilesFS'
+class in the module fs.wrapfs.hidedotfilesfs.  This wrapper implements the
+standard unix shell functionality of hiding dot-files in directory listings.
 
 """
 
@@ -22,6 +22,7 @@ from fs.base import FS, threading, synchronize
 from fs.errors import *
 
 def rewrite_errors(func):
+    """Re-write paths in errors raised by wrapped FS objects."""
     @wraps(func)
     def wrapper(self,*args,**kwds):
         try:
@@ -276,276 +277,3 @@ def wrap_fs_methods(decorator,cls=None,exclude=[]):
     else:
         return apply_decorator
 
-
-class LazyFS(WrapFS):
-    """Simple 'lazy initialization' for FS objects.
-
-    This FS wrapper can be created with an FS instance, an FS class, or a
-    (class,args,kwds) tuple.  The actual FS instance will be created on demand
-    the first time it is accessed.
-    """
-
-    def __init__(self,fs):
-        super(LazyFS,self).__init__(fs)
-        self._lazy_creation_lock = threading.RLock()
-
-    def __getstate__(self):
-        state = super(LazyFS,self).__getstate__()
-        del state["_lazy_creation_lock"]
-        return state
-
-    def __setstate__(self,state):
-        self.__dict__.update(state)
-        self._lazy_creation_lock = threading.RLock()
-
-    def _get_wrapped_fs(self):
-        """Obtain the wrapped FS instance, created it if necessary."""
-        try:
-            return self.__dict__["wrapped_fs"]
-        except KeyError:
-            self._lazy_creation_lock.acquire()
-            try:
-                try:
-                    return self.__dict__["wrapped_fs"]
-                except KeyError:
-                    fs = self._fsclass(*self._fsargs,**self._fskwds)
-                    self.__dict__["wrapped_fs"] = fs
-                    return fs
-            finally:
-                self._lazy_creation_lock.release()
-
-    def _set_wrapped_fs(self,fs):
-        if isinstance(fs,FS):
-            self.__dict__["wrapped_fs"] = fs
-        elif isinstance(fs,type):
-            self._fsclass = fs
-            self._fsargs = []
-            self._fskwds = {}
-        else:
-            self._fsclass = fs[0]
-            try:
-                self._fsargs = fs[1]
-            except IndexError:
-                self._fsargs = []
-            try:
-                self._fskwds = fs[2]
-            except IndexError:
-                self._fskwds = {}
-
-    wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs)
-
-
-    def close(self):
-        if not self.closed:
-            #  If it was never initialized, create a fake one to close.
-            if "wrapped_fs" not in self.__dict__:
-                self.__dict__["wrapped_fs"] = FS()
-            super(LazyFS,self).close()
-
-
-
-class HideDotFiles(WrapFS):
-    """FS wrapper class that hides dot-files in directory listings.
-
-    The listdir() function takes an extra keyword argument 'hidden'
-    indicating whether hidden dotfiles shoud be included in the output.
-    It is False by default.
-    """
-
-    def is_hidden(self,path):
-        """Check whether the given path should be hidden."""
-        return path and basename(path)[0] == "."
-
-    def _encode(self,path):
-        return path
-
-    def _decode(self,path):
-        return path
-
-    def listdir(self,path="",**kwds):
-        hidden = kwds.pop("hidden",True)
-        entries = self.wrapped_fs.listdir(path,**kwds)
-        if not hidden:
-            entries = [e for e in entries if not self.is_hidden(e)]
-        return entries
-
-    def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",hidden=False):
-        if search == "breadth":
-            dirs = [path]
-            while dirs:
-                current_path = dirs.pop()
-                paths = []
-                for filename in self.listdir(current_path,hidden=hidden):
-                    path = pathjoin(current_path, filename)
-                    if self.isdir(path):
-                        if dir_wildcard is not None:
-                            if fnmatch(path, dir_wilcard):
-                                dirs.append(path)
-                        else:
-                            dirs.append(path)
-                    else:
-                        if wildcard is not None:
-                            if fnmatch(path, wildcard):
-                                paths.append(filename)
-                        else:
-                            paths.append(filename)
-                yield (current_path, paths)
-        elif search == "depth":
-            def recurse(recurse_path):
-                for path in self.listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True,hidden=hidden):
-                    for p in recurse(path):
-                        yield p
-                yield (recurse_path, self.listdir(recurse_path, wildcard=wildcard, files_only=True,hidden=hidden))
-            for p in recurse(path):
-                yield p
-        else:
-            raise ValueError("Search should be 'breadth' or 'depth'")
-
-
-    def isdirempty(self, path):
-        path = normpath(path)
-        iter_dir = iter(self.listdir(path,hidden=True))
-        try:
-            iter_dir.next()
-        except StopIteration:
-            return True
-        return False
-
-
-class LimitSize(WrapFS):
-
-    def __init__(self,fs,max_size):
-        self.max_size = max_size
-        super(LimitSize,self).__init__(fs)
-        self.cur_size = sum(self.getsize(f) for f in self.walkfiles())
-
-    @synchronize
-    def open(self,path,mode="r"):
-        try:
-            size = self.getsize(path)
-        except ResourceNotFoundError:
-            size = 0
-        f = super(LimitSize,self).open(path,mode)
-        if "w" in mode:
-            self.cur_size -= size
-            size = 0
-        return LimitSizeFile(self,f,mode,size)
-
-    def _acquire_space(self,size):
-        new_size = self.cur_size + size
-        if new_size > self.max_size:
-            raise StorageSpaceError("write")
-        self.cur_size = new_size
-
-    #  Disable use of system-level paths, so that copy/copydir have to
-    #  fall back to manual writes and pass through _acquire_space.
-    getsyspath = FS.getsyspath
-    copy = FS.copy
-    copydir = FS.copydir
-
-    @synchronize
-    def remove(self,path):
-        size = self.getsize(path)
-        super(LimitSize,self).remove(path)
-        self.cur_size -= size
-
-    @synchronize
-    def removedir(self,path,recursive=False,force=False):
-        size = sum(self.getsize(f) for f in self.walkfiles(path))
-        super(LimitSize,self).removedir(path,recursive=recursive,force=force)
-        self.cur_size -= size
-
-
-class LimitSizeFile(object):
-
-    def __init__(self,fs,file,mode,size):
-        self._lock = fs._lock
-        self.fs = fs
-        self.file = file
-        self.mode = mode
-        self.size = size
-        self.closed = False
-
-    @synchronize
-    def write(self,data):
-        pos = self.file.tell()
-        if pos > self.size:
-            self.size = pos
-        diff = pos + len(data) - self.size
-        if diff <= 0:
-            self.file.write(data)
-        else:
-            self.fs._acquire_space(diff)
-            self.file.write(data)
-            self.size += diff
-
-    def writelines(self,lines):
-        for line in lines:
-            self.write(line)
-
-    @synchronize
-    def truncate(self,size=None):
-        pos = self.file.tell()
-        if size is None:
-            size = pos
-        self.fs._acquire_space(size - self.size)
-        self.file.truncate(size)
-        self.size = size
-
-    #  This is lifted straight from the stdlib's tempfile.py
-    def __getattr__(self,name):
-        file = self.__dict__['file']
-        a = getattr(file, name)
-        if not issubclass(type(a), type(0)):
-            setattr(self, name, a)
-        return a
-
-    def __enter__(self):
-        self.file.__enter__()
-        return self
-
-    def __exit__(self,exc,value,tb):
-        self.close()
-        return False
-
-    def __iter__(self):
-        return iter(self.file)
-
-
-class ReadOnlyFS(WrapFS):
-    """ Makes a FS object read only. Any operation that could potentially modify
-    the underlying file system will throw an UnsupportedError
-    
-    Note that this isn't a secure sandbox, untrusted code could work around the
-    read-only restrictions by getting the base class. Its main purpose is to
-    provide a degree of safety if you want to protect an FS object from
-    modification.
-    
-    """
-        
-    def getsyspath(self, path, allow_none=False):
-        """ Doesn't technically modify the filesystem but could be used to work
-        around read-only restrictions. """
-        if allow_none:
-            return None
-        raise NoSysPathError(path)
-    
-    def open(self, path, mode='r', **kwargs):
-        """ Only permit read access """
-        if 'w' in mode or 'a' in mode:
-            raise UnsupportedError('write')
-        return super(ReadOnlyFS, self).open(path, mode, **kwargs)
-        
-    def _no_can_do(self, *args, **kwargs):
-        """ Replacement method for methods that can modify the file system """
-        raise UnsupportedError('write')
-      
-    move = _no_can_do
-    movedir = _no_can_do
-    copy = _no_can_do
-    copydir = _no_can_do
-    makedir = _no_can_do
-    rename = _no_can_do
-    setxattr = _no_can_do
-    delattr = _no_can_do
-    
\ No newline at end of file

fs/wrapfs/hidedotfilesfs.py

+"""
+fs.wrapfs.hidedotfiles
+======================
+
+An FS wrapper class for hiding dot-files in directory listings.
+
+"""
+
+from fs.wrapfs import WrapFS
+
+
+class HideDotFilesFS(WrapFS):
+    """FS wrapper class that hides dot-files in directory listings.
+
+    The listdir() function takes an extra keyword argument 'hidden'
+    indicating whether hidden dotfiles shoud be included in the output.
+    It is False by default.
+    """
+
+    def is_hidden(self,path):
+        """Check whether the given path should be hidden."""
+        return path and basename(path)[0] == "."
+
+    def _encode(self,path):
+        return path
+
+    def _decode(self,path):
+        return path
+
+    def listdir(self,path="",**kwds):
+        hidden = kwds.pop("hidden",True)
+        entries = self.wrapped_fs.listdir(path,**kwds)
+        if not hidden:
+            entries = [e for e in entries if not self.is_hidden(e)]
+        return entries
+
+    def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",hidden=False):
+        if search == "breadth":
+            dirs = [path]
+            while dirs:
+                current_path = dirs.pop()
+                paths = []
+                for filename in self.listdir(current_path,hidden=hidden):
+                    path = pathjoin(current_path, filename)
+                    if self.isdir(path):
+                        if dir_wildcard is not None:
+                            if fnmatch(path, dir_wilcard):
+                                dirs.append(path)
+                        else:
+                            dirs.append(path)
+                    else:
+                        if wildcard is not None:
+                            if fnmatch(path, wildcard):
+                                paths.append(filename)
+                        else:
+                            paths.append(filename)
+                yield (current_path, paths)
+        elif search == "depth":
+            def recurse(recurse_path):
+                for path in self.listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True,hidden=hidden):
+                    for p in recurse(path):
+                        yield p
+                yield (recurse_path, self.listdir(recurse_path, wildcard=wildcard, files_only=True,hidden=hidden))
+            for p in recurse(path):
+                yield p
+        else:
+            raise ValueError("Search should be 'breadth' or 'depth'")
+
+
+    def isdirempty(self, path):
+        path = normpath(path)
+        iter_dir = iter(self.listdir(path,hidden=True))
+        try:
+            iter_dir.next()
+        except StopIteration:
+            return True
+        return False
+
+

fs/wrapfs/lazyfs.py

+"""
+fs.wrapfs.lazyfs
+================
+
+A class for lazy initialisation of an FS object.
+
+This module provides the class LazyFS, an FS wrapper class that can lazily
+initialise its underlying FS object.
+
+"""
+
+try:
+    from threading import Lock
+except ImportError:
+    from fs.base import DummyLock as Lock
+
+from fs.base import FS
+from fs.wrapfs import WrapFS
+
+
+class LazyFS(WrapFS):
+    """Simple 'lazy initialization' for FS objects.
+
+    This FS wrapper can be created with an FS instance, an FS class, or a
+    (class,args,kwds) tuple.  The actual FS instance will be created on demand
+    the first time it is accessed.
+    """
+
+    def __init__(self,fs):
+        super(LazyFS,self).__init__(fs)
+        self._lazy_creation_lock = Lock()
+
+    def __getstate__(self):
+        state = super(LazyFS,self).__getstate__()
+        del state["_lazy_creation_lock"]
+        return state
+
+    def __setstate__(self,state):
+        self.__dict__.update(state)
+        self._lazy_creation_lock = Lock()
+
+    def _get_wrapped_fs(self):
+        """Obtain the wrapped FS instance, created it if necessary."""
+        try:
+            return self.__dict__["wrapped_fs"]
+        except KeyError:
+            self._lazy_creation_lock.acquire()
+            try:
+                try:
+                    return self.__dict__["wrapped_fs"]
+                except KeyError:
+                    fs = self._fsclass(*self._fsargs,**self._fskwds)
+                    self.__dict__["wrapped_fs"] = fs
+                    return fs
+            finally:
+                self._lazy_creation_lock.release()
+
+    def _set_wrapped_fs(self,fs):
+        if isinstance(fs,FS):
+            self.__dict__["wrapped_fs"] = fs
+        elif isinstance(fs,type):
+            self._fsclass = fs
+            self._fsargs = []
+            self._fskwds = {}
+        else:
+            self._fsclass = fs[0]
+            try:
+                self._fsargs = fs[1]
+            except IndexError:
+                self._fsargs = []
+            try:
+                self._fskwds = fs[2]
+            except IndexError:
+                self._fskwds = {}
+
+    wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs)
+
+
+    def close(self):
+        if not self.closed:
+            #  If it was never initialized, create a fake one to close.
+            if "wrapped_fs" not in self.__dict__:
+                self.__dict__["wrapped_fs"] = FS()
+            super(LazyFS,self).close()
+
+

fs/wrapfs/limitsizefs.py

+"""
+fs.wrapfs.limitsizefs
+=====================
+
+An FS wrapper class for limiting the size of the underlying FS.
+
+This module provides the class LimitSizeFS, an FS wrapper that can limit the
+total size of files stored in the wrapped FS.
+
+"""
+
+from fs.wrapfs import WrapFS
+
+
+class LimitSizeFS(WrapFS):
+    """FS wrapper class to limit total size of files stored."""
+
+    def __init__(self,fs,max_size):
+        self.max_size = max_size
+        super(LimitSize,self).__init__(fs)
+        self.cur_size = sum(self.getsize(f) for f in self.walkfiles())
+
+    @synchronize
+    def open(self,path,mode="r"):
+        try:
+            size = self.getsize(path)
+        except ResourceNotFoundError:
+            size = 0
+        f = super(LimitSize,self).open(path,mode)
+        if "w" in mode:
+            self.cur_size -= size
+            size = 0
+        return LimitSizeFile(self,f,mode,size)
+
+    def _acquire_space(self,size):
+        new_size = self.cur_size + size
+        if new_size > self.max_size:
+            raise StorageSpaceError("write")
+        self.cur_size = new_size
+
+    #  Disable use of system-level paths, so that copy/copydir have to
+    #  fall back to manual writes and pass through _acquire_space.
+    getsyspath = FS.getsyspath
+    copy = FS.copy
+    copydir = FS.copydir
+
+    @synchronize
+    def remove(self,path):
+        size = self.getsize(path)
+        super(LimitSize,self).remove(path)
+        self.cur_size -= size
+
+    @synchronize
+    def removedir(self,path,recursive=False,force=False):
+        size = sum(self.getsize(f) for f in self.walkfiles(path))
+        super(LimitSize,self).removedir(path,recursive=recursive,force=force)
+        self.cur_size -= size
+
+
+class LimitSizeFile(object):
+    """Filelike wrapper class for use by LimitSizeFS."""
+
+    def __init__(self,fs,file,mode,size):
+        self._lock = fs._lock
+        self.fs = fs
+        self.file = file
+        self.mode = mode
+        self.size = size
+        self.closed = False
+
+    @synchronize
+    def write(self,data):
+        pos = self.file.tell()
+        if pos > self.size:
+            self.size = pos
+        diff = pos + len(data) - self.size
+        if diff <= 0:
+            self.file.write(data)
+        else:
+            self.fs._acquire_space(diff)
+            self.file.write(data)
+            self.size += diff
+
+    def writelines(self,lines):
+        for line in lines:
+            self.write(line)
+
+    @synchronize
+    def truncate(self,size=None):
+        pos = self.file.tell()
+        if size is None:
+            size = pos
+        self.fs._acquire_space(size - self.size)
+        self.file.truncate(size)
+        self.size = size
+
+    #  This is lifted straight from the stdlib's tempfile.py
+    def __getattr__(self,name):
+        file = self.__dict__['file']
+        a = getattr(file, name)
+        if not issubclass(type(a), type(0)):
+            setattr(self, name, a)
+        return a
+
+    def __enter__(self):
+        self.file.__enter__()
+        return self
+
+    def __exit__(self,exc,value,tb):
+        self.close()
+        return False
+
+    def __iter__(self):
+        return iter(self.file)
+

fs/wrapfs/readonlyfs.py

+"""
+fs.wrapfs.readonlyfs
+====================
+
+An FS wrapper class for blocking operations that would modify the FS.
+
+"""
+
+from fs.wrapfs import WrapFS
+
+class ReadOnlyFS(WrapFS):
+    """ Makes a FS object read only. Any operation that could potentially modify
+    the underlying file system will throw an UnsupportedError
+    
+    Note that this isn't a secure sandbox, untrusted code could work around the
+    read-only restrictions by getting the base class. Its main purpose is to
+    provide a degree of safety if you want to protect an FS object from
+    modification.
+    
+    """
+        
+    def getsyspath(self, path, allow_none=False):
+        """ Doesn't technically modify the filesystem but could be used to work
+        around read-only restrictions. """
+        if allow_none:
+            return None
+        raise NoSysPathError(path)
+    
+    def open(self, path, mode='r', **kwargs):
+        """ Only permit read access """
+        if 'w' in mode or 'a' in mode:
+            raise UnsupportedError('write')
+        return super(ReadOnlyFS, self).open(path, mode, **kwargs)
+        
+    def _no_can_do(self, *args, **kwargs):
+        """ Replacement method for methods that can modify the file system """
+        raise UnsupportedError('write')
+      
+    move = _no_can_do
+    movedir = _no_can_do
+    copy = _no_can_do
+    copydir = _no_can_do
+    makedir = _no_can_do
+    rename = _no_can_do
+    setxattr = _no_can_do
+    delattr = _no_can_do
+    

setup.py

@@ -23,6 +23,7 @@ setup(name='fs',
       url="http://code.google.com/p/pyfilesystem/",
       download_url="http://code.google.com/p/pyfilesystem/downloads/list",
       platforms = ['any'],
-      packages=['fs','fs.expose','fs.expose.fuse','fs.tests'],
+      packages=['fs','fs.expose','fs.expose.fuse','fs.tests','fs.wrapfs'],
       classifiers=classifiers,
       )
+