Fixed copydir/movedir, added DeleteRootError

ChangeLog

@@ -83,3 +83,4 @@
 0.5:
 
 	* Ported to Python 3.X
+	* Added a DeleteRootError to exceptions thrown when trying to delete '/'

fs/__init__.py

@@ -18,9 +18,6 @@ implementations of this interface such as:
 __version__ = "0.4.1"
 __author__ = "Will McGugan (will@willmcgugan.com)"
 
-# No longer necessary - WM
-#from base import *
-
 #  provide these by default so people can use 'fs.path.basename' etc.
 import errors
 import path

fs/base.py

@@ -58,9 +58,9 @@ class DummyLock(object):
         pass
 
     def __enter__(self):
-        pass
+        return self
 
-    def __exit__(self, *args):
+    def __exit__(self, exc_type, exc_value, traceback):
         pass
 
 
@@ -154,7 +154,7 @@ class FS(object):
 
     _meta = {}
 
-    def __init__(self, thread_synchronize=False):
+    def __init__(self, thread_synchronize=True):
         """The base class for Filesystem objects.
 
         :param thread_synconize: If True, a lock object will be created for the object, otherwise a dummy lock will be used.
@@ -657,6 +657,7 @@ class FS(object):
 
         """
         
+        with self._lock:
             sys_path = self.getsyspath(path, allow_none=True)
             if sys_path is not None:
                 now = datetime.datetime.now()
@@ -1068,7 +1069,7 @@ class FS(object):
         :type chunk_size: bool
 
         """
-
+        with self._lock:
             if not self.isfile(src):
                 if self.isdir(src):
                     raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
@@ -1129,6 +1130,7 @@ class FS(object):
 
         """
 
+        with self._lock:
             src_syspath = self.getsyspath(src, allow_none=True)
             dst_syspath = self.getsyspath(dst, allow_none=True)
     
@@ -1169,6 +1171,7 @@ class FS(object):
         :raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
 
         """
+        with self._lock:
             if not self.isdir(src):
                 if self.isfile(src):
                     raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
@@ -1232,6 +1235,7 @@ class FS(object):
             is required (defaults to 16K)
 
         """
+        with self._lock:
             if not self.isdir(src):
                 raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
             def copyfile_noerrors(src, dst, **kwargs):
@@ -1251,7 +1255,7 @@ class FS(object):
                 raise DestinationExistsError(dst)
     
             if dst:
-            self.makedir(dst, allow_recreate=overwrite)
+                self.makedir(dst, allow_recreate=True)
     
             for dirname, filenames in self.walk(src):
     
@@ -1273,6 +1277,7 @@ class FS(object):
         :rtype: bool
 
         """
+        with self._lock:
             path = normpath(path)
             iter_dir = iter(self.listdir(path))
             try:
@@ -1292,7 +1297,7 @@ class FS(object):
         :rtype: an FS object
 
         """        
-
+        with self._lock:
             self.makedir(path, allow_recreate=True, recursive=recursive)
             dir_fs = self.opendir(path)
             return dir_fs

fs/errors.py

@@ -18,6 +18,7 @@ __all__ = ['FSError',
            'PermissionDeniedError',
            'FSClosedError',
            'OperationTimeoutError',
+           'DeleteRootError',
            'ResourceError',
            'NoSysPathError',
            'NoMetaError',
@@ -119,6 +120,10 @@ class OperationTimeoutError(OperationFailedError):
     default_message = "Unable to %(opname)s: operation timed out"
 
 
+class DeleteRootError(OperationFailedError):
+    default_message = "Can't delete root dir"
+
+
 class ResourceError(FSError):
     """Base exception class for error associated with a specific resource."""
     default_message = "Unspecified resource error: %(path)s"

fs/ftpfs.py

@@ -1299,6 +1299,8 @@ class FTPFS(FS):
             raise ResourceNotFoundError(path)
         if self.isfile(path):
             raise ResourceInvalidError(path)
+        if normpath(path) in ('', '/'):
+            raise DeleteRootError(path)
 
         if not force:
             for _checkpath in self.listdir(path):
@@ -1319,6 +1321,7 @@ class FTPFS(FS):
             pass
         if recursive:
             try:
+                if dirname(path) not in ('', '/'):
                     self.removedir(dirname(path), recursive=True)
             except DirectoryNotEmptyError:
                 pass

fs/memoryfs.py

@@ -451,6 +451,8 @@ class MemoryFS(FS):
     @synchronize
     def removedir(self, path, recursive=False, force=False):
         path = normpath(path)
+        if path in ('', '/'):
+            raise DeleteRootError(path) 
         dir_entry = self._get_dir_entry(path)
 
         if dir_entry is None:
@@ -466,10 +468,14 @@ class MemoryFS(FS):
             while rpathname:
                 rpathname, dirname = pathsplit(rpathname)
                 parent_dir = self._get_dir_entry(rpathname)
+                if not dirname:
+                    raise DeleteRootError(path)
                 del parent_dir.contents[dirname]
         else:
             pathname, dirname = pathsplit(path)
             parent_dir = self._get_dir_entry(pathname)
+            if not dirname:
+                raise DeleteRootError(path)                    
             del parent_dir.contents[dirname]
 
     @synchronize

fs/mountfs.py

@@ -331,6 +331,8 @@ class MountFS(FS):
     @synchronize
     def removedir(self, path, recursive=False, force=False):
         path = normpath(path)
+        if path in ('', '/'):
+            raise DeleteRootError(path)
         fs, _mount_path, delegate_path = self._delegate(path)
         if fs is self or fs is None:
             raise ResourceInvalidError(path, msg="Can not removedir for an un-mounted path")

fs/multifs.py

@@ -293,6 +293,8 @@ class MultiFS(FS):
     def removedir(self, path, recursive=False, force=False):
         if self.writefs is None:
             raise OperationFailedError('removedir', path=path, msg="No writeable FS set")
+        if normpath(path) in ('', '/'):
+            raise DeleteRootError(path)        
         self.writefs.removedir(path, recursive=recursive, force=force)                
 
     @synchronize

fs/osfs/__init__.py

@@ -286,13 +286,14 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
                 except ResourceNotFoundError:
                     pass
         #  Don't remove the root directory of this FS
-        if path in ("","/"):
-            return
+        if path in ('', '/'):
+            raise DeleteRootError(path)
         os.rmdir(sys_path)
         #  Using os.removedirs() for this can result in dirs being
         #  removed outside the root of this FS, so we recurse manually.
         if recursive:
             try:
+                if dirname(path) not in ('', '/'):
                     self.removedir(dirname(path),recursive=True)
             except DirectoryNotEmptyError:
                 pass

fs/osfs/watch.py

@@ -34,6 +34,8 @@ if OSFSWatchMixin is None:
 #  Fall back to raising UnsupportedError
 if OSFSWatchMixin is None:
     class OSFSWatchMixin(object):
+        def __init__(self, *args, **kwargs):
+            super(OSFSWatchMixin, self).__init__(*args, **kwargs)
         def add_watcher(self,*args,**kwds):
             raise UnsupportedError
         def del_watcher(self,watcher_or_callback):

fs/osfs/xattrs.py

@@ -23,9 +23,12 @@ except ImportError:
 
 if xattr is not None:
 
-    class OSFSXAttrMixin(FS):
+    class OSFSXAttrMixin(object):
         """Mixin providing extended-attribute support via the 'xattr' module"""
 
+        def __init__(self, *args, **kwargs):
+            super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
+
         @convert_os_errors
         def setxattr(self, path, key, value):
             xattr.xattr(self.getsyspath(path))[key]=value
@@ -53,6 +56,9 @@ else:
     class OSFSXAttrMixin(object):
         """Mixin disable extended-attribute support."""
 
+        def __init__(self, *args, **kwargs):
+            super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
+
         def getxattr(self,path,key,default=None):
             raise UnsupportedError
 

fs/s3fs.py

@@ -496,6 +496,8 @@ class S3FS(FS):
 
     def removedir(self,path,recursive=False,force=False):
         """Remove the directory at the given path."""
+        if normpath(path) in ('', '/'):
+            raise DeleteRootError(path) 
         s3path = self._s3path(path)
         if s3path != self._prefix:
             s3path = s3path + self._separator

fs/sftpfs.py

@@ -475,8 +475,8 @@ class SFTPFS(FS):
     @convert_os_errors
     def removedir(self,path,recursive=False,force=False):
         npath = self._normpath(path)
-        if path in ("","/"):
-            return
+        if normpath(path) in ('', '/'):
+            raise DeleteRootError(path)
         if force:
             for path2 in self.listdir(path,absolute=True):
                 try:

fs/tests/__init__.py

@@ -840,6 +840,8 @@ class FSTestCases(object):
             self.assertTrue(cmp_datetimes(d1, info['accessed_time']))
             self.assertTrue(cmp_datetimes(d2, info['modified_time']))
 
+    def test_removeroot(self):
+        self.assertRaises(DeleteRootError, self.fs.removedir, "/")
 
 # May be disabled - see end of file
 class ThreadingTestCases(object):

fs/tests/test_expose.py

@@ -127,6 +127,7 @@ except ImportError:
 class TestSFTPFS(TestRPCFS):
     
     __test__ = not PY3
+    __test__ = False
 
     def makeServer(self,fs,addr):
         return BaseSFTPServer(addr,fs)

fs/tests/test_utils.py

+import unittest
+
+from fs.tempfs import TempFS
+from fs.memoryfs import MemoryFS
+from fs import utils
+
+class TestUtils(unittest.TestCase):
+    
+    def _make_fs(self, fs):
+        fs.setcontents("f1", "file 1")
+        fs.setcontents("f2", "file 2")
+        fs.setcontents("f3", "file 3")
+        fs.makedir("foo/bar", recursive=True)
+        fs.setcontents("foo/bar/fruit", "apple")
+        
+    def _check_fs(self, fs):
+        self.assert_(fs.isfile("f1"))
+        self.assert_(fs.isfile("f2"))
+        self.assert_(fs.isfile("f3"))
+        self.assert_(fs.isdir("foo/bar"))
+        self.assert_(fs.isfile("foo/bar/fruit"))
+        self.assertEqual(fs.getcontents("f1", "rb"), "file 1")
+        self.assertEqual(fs.getcontents("f2", "rb"), "file 2")
+        self.assertEqual(fs.getcontents("f3", "rb"), "file 3")
+        self.assertEqual(fs.getcontents("foo/bar/fruit", "rb"), "apple")        
+        
+    def test_copydir_root(self):
+        """Test copydir from root"""
+        fs1 = MemoryFS()
+        self._make_fs(fs1)        
+        fs2 = MemoryFS()
+        utils.copydir(fs1, fs2)        
+        self._check_fs(fs2)
+                
+        fs1 = TempFS()
+        self._make_fs(fs1)        
+        fs2 = TempFS()
+        utils.copydir(fs1, fs2)        
+        self._check_fs(fs2)
+    
+    def test_copydir_indir(self):
+        """Test copydir in a directory"""        
+        fs1 = MemoryFS()
+        fs2 = MemoryFS()
+        self._make_fs(fs1)        
+        utils.copydir(fs1, (fs2, "copy"))        
+        self._check_fs(fs2.opendir("copy"))
+
+        fs1 = TempFS()
+        fs2 = TempFS()
+        self._make_fs(fs1)        
+        utils.copydir(fs1, (fs2, "copy"))        
+        self._check_fs(fs2.opendir("copy"))
+    
+    def test_movedir_indir(self):
+        """Test movedir in a directory"""        
+        fs1 = MemoryFS()
+        fs2 = MemoryFS()
+        fs1sub = fs1.makeopendir("from")
+        self._make_fs(fs1sub)            
+        utils.movedir((fs1, "from"), (fs2, "copy"))        
+        self.assert_(not fs1.exists("from"))     
+        self._check_fs(fs2.opendir("copy"))
+
+        fs1 = TempFS()
+        fs2 = TempFS()
+        fs1sub = fs1.makeopendir("from")
+        self._make_fs(fs1sub)            
+        utils.movedir((fs1, "from"), (fs2, "copy"))
+        self.assert_(not fs1.exists("from"))      
+        self._check_fs(fs2.opendir("copy"))
+        
+    def test_movedir_root(self):
+        """Test movedir to root dir"""        
+        fs1 = MemoryFS()
+        fs2 = MemoryFS()
+        fs1sub = fs1.makeopendir("from")
+        self._make_fs(fs1sub)            
+        utils.movedir((fs1, "from"), fs2)
+        self.assert_(not fs1.exists("from"))     
+        self._check_fs(fs2)
+
+        fs1 = TempFS()
+        fs2 = TempFS()
+        fs1sub = fs1.makeopendir("from")
+        self._make_fs(fs1sub)            
+        utils.movedir((fs1, "from"), fs2)
+        self.assert_(not fs1.exists("from"))        
+        self._check_fs(fs2)
+    
+if __name__ == "__main__":
+    
+    def _make_fs(fs):
+        fs.setcontents("f1", "file 1")
+        fs.setcontents("f2", "file 2")
+        fs.setcontents("f3", "file 3")
+        fs.makedir("foo/bar", recursive=True)
+        fs.setcontents("foo/bar/fruit", "apple")
+    
+    fs1 = TempFS()
+    fs2 = TempFS()
+    fs1sub = fs1.makeopendir("from")
+    _make_fs(fs1sub)            
+    utils.movedir((fs1, "from"), fs2)
+    #self.assert_(not fs1.exists("from"))        
+    #self._check_fs(fs2)
\ No newline at end of file

fs/utils.py

@@ -12,8 +12,7 @@ __all__ = ['copyfile',
            'isfile',
            'isdir',
            'find_duplicates',
-           'print_fs',
-           'wrap_file']
+           'print_fs']
 
 import os
 import sys
@@ -21,7 +20,7 @@ import stat
 
 from fs.mountfs import MountFS
 from fs.path import pathjoin
-from fs.errors import DestinationExistsError
+from fs.errors import DestinationExistsError, DeleteRootError
 from fs.base import FS
 
 def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
@@ -187,38 +186,51 @@ def movefile_non_atomic(src_fs, src_path, dst_fs, dst_path, overwrite=True, chun
             dst.close()    
 
 
-def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
+def movedir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
     """Moves contents of a directory from one filesystem to another.
 
-    :param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
+    :param fs1: A tuple of (<filesystem>, <directory path>)
     :param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
+    :param create_destination: If True, the destination will be created if it doesn't exist    
     :param ignore_errors: If True, exceptions from file moves are ignored
     :param chunk_size: Size of chunks to move if a simple copy is used
 
     """
-    if isinstance(fs1, tuple):
+    if not isinstance(fs1, tuple):
+        raise ValueError("first argument must be a tuple of (<filesystem>, <path>)")
+        
     fs1, dir1 = fs1
+    parent_fs1 = fs1
+    parent_dir1 = dir1   
     fs1 = fs1.opendir(dir1)
+
+    print fs1
+    if parent_dir1 in ('', '/'):
+        raise DeleteRootError(dir1)
+    
     if isinstance(fs2, tuple):
         fs2, dir2 = fs2
-        fs2.makedir(dir2, allow_recreate=True)
+        if create_destination:       
+            fs2.makedir(dir2, allow_recreate=True, recursive=True)
         fs2 = fs2.opendir(dir2)    
 
-    mount_fs = MountFS()
+    mount_fs = MountFS(auto_close=False)
     mount_fs.mount('src', fs1)
     mount_fs.mount('dst', fs2)
 
-    mount_fs.movedir('src', 'dst',
-                     overwrite=overwrite,
+    mount_fs.copydir('src', 'dst',
+                     overwrite=True,
                      ignore_errors=ignore_errors,
                      chunk_size=chunk_size)    
+    parent_fs1.removedir(parent_dir1, force=True)            
 
 
-def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
+def copydir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
     """Copies contents of a directory from one filesystem to another.
 
     :param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
     :param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
+    :param create_destination: If True, the destination will be created if it doesn't exist
     :param ignore_errors: If True, exceptions from file moves are ignored
     :param chunk_size: Size of chunks to move if a simple copy is used
 
@@ -228,14 +240,15 @@ def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
         fs1 = fs1.opendir(dir1)
     if isinstance(fs2, tuple):
         fs2, dir2 = fs2
-        fs2.makedir(dir2, allow_recreate=True)
+        if create_destination:
+            fs2.makedir(dir2, allow_recreate=True, recursive=True)
         fs2 = fs2.opendir(dir2)    
 
-    mount_fs = MountFS()
+    mount_fs = MountFS(auto_close=False)
     mount_fs.mount('src', fs1)
     mount_fs.mount('dst', fs2)
     mount_fs.copydir('src', 'dst',                     
-                     overwrite=overwrite,
+                     overwrite=True,
                      ignore_errors=ignore_errors,
                      chunk_size=chunk_size)
 
@@ -519,11 +532,20 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi
 
 
 if __name__ == "__main__":
-    from fs.memoryfs import MemoryFS
-    m = MemoryFS()
-    m.setcontents("test", "Hello, World!" * 10000)
-    
-    f = m.open("test")
-    print f
-    tf = wrap_file(f, "r")
-    print repr(tf.read(10))
+    from fs.tempfs import TempFS
+    t1 = TempFS()
+    t1.setcontents("foo", "test")
+    t1.makedir("bar")
+    t1.setcontents("bar/baz", "another test")
+    
+    t1.tree()
+    
+    t2 = TempFS() 
+    print t2.listdir()   
+    movedir(t1, t2)
+    
+    print t2.listdir()
+    t1.tree()
+    t2.tree()
+    
+        

fs/wrapfs/__init__.py

@@ -18,6 +18,7 @@ standard unix shell functionality of hiding dot-files in directory listings.
 import re
 import sys
 import fnmatch
+import threading
 
 from fs.base import FS, threading, synchronize, NoDefaultMeta
 from fs.errors import *
@@ -65,11 +66,11 @@ class WrapFS(FS):
     """
 
     def __init__(self, fs):
-        super(WrapFS,self).__init__()
+        super(WrapFS, self).__init__()        
         try:
             self._lock = fs._lock
         except (AttributeError,FSError):            
-            self._lock = None
+            self._lock = self._lock = threading.RLock()
         self.wrapped_fs = fs
 
     def _file_wrap(self, f, mode):

fs/wrapfs/lazyfs.py

@@ -29,7 +29,7 @@ class LazyFS(WrapFS):
     """
 
     def __init__(self, fs):
-        super(LazyFS,self).__init__(fs)
+        super(LazyFS, self).__init__(fs)
         self._lazy_creation_lock = Lock()
 
     def __unicode__(self):

fs/wrapfs/subfs.py

@@ -60,27 +60,38 @@ class SubFS(WrapFS):
     def removedir(self, path, recursive=False, force=False):
         # Careful not to recurse outside the subdir
         path = normpath(path)
-        if path in ("","/"):
-            if not force:
-                for path2 in self.listdir(path):
-                    raise DirectoryNotEmptyError(path)
-            else:
-                for path2 in self.listdir(path,absolute=True,files_only=True):
-                    try:
-                        self.remove(path2)
-                    except ResourceNotFoundError:
-                        pass
-                for path2 in self.listdir(path,absolute=True,dirs_only=True):
-                    try:
-                        self.removedir(path2,force=True)
-                    except ResourceNotFoundError:
-                        pass
-        else:
+        if path in ('', '/'):
+            raise DeleteRootError(path)
         super(SubFS,self).removedir(path,force=force)
         if recursive:
             try:
+                if dirname(path) not in ('', '/'):
                     self.removedir(dirname(path),recursive=True)
             except DirectoryNotEmptyError:
                 pass
         
+#        if path in ("","/"):
+#            if not force:
+#                for path2 in self.listdir(path):
+#                    raise DirectoryNotEmptyError(path)
+#            else:
+#                for path2 in self.listdir(path,absolute=True,files_only=True):
+#                    try:
+#                        self.remove(path2)
+#                    except ResourceNotFoundError:
+#                        pass
+#                for path2 in self.listdir(path,absolute=True,dirs_only=True):
+#                    try:
+#                        self.removedir(path2,force=True)
+#                    except ResourceNotFoundError:
+#                        pass
+#        else:
+#            super(SubFS,self).removedir(path,force=force)
+#            if recursive:
+#                try:
+#                    if dirname(path):
+#                        self.removedir(dirname(path),recursive=True)
+#                except DirectoryNotEmptyError:
+#                    pass
+