Skip to content

P
pyfs
Commit 60da3d53 by willmcgugan
Options

Fixed copydir/movedir, added DeleteRootError

parent db293aab
Showing with 231 additions and 55 deletions
ChangeLog
......@@ -83,3 +83,4 @@
0.5:
* Ported to Python 3.X
* Added a DeleteRootError to exceptions thrown when trying to delete '/'
fs/__init__.py
......@@ -18,9 +18,6 @@ implementations of this interface such as:
__version__ = "0.4.1"
__author__ = "Will McGugan (will@willmcgugan.com)"
# No longer necessary - WM
#from base import *
# provide these by default so people can use 'fs.path.basename' etc.
import errors
import path
......
fs/base.py
......@@ -58,9 +58,9 @@ class DummyLock(object):
pass
def __enter__(self):
pass
return self
def __exit__(self, *args):
def __exit__(self, exc_type, exc_value, traceback):
pass
......@@ -154,7 +154,7 @@ class FS(object):
_meta = {}
def __init__(self, thread_synchronize=False):
def __init__(self, thread_synchronize=True):
"""The base class for Filesystem objects.
:param thread_synconize: If True, a lock object will be created for the object, otherwise a dummy lock will be used.
......@@ -657,6 +657,7 @@ class FS(object):
"""
with self._lock:
sys_path = self.getsyspath(path, allow_none=True)
if sys_path is not None:
now = datetime.datetime.now()
......@@ -1068,7 +1069,7 @@ class FS(object):
:type chunk_size: bool
"""
with self._lock:
if not self.isfile(src):
if self.isdir(src):
raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
......@@ -1129,6 +1130,7 @@ class FS(object):
"""
with self._lock:
src_syspath = self.getsyspath(src, allow_none=True)
dst_syspath = self.getsyspath(dst, allow_none=True)
......@@ -1169,6 +1171,7 @@ class FS(object):
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
"""
with self._lock:
if not self.isdir(src):
if self.isfile(src):
raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
......@@ -1232,6 +1235,7 @@ class FS(object):
is required (defaults to 16K)
"""
with self._lock:
if not self.isdir(src):
raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
def copyfile_noerrors(src, dst, **kwargs):
......@@ -1251,7 +1255,7 @@ class FS(object):
raise DestinationExistsError(dst)
if dst:
self.makedir(dst, allow_recreate=overwrite)
self.makedir(dst, allow_recreate=True)
for dirname, filenames in self.walk(src):
......@@ -1273,6 +1277,7 @@ class FS(object):
:rtype: bool
"""
with self._lock:
path = normpath(path)
iter_dir = iter(self.listdir(path))
try:
......@@ -1292,7 +1297,7 @@ class FS(object):
:rtype: an FS object
"""
with self._lock:
self.makedir(path, allow_recreate=True, recursive=recursive)
dir_fs = self.opendir(path)
return dir_fs
......
fs/errors.py
......@@ -18,6 +18,7 @@ __all__ = ['FSError',
'PermissionDeniedError',
'FSClosedError',
'OperationTimeoutError',
'DeleteRootError',
'ResourceError',
'NoSysPathError',
'NoMetaError',
......@@ -119,6 +120,10 @@ class OperationTimeoutError(OperationFailedError):
default_message = "Unable to %(opname)s: operation timed out"
class DeleteRootError(OperationFailedError):
default_message = "Can't delete root dir"
class ResourceError(FSError):
"""Base exception class for error associated with a specific resource."""
default_message = "Unspecified resource error: %(path)s"
......
fs/ftpfs.py
......@@ -1299,6 +1299,8 @@ class FTPFS(FS):
raise ResourceNotFoundError(path)
if self.isfile(path):
raise ResourceInvalidError(path)
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
if not force:
for _checkpath in self.listdir(path):
......@@ -1319,6 +1321,7 @@ class FTPFS(FS):
pass
if recursive:
try:
if dirname(path) not in ('', '/'):
self.removedir(dirname(path), recursive=True)
except DirectoryNotEmptyError:
pass
......
fs/memoryfs.py
......@@ -451,6 +451,8 @@ class MemoryFS(FS):
@synchronize
def removedir(self, path, recursive=False, force=False):
path = normpath(path)
if path in ('', '/'):
raise DeleteRootError(path)
dir_entry = self._get_dir_entry(path)
if dir_entry is None:
......@@ -466,10 +468,14 @@ class MemoryFS(FS):
while rpathname:
rpathname, dirname = pathsplit(rpathname)
parent_dir = self._get_dir_entry(rpathname)
if not dirname:
raise DeleteRootError(path)
del parent_dir.contents[dirname]
else:
pathname, dirname = pathsplit(path)
parent_dir = self._get_dir_entry(pathname)
if not dirname:
raise DeleteRootError(path)
del parent_dir.contents[dirname]
@synchronize
......
fs/mountfs.py
......@@ -331,6 +331,8 @@ class MountFS(FS):
@synchronize
def removedir(self, path, recursive=False, force=False):
path = normpath(path)
if path in ('', '/'):
raise DeleteRootError(path)
fs, _mount_path, delegate_path = self._delegate(path)
if fs is self or fs is None:
raise ResourceInvalidError(path, msg="Can not removedir for an un-mounted path")
......
fs/multifs.py
......@@ -293,6 +293,8 @@ class MultiFS(FS):
def removedir(self, path, recursive=False, force=False):
if self.writefs is None:
raise OperationFailedError('removedir', path=path, msg="No writeable FS set")
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
self.writefs.removedir(path, recursive=recursive, force=force)
@synchronize
......
fs/osfs/__init__.py
......@@ -286,13 +286,14 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
except ResourceNotFoundError:
pass
# Don't remove the root directory of this FS
if path in ("","/"):
return
if path in ('', '/'):
raise DeleteRootError(path)
os.rmdir(sys_path)
# Using os.removedirs() for this can result in dirs being
# removed outside the root of this FS, so we recurse manually.
if recursive:
try:
if dirname(path) not in ('', '/'):
self.removedir(dirname(path),recursive=True)
except DirectoryNotEmptyError:
pass
......
fs/osfs/watch.py
......@@ -34,6 +34,8 @@ if OSFSWatchMixin is None:
# Fall back to raising UnsupportedError
if OSFSWatchMixin is None:
class OSFSWatchMixin(object):
def __init__(self, *args, **kwargs):
super(OSFSWatchMixin, self).__init__(*args, **kwargs)
def add_watcher(self,*args,**kwds):
raise UnsupportedError
def del_watcher(self,watcher_or_callback):
......
fs/osfs/xattrs.py
......@@ -23,9 +23,12 @@ except ImportError:
if xattr is not None:
class OSFSXAttrMixin(FS):
class OSFSXAttrMixin(object):
"""Mixin providing extended-attribute support via the 'xattr' module"""
def __init__(self, *args, **kwargs):
super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
@convert_os_errors
def setxattr(self, path, key, value):
xattr.xattr(self.getsyspath(path))[key]=value
......@@ -53,6 +56,9 @@ else:
class OSFSXAttrMixin(object):
"""Mixin disable extended-attribute support."""
def __init__(self, *args, **kwargs):
super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
def getxattr(self,path,key,default=None):
raise UnsupportedError
......
fs/s3fs.py
......@@ -496,6 +496,8 @@ class S3FS(FS):
def removedir(self,path,recursive=False,force=False):
"""Remove the directory at the given path."""
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
s3path = self._s3path(path)
if s3path != self._prefix:
s3path = s3path + self._separator
......
fs/sftpfs.py
......@@ -475,8 +475,8 @@ class SFTPFS(FS):
@convert_os_errors
def removedir(self,path,recursive=False,force=False):
npath = self._normpath(path)
if path in ("","/"):
return
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
if force:
for path2 in self.listdir(path,absolute=True):
try:
......
fs/tests/__init__.py
......@@ -840,6 +840,8 @@ class FSTestCases(object):
self.assertTrue(cmp_datetimes(d1, info['accessed_time']))
self.assertTrue(cmp_datetimes(d2, info['modified_time']))
def test_removeroot(self):
self.assertRaises(DeleteRootError, self.fs.removedir, "/")
# May be disabled - see end of file
class ThreadingTestCases(object):
......
fs/tests/test_expose.py
......@@ -127,6 +127,7 @@ except ImportError:
class TestSFTPFS(TestRPCFS):
__test__ = not PY3
__test__ = False
def makeServer(self,fs,addr):
return BaseSFTPServer(addr,fs)
......
fs/tests/test_utils.py 0 → 100644
import unittest
from fs.tempfs import TempFS
from fs.memoryfs import MemoryFS
from fs import utils
class TestUtils(unittest.TestCase):
def _make_fs(self, fs):
fs.setcontents("f1", "file 1")
fs.setcontents("f2", "file 2")
fs.setcontents("f3", "file 3")
fs.makedir("foo/bar", recursive=True)
fs.setcontents("foo/bar/fruit", "apple")
def _check_fs(self, fs):
self.assert_(fs.isfile("f1"))
self.assert_(fs.isfile("f2"))
self.assert_(fs.isfile("f3"))
self.assert_(fs.isdir("foo/bar"))
self.assert_(fs.isfile("foo/bar/fruit"))
self.assertEqual(fs.getcontents("f1", "rb"), "file 1")
self.assertEqual(fs.getcontents("f2", "rb"), "file 2")
self.assertEqual(fs.getcontents("f3", "rb"), "file 3")
self.assertEqual(fs.getcontents("foo/bar/fruit", "rb"), "apple")
def test_copydir_root(self):
"""Test copydir from root"""
fs1 = MemoryFS()
self._make_fs(fs1)
fs2 = MemoryFS()
utils.copydir(fs1, fs2)
self._check_fs(fs2)
fs1 = TempFS()
self._make_fs(fs1)
fs2 = TempFS()
utils.copydir(fs1, fs2)
self._check_fs(fs2)
def test_copydir_indir(self):
"""Test copydir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
def test_movedir_indir(self):
"""Test movedir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
def test_movedir_root(self):
"""Test movedir to root dir"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
if __name__ == "__main__":
def _make_fs(fs):
fs.setcontents("f1", "file 1")
fs.setcontents("f2", "file 2")
fs.setcontents("f3", "file 3")
fs.makedir("foo/bar", recursive=True)
fs.setcontents("foo/bar/fruit", "apple")
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
_make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
#self.assert_(not fs1.exists("from"))
#self._check_fs(fs2)
\ No newline at end of file
fs/utils.py
......@@ -12,8 +12,7 @@ __all__ = ['copyfile',
'isfile',
'isdir',
'find_duplicates',
'print_fs',
'wrap_file']
'print_fs']
import os
import sys
......@@ -21,7 +20,7 @@ import stat
from fs.mountfs import MountFS
from fs.path import pathjoin
from fs.errors import DestinationExistsError
from fs.errors import DestinationExistsError, DeleteRootError
from fs.base import FS
def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
......@@ -187,38 +186,51 @@ def movefile_non_atomic(src_fs, src_path, dst_fs, dst_path, overwrite=True, chun
dst.close()
def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
def movedir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
"""Moves contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
:param fs1: A tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used
"""
if isinstance(fs1, tuple):
if not isinstance(fs1, tuple):
raise ValueError("first argument must be a tuple of (<filesystem>, <path>)")
fs1, dir1 = fs1
parent_fs1 = fs1
parent_dir1 = dir1
fs1 = fs1.opendir(dir1)
print fs1
if parent_dir1 in ('', '/'):
raise DeleteRootError(dir1)
if isinstance(fs2, tuple):
fs2, dir2 = fs2
fs2.makedir(dir2, allow_recreate=True)
if create_destination:
fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS()
mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2)
mount_fs.movedir('src', 'dst',
overwrite=overwrite,
mount_fs.copydir('src', 'dst',
overwrite=True,
ignore_errors=ignore_errors,
chunk_size=chunk_size)
parent_fs1.removedir(parent_dir1, force=True)
def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
def copydir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
"""Copies contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used
......@@ -228,14 +240,15 @@ def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
fs1 = fs1.opendir(dir1)
if isinstance(fs2, tuple):
fs2, dir2 = fs2
fs2.makedir(dir2, allow_recreate=True)
if create_destination:
fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS()
mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2)
mount_fs.copydir('src', 'dst',
overwrite=overwrite,
overwrite=True,
ignore_errors=ignore_errors,
chunk_size=chunk_size)
......@@ -519,11 +532,20 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi
if __name__ == "__main__":
from fs.memoryfs import MemoryFS
m = MemoryFS()
m.setcontents("test", "Hello, World!" * 10000)
f = m.open("test")
print f
tf = wrap_file(f, "r")
print repr(tf.read(10))
from fs.tempfs import TempFS
t1 = TempFS()
t1.setcontents("foo", "test")
t1.makedir("bar")
t1.setcontents("bar/baz", "another test")
t1.tree()
t2 = TempFS()
print t2.listdir()
movedir(t1, t2)
print t2.listdir()
t1.tree()
t2.tree()
fs/wrapfs/__init__.py
......@@ -18,6 +18,7 @@ standard unix shell functionality of hiding dot-files in directory listings.
import re
import sys
import fnmatch
import threading
from fs.base import FS, threading, synchronize, NoDefaultMeta
from fs.errors import *
......@@ -65,11 +66,11 @@ class WrapFS(FS):
"""
def __init__(self, fs):
super(WrapFS,self).__init__()
super(WrapFS, self).__init__()
try:
self._lock = fs._lock
except (AttributeError,FSError):
self._lock = None
self._lock = self._lock = threading.RLock()
self.wrapped_fs = fs
def _file_wrap(self, f, mode):
......
fs/wrapfs/lazyfs.py
......@@ -29,7 +29,7 @@ class LazyFS(WrapFS):
"""
def __init__(self, fs):
super(LazyFS,self).__init__(fs)
super(LazyFS, self).__init__(fs)
self._lazy_creation_lock = Lock()
def __unicode__(self):
......
fs/wrapfs/subfs.py
......@@ -60,27 +60,38 @@ class SubFS(WrapFS):
def removedir(self, path, recursive=False, force=False):
# Careful not to recurse outside the subdir
path = normpath(path)
if path in ("","/"):
if not force:
for path2 in self.listdir(path):
raise DirectoryNotEmptyError(path)
else:
for path2 in self.listdir(path,absolute=True,files_only=True):
try:
self.remove(path2)
except ResourceNotFoundError:
pass
for path2 in self.listdir(path,absolute=True,dirs_only=True):
try:
self.removedir(path2,force=True)
except ResourceNotFoundError:
pass
else:
if path in ('', '/'):
raise DeleteRootError(path)
super(SubFS,self).removedir(path,force=force)
if recursive:
try:
if dirname(path) not in ('', '/'):
self.removedir(dirname(path),recursive=True)
except DirectoryNotEmptyError:
pass
# if path in ("","/"):
# if not force:
# for path2 in self.listdir(path):
# raise DirectoryNotEmptyError(path)
# else:
# for path2 in self.listdir(path,absolute=True,files_only=True):
# try:
# self.remove(path2)
# except ResourceNotFoundError:
# pass
# for path2 in self.listdir(path,absolute=True,dirs_only=True):
# try:
# self.removedir(path2,force=True)
# except ResourceNotFoundError:
# pass
# else:
# super(SubFS,self).removedir(path,force=force)
# if recursive:
# try:
# if dirname(path):
# self.removedir(dirname(path),recursive=True)
# except DirectoryNotEmptyError:
# pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment