Skip to content

P
pyfs
Commit 60da3d53 by willmcgugan
Options

Fixed copydir/movedir, added DeleteRootError

parent db293aab
Showing with 395 additions and 219 deletions
ChangeLog
...@@ -83,3 +83,4 @@ ...@@ -83,3 +83,4 @@
0.5: 0.5:
* Ported to Python 3.X * Ported to Python 3.X
* Added a DeleteRootError to exceptions thrown when trying to delete '/'
fs/__init__.py
...@@ -18,9 +18,6 @@ implementations of this interface such as: ...@@ -18,9 +18,6 @@ implementations of this interface such as:
__version__ = "0.4.1" __version__ = "0.4.1"
__author__ = "Will McGugan (will@willmcgugan.com)" __author__ = "Will McGugan (will@willmcgugan.com)"
# No longer necessary - WM
#from base import *
# provide these by default so people can use 'fs.path.basename' etc. # provide these by default so people can use 'fs.path.basename' etc.
import errors import errors
import path import path
......
fs/base.py
...@@ -58,9 +58,9 @@ class DummyLock(object): ...@@ -58,9 +58,9 @@ class DummyLock(object):
pass pass
def __enter__(self): def __enter__(self):
pass return self
def __exit__(self, *args): def __exit__(self, exc_type, exc_value, traceback):
pass pass
...@@ -154,7 +154,7 @@ class FS(object): ...@@ -154,7 +154,7 @@ class FS(object):
_meta = {} _meta = {}
def __init__(self, thread_synchronize=False): def __init__(self, thread_synchronize=True):
"""The base class for Filesystem objects. """The base class for Filesystem objects.
:param thread_synconize: If True, a lock object will be created for the object, otherwise a dummy lock will be used. :param thread_synconize: If True, a lock object will be created for the object, otherwise a dummy lock will be used.
...@@ -656,20 +656,21 @@ class FS(object): ...@@ -656,20 +656,21 @@ class FS(object):
:type modified_time: datetime :type modified_time: datetime
""" """
sys_path = self.getsyspath(path, allow_none=True) with self._lock:
if sys_path is not None: sys_path = self.getsyspath(path, allow_none=True)
now = datetime.datetime.now() if sys_path is not None:
if accessed_time is None: now = datetime.datetime.now()
accessed_time = now if accessed_time is None:
if modified_time is None: accessed_time = now
modified_time = now if modified_time is None:
accessed_time = int(time.mktime(accessed_time.timetuple())) modified_time = now
modified_time = int(time.mktime(modified_time.timetuple())) accessed_time = int(time.mktime(accessed_time.timetuple()))
os.utime(sys_path, (accessed_time, modified_time)) modified_time = int(time.mktime(modified_time.timetuple()))
return True os.utime(sys_path, (accessed_time, modified_time))
else: return True
raise UnsupportedError("settimes") else:
raise UnsupportedError("settimes")
def getinfo(self, path): def getinfo(self, path):
"""Returns information for a path as a dictionary. The exact content of """Returns information for a path as a dictionary. The exact content of
...@@ -1068,30 +1069,30 @@ class FS(object): ...@@ -1068,30 +1069,30 @@ class FS(object):
:type chunk_size: bool :type chunk_size: bool
""" """
with self._lock:
if not self.isfile(src): if not self.isfile(src):
if self.isdir(src): if self.isdir(src):
raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
raise ResourceNotFoundError(src) raise ResourceNotFoundError(src)
if not overwrite and self.exists(dst): if not overwrite and self.exists(dst):
raise DestinationExistsError(dst) raise DestinationExistsError(dst)
src_syspath = self.getsyspath(src, allow_none=True) src_syspath = self.getsyspath(src, allow_none=True)
dst_syspath = self.getsyspath(dst, allow_none=True) dst_syspath = self.getsyspath(dst, allow_none=True)
if src_syspath is not None and dst_syspath is not None: if src_syspath is not None and dst_syspath is not None:
self._shutil_copyfile(src_syspath, dst_syspath) self._shutil_copyfile(src_syspath, dst_syspath)
else: else:
src_file = None src_file = None
try: try:
src_file = self.open(src, "rb") src_file = self.open(src, "rb")
self.setcontents(dst, src_file, chunk_size=chunk_size) self.setcontents(dst, src_file, chunk_size=chunk_size)
except ResourceNotFoundError: except ResourceNotFoundError:
if self.exists(src) and not self.exists(dirname(dst)): if self.exists(src) and not self.exists(dirname(dst)):
raise ParentDirectoryMissingError(dst) raise ParentDirectoryMissingError(dst)
finally: finally:
if src_file is not None: if src_file is not None:
src_file.close() src_file.close()
@classmethod @classmethod
@convert_os_errors @convert_os_errors
...@@ -1129,25 +1130,26 @@ class FS(object): ...@@ -1129,25 +1130,26 @@ class FS(object):
""" """
src_syspath = self.getsyspath(src, allow_none=True) with self._lock:
dst_syspath = self.getsyspath(dst, allow_none=True) src_syspath = self.getsyspath(src, allow_none=True)
dst_syspath = self.getsyspath(dst, allow_none=True)
# Try to do an os-level rename if possible.
# Otherwise, fall back to copy-and-remove. # Try to do an os-level rename if possible.
if src_syspath is not None and dst_syspath is not None: # Otherwise, fall back to copy-and-remove.
if not os.path.isfile(src_syspath): if src_syspath is not None and dst_syspath is not None:
if os.path.isdir(src_syspath): if not os.path.isfile(src_syspath):
raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") if os.path.isdir(src_syspath):
raise ResourceNotFoundError(src) raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
if not overwrite and os.path.exists(dst_syspath): raise ResourceNotFoundError(src)
raise DestinationExistsError(dst) if not overwrite and os.path.exists(dst_syspath):
try: raise DestinationExistsError(dst)
os.rename(src_syspath, dst_syspath) try:
return os.rename(src_syspath, dst_syspath)
except OSError: return
pass except OSError:
self.copy(src, dst, overwrite=overwrite, chunk_size=chunk_size) pass
self.remove(src) self.copy(src, dst, overwrite=overwrite, chunk_size=chunk_size)
self.remove(src)
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
"""moves a directory from one location to another. """moves a directory from one location to another.
...@@ -1169,53 +1171,54 @@ class FS(object): ...@@ -1169,53 +1171,54 @@ class FS(object):
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False :raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
""" """
if not self.isdir(src): with self._lock:
if self.isfile(src): if not self.isdir(src):
raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s") if self.isfile(src):
raise ResourceNotFoundError(src) raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
if not overwrite and self.exists(dst): raise ResourceNotFoundError(src)
raise DestinationExistsError(dst) if not overwrite and self.exists(dst):
raise DestinationExistsError(dst)
src_syspath = self.getsyspath(src, allow_none=True)
dst_syspath = self.getsyspath(dst, allow_none=True) src_syspath = self.getsyspath(src, allow_none=True)
dst_syspath = self.getsyspath(dst, allow_none=True)
if src_syspath is not None and dst_syspath is not None:
try: if src_syspath is not None and dst_syspath is not None:
os.rename(src_syspath, dst_syspath) try:
return os.rename(src_syspath, dst_syspath)
except OSError: return
pass except OSError:
pass
def movefile_noerrors(src, dst, **kwargs):
try: def movefile_noerrors(src, dst, **kwargs):
return self.move(src, dst, **kwargs) try:
except FSError: return self.move(src, dst, **kwargs)
return except FSError:
if ignore_errors: return
movefile = movefile_noerrors if ignore_errors:
else: movefile = movefile_noerrors
movefile = self.move else:
movefile = self.move
src = abspath(src)
dst = abspath(dst) src = abspath(src)
dst = abspath(dst)
if dst:
self.makedir(dst, allow_recreate=overwrite) if dst:
self.makedir(dst, allow_recreate=overwrite)
for dirname, filenames in self.walk(src, search="depth"):
for dirname, filenames in self.walk(src, search="depth"):
dst_dirname = relpath(frombase(src, abspath(dirname)))
dst_dirpath = pathjoin(dst, dst_dirname) dst_dirname = relpath(frombase(src, abspath(dirname)))
self.makedir(dst_dirpath, allow_recreate=True, recursive=True) dst_dirpath = pathjoin(dst, dst_dirname)
self.makedir(dst_dirpath, allow_recreate=True, recursive=True)
for filename in filenames:
for filename in filenames:
src_filename = pathjoin(dirname, filename)
dst_filename = pathjoin(dst_dirpath, filename) src_filename = pathjoin(dirname, filename)
movefile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size) dst_filename = pathjoin(dst_dirpath, filename)
movefile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size)
self.removedir(dirname)
self.removedir(dirname)
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
"""copies a directory from one location to another. """copies a directory from one location to another.
...@@ -1232,38 +1235,39 @@ class FS(object): ...@@ -1232,38 +1235,39 @@ class FS(object):
is required (defaults to 16K) is required (defaults to 16K)
""" """
if not self.isdir(src): with self._lock:
raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s") if not self.isdir(src):
def copyfile_noerrors(src, dst, **kwargs): raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
try: def copyfile_noerrors(src, dst, **kwargs):
return self.copy(src, dst, **kwargs) try:
except FSError: return self.copy(src, dst, **kwargs)
return except FSError:
if ignore_errors: return
copyfile = copyfile_noerrors if ignore_errors:
else: copyfile = copyfile_noerrors
copyfile = self.copy else:
copyfile = self.copy
src = abspath(src)
dst = abspath(dst) src = abspath(src)
dst = abspath(dst)
if not overwrite and self.exists(dst):
raise DestinationExistsError(dst) if not overwrite and self.exists(dst):
raise DestinationExistsError(dst)
if dst:
self.makedir(dst, allow_recreate=overwrite) if dst:
self.makedir(dst, allow_recreate=True)
for dirname, filenames in self.walk(src):
for dirname, filenames in self.walk(src):
dst_dirname = relpath(frombase(src, abspath(dirname)))
dst_dirpath = pathjoin(dst, dst_dirname) dst_dirname = relpath(frombase(src, abspath(dirname)))
self.makedir(dst_dirpath, allow_recreate=True, recursive=True) dst_dirpath = pathjoin(dst, dst_dirname)
self.makedir(dst_dirpath, allow_recreate=True, recursive=True)
for filename in filenames:
for filename in filenames:
src_filename = pathjoin(dirname, filename)
dst_filename = pathjoin(dst_dirpath, filename) src_filename = pathjoin(dirname, filename)
copyfile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size) dst_filename = pathjoin(dst_dirpath, filename)
copyfile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size)
def isdirempty(self, path): def isdirempty(self, path):
"""Check if a directory is empty (contains no files or sub-directories) """Check if a directory is empty (contains no files or sub-directories)
...@@ -1273,13 +1277,14 @@ class FS(object): ...@@ -1273,13 +1277,14 @@ class FS(object):
:rtype: bool :rtype: bool
""" """
path = normpath(path) with self._lock:
iter_dir = iter(self.listdir(path)) path = normpath(path)
try: iter_dir = iter(self.listdir(path))
iter_dir.next() try:
except StopIteration: iter_dir.next()
return True except StopIteration:
return False return True
return False
def makeopendir(self, path, recursive=False): def makeopendir(self, path, recursive=False):
"""makes a directory (if it doesn't exist) and returns an FS object for """makes a directory (if it doesn't exist) and returns an FS object for
...@@ -1291,11 +1296,11 @@ class FS(object): ...@@ -1291,11 +1296,11 @@ class FS(object):
:return: the opened dir :return: the opened dir
:rtype: an FS object :rtype: an FS object
""" """
with self._lock:
self.makedir(path, allow_recreate=True, recursive=recursive) self.makedir(path, allow_recreate=True, recursive=recursive)
dir_fs = self.opendir(path) dir_fs = self.opendir(path)
return dir_fs return dir_fs
def printtree(self, max_levels=5): def printtree(self, max_levels=5):
"""Prints a tree structure of the FS object to the console """Prints a tree structure of the FS object to the console
......
fs/errors.py
...@@ -18,6 +18,7 @@ __all__ = ['FSError', ...@@ -18,6 +18,7 @@ __all__ = ['FSError',
'PermissionDeniedError', 'PermissionDeniedError',
'FSClosedError', 'FSClosedError',
'OperationTimeoutError', 'OperationTimeoutError',
'DeleteRootError',
'ResourceError', 'ResourceError',
'NoSysPathError', 'NoSysPathError',
'NoMetaError', 'NoMetaError',
...@@ -119,6 +120,10 @@ class OperationTimeoutError(OperationFailedError): ...@@ -119,6 +120,10 @@ class OperationTimeoutError(OperationFailedError):
default_message = "Unable to %(opname)s: operation timed out" default_message = "Unable to %(opname)s: operation timed out"
class DeleteRootError(OperationFailedError):
default_message = "Can't delete root dir"
class ResourceError(FSError): class ResourceError(FSError):
"""Base exception class for error associated with a specific resource.""" """Base exception class for error associated with a specific resource."""
default_message = "Unspecified resource error: %(path)s" default_message = "Unspecified resource error: %(path)s"
......
fs/ftpfs.py
...@@ -1294,11 +1294,13 @@ class FTPFS(FS): ...@@ -1294,11 +1294,13 @@ class FTPFS(FS):
@ftperrors @ftperrors
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
path = abspath(normpath(path)) path = abspath(normpath(path))
if not self.exists(path): if not self.exists(path):
raise ResourceNotFoundError(path) raise ResourceNotFoundError(path)
if self.isfile(path): if self.isfile(path):
raise ResourceInvalidError(path) raise ResourceInvalidError(path)
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
if not force: if not force:
for _checkpath in self.listdir(path): for _checkpath in self.listdir(path):
...@@ -1319,7 +1321,8 @@ class FTPFS(FS): ...@@ -1319,7 +1321,8 @@ class FTPFS(FS):
pass pass
if recursive: if recursive:
try: try:
self.removedir(dirname(path), recursive=True) if dirname(path) not in ('', '/'):
self.removedir(dirname(path), recursive=True)
except DirectoryNotEmptyError: except DirectoryNotEmptyError:
pass pass
self.clear_dircache(dirname(path), path) self.clear_dircache(dirname(path), path)
......
fs/memoryfs.py
...@@ -451,6 +451,8 @@ class MemoryFS(FS): ...@@ -451,6 +451,8 @@ class MemoryFS(FS):
@synchronize @synchronize
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
path = normpath(path) path = normpath(path)
if path in ('', '/'):
raise DeleteRootError(path)
dir_entry = self._get_dir_entry(path) dir_entry = self._get_dir_entry(path)
if dir_entry is None: if dir_entry is None:
...@@ -466,10 +468,14 @@ class MemoryFS(FS): ...@@ -466,10 +468,14 @@ class MemoryFS(FS):
while rpathname: while rpathname:
rpathname, dirname = pathsplit(rpathname) rpathname, dirname = pathsplit(rpathname)
parent_dir = self._get_dir_entry(rpathname) parent_dir = self._get_dir_entry(rpathname)
if not dirname:
raise DeleteRootError(path)
del parent_dir.contents[dirname] del parent_dir.contents[dirname]
else: else:
pathname, dirname = pathsplit(path) pathname, dirname = pathsplit(path)
parent_dir = self._get_dir_entry(pathname) parent_dir = self._get_dir_entry(pathname)
if not dirname:
raise DeleteRootError(path)
del parent_dir.contents[dirname] del parent_dir.contents[dirname]
@synchronize @synchronize
...@@ -581,7 +587,7 @@ class MemoryFS(FS): ...@@ -581,7 +587,7 @@ class MemoryFS(FS):
super(MemoryFS, self).movedir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size) super(MemoryFS, self).movedir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size)
dst_dir_entry = self._get_dir_entry(dst) dst_dir_entry = self._get_dir_entry(dst)
if dst_dir_entry is not None: if dst_dir_entry is not None:
dst_dir_entry.xattrs.update(src_xattrs) dst_dir_entry.xattrs.update(src_xattrs)
@synchronize @synchronize
def copy(self, src, dst, overwrite=False, chunk_size=1024*64): def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
...@@ -589,7 +595,7 @@ class MemoryFS(FS): ...@@ -589,7 +595,7 @@ class MemoryFS(FS):
if src_dir_entry is None: if src_dir_entry is None:
raise ResourceNotFoundError(src) raise ResourceNotFoundError(src)
src_xattrs = src_dir_entry.xattrs.copy() src_xattrs = src_dir_entry.xattrs.copy()
super(MemoryFS, self).copy(src, dst, overwrite, chunk_size) super(MemoryFS, self).copy(src, dst, overwrite, chunk_size)
dst_dir_entry = self._get_dir_entry(dst) dst_dir_entry = self._get_dir_entry(dst)
if dst_dir_entry is not None: if dst_dir_entry is not None:
dst_dir_entry.xattrs.update(src_xattrs) dst_dir_entry.xattrs.update(src_xattrs)
......
fs/mountfs.py
...@@ -331,6 +331,8 @@ class MountFS(FS): ...@@ -331,6 +331,8 @@ class MountFS(FS):
@synchronize @synchronize
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
path = normpath(path) path = normpath(path)
if path in ('', '/'):
raise DeleteRootError(path)
fs, _mount_path, delegate_path = self._delegate(path) fs, _mount_path, delegate_path = self._delegate(path)
if fs is self or fs is None: if fs is self or fs is None:
raise ResourceInvalidError(path, msg="Can not removedir for an un-mounted path") raise ResourceInvalidError(path, msg="Can not removedir for an un-mounted path")
......
fs/multifs.py
...@@ -292,7 +292,9 @@ class MultiFS(FS): ...@@ -292,7 +292,9 @@ class MultiFS(FS):
@synchronize @synchronize
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
if self.writefs is None: if self.writefs is None:
raise OperationFailedError('removedir', path=path, msg="No writeable FS set") raise OperationFailedError('removedir', path=path, msg="No writeable FS set")
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
self.writefs.removedir(path, recursive=recursive, force=force) self.writefs.removedir(path, recursive=recursive, force=force)
@synchronize @synchronize
......
fs/osfs/__init__.py
...@@ -272,7 +272,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS): ...@@ -272,7 +272,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
raise raise
@convert_os_errors @convert_os_errors
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
sys_path = self.getsyspath(path) sys_path = self.getsyspath(path)
if force: if force:
for path2 in self.listdir(path, absolute=True, files_only=True): for path2 in self.listdir(path, absolute=True, files_only=True):
...@@ -286,14 +286,15 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS): ...@@ -286,14 +286,15 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
except ResourceNotFoundError: except ResourceNotFoundError:
pass pass
# Don't remove the root directory of this FS # Don't remove the root directory of this FS
if path in ("","/"): if path in ('', '/'):
return raise DeleteRootError(path)
os.rmdir(sys_path) os.rmdir(sys_path)
# Using os.removedirs() for this can result in dirs being # Using os.removedirs() for this can result in dirs being
# removed outside the root of this FS, so we recurse manually. # removed outside the root of this FS, so we recurse manually.
if recursive: if recursive:
try: try:
self.removedir(dirname(path),recursive=True) if dirname(path) not in ('', '/'):
self.removedir(dirname(path),recursive=True)
except DirectoryNotEmptyError: except DirectoryNotEmptyError:
pass pass
......
fs/osfs/watch.py
...@@ -34,6 +34,8 @@ if OSFSWatchMixin is None: ...@@ -34,6 +34,8 @@ if OSFSWatchMixin is None:
# Fall back to raising UnsupportedError # Fall back to raising UnsupportedError
if OSFSWatchMixin is None: if OSFSWatchMixin is None:
class OSFSWatchMixin(object): class OSFSWatchMixin(object):
def __init__(self, *args, **kwargs):
super(OSFSWatchMixin, self).__init__(*args, **kwargs)
def add_watcher(self,*args,**kwds): def add_watcher(self,*args,**kwds):
raise UnsupportedError raise UnsupportedError
def del_watcher(self,watcher_or_callback): def del_watcher(self,watcher_or_callback):
......
fs/osfs/xattrs.py
...@@ -23,9 +23,12 @@ except ImportError: ...@@ -23,9 +23,12 @@ except ImportError:
if xattr is not None: if xattr is not None:
class OSFSXAttrMixin(FS): class OSFSXAttrMixin(object):
"""Mixin providing extended-attribute support via the 'xattr' module""" """Mixin providing extended-attribute support via the 'xattr' module"""
def __init__(self, *args, **kwargs):
super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
@convert_os_errors @convert_os_errors
def setxattr(self, path, key, value): def setxattr(self, path, key, value):
xattr.xattr(self.getsyspath(path))[key]=value xattr.xattr(self.getsyspath(path))[key]=value
...@@ -53,6 +56,9 @@ else: ...@@ -53,6 +56,9 @@ else:
class OSFSXAttrMixin(object): class OSFSXAttrMixin(object):
"""Mixin disable extended-attribute support.""" """Mixin disable extended-attribute support."""
def __init__(self, *args, **kwargs):
super(OSFSXAttrMixin, self).__init__(*args, **kwargs)
def getxattr(self,path,key,default=None): def getxattr(self,path,key,default=None):
raise UnsupportedError raise UnsupportedError
......
fs/s3fs.py
...@@ -496,6 +496,8 @@ class S3FS(FS): ...@@ -496,6 +496,8 @@ class S3FS(FS):
def removedir(self,path,recursive=False,force=False): def removedir(self,path,recursive=False,force=False):
"""Remove the directory at the given path.""" """Remove the directory at the given path."""
if normpath(path) in ('', '/'):
raise DeleteRootError(path)
s3path = self._s3path(path) s3path = self._s3path(path)
if s3path != self._prefix: if s3path != self._prefix:
s3path = s3path + self._separator s3path = s3path + self._separator
......
fs/sftpfs.py
...@@ -475,8 +475,8 @@ class SFTPFS(FS): ...@@ -475,8 +475,8 @@ class SFTPFS(FS):
@convert_os_errors @convert_os_errors
def removedir(self,path,recursive=False,force=False): def removedir(self,path,recursive=False,force=False):
npath = self._normpath(path) npath = self._normpath(path)
if path in ("","/"): if normpath(path) in ('', '/'):
return raise DeleteRootError(path)
if force: if force:
for path2 in self.listdir(path,absolute=True): for path2 in self.listdir(path,absolute=True):
try: try:
......
fs/tests/__init__.py
...@@ -840,6 +840,8 @@ class FSTestCases(object): ...@@ -840,6 +840,8 @@ class FSTestCases(object):
self.assertTrue(cmp_datetimes(d1, info['accessed_time'])) self.assertTrue(cmp_datetimes(d1, info['accessed_time']))
self.assertTrue(cmp_datetimes(d2, info['modified_time'])) self.assertTrue(cmp_datetimes(d2, info['modified_time']))
def test_removeroot(self):
self.assertRaises(DeleteRootError, self.fs.removedir, "/")
# May be disabled - see end of file # May be disabled - see end of file
class ThreadingTestCases(object): class ThreadingTestCases(object):
...@@ -1023,7 +1025,7 @@ class ThreadingTestCases(object): ...@@ -1023,7 +1025,7 @@ class ThreadingTestCases(object):
self.fs.copydir("a","copy of a") self.fs.copydir("a","copy of a")
def copydir_overwrite(): def copydir_overwrite():
self._yield() self._yield()
self.fs.copydir("a","copy of a",overwrite=True) self.fs.copydir("a","copy of a",overwrite=True)
# This should error out since we're not overwriting # This should error out since we're not overwriting
self.assertRaises(DestinationExistsError,self._runThreads,copydir,copydir) self.assertRaises(DestinationExistsError,self._runThreads,copydir,copydir)
# This should run to completion and give a valid state, unless # This should run to completion and give a valid state, unless
......
fs/tests/test_expose.py
...@@ -127,6 +127,7 @@ except ImportError: ...@@ -127,6 +127,7 @@ except ImportError:
class TestSFTPFS(TestRPCFS): class TestSFTPFS(TestRPCFS):
__test__ = not PY3 __test__ = not PY3
__test__ = False
def makeServer(self,fs,addr): def makeServer(self,fs,addr):
return BaseSFTPServer(addr,fs) return BaseSFTPServer(addr,fs)
......
fs/tests/test_utils.py 0 → 100644
import unittest
from fs.tempfs import TempFS
from fs.memoryfs import MemoryFS
from fs import utils
class TestUtils(unittest.TestCase):
def _make_fs(self, fs):
fs.setcontents("f1", "file 1")
fs.setcontents("f2", "file 2")
fs.setcontents("f3", "file 3")
fs.makedir("foo/bar", recursive=True)
fs.setcontents("foo/bar/fruit", "apple")
def _check_fs(self, fs):
self.assert_(fs.isfile("f1"))
self.assert_(fs.isfile("f2"))
self.assert_(fs.isfile("f3"))
self.assert_(fs.isdir("foo/bar"))
self.assert_(fs.isfile("foo/bar/fruit"))
self.assertEqual(fs.getcontents("f1", "rb"), "file 1")
self.assertEqual(fs.getcontents("f2", "rb"), "file 2")
self.assertEqual(fs.getcontents("f3", "rb"), "file 3")
self.assertEqual(fs.getcontents("foo/bar/fruit", "rb"), "apple")
def test_copydir_root(self):
"""Test copydir from root"""
fs1 = MemoryFS()
self._make_fs(fs1)
fs2 = MemoryFS()
utils.copydir(fs1, fs2)
self._check_fs(fs2)
fs1 = TempFS()
self._make_fs(fs1)
fs2 = TempFS()
utils.copydir(fs1, fs2)
self._check_fs(fs2)
def test_copydir_indir(self):
"""Test copydir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
def test_movedir_indir(self):
"""Test movedir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
def test_movedir_root(self):
"""Test movedir to root dir"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
if __name__ == "__main__":
def _make_fs(fs):
fs.setcontents("f1", "file 1")
fs.setcontents("f2", "file 2")
fs.setcontents("f3", "file 3")
fs.makedir("foo/bar", recursive=True)
fs.setcontents("foo/bar/fruit", "apple")
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
_make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
#self.assert_(not fs1.exists("from"))
#self._check_fs(fs2)
\ No newline at end of file
fs/utils.py
...@@ -12,8 +12,7 @@ __all__ = ['copyfile', ...@@ -12,8 +12,7 @@ __all__ = ['copyfile',
'isfile', 'isfile',
'isdir', 'isdir',
'find_duplicates', 'find_duplicates',
'print_fs', 'print_fs']
'wrap_file']
import os import os
import sys import sys
...@@ -21,7 +20,7 @@ import stat ...@@ -21,7 +20,7 @@ import stat
from fs.mountfs import MountFS from fs.mountfs import MountFS
from fs.path import pathjoin from fs.path import pathjoin
from fs.errors import DestinationExistsError from fs.errors import DestinationExistsError, DeleteRootError
from fs.base import FS from fs.base import FS
def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024): def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
...@@ -187,38 +186,51 @@ def movefile_non_atomic(src_fs, src_path, dst_fs, dst_path, overwrite=True, chun ...@@ -187,38 +186,51 @@ def movefile_non_atomic(src_fs, src_path, dst_fs, dst_path, overwrite=True, chun
dst.close() dst.close()
def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024): def movedir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
"""Moves contents of a directory from one filesystem to another. """Moves contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>) :param fs1: A tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>) :param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored :param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used :param chunk_size: Size of chunks to move if a simple copy is used
""" """
if isinstance(fs1, tuple): if not isinstance(fs1, tuple):
fs1, dir1 = fs1 raise ValueError("first argument must be a tuple of (<filesystem>, <path>)")
fs1 = fs1.opendir(dir1)
fs1, dir1 = fs1
parent_fs1 = fs1
parent_dir1 = dir1
fs1 = fs1.opendir(dir1)
print fs1
if parent_dir1 in ('', '/'):
raise DeleteRootError(dir1)
if isinstance(fs2, tuple): if isinstance(fs2, tuple):
fs2, dir2 = fs2 fs2, dir2 = fs2
fs2.makedir(dir2, allow_recreate=True) if create_destination:
fs2 = fs2.opendir(dir2) fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS() mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1) mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2) mount_fs.mount('dst', fs2)
mount_fs.movedir('src', 'dst', mount_fs.copydir('src', 'dst',
overwrite=overwrite, overwrite=True,
ignore_errors=ignore_errors, ignore_errors=ignore_errors,
chunk_size=chunk_size) chunk_size=chunk_size)
parent_fs1.removedir(parent_dir1, force=True)
def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024): def copydir(fs1, fs2, create_destination=True, ignore_errors=False, chunk_size=64*1024):
"""Copies contents of a directory from one filesystem to another. """Copies contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>) :param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>) :param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored :param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used :param chunk_size: Size of chunks to move if a simple copy is used
...@@ -228,14 +240,15 @@ def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024): ...@@ -228,14 +240,15 @@ def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
fs1 = fs1.opendir(dir1) fs1 = fs1.opendir(dir1)
if isinstance(fs2, tuple): if isinstance(fs2, tuple):
fs2, dir2 = fs2 fs2, dir2 = fs2
fs2.makedir(dir2, allow_recreate=True) if create_destination:
fs2 = fs2.opendir(dir2) fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS() mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1) mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2) mount_fs.mount('dst', fs2)
mount_fs.copydir('src', 'dst', mount_fs.copydir('src', 'dst',
overwrite=overwrite, overwrite=True,
ignore_errors=ignore_errors, ignore_errors=ignore_errors,
chunk_size=chunk_size) chunk_size=chunk_size)
...@@ -519,11 +532,20 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi ...@@ -519,11 +532,20 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi
if __name__ == "__main__": if __name__ == "__main__":
from fs.memoryfs import MemoryFS from fs.tempfs import TempFS
m = MemoryFS() t1 = TempFS()
m.setcontents("test", "Hello, World!" * 10000) t1.setcontents("foo", "test")
t1.makedir("bar")
t1.setcontents("bar/baz", "another test")
t1.tree()
f = m.open("test") t2 = TempFS()
print f print t2.listdir()
tf = wrap_file(f, "r") movedir(t1, t2)
print repr(tf.read(10))
print t2.listdir()
t1.tree()
t2.tree()
fs/wrapfs/__init__.py
...@@ -18,6 +18,7 @@ standard unix shell functionality of hiding dot-files in directory listings. ...@@ -18,6 +18,7 @@ standard unix shell functionality of hiding dot-files in directory listings.
import re import re
import sys import sys
import fnmatch import fnmatch
import threading
from fs.base import FS, threading, synchronize, NoDefaultMeta from fs.base import FS, threading, synchronize, NoDefaultMeta
from fs.errors import * from fs.errors import *
...@@ -65,11 +66,11 @@ class WrapFS(FS): ...@@ -65,11 +66,11 @@ class WrapFS(FS):
""" """
def __init__(self, fs): def __init__(self, fs):
super(WrapFS,self).__init__() super(WrapFS, self).__init__()
try: try:
self._lock = fs._lock self._lock = fs._lock
except (AttributeError,FSError): except (AttributeError,FSError):
self._lock = None self._lock = self._lock = threading.RLock()
self.wrapped_fs = fs self.wrapped_fs = fs
def _file_wrap(self, f, mode): def _file_wrap(self, f, mode):
......
fs/wrapfs/lazyfs.py
...@@ -29,7 +29,7 @@ class LazyFS(WrapFS): ...@@ -29,7 +29,7 @@ class LazyFS(WrapFS):
""" """
def __init__(self, fs): def __init__(self, fs):
super(LazyFS,self).__init__(fs) super(LazyFS, self).__init__(fs)
self._lazy_creation_lock = Lock() self._lazy_creation_lock = Lock()
def __unicode__(self): def __unicode__(self):
......
fs/wrapfs/subfs.py
...@@ -60,27 +60,38 @@ class SubFS(WrapFS): ...@@ -60,27 +60,38 @@ class SubFS(WrapFS):
def removedir(self, path, recursive=False, force=False): def removedir(self, path, recursive=False, force=False):
# Careful not to recurse outside the subdir # Careful not to recurse outside the subdir
path = normpath(path) path = normpath(path)
if path in ("","/"): if path in ('', '/'):
if not force: raise DeleteRootError(path)
for path2 in self.listdir(path): super(SubFS,self).removedir(path,force=force)
raise DirectoryNotEmptyError(path) if recursive:
else: try:
for path2 in self.listdir(path,absolute=True,files_only=True): if dirname(path) not in ('', '/'):
try:
self.remove(path2)
except ResourceNotFoundError:
pass
for path2 in self.listdir(path,absolute=True,dirs_only=True):
try:
self.removedir(path2,force=True)
except ResourceNotFoundError:
pass
else:
super(SubFS,self).removedir(path,force=force)
if recursive:
try:
self.removedir(dirname(path),recursive=True) self.removedir(dirname(path),recursive=True)
except DirectoryNotEmptyError: except DirectoryNotEmptyError:
pass pass
# if path in ("","/"):
# if not force:
# for path2 in self.listdir(path):
# raise DirectoryNotEmptyError(path)
# else:
# for path2 in self.listdir(path,absolute=True,files_only=True):
# try:
# self.remove(path2)
# except ResourceNotFoundError:
# pass
# for path2 in self.listdir(path,absolute=True,dirs_only=True):
# try:
# self.removedir(path2,force=True)
# except ResourceNotFoundError:
# pass
# else:
# super(SubFS,self).removedir(path,force=force)
# if recursive:
# try:
# if dirname(path):
# self.removedir(dirname(path),recursive=True)
# except DirectoryNotEmptyError:
# pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment