Test fixes

LICENSE.txt

-Copyright (c) 2009-2012, Will McGugan <will@willmcgugan.com> and contributors.
+Copyright (c) 2009-2014, Will McGugan <will@willmcgugan.com> and contributors.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-    1. Redistributions of source code must retain the above copyright notice, 
+    1. Redistributions of source code must retain the above copyright notice,
        this list of conditions and the following disclaimer.
-    
-    2. Redistributions in binary form must reproduce the above copyright 
+
+    2. Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 

fs/__init__.py

@@ -15,7 +15,7 @@ implementations of this interface such as:
 
 """
 
-__version__ = "0.4.1"
+__version__ = "0.5.0-dev"
 __author__ = "Will McGugan (will@willmcgugan.com)"
 
 #  provide these by default so people can use 'fs.path.basename' etc.

fs/errors.py

@@ -266,6 +266,10 @@ def convert_os_errors(func):
                 raise OperationFailedError(opname,details=e),None,tb
             if e.errno == errno.ENOENT:
                 raise ResourceNotFoundError(path,opname=opname,details=e),None,tb
+            if e.errno == errno.EFAULT:
+                # This can happen when listdir a directory that is deleted by another thread
+                # Best to interpret it as a resource not found
+                raise ResourceNotFoundError(path,opname=opname,details=e),None,tb
             if e.errno == errno.ESRCH:
                 raise ResourceNotFoundError(path,opname=opname,details=e),None,tb
             if e.errno == errno.ENOTEMPTY:

fs/expose/fuse/fuse3.py

 # Copyright (c) 2008 Giorgos Verigakis <verigak@gmail.com>
-# 
+#
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
 # copyright notice and this permission notice appear in all copies.
-# 
+#
 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
@@ -74,7 +74,7 @@ elif _system == 'Linux':
     c_uid_t = c_uint
     setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int)
     getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t)
-    
+
     _machine = machine()
     if _machine == 'x86_64':
         c_stat._fields_ = [
@@ -256,12 +256,12 @@ class FUSE(object):
     """This class is the lower level interface and should not be subclassed
        under normal use. Its methods are called by fuse.
        Assumes API version 2.6 or later."""
-    
+
     def __init__(self, operations, mountpoint, raw_fi=False, **kwargs):
         """Setting raw_fi to True will cause FUSE to pass the fuse_file_info
            class as is to Operations, instead of just the fh field.
            This gives you access to direct_io, keep_cache, etc."""
-        
+
         self.operations = operations
         self.raw_fi = raw_fi
         args = ['fuse']
@@ -277,7 +277,7 @@ class FUSE(object):
             for key, val in kwargs.items()))
         args.append(mountpoint)
         argv = (c_char_p * len(args))(*args)
-        
+
         fuse_ops = fuse_operations()
         for name, prototype in fuse_operations._fields_:
             if prototype != c_voidp and getattr(operations, name, None):
@@ -286,7 +286,7 @@ class FUSE(object):
         _libfuse.fuse_main_real(len(args), argv, pointer(fuse_ops),
             sizeof(fuse_ops), None)
         del self.operations     # Invoke the destructor
-    
+
     def _wrapper_(self, func, *args, **kwargs):
         """Decorator for the methods that follow"""
         try:
@@ -296,46 +296,46 @@ class FUSE(object):
         except:
             print_exc()
             return -EFAULT
-    
+
     def getattr(self, path, buf):
         return self.fgetattr(path, buf, None)
-    
+
     def readlink(self, path, buf, bufsize):
         ret = self.operations('readlink', path).encode('utf-8')
         data = create_string_buffer(ret[:bufsize - 1])
         memmove(buf, data, len(data))
         return 0
-    
+
     def mknod(self, path, mode, dev):
         return self.operations('mknod', path, mode, dev)
-    
+
     def mkdir(self, path, mode):
         return self.operations('mkdir', path, mode)
-    
+
     def unlink(self, path):
         return self.operations('unlink', path)
-    
+
     def rmdir(self, path):
         return self.operations('rmdir', path)
-    
+
     def symlink(self, source, target):
         return self.operations('symlink', target, source)
-    
+
     def rename(self, old, new):
         return self.operations('rename', old, new)
-    
+
     def link(self, source, target):
         return self.operations('link', target, source)
-    
+
     def chmod(self, path, mode):
         return self.operations('chmod', path, mode)
-    
+
     def chown(self, path, uid, gid):
         return self.operations('chown', path, uid, gid)
-    
+
     def truncate(self, path, length):
         return self.operations('truncate', path, length)
-    
+
     def open(self, path, fip):
         fi = fip.contents
         if self.raw_fi:
@@ -343,7 +343,7 @@ class FUSE(object):
         else:
             fi.fh = self.operations('open', path, fi.flags)
             return 0
-    
+
     def read(self, path, buf, size, offset, fip):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         ret = self.operations('read', path, size, offset, fh)
@@ -352,12 +352,12 @@ class FUSE(object):
         data = create_string_buffer(ret[:size], size)
         memmove(buf, data, size)
         return size
-    
+
     def write(self, path, buf, size, offset, fip):
         data = string_at(buf, size)
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('write', path, data, offset, fh)
-    
+
     def statfs(self, path, buf):
         stv = buf.contents
         attrs = self.operations('statfs', path)
@@ -365,23 +365,23 @@ class FUSE(object):
             if hasattr(stv, key):
                 setattr(stv, key, val)
         return 0
-    
+
     def flush(self, path, fip):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('flush', path, fh)
-    
+
     def release(self, path, fip):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('release', path, fh)
-    
+
     def fsync(self, path, datasync, fip):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('fsync', path, datasync, fh)
-    
+
     def setxattr(self, path, name, value, size, options, *args):
         data = string_at(value, size)
         return self.operations('setxattr', path, name, data, options, *args)
-    
+
     def getxattr(self, path, name, value, size, *args):
         ret = self.operations('getxattr', path, name, *args)
         retsize = len(ret)
@@ -391,7 +391,7 @@ class FUSE(object):
                 return -ERANGE
             memmove(value, buf, retsize)
         return retsize
-    
+
     def listxattr(self, path, namebuf, size):
         ret = self.operations('listxattr', path)
         buf = create_string_buffer('\x00'.join(ret)) if ret else ''
@@ -401,20 +401,21 @@ class FUSE(object):
                 return -ERANGE
             memmove(namebuf, buf, bufsize)
         return bufsize
-    
+
     def removexattr(self, path, name):
         return self.operations('removexattr', path, name)
-    
+
     def opendir(self, path, fip):
         # Ignore raw_fi
         fip.contents.fh = self.operations('opendir', path)
         return 0
-    
+
     def readdir(self, path, buf, filler, offset, fip):
         # Ignore raw_fi
         for item in self.operations('readdir', path, fip.contents.fh):
             if isinstance(item, str):
                 name, st, offset = item, None, 0
+                name = name.encode('utf-8')
             else:
                 name, attrs, offset = item
                 if attrs:
@@ -422,27 +423,27 @@ class FUSE(object):
                     set_st_attrs(st, attrs)
                 else:
                     st = None
-            if filler(buf, name.encode('utf-8'), st, offset) != 0:
+            if filler(buf, name, st, offset) != 0:
                 break
         return 0
-    
+
     def releasedir(self, path, fip):
         # Ignore raw_fi
         return self.operations('releasedir', path, fip.contents.fh)
-    
+
     def fsyncdir(self, path, datasync, fip):
         # Ignore raw_fi
         return self.operations('fsyncdir', path, datasync, fip.contents.fh)
-    
+
     def init(self, conn):
         return self.operations('init', '/')
-    
+
     def destroy(self, private_data):
         return self.operations('destroy', '/')
-    
+
     def access(self, path, amode):
         return self.operations('access', path, amode)
-    
+
     def create(self, path, mode, fip):
         fi = fip.contents
         if self.raw_fi:
@@ -450,11 +451,11 @@ class FUSE(object):
         else:
             fi.fh = self.operations('create', path, mode)
             return 0
-    
+
     def ftruncate(self, path, length, fip):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('truncate', path, length, fh)
-    
+
     def fgetattr(self, path, buf, fip):
         memset(buf, 0, sizeof(c_stat))
         st = buf.contents
@@ -462,11 +463,11 @@ class FUSE(object):
         attrs = self.operations('getattr', path, fh)
         set_st_attrs(st, attrs)
         return 0
-    
+
     def lock(self, path, fip, cmd, lock):
         fh = fip.contents if self.raw_fi else fip.contents.fh
         return self.operations('lock', path, fh, cmd, lock)
-    
+
     def utimens(self, path, buf):
         if buf:
             atime = time_of_timespec(buf.contents.actime)
@@ -475,7 +476,7 @@ class FUSE(object):
         else:
             times = None
         return self.operations('utimens', path, times)
-    
+
     def bmap(self, path, blocksize, idx):
         return self.operations('bmap', path, blocksize, idx)
 
@@ -484,46 +485,46 @@ class Operations(object):
     """This class should be subclassed and passed as an argument to FUSE on
        initialization. All operations should raise an OSError exception on
        error.
-       
+
        When in doubt of what an operation should do, check the FUSE header
        file or the corresponding system call man page."""
-    
+
     def __call__(self, op, *args):
         if not hasattr(self, op):
             raise OSError(EFAULT, '')
         return getattr(self, op)(*args)
-        
+
     def access(self, path, amode):
         return 0
-    
+
     bmap = None
-    
+
     def chmod(self, path, mode):
         raise OSError(EROFS, '')
-    
+
     def chown(self, path, uid, gid):
         raise OSError(EROFS, '')
-    
+
     def create(self, path, mode, fi=None):
         """When raw_fi is False (default case), fi is None and create should
            return a numerical file handle.
            When raw_fi is True the file handle should be set directly by create
            and return 0."""
         raise OSError(EROFS, '')
-    
+
     def destroy(self, path):
         """Called on filesystem destruction. Path is always /"""
         pass
-    
+
     def flush(self, path, fh):
         return 0
-    
+
     def fsync(self, path, datasync, fh):
         return 0
-    
+
     def fsyncdir(self, path, datasync, fh):
         return 0
-    
+
     def getattr(self, path, fh=None):
         """Returns a dictionary with keys identical to the stat C structure
            of stat(2).
@@ -531,33 +532,33 @@ class Operations(object):
            NOTE: There is an incombatibility between Linux and Mac OS X concerning
            st_nlink of directories. Mac OS X counts all files inside the directory,
            while Linux counts only the subdirectories."""
-        
+
         if path != '/':
             raise OSError(ENOENT, '')
         return dict(st_mode=(S_IFDIR | 0o755), st_nlink=2)
-    
+
     def getxattr(self, path, name, position=0):
         raise OSError(ENOTSUP, '')
-    
+
     def init(self, path):
         """Called on filesystem initialization. Path is always /
            Use it instead of __init__ if you start threads on initialization."""
         pass
-    
+
     def link(self, target, source):
         raise OSError(EROFS, '')
-    
+
     def listxattr(self, path):
         return []
-        
+
     lock = None
-    
+
     def mkdir(self, path, mode):
         raise OSError(EROFS, '')
-    
+
     def mknod(self, path, mode, dev):
         raise OSError(EROFS, '')
-    
+
     def open(self, path, flags):
         """When raw_fi is False (default case), open should return a numerical
            file handle.
@@ -565,60 +566,60 @@ class Operations(object):
                open(self, path, fi)
            and the file handle should be set directly."""
         return 0
-    
+
     def opendir(self, path):
         """Returns a numerical file handle."""
         return 0
-    
+
     def read(self, path, size, offset, fh):
         """Returns a string containing the data requested."""
         raise OSError(ENOENT, '')
-    
+
     def readdir(self, path, fh):
         """Can return either a list of names, or a list of (name, attrs, offset)
            tuples. attrs is a dict as in getattr."""
         return ['.', '..']
-    
+
     def readlink(self, path):
         raise OSError(ENOENT, '')
-    
+
     def release(self, path, fh):
         return 0
-    
+
     def releasedir(self, path, fh):
         return 0
-    
+
     def removexattr(self, path, name):
         raise OSError(ENOTSUP, '')
-    
+
     def rename(self, old, new):
         raise OSError(EROFS, '')
-    
+
     def rmdir(self, path):
         raise OSError(EROFS, '')
-    
+
     def setxattr(self, path, name, value, options, position=0):
         raise OSError(ENOTSUP, '')
-    
+
     def statfs(self, path):
         """Returns a dictionary with keys identical to the statvfs C structure
            of statvfs(3).
            On Mac OS X f_bsize and f_frsize must be a power of 2 (minimum 512)."""
         return {}
-    
+
     def symlink(self, target, source):
         raise OSError(EROFS, '')
-    
+
     def truncate(self, path, length, fh=None):
         raise OSError(EROFS, '')
-    
+
     def unlink(self, path):
         raise OSError(EROFS, '')
-    
+
     def utimens(self, path, times=None):
         """Times is a (atime, mtime) tuple. If None use current time."""
         return 0
-    
+
     def write(self, path, data, offset, fh):
         raise OSError(EROFS, '')
 

fs/osfs/__init__.py

@@ -21,6 +21,7 @@ import errno
 import datetime
 import platform
 import io
+import shutil
 
 from fs.base import *
 from fs.path import *
@@ -246,7 +247,9 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
     @convert_os_errors
     def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
         _decode_path = self._decode_path
-        paths = [_decode_path(p) for p in os.listdir(self.getsyspath(path))]
+        sys_path = self.getsyspath(path)
+        listing = os.listdir(sys_path)
+        paths = [_decode_path(p) for p in listing]
         return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
 
     @convert_os_errors
@@ -283,22 +286,15 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
 
     @convert_os_errors
     def removedir(self, path, recursive=False, force=False):
-        sys_path = self.getsyspath(path)
-        if force:
-            for path2 in self.listdir(path, absolute=True, files_only=True):
-                try:
-                    self.remove(path2)
-                except ResourceNotFoundError:
-                    pass
-            for path2 in self.listdir(path, absolute=True, dirs_only=True):
-                try:
-                    self.removedir(path2, force=True)
-                except ResourceNotFoundError:
-                    pass
         #  Don't remove the root directory of this FS
         if path in ('', '/'):
             raise RemoveRootError(path)
-        os.rmdir(sys_path)
+        sys_path = self.getsyspath(path)
+        if force:
+            # shutil implementation handles concurrency better
+            shutil.rmtree(sys_path, ignore_errors=True)
+        else:
+            os.rmdir(sys_path)
         #  Using os.removedirs() for this can result in dirs being
         #  removed outside the root of this FS, so we recurse manually.
         if recursive:

fs/tests/__init__.py

@@ -137,7 +137,6 @@ class FSTestCases(object):
         f.close()
 
     def test_createfile(self):
-        """Test createfile"""
         test = b('now with content')
         self.fs.createfile("test.txt")
         self.assert_(self.fs.exists("test.txt"))
@@ -396,8 +395,8 @@ class FSTestCases(object):
         alpha = u"\N{GREEK SMALL LETTER ALPHA}"
         beta = u"\N{GREEK SMALL LETTER BETA}"
         self.fs.makedir(alpha)
-        self.fs.setcontents(alpha+"/a", b(''))
-        self.fs.setcontents(alpha+"/"+beta, b(''))
+        self.fs.setcontents(alpha + "/a", b(''))
+        self.fs.setcontents(alpha + "/" + beta, b(''))
         self.assertTrue(self.check(alpha))
         self.assertEquals(sorted(self.fs.listdir(alpha)), ["a", beta])
 

fs/watch.py

@@ -315,7 +315,7 @@ class WatchableFS(WatchableFSMixin,WrapFS):
 
     def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=64*1024):
         existed = self.wrapped_fs.isfile(path)
-        ret = super(WatchableFS, self).setcontents(path, data, chunk_size=chunk_size)
+        ret = super(WatchableFS, self).setcontents(path, data=data, encoding=encoding, errors=errors, chunk_size=chunk_size)
         if not existed:
             self.notify_watchers(CREATED, path)
         self.notify_watchers(ACCESSED, path)
@@ -325,7 +325,7 @@ class WatchableFS(WatchableFSMixin,WrapFS):
 
     def createfile(self, path, wipe=False):
         existed = self.wrapped_fs.isfile(path)
-        ret = super(WatchableFS, self).createfile(path, wipe=False)
+        ret = super(WatchableFS, self).createfile(path, wipe=wipe)
         if not existed:
             self.notify_watchers(CREATED,path)
         self.notify_watchers(ACCESSED,path)