Changed syntax for commands to be more url like, optimized sftps to use fewer queries for listdir

fs/commands/fscp.py

 from fs.opener import opener
 from fs.utils import copyfile, copystructure
-from fs.path import pathjoin
+from fs.path import pathjoin, iswildcard
 from fs.errors import FSError
 from fs.commands.runner import Command
 import sys
@@ -92,7 +92,7 @@ Copy SOURCE to DESTINATION"""
             if src_path is None:
                 src_path = '/'
 
-            if self.is_wildcard(src_path):
+            if iswildcard(src_path):
                 for file_path in src_fs.listdir(wildcard=src_path, full=True):
                     copy_fs_paths.append((self.FILE, src_fs, file_path, file_path))
                     

fs/commands/fsls.py

 #!/usr/bin/env python
 
 from fs.opener import opener
-from fs.path import pathsplit, abspath, isdotfile
+from fs.path import pathsplit, abspath, isdotfile, iswildcard
 from fs.commands.runner import Command
 from collections import defaultdict
 import sys
@@ -43,10 +43,10 @@ List contents of [PATH]"""
             path = path or '.'
             wildcard = None
             
-            if self.is_wildcard(path):
+            if iswildcard(path):
                 path, wildcard = pathsplit(path)
                 
-            if fs.isfile(path):                
+            if path != '.' and fs.isfile(path):                
                 if not options.dirsonly:                    
                     file_paths.append(path)
             else:                  

fs/commands/fstree.py

@@ -14,7 +14,9 @@ Recursively display the contents of PATH in an ascii tree"""
     def get_optparse(self):
         optparse = super(FSTree, self).get_optparse()        
         optparse.add_option('-d', '--depth', dest='depth', type="int", default=5,
-                            help="Maximum depth to display", metavar="DEPTH")        
+                            help="Maximum depth to display", metavar="DEPTH")
+        optparse.add_option('-a', '--all', dest='all', action='store_true', default=False,
+                            help="do not hide dot files")        
         return optparse
         
     def do_run(self, options, args):        
@@ -31,7 +33,8 @@ Recursively display the contents of PATH in an ascii tree"""
             print_fs(fs, path or '',
                      file_out=self.output_file,
                      max_levels=options.depth,
-                     terminal_colors=self.is_terminal())        
+                     terminal_colors=self.is_terminal(),
+                     hide_dotfiles=not options.all)        
    
 def run():
     return FSTree().run()          

fs/commands/runner.py

@@ -2,7 +2,7 @@ import sys
 from optparse import OptionParser
 from fs.opener import opener, OpenerError
 from fs.errors import FSError
-from fs.path import splitext, pathsplit, isdotfile
+from fs.path import splitext, pathsplit, isdotfile, iswildcard
 import platform
 from collections import defaultdict
 
@@ -55,11 +55,6 @@ class Command(object):
         self.terminal_width = w
         self.name = self.__class__.__name__.lower()
     
-    def is_wildcard(self, path):
-        if path is None:
-            return False
-        return '*' in path or '?' in path
-    
     def is_terminal(self):
         try:
             return self.output_file.isatty()
@@ -111,7 +106,7 @@ class Command(object):
         if path is None:
             return [], []
         pathname, resourcename = pathsplit(path)
-        if self.is_wildcard(resourcename):
+        if iswildcard(resourcename):
             dir_paths = fs.listdir(pathname,
                                    wildcard=resourcename,
                                    absolute=True,
@@ -137,7 +132,7 @@ class Command(object):
         resources = []
         
         for fs, path in fs_paths:
-            if self.is_wildcard(path):
+            if path and iswildcard(path):
                 if not files_only:
                     dir_paths = fs.listdir(wildcard=path, dirs_only=True)
                     for path in dir_paths:
@@ -227,8 +222,8 @@ class Command(object):
             if self.is_terminal():
                 self.output("\n")
             return 0
-        except ValueError:
-            pass
+        #except ValueError:
+        #    pass
         except SystemExit:
             return 0
         except IOError:

fs/ftpfs.py

@@ -751,8 +751,7 @@ class FTPFS(FS):
 
     def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT,
                  port=21,
-                 dircache=True,
-                 max_buffer_size=128*1024*1024):
+                 dircache=True):
         """ Connect to a FTP server.
         
         :param host: Host to connect to
@@ -765,8 +764,7 @@ class FTPFS(FS):
         which will speed up operations such as getinfo, isdi, isfile, but
         changes to the ftp file structure will not be visible until
         `~fs.ftpfs.FTPFS.clear_dircache` is called        
-        :param dircache: If True directory information will be cached for fast access
-        :param max_buffer_size: Number of bytes to hold before blocking write operations
+        :param dircache: If True directory information will be cached for fast access        
 
         """
 
@@ -780,9 +778,7 @@ class FTPFS(FS):
         self.timeout = timeout
 
         self.use_dircache = dircache
-        self.get_dircache()
-
-        self.max_buffer_size = max_buffer_size
+        self.get_dircache()        
 
         self._cache_hint = False
         self._locals._ftp = None

fs/memoryfs.py

@@ -156,7 +156,7 @@ class DirEntry(object):
         self.locks += 1
 
     def unlock(self):
-        self.locks -=1
+        self.locks -= 1
         assert self.locks >=0, "Lock / Unlock mismatch!"
 
     def desc_contents(self):
@@ -494,7 +494,7 @@ class MemoryFS(FS):
         if dir_entry is None:
             raise ResourceNotFoundError(path)
         if dir_entry.isfile():
-            raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
+            raise ResourceInvalidError(path, msg="not a directory: %(path)s")
         paths = dir_entry.contents.keys()
         for (i,p) in enumerate(paths):
             if not isinstance(p,unicode):
@@ -522,7 +522,7 @@ class MemoryFS(FS):
         return info    
     
     @synchronize
-    def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
+    def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
         src_dir_entry = self._get_dir_entry(src)
         if src_dir_entry is None:
             raise ResourceNotFoundError(src)
@@ -533,7 +533,7 @@ class MemoryFS(FS):
             dst_dir_entry.xattrs.update(src_xattrs)
     
     @synchronize
-    def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
+    def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
         src_dir_entry = self._get_dir_entry(src)
         if src_dir_entry is None:
             raise ResourceNotFoundError(src)
@@ -544,7 +544,7 @@ class MemoryFS(FS):
             dst_dir_entry.xattrs.update(src_xattrs)
     
     @synchronize
-    def copy(self, src, dst, overwrite=False, chunk_size=16384):
+    def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
         src_dir_entry = self._get_dir_entry(src)
         if src_dir_entry is None:
             raise ResourceNotFoundError(src)
@@ -555,7 +555,7 @@ class MemoryFS(FS):
             dst_dir_entry.xattrs.update(src_xattrs)
     
     @synchronize
-    def move(self, src, dst, overwrite=False, chunk_size=16384):
+    def move(self, src, dst, overwrite=False, chunk_size=1024*64):
         src_dir_entry = self._get_dir_entry(src)
         if src_dir_entry is None:
             raise ResourceNotFoundError(src)
@@ -566,6 +566,27 @@ class MemoryFS(FS):
             dst_dir_entry.xattrs.update(src_xattrs)        
     
     @synchronize
+    def getcontents(self, path):        
+        dir_entry = self._get_dir_entry(path)
+        if dir_entry is None:
+            raise ResourceNotFoundError(path)
+        if not dir_entry.isfile():
+            raise ResourceInvalidError(path, msg="not a directory: %(path)s")
+        return dir_entry.data or ''
+    
+    @synchronize
+    def setcontents(self, path, data, chunk_size=1024*64):
+        if not isinstance(data, str):
+            return super(MemoryFS, self).setcontents(path, data, chunk_size)        
+        if not self.exists(path):      
+            self.open(path, 'w').close()                    
+            
+        dir_entry = self._get_dir_entry(path)
+        if not dir_entry.isfile():
+            raise ResourceInvalidError('Not a directory %(path)s', path)
+        dir_entry.data = data                
+    
+    @synchronize
     def setxattr(self, path, key, value):                
         dir_entry = self._dir_entry(path)
         key = unicode(key)        

fs/opener.py

 import sys
 from fs.osfs import OSFS
-from fs.path import pathsplit
+from fs.path import pathsplit, basename, join, iswildcard
+import os
 import os.path
 import re
+from urlparse import urlparse
 
 class OpenerError(Exception):
     pass
@@ -31,11 +33,15 @@ def _expand_syspath(path):
     return path
 
 
+
+
 class OpenerRegistry(object):
      
+
     re_fs_url = re.compile(r'''
 ^
-(?:\[(.*?)\])*
+(.*?)
+:\/\/
 
 (?:
 \((.*?)\)
@@ -46,6 +52,8 @@ class OpenerRegistry(object):
 \+(.*?)$
 )*$
 ''', re.VERBOSE)
+        
+    
      
     def __init__(self, openers=[]):
         self.registry = {}
@@ -56,13 +64,12 @@ class OpenerRegistry(object):
     
     @classmethod
     def split_segments(self, fs_url):        
-        match = self.re_fs_url.match(fs_url)
-        assert match is not None, "broken re?"
-        return match.groups()        
+        match = self.re_fs_url.match(fs_url)        
+        return match        
     
     def get_opener(self, name):
         if name not in self.registry:
-            raise NoOpenerError("No opener for [%s]" % name)
+            raise NoOpenerError("No opener for %s" % name)
         index = self.registry[name]
         return self.openers[index]        
     
@@ -73,34 +80,56 @@ class OpenerRegistry(object):
             self.registry[name] = index
     
     def parse(self, fs_url, default_fs_name=None, writeable=False, create=False):
-                        
-        fs_name, paren_url, fs_url, path = self.split_segments(fs_url)
+                   
+        orig_url = fs_url     
+        match = self.split_segments(fs_url)
         
-        fs_url = fs_url or paren_url
-        if fs_name is None and path is None:
-            fs_url = os.path.expanduser(os.path.expandvars(fs_url))
-            fs_url = os.path.normpath(os.path.abspath(fs_url))
-            fs_url, path = pathsplit(fs_url)
-            if not fs_url:
-                fs_url = '/'
+        if match:
+            fs_name, paren_url, fs_url, path = match.groups()                                
+            fs_url = fs_url or paren_url or ''          
+            if ':' in fs_name:
+                fs_name, sub_protocol = fs_name.split(':', 1)
+                fs_url = '%s://%s' % (sub_protocol, fs_url)
+            
+            fs_name = fs_name or self.default_opener                                                                    
+                
+        else:
+            fs_name = default_fs_name or self.default_opener
+            fs_url = _expand_syspath(fs_url) 
+            path = ''           
+
+    
+        fs_name,  fs_name_params = self.parse_name(fs_name)        
+        opener = self.get_opener(fs_name)
         
-        fs_name = fs_name or self.default_opener                                
+        if fs_url is None:
+            raise OpenerError("Unable to parse '%s'" % orig_url)
         
-        if fs_name is None:
-            fs_name = fs_default_name
+        wildcard = None
+        if iswildcard(fs_url):
+            fs_url, wildcard = pathsplit(fs_url)
         
-        fs_name,  fs_name_params = self.parse_name(fs_name)        
-        opener = self.get_opener(fs_name)
-                   
-        fs = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
+        fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
+                
+        if wildcard:
+            fs_path = join(fs_path or '', wildcard)
+        else:
+            path = join(fs_path or '', path)
         
         if path:
             pathname, resourcename = pathsplit(path)
             if pathname:
                 fs = fs.opendir(pathname)
                 path = resourcename
+            if not iswildcard(path):
+                if fs.isdir(path):
+                    fs = fs.opendir(path)
+                    fs_path = ''
+                else:
+                    fs_path = path
+                
         
-        return fs, path
+        return fs, fs_path
     
     def parse_credentials(self, url):
         
@@ -145,11 +174,21 @@ class OSFSOpener(Opener):
     names = ['osfs', 'file']
     
     @classmethod
-    def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): 
-        username, password, fs_path = registry.parse_credentials(fs_path)       
-        from fs.osfs import OSFS
-        osfs = OSFS(fs_path, create=create)
-        return osfs
+    def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
+        from fs.osfs import OSFS 
+        username, password, fs_path = registry.parse_credentials(fs_path)                
+        
+                
+        path = _expand_syspath(fs_path)
+        if create:
+            sys.makedirs(fs_path)
+        if os.path.isdir(path):
+            osfs = OSFS(path)
+            filepath = None
+        else:
+            path, filepath = pathsplit(path)
+            osfs = OSFS(path, create=create)
+        return osfs, filepath
         
         
 class ZipOpener(Opener):
@@ -157,22 +196,21 @@ class ZipOpener(Opener):
     
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
-                        
-        create_zip = fs_name_params == 'new' 
-        append_zip = fs_name_params == 'append'       
-        
-        zip_file = None
-        if fs_path.startswith('['):
-            container_fs, container_path = registry.parse(fs_path)
-            if not container_path:
-                raise OpenerError("Not a file")
-            container_mode = 'r+b'
-            if create_zip:
-                container_mode = 'w+b'
-            elif writeable:
-                container_mode = 'w+b'
+                                
+        append_zip = fs_name_params == 'add'     
                 
-            zip_file = container_fs.open(container_path, mode=container_mode)                                      
+        zip_fs, zip_path = registry.parse(fs_path)
+        if zip_path is None:
+            raise OpenerError('File required for zip opener')
+        if create:
+            open_mode = 'wb'        
+            if append_zip:
+                open_mode = 'r+b'
+        else:
+            open_mode = 'rb'
+                    
+        zip_file = zip_fs.open(zip_path, mode=open_mode)         
+                                            
                         
         username, password, fs_path = registry.parse_credentials(fs_path)
         
@@ -182,21 +220,18 @@ class ZipOpener(Opener):
            
         if append_zip:
             mode = 'a'            
-        elif create_zip or create:
+        elif create:
             mode = 'w'
         else:
             if writeable:
                 mode = 'w'
             else:
                 mode = 'a'        
-                
-        if fs_name == 'zip64':
-            allow_zip_64 = True
-        else:
-            allow_zip_64 = False  
+         
+        allow_zip_64 = fs_name == 'zip64'                
               
         zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64)
-        return zipfs
+        return zipfs, None
     
 class RPCOpener(Opener):
     names = ['rpc']
@@ -206,9 +241,16 @@ class RPCOpener(Opener):
         from fs.rpcfs import RPCFS             
         username, password, fs_path = registry.parse_credentials(fs_path)
         if not fs_path.startswith('http://'):
-            fs_path = 'http://' + fs_path                
-        rpcfs = RPCFS(fs_path)
-        return rpcfs
+            fs_path = 'http://' + fs_path
+            
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+
+        rpcfs = RPCFS('%s://%s' % (scheme, netloc))
+        
+        if create and path:
+            rpcfs.makedir(path, recursive=True, allow_recreate=True)
+        
+        return rpcfs, path or None
 
 class FTPOpener(Opener):
     names = ['ftp']
@@ -217,22 +259,28 @@ class FTPOpener(Opener):
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
         from fs.ftpfs import FTPFS
         username, password, fs_path = registry.parse_credentials(fs_path)
-        
-        if '/' in fs_path:
-            url, root_path = fs_path.split('/', 1)
-        else:
-            url = fs_path
-            root_path = ''
-                                                                
+                    
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+        if not scheme:
+            fs_path = 'ftp://' + fs_path
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+                 
+        dirpath, resourcepath = pathsplit(path)        
+        url = netloc
+                                
         ftpfs = FTPFS(url, user=username or '', passwd=password or '')
         ftpfs.cache_hint(True)
         
-        if root_path not in ('', '/'):
-            if not ftpfs.isdir(root_path):
-                raise OpenerError("'%s' is not a directory on the server" % root_path)
-            return ftpfs.opendir(root_path)
+        if create and path:
+            ftpfs.makedir(path, recursive=True, allow_recreate=True)
         
-        return ftpfs
+        if dirpath:
+            ftpfs = ftpfs.opendir(dirpath)
+                            
+        if not resourcepath:
+            return ftpfs, None        
+        else:
+            return ftpfs, resourcepath
 
 
 class SFTPOpener(Opener):
@@ -256,6 +304,8 @@ class SFTPOpener(Opener):
             addr = fs_path
             fs_path = '/'
             
+        fs_path, resourcename = pathsplit(fs_path)
+            
         host = addr
         port = None
         if ':' in host:
@@ -265,10 +315,25 @@ class SFTPOpener(Opener):
             except ValueError:
                 pass
             else:
-                host = (addr, port)                            
+                host = (addr, port)
+            
+        #if not username or not password:
+        #    raise OpenerError('SFTP requires authentication')
+            
+        if create:
+            sftpfs = SFTPFS(host, root_path='/', **credentials)
+            if not sftpfs._transport.is_authenticated():
+                sftpfs.close()
+                raise OpenerError('SFTP requires authentication')
+            sftpfs = sfspfs.makeopendir(fs_path)
+            return sftpfs, None
                 
         sftpfs = SFTPFS(host, root_path=fs_path, **credentials)
-        return sftpfs
+        if not sftpfs._transport.is_authenticated():
+            sftpfs.close()
+            raise OpenerError('SFTP requires authentication')            
+            
+        return sftpfs, resourcename
     
     
 class MemOpener(Opener):
@@ -277,7 +342,10 @@ class MemOpener(Opener):
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create):
         from fs.memoryfs import MemoryFS
-        return MemoryFS()
+        memfs = MemoryFS()
+        if create:
+            memfs = memfs.makeopendir(fs_path)
+        return memfs, None
     
 class DebugOpener(Opener):
     names = ['debug']
@@ -287,13 +355,13 @@ class DebugOpener(Opener):
         from fs.wrapfs.debugfs import DebugFS
         if fs_path:
             fs, path = registry.parse(fs_path, writeable=writeable, create=create)
-            return DebugFS(fs, verbose=False)     
+            return DebugFS(fs, verbose=False), None     
         if fs_name_params == 'ram':
             from fs.memoryfs import MemoryFS
-            return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False)
+            return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False), None
         else:
             from fs.tempfs import TempFS
-            return DebugFS(TempFS(), identifier=fs_name_params, verbose=False)
+            return DebugFS(TempFS(), identifier=fs_name_params, verbose=False), None
     
 class TempOpener(Opener):
     names = ['temp']
@@ -301,7 +369,7 @@ class TempOpener(Opener):
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create):
         from fs.tempfs import TempFS        
-        return TempFS(identifier=fs_name_params, temp_dir=fs_path)
+        return TempFS(identifier=fs_name_params, temp_dir=fs_path), None
     
 
 opener = OpenerRegistry([OSFSOpener,
@@ -317,7 +385,7 @@ opener = OpenerRegistry([OSFSOpener,
 
 def main():
     
-    fs, path = opener.parse('galleries.zip')
+    fs, path = opener.parse('sftp://willmcgugan.com')
     print fs, path
     
 if __name__ == "__main__":

fs/path.py

@@ -515,3 +515,16 @@ class PathMap(object):
     def names(self,root="/"):
         return list(self.iternames(root))
 
+_wild_chars = frozenset('*?[]!{}')
+def iswildcard(path):
+    """Check if a path ends with a wildcard
+    
+    >>> is_wildcard('foo/bar/baz.*')
+    True
+    >>> is_wildcard('foo/bar')
+    False
+    
+    """
+    assert path is not None
+    base_chars = frozenset(basename(path))    
+    return not base_chars.isdisjoint(_wild_chars)    

fs/sftpfs.py

@@ -9,12 +9,15 @@ Filesystem accessing an SFTP server (via paramiko)
 import datetime
 import stat as statinfo
 import threading
-
+import os
 import paramiko
+from getpass import getuser
+from binascii import hexlify
 
 from fs.base import *
 from fs.path import *
 from fs.errors import *
+from fs.utils import isdir, isfile
 
 # SFTPClient appears to not be thread-safe, so we use an instance per thread
 if hasattr(threading, "local"):
@@ -58,6 +61,7 @@ class SFTPFS(FS):
               'atomic.setcontents' : False
               }
 
+
     def __init__(self, connection, root_path="/", encoding=None, **credentials):
         """SFTPFS constructor.
 
@@ -88,18 +92,81 @@ class SFTPFS(FS):
         self._tlocal = thread_local()
         self._transport = None
         self._client = None
+        
+        
+        hostname = None
+        if isinstance(connection, basestring):
+            hostname = connection
+        else:
+            try:
+                hostname, port = connection
+            except ValueError:
+                pass
+            
+        
+        hostkeytype = None
+        hostkey = None
+                
+        if hostname is not None:
+            try:
+                host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
+            except IOError:
+                try:
+                    # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
+                    host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts'))
+                except IOError:
+                    
+                    host_keys = {}
+                        
+            if host_keys.has_key(hostname):
+                hostkeytype = host_keys[hostname].keys()[0]
+                hostkey = host_keys[hostname][hostkeytype]
+                credentials['hostkey'] = hostkey                
+        
+        if not credentials.get('username'):
+            credentials['username'] = getuser()                    
+        
+        super(SFTPFS, self).__init__()
+        
         if isinstance(connection,paramiko.Channel):
             self._transport = None
             self._client = paramiko.SFTPClient(connection)
-        else:
-            if not isinstance(connection,paramiko.Transport):
+        else:            
+            if not isinstance(connection, paramiko.Transport):
                 connection = paramiko.Transport(connection)
                 self._owns_transport = True
-            if not connection.is_authenticated():
-                connection.connect(**credentials)
+            try:           
+                if not connection.is_authenticated():                                            
+                    connection.connect(**credentials)
+                if not connection.is_authenticated():
+                    self._agent_auth(connection, credentials.get('username'))
+                if not connection.is_authenticated():
+                    connection.close()
+                    raise RemoteConnectionError('No auth')
+            except paramiko.AuthenticationException:
+                raise RemoteConnectionError('Auth rejected')
             self._transport = connection
         self.root_path = abspath(normpath(root_path))
-        super(SFTPFS, self).__init__()
+
+    @classmethod
+    def _agent_auth(cls, transport, username):
+        """
+        Attempt to authenticate to the given transport using any of the private
+        keys available from an SSH agent.
+        """
+        
+        agent = paramiko.Agent()
+        agent_keys = agent.get_keys()
+        if len(agent_keys) == 0:
+            return False
+            
+        for key in agent_keys:            
+            try:
+                transport.auth_publickey(username, key)                
+                return key
+            except paramiko.SSHException:
+                pass
+        return None       
 
     def __del__(self):
         self.close()
@@ -184,6 +251,8 @@ class SFTPFS(FS):
 
     @convert_os_errors
     def isdir(self,path):
+        if path == '/':
+            return True
         npath = self._normpath(path)
         try:
             stat = self.client.stat(npath)
@@ -209,6 +278,10 @@ class SFTPFS(FS):
         npath = self._normpath(path)
         try:
             paths = self.client.listdir(npath)
+            if dirs_only or files_only:
+                path_attrs = self.client.listdir_attr(npath)
+            else:
+                path_attrs = None
         except IOError, e:
             if getattr(e,"errno",None) == 2:
                 if self.isfile(path):
@@ -217,10 +290,72 @@ class SFTPFS(FS):
             elif self.isfile(path):
                 raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
             raise
+        
+        if path_attrs is not None:
+            if dirs_only:
+                filter_paths = []
+                for path, attr in zip(paths, path_attrs):
+                    if isdir(self, path, attr.__dict__):
+                        filter_paths.append(path)
+                paths = filter_paths
+            elif files_only:
+                filter_paths = []
+                for path, attr in zip(paths, path_attrs):
+                    if isfile(self, path, attr.__dict__):
+                        filter_paths.append(path)
+                paths = filter_paths
+        
         for (i,p) in enumerate(paths):
             if not isinstance(p,unicode):
                 paths[i] = p.decode(self.encoding)
-        return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
+        return self._listdir_helper(path, paths, wildcard, full, absolute, False, False)
+
+
+    @convert_os_errors
+    def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        npath = self._normpath(path)
+        try:
+            paths = self.client.listdir(npath)
+            attrs = self.client.listdir_attr(npath)
+            attrs_map = dict(zip(paths, attrs))            
+        except IOError, e:
+            if getattr(e,"errno",None) == 2:
+                if self.isfile(path):
+                    raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
+                raise ResourceNotFoundError(path)
+            elif self.isfile(path):
+                raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
+            raise
+        
+    
+        if dirs_only:
+            filter_paths = []
+            for path, attr in zip(paths, attrs):
+                if isdir(self, path, attr.__dict__):
+                    filter_paths.append(path)
+            paths = filter_paths
+        elif files_only:
+            filter_paths = []
+            for path, attr in zip(paths, attrs):
+                if isfile(self, path, attr.__dict__):
+                    filter_paths.append(path)
+            paths = filter_paths
+            
+        for (i, p) in enumerate(paths):
+            if not isinstance(p, unicode):
+                paths[i] = p.decode(self.encoding)
+                
+        def getinfo(p):
+            resourcename = basename(p)
+            info = attrs_map.get(resourcename)
+            if info is None:
+                return self.getinfo(pathjoin(path, p))
+            return self._extract_info(info.__dict__)
+                
+        return [(p, getinfo(p)) for p in 
+                    self._listdir_helper(path, paths, wildcard, full, absolute, False, False)]
+
+
 
     @convert_os_errors
     def makedir(self,path,recursive=False,allow_recreate=False):
@@ -335,6 +470,23 @@ class SFTPFS(FS):
                 raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s")
             raise
 
+    _info_vars = frozenset('st_size st_uid st_gid st_mode st_atime st_mtime'.split())
+    @classmethod
+    def _extract_info(cls, stats):
+        fromtimestamp = datetime.datetime.fromtimestamp
+        info = dict((k, v) for k, v in stats.iteritems() if k in cls._info_vars)        
+        info['size'] = info['st_size']
+        ct = info.get('st_ctime')
+        if ct is not None:
+            info['created_time'] = fromtimestamp(ct)
+        at = info.get('st_atime')
+        if at is not None:
+            info['accessed_time'] = fromtimestamp(at)
+        mt = info.get('st_mtime')
+        if mt is not None:
+            info['modified_time'] = fromtimestamp(mt)
+        return info
+
     @convert_os_errors
     def getinfo(self, path):        
         npath = self._normpath(path)

fs/utils.py

@@ -56,10 +56,13 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
     src = None        
     try:
         # Chunk copy
-        src = src_fs.open(src_path, 'rb')        
+        if src_fs.getsize(src_path) < chunk_size:
+            src = src_fs.getcontents(src_path)
+        else:
+            src = src_fs.open(src_path, 'rb')        
         dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
     finally:
-        if src is not None:
+        if src is not None and hasattr(src, 'close'):
             src.close()
 
 
@@ -89,14 +92,18 @@ def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
         FS._shutil_movefile(src_syspath, dst_syspath)        
         return
 
-    src = None
-    try:        
-        src = src_fs.open(src_path, 'rb')
-        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)        
-        src_fs.remove(src_path)
 
+    src = None        
+    try:
+        # Chunk copy
+        if src_fs.getsize(src_path) < chunk_size:
+            src = src_fs.getcontents(src_path)
+        else:
+            src = src_fs.open(src_path, 'rb')        
+        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
+        src_fs.remove(src_path)
     finally:
-        if src is not None:
+        if src is not None and hasattr(src, 'close'):
             src.close()
 
 def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
@@ -324,7 +331,7 @@ def find_duplicates(fs,
             paths = list(set(paths).difference(dups))
 
 
-def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
+def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hide_dotfiles=False):
     """Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid.
     Be careful about printing a OSFS, or any other large filesystem.
     Without max_levels set, this function will traverse the entire directory tree.
@@ -343,13 +350,14 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
     :param file_out: File object to write output to (defaults to sys.stdout)
     :param terminal_colors: If True, terminal color codes will be written, set to False for non-console output.
         The default (None) will select an appropriate setting for the platform.
+    :param hide_dotfiles: if True, files or directories begining with '.' will be removed
 
     """
 
     if file_out is None:
         file_out = sys.stdout
 
-    file_encoding = getattr(file_out, 'encoding', 'utf-8')
+    file_encoding = getattr(file_out, 'encoding', 'utf-8') or 'utf-8'
 
     if terminal_colors is None:
         if sys.platform.startswith('win'):
@@ -388,12 +396,16 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
     def print_dir(fs, path, levels=[]):
         
         try:
-            dir_listing = [(fs.isdir(pathjoin(path,p)), p) for p in fs.listdir(path)]
+            dir_listing = ( [(True, p) for p in fs.listdir(path, dirs_only=True)] +
+                            [(False, p) for p in fs.listdir(path, files_only=True)] )            
         except Exception, e:
             prefix = ''.join([('|   ', '    ')[last] for last in levels]) + '   '
             write(wrap_prefix(prefix[:-1] + '    ') + wrap_error("unabled to retrieve directory list (%s) ..." % str(e)))
             return 0        
         
+        if hide_dotfiles:
+            dir_listing = [(isdir, p) for isdir, p in dir_listing if not p.startswith('.')]
+        
         dir_listing.sort(key = lambda (isdir, p):(not isdir, p.lower()))
                             
         for i, (is_dir, item) in enumerate(dir_listing):