More changes to the FS url syntax

fs/commands/fstree.py

@@ -13,10 +13,12 @@ Recursively display the contents of PATH in an ascii tree"""
     
     def get_optparse(self):
         optparse = super(FSTree, self).get_optparse()        
-        optparse.add_option('-d', '--depth', dest='depth', type="int", default=5,
-                            help="Maximum depth to display", metavar="DEPTH")
+        optparse.add_option('-L', '--level', dest='depth', type="int", default=5,
+                            help="Descend only LEVEL directories deep", metavar="LEVEL")
         optparse.add_option('-a', '--all', dest='all', action='store_true', default=False,
                             help="do not hide dot files")        
+        optparse.add_option('-d', '--dirsfirst', dest='dirsfirst', action='store_true', default=False,
+                            help="List directories before files")
         return optparse
         
     def do_run(self, options, args):        
@@ -34,7 +36,8 @@ Recursively display the contents of PATH in an ascii tree"""
                      file_out=self.output_file,
                      max_levels=options.depth,
                      terminal_colors=self.is_terminal(),
-                     hide_dotfiles=not options.all)        
+                     hide_dotfiles=not options.all,
+                     dirs_first=options.dirsfirst)        
    
 def run():
     return FSTree().run()          

fs/opener.py

@@ -44,12 +44,12 @@ class OpenerRegistry(object):
 :\/\/
 
 (?:
-\((.*?)\)
 |(.*?)
+|(?:(.*?)!)
 )
 
 (?:
-\!(.*?)$
+!(.*?)$
 )*$
 ''', re.VERBOSE)
         
@@ -85,11 +85,16 @@ class OpenerRegistry(object):
         match = self.split_segments(fs_url)
         
         if match:
-            fs_name, paren_url, fs_url, path = match.groups()                                
-            fs_url = fs_url or paren_url or ''          
+            fs_name, fs_url, _, path = match.groups()
+            path = path or ''
+            fs_url = fs_url or ''
             if ':' in fs_name:
                 fs_name, sub_protocol = fs_name.split(':', 1)
                 fs_url = '%s://%s' % (sub_protocol, fs_url)
+            if '!' in path:
+                paths = path.split('!')
+                path = paths.pop()
+                fs_url = '%s!%s' % (fs_url, '!'.join(paths))
             
             fs_name = fs_name or self.default_opener                                                                    
                 
@@ -105,32 +110,30 @@ class OpenerRegistry(object):
         if fs_url is None:
             raise OpenerError("Unable to parse '%s'" % orig_url)
         
-        wildcard = None
-        if iswildcard(fs_url):
-            fs_url, wildcard = pathsplit(fs_url)
+        #wildcard = None
+        #if iswildcard(fs_url):
+        #    fs_url, wildcard = pathsplit(fs_url)
         
         fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
         
-        if wildcard:
-            fs_path = join(fs_path or '', wildcard)
-        else:
-            path = join(fs_path or '', path)
-        
-        if path:
-            pathname, resourcename = pathsplit(path)
+        pathname, resourcename = pathsplit(fs_path or '')
         if pathname:
             fs = fs.opendir(pathname)
-                path = resourcename
-            if not iswildcard(path):
-                if fs.isdir(path):
-                    fs = fs.opendir(path)
-                    fs_path = ''
-                else:
-                    fs_path = path
+            fs_path = resourcename
+        
+        if fs_path and iswildcard(fs_path):
+            return fs, fs_path     
         
+        fs_path = join(fs_path, path)
+        
+        pathname, resourcename = pathsplit(fs_path or '')
+        if pathname:
+            fs = fs.opendir(pathname)
+            fs_path = resourcename
                 
         return fs, fs_path
         
+    
     def parse_credentials(self, url):
         
         username = None
@@ -385,7 +388,7 @@ opener = OpenerRegistry([OSFSOpener,
 
 def main():
     
-    fs, path = opener.parse('sftp://willmcgugan.com')
+    fs, path = opener.parse('zip:zip://~/zips.zip!t.zip!')
     print fs, path
     
 if __name__ == "__main__":

fs/sftpfs.py

@@ -19,6 +19,9 @@ from fs.path import *
 from fs.errors import *
 from fs.utils import isdir, isfile
 
+class WrongHostKeyError(RemoteConnectionError):
+    pass
+
 # SFTPClient appears to not be thread-safe, so we use an instance per thread
 if hasattr(threading, "local"):
     thread_local = threading.local
@@ -62,7 +65,7 @@ class SFTPFS(FS):
               }
 
 
-    def __init__(self, connection, root_path="/", encoding=None, username='', password=None, pkey=None):
+    def __init__(self, connection, root_path="/", encoding=None, hostkey=None, username='', password=None, pkey=None):
         """SFTPFS constructor.
 
         The only required argument is 'connection', which must be something
@@ -80,6 +83,7 @@ class SFTPFS(FS):
         :param connection: a connection string
         :param root_path: The root path to open
         :param encoding: String encoding of paths (defaults to UTF-8)
+        :param hostkey: the host key expected from the server or None if you don't require server validation
         :param username: Name of SFTP user
         :param password: Password for SFTP user
         :param pkey: Public key
@@ -113,11 +117,16 @@ class SFTPFS(FS):
                 connection.daemon = True
                 self._owns_transport = True
             
-        if not connection.is_authenticated():
+        if hostkey is not None:
+            key = self.get_remote_server_key()
+            if hostkey != key:
+                raise WrongHostKeyError('Host keys do not match')
             
-            try:                                
         connection.start_client()
         
+        if (password or pkey) and not connection.is_authenticated():
+            
+            try:                
                 if pkey:                    
                     connection.auth_publickey(username, pkey)
                 
@@ -312,7 +321,7 @@ class SFTPFS(FS):
         try:            
             attrs = self.client.listdir_attr(npath)
             attrs_map = dict((a.filename, a) for a in attrs)                        
-            paths = attrs.keys()            
+            paths = attrs_map.keys()            
         except IOError, e:
             if getattr(e,"errno",None) == 2:
                 if self.isfile(path):

fs/utils.py

@@ -331,7 +331,7 @@ def find_duplicates(fs,
             paths = list(set(paths).difference(dups))
 
 
-def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hide_dotfiles=False):
+def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hide_dotfiles=False, dirs_first=False):
     """Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid.
     Be careful about printing a OSFS, or any other large filesystem.
     Without max_levels set, this function will traverse the entire directory tree.
@@ -406,7 +406,10 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi
         if hide_dotfiles:
             dir_listing = [(isdir, p) for isdir, p in dir_listing if not p.startswith('.')]
         
+        if dirs_first:
             dir_listing.sort(key = lambda (isdir, p):(not isdir, p.lower()))
+        else:
+            dir_listing.sort(key = lambda (isdir, p):p.lower())
                             
         for i, (is_dir, item) in enumerate(dir_listing):