Major ftp improvments

fs/base.py

@@ -419,7 +419,7 @@ class FS(object):
         :raises ResourceInvalidError: If the path exists, but is not a directory
 
         """
-
+        path = normpath(path)
         def getinfo(p):
             try:
                 if full or absolute:
@@ -783,7 +783,7 @@ class FS(object):
             raise OperationFailedError("get size of resource", path)
         return size
 
-    def copy(self, src, dst, overwrite=False, chunk_size=16384):
+    def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
         """Copies a file from src to dst.
 
         :param src: the source path

fs/commands/fscat.py

@@ -13,10 +13,10 @@ Concetanate FILE(s)"""
     
     def do_run(self, options, args):
         count = 0            
-        for fs, path, is_dir in self.get_resources(args):
+        for fs, path, is_dir in self.get_resources(args):            
             if is_dir:
                 self.error('%s is a directory\n' % path)
-                return 1            
+                return 1                      
             self.output(fs.getcontents(path))
             count += 1              
         if self.is_terminal() and count:

fs/commands/fsls.py

@@ -6,14 +6,14 @@ from fs.commands.runner import Command
 from collections import defaultdict
 import sys
 
-class FSList(Command):
+class FSls(Command):
     
     usage = """fsls [OPTIONS]... [PATH]
 List contents of [PATH]"""
     
     
     def get_optparse(self):
-        optparse = super(FSList, self).get_optparse()        
+        optparse = super(FSls, self).get_optparse()        
         optparse.add_option('-u', '--full', dest='fullpath', action="store_true", default=False,
                             help="output full path", metavar="FULL")
         optparse.add_option('-s', '--syspath', dest='syspath', action="store_true", default=False,
@@ -37,9 +37,11 @@ List contents of [PATH]"""
             args = [u'.']
             
         dir_paths = []   
-        file_paths = []             
+        file_paths = []
+        fs_used = set()             
         for fs_url in args:                        
-            fs, path = self.open_fs(fs_url)                
+            fs, path = self.open_fs(fs_url) 
+            fs_used.add(fs)               
             path = path or '.'
             wildcard = None
             
@@ -61,7 +63,13 @@ List contents of [PATH]"""
                                              wildcard=wildcard,
                                              full=options.fullpath,                                   
                                              files_only=True)
-                        
+        
+        try:
+            for fs in fs_used:
+                fs.close()
+        except FSError:
+            pass
+        
         if options.syspath:
             dir_paths = [fs.getsyspath(path, allow_none=True) or path for path in dir_paths]
             file_paths = [fs.getsyspath(path, allow_none=True) or path for path in file_paths]
@@ -154,7 +162,7 @@ List contents of [PATH]"""
             output('\n')
 
 def run():
-    return FSList().run()            
+    return FSls().run()            
     
 if __name__ == "__main__":
     sys.exit(run())

fs/commands/fsmount.py

@@ -22,6 +22,8 @@ Mounts a file system on a system path"""
                             help="run the mount process in the foreground", metavar="FOREGROUND")
         optparse.add_option('-u', '--unmount', dest='unmount', action="store_true", default=False,
                             help="unmount path", metavar="UNMOUNT")
+        optparse.add_option('-n', '--nocache', dest='nocache', action="store_true", default=False,
+                            help="do not cache network filesystems", metavar="NOCACHE")
         
         return optparse
     
@@ -59,17 +61,20 @@ Mounts a file system on a system path"""
                     return 1
                 fs = fs.opendir(path)
                 path = '/'
+            if not options.nocache:
+                fs.cache_hint(True)
             if not os.path.exists(mount_path):
                os.makedirs(mount_path)
             from fs.expose import fuse
-            if options.foreground:                            
+            if options.foreground:                                            
                 fuse_process = fuse.mount(fs,
                                           mount_path,
                                           foreground=True)                                                    
-            else:                
-                mp = fuse.mount(fs,
-                                mount_path,
-                                foreground=False)
+            else:
+                if not os.fork():                
+                    mp = fuse.mount(fs,
+                                    mount_path,
+                                    foreground=True)
                 
 
     

fs/commands/runner.py

@@ -131,7 +131,7 @@ class Command(object):
         
         resources = []
         
-        for fs, path in fs_paths:
+        for fs, path in fs_paths:            
             if path and iswildcard(path):
                 if not files_only:
                     dir_paths = fs.listdir(wildcard=path, dirs_only=True)

fs/expose/fuse/__init__.py

@@ -245,14 +245,10 @@ class FSOperations(Operations):
     @handle_fs_errors
     def readdir(self, path, fh=None):
         path = path.decode(NATIVE_ENCODING)
-        entries = ['.', '..']
-        #print
-        #print self.fs
-        for (nm,info) in self.fs.listdirinfo(path):
-            #print "*", repr(nm), info
+        entries = ['.', '..']        
+        for (nm,info) in self.fs.listdirinfo(path):            
             self._fill_stat_dict(pathjoin(path,nm),info)
-            entries.append((nm.encode(NATIVE_ENCODING),info,0))  
-        #print      
+            entries.append((nm.encode(NATIVE_ENCODING),info,0))              
         return entries
 
     @handle_fs_errors

fs/opener.py

@@ -102,14 +102,12 @@ class OpenerRegistry(object):
             fs_name = default_fs_name or self.default_opener
             fs_url = _expand_syspath(fs_url) 
             path = ''           
-
     
         fs_name,  fs_name_params = self.parse_name(fs_name)        
         opener = self.get_opener(fs_name)
         
         if fs_url is None:
-            raise OpenerError("Unable to parse '%s'" % orig_url)
-        
+            raise OpenerError("Unable to parse '%s'" % orig_url)        
         
         fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
         
@@ -118,19 +116,14 @@ class OpenerRegistry(object):
             if pathname:
                 fs = fs.opendir(pathname)
             return fs, resourcename
-            
-        #pathname, resourcename = pathsplit(fs_path or '')
-        #if pathname and resourcename:
-        #    fs = fs.opendir(pathname)
-        #    fs_path = resourcename
-                
+                            
         fs_path = join(fs_path, path)
                 
         pathname, resourcename = pathsplit(fs_path or '')        
         if pathname and resourcename:
             fs = fs.opendir(pathname)
             fs_path = resourcename
-                       
+                               
         return fs, fs_path        
     
     def parse_credentials(self, url):
@@ -198,13 +191,10 @@ class ZipOpener(Opener):
         zip_fs, zip_path = registry.parse(fs_path)
         if zip_path is None:
             raise OpenerError('File required for zip opener')
-        if create:
-            open_mode = 'wb'        
-            if append_zip:
-                open_mode = 'r+b'
+        if writeable:
+            open_mode = 'r+b'
         else:
             open_mode = 'rb'
-                    
         zip_file = zip_fs.open(zip_path, mode=open_mode)                                            
                         
         username, password, fs_path = registry.parse_credentials(fs_path)
@@ -212,18 +202,12 @@ class ZipOpener(Opener):
         from fs.zipfs import ZipFS
         if zip_file is None:            
             zip_file = fs_path
-           
-        if append_zip:
+        
+        mode = 'r'
+        if writeable:
             mode = 'a'            
-        elif create:
-            mode = 'w'
-        else:
-            if writeable:
-                mode = 'w'
-            else:
-                mode = 'a'        
          
-        allow_zip_64 = fs_name == 'zip64'                
+        allow_zip_64 = fs_name.endswith('64')                
               
         zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64)
         return zipfs, None
@@ -256,7 +240,7 @@ class FTPOpener(Opener):
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
         from fs.ftpfs import FTPFS
         username, password, fs_path = registry.parse_credentials(fs_path)
-                    
+                                        
         scheme, netloc, path, params, query, fragment = urlparse(fs_path)
         if not scheme:
             fs_path = 'ftp://' + fs_path
@@ -365,7 +349,11 @@ class TempOpener(Opener):
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create):
         from fs.tempfs import TempFS        
-        return TempFS(identifier=fs_name_params, temp_dir=fs_path), None
+        fs = TempFS(identifier=fs_name_params)
+        if create and fs_path:
+            fs = fs.makeopendir(fs_path)
+            fs_path = pathsplit(fs_path)
+        return fs, fs_path
     
 
 opener = OpenerRegistry([OSFSOpener,
@@ -382,7 +370,7 @@ opener = OpenerRegistry([OSFSOpener,
 def main():
     
     #fs, path = opener.parse('zip:zip://~/zips.zip!t.zip!')
-    fs, path = opener.parse('rpc://127.0.0.1/a/*.JPG')
+    fs, path = opener.parse('ftp://releases.mozilla.org/welcome.msg')
     
     print fs, path
     

fs/path.py

@@ -266,6 +266,10 @@ def issamedir(path1, path2):
     """
     return pathsplit(normpath(path1))[0] == pathsplit(normpath(path2))[0]
 
+def isbase(path1, path2):
+    p1 = forcedir(abspath(path1))
+    p2 = forcedir(abspath(path2))
+    return p1 == p2 or p1.startswith(p2) 
 
 def isprefix(path1, path2):
     """Return true is path1 is a prefix of path2.

fs/rpcfs.py

@@ -108,6 +108,7 @@ class RPCFS(FS):
         self._transport = transport
         self.proxy = self._make_proxy()
         FS.__init__(self,thread_synchronize=False)
+        self.isdir('/')
 
     def _make_proxy(self):
         kwds = dict(allow_none=True)

fs/tempfs.py

@@ -39,6 +39,9 @@ class TempFS(OSFS):
         default uses "TempFS"
 
         """
+        self.identifier = identifier
+        self.temp_dir = temp_dir
+        self.dir_mode = dir_mode
         self._temp_dir = tempfile.mkdtemp(identifier or "TempFS",dir=temp_dir)
         self._cleaned = False
         super(TempFS, self).__init__(self._temp_dir, dir_mode=dir_mode, thread_synchronize=thread_synchronize)
@@ -50,6 +53,13 @@ class TempFS(OSFS):
 
     def __unicode__(self):
         return u'<TempFS: %s>' % self._temp_dir
+    
+    def __setstate__(self, state):
+        state = super(TempFS, self).__setstate__(state)
+        self._temp_dir = tempfile.mkdtemp(self.identifier or "TempFS", dir=self.temp_dir)  
+        super(TempFS, self).__init__(self._temp_dir,
+                                     dir_mode=self.dir_mode,
+                                     thread_synchronize=self.thread_synchronize)      
 
     def close(self):
         """Removes the temporary directory.

fs/tests/__init__.py

@@ -486,6 +486,7 @@ class FSTestCases(object):
         makefile("foo/bar/a.txt")
         self.assert_(check("foo/bar/a.txt"))
         self.assert_(checkcontents("foo/bar/a.txt"))
+        #import rpdb2; rpdb2.start_embedded_debugger('password');
         self.fs.copy("foo/bar/a.txt", "foo/b.txt")
         self.assert_(check("foo/bar/a.txt"))
         self.assert_(check("foo/b.txt"))

fs/tests/test_ftpfs.py

@@ -32,8 +32,9 @@ class TestFTPFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
 
         self.ftp_server = subprocess.Popen([sys.executable, abspath(__file__), self.temp_dir, str(use_port)])
         # Need to sleep to allow ftp server to start
-        time.sleep(.1)
-        self.fs = ftpfs.FTPFS('127.0.0.1', 'user', '12345', port=use_port, timeout=5.0)
+        time.sleep(.2)
+        self.fs = ftpfs.FTPFS('127.0.0.1', 'user', '12345', dircache=True, port=use_port, timeout=5.0)
+        self.fs.cache_hint(True)
 
 
     def tearDown(self):

fs/utils.py

@@ -53,14 +53,23 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
         FS._shutil_copyfile(src_syspath, dst_syspath)
         return
 
-    src = None        
+    src = None
+    dst = None        
     try:
         # Chunk copy
         src = src_fs.open(src_path, 'rb')        
-        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
+        dst = src_fs.open(dst_path, 'wb')
+        write = dst.write
+        read = src.read
+        chunk = read(chunk_size)
+        while chunk:            
+            write(chunk)
+            chunk = read(chunk_size)                    
     finally:
-        if src is not None and hasattr(src, 'close'):
+        if src is not None:
             src.close()
+        if dst is not None:
+            dst.close()
 
 
 def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
@@ -89,14 +98,23 @@ def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
         FS._shutil_movefile(src_syspath, dst_syspath)        
         return
 
-    src = None        
+    src = None
+    dst = None        
     try:
         # Chunk copy
         src = src_fs.open(src_path, 'rb')        
-        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)            
+        dst = src_fs.open(dst_path, 'wb')
+        write = dst.write
+        read = src.read
+        chunk = read(chunk_size)
+        while chunk:            
+            write(chunk)
+            chunk = read(chunk_size)                    
     finally:
-        if src is not None and hasattr(src, 'close'):
+        if src is not None:
             src.close()
+        if dst is not None:
+            dst.close()
     src_fs.remove(src_path)
 
 
@@ -416,7 +434,7 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hi
                                             
             if is_dir:                
                 write('%s %s' % (wrap_prefix(prefix + '--'), wrap_dirname(item)))
-                if max_levels is not None and len(levels) >= max_levels:
+                if max_levels is not None and len(levels) + 1 >= max_levels:
                     pass
                     #write(wrap_prefix(prefix[:-1] + '       ') + wrap_error('max recursion levels reached'))
                 else: