Added FS command line scripts

ChangeLog

@@ -60,4 +60,7 @@
     * Added utility module 'fs.filelike' with some helpers for building and
       manipulating file-like objects.
     * Added getmeta and hasmeta methods
+    * Separated behaviour of setcontents and createfile
+    * Added a getmmap to base
+    * Added command line scripts fsls, fstree, fscat, fscp, fsmv
 

fs/base.py

@@ -207,8 +207,8 @@ class FS(object):
          * *unicode_paths* True if the file system can use unicode paths
          * *case_insensitive_paths* True if the file system ignores the case of paths        
          * *atomic.makedir* True if making a directory is an atomic operation
-         * *atomic.rename" True if rename is an atomic operation, (and not implemented as a copy followed by a delete)
-         * *atomic.setcontents" True if the implementation supports setting the contents of a file as an atomic operation (without opening a file)
+         * *atomic.rename* True if rename is an atomic operation, (and not implemented as a copy followed by a delete)
+         * *atomic.setcontents* True if the implementation supports setting the contents of a file as an atomic operation (without opening a file)
          
         The following are less common:
         
@@ -569,11 +569,9 @@ class FS(object):
         try:
             sys_path = self.getsyspath(path)
         except NoSysPathError:
-            return "No description available"
-        if self.isdir(path):
-            return "OS dir, maps to %s" % sys_path
-        else:
-            return "OS file, maps to %s" % sys_path
+            return "No description available"        
+        return "OS resource, maps to %s" % sys_path
+        
 
     def getcontents(self, path):
         """Returns the contents of a file as a string.
@@ -591,28 +589,52 @@ class FS(object):
             if f is not None:
                 f.close()
 
-    def createfile(self, path, data=""):
-        """A convenience method to create a new file from a string.
+    def setcontents(self, path, data, chunk_size=1024*64):
+        """A convenience method to create a new file from a string or file-like object
 
         :param path: a path of the file to create
         :param data: a string or a file-like object containing the contents for the new file
+        :param chunk_size: Number of bytes to read in a chunk, if the implementation has to resort to a read / copy loop
+        """
+        
+        if not data:
+            self.createfile(path)
+        else:
+            f = None
+            try:
+                f = self.open(path, 'wb')
+                if hasattr(data, "read"):
+                    read = data.read
+                    write = f.write
+                    chunk = read(chunk_size)
+                    while chunk:
+                        write(chunk)
+                        chunk = read(chunk_size)
+                else:
+                    f.write(data)
+                if hasattr(f, 'flush'):
+                    f.flush()
+            finally:
+                if f is not None:
+                    f.close()
+        
+    def createfile(self, path, wipe=False):
+        """Creates an empty file if it doesn't exist
+        
+        :param path: path to the file to create
+        :param wipe: If True, the contents of the file will be erased 
+        
         """
+        if not wipe and self.isfile(path):
+            return
+            
         f = None
         try:
-            f = self.open(path, 'wb')
-            if hasattr(data, "read"):
-                chunk = data.read(1024*512)
-                while chunk:
-                    f.write(chunk)
-                    chunk = data.read(1024*512)
-            else:
-                f.write(data)
-            if hasattr(f, 'flush'):
-                f.flush()
+            f = self.open(path, 'w')
         finally:
             if f is not None:
                 f.close()
-    setcontents = createfile
+            
 
     def opendir(self, path):
         """Opens a directory and returns a FS object representing its contents.
@@ -775,24 +797,17 @@ class FS(object):
         if src_syspath is not None and dst_syspath is not None:
             self._shutil_copyfile(src_syspath, dst_syspath)
         else:
-            src_file, dst_file = None, None
+            src_file = None            
             try:
-                src_file = self.open(src, "rb")
-                dst_file = self.open(dst, "wb")
-
-                while True:
-                    chunk = src_file.read(chunk_size)
-                    dst_file.write(chunk)
-                    if len(chunk) != chunk_size:
-                        break
+                src_file = self.open(src, "rb")                
+                self.setcontents(dst, src_file, chunk_size=chunk_size)                
             finally:
                 if src_file is not None:
-                    src_file.close()
-                if dst_file is not None:
-                    dst_file.close()
-
-    @convert_os_errors
-    def _shutil_copyfile(self, src_syspath, dst_syspath):
+                    src_file.close()                
+    
+    @classmethod
+    @convert_os_errors    
+    def _shutil_copyfile(cls, src_syspath, dst_syspath):
         try:
             shutil.copyfile(src_syspath, dst_syspath)
         except IOError, e:
@@ -801,6 +816,12 @@ class FS(object):
                 if not os.path.exists(dirname(dst_syspath)):
                     raise ParentDirectoryMissingError(dst_syspath)
             raise
+        
+    @classmethod
+    @convert_os_errors
+    def _shutil_movefile(cls, src_syspath, dst_syspath):
+        shutil.move(src_syspath, dst_syspath)
+        
 
     def move(self, src, dst, overwrite=False, chunk_size=16384):
         """moves a file from one location to another.
@@ -984,6 +1005,36 @@ class FS(object):
         from fs.browsewin import browse
         browse(self)
 
+    def getmmap(self, path, read_only=False, copy=False):
+        """Returns a mmap object for this path.
+        
+        See http://docs.python.org/library/mmap.html for more details on the mmap module.
+        
+        :param path: A path on this filesystem
+        :param read_only: If True, the mmap may not be modified
+        :param copy: If False then changes wont be written back to the file
+        :raises NoMMapError: Only paths that have a syspath can be opened as a mmap 
+         
+        """        
+        syspath = self.getsyspath(path, allow_none=True)
+        if syspath is None:
+            raise NoMMapError(path)
+        
+        import mmap
+        
+        if read_only:
+            f = open(syspath, 'rb')
+            access = mmap.ACCESS_READ
+        else:
+            if copy:
+                f = open(syspath, 'rb')
+                access = mmap.ACCESS_COPY
+            else:
+                f = open(syspath, 'r+b')
+                access = mmap.ACCESS_WRITE      
+                  
+        m = mmap.mmap(f.fileno, 0, access=access)
+        return m
 
 
 def flags_to_mode(flags):

fs/commands/fscat

+#!/usr/bin/env python
+import sys
+from fs.commands.fscat import run
+sys.exit(run())
+
+

fs/commands/fscat.py

+#!/usr/bin/env python
+
+from fs.opener import opener
+from fs.commands.runner import Command
+import sys
+
+class FSCat(Command):
+    
+    def do_run(self, options, args):
+        count = 0            
+        for fs, path, is_dir in self.get_resources(args):
+            if is_dir:
+                self.error('%s is a directory\n' % path)
+                return 1            
+            self.output(fs.getcontents(path))
+            count += 1              
+        if self.is_terminal() and count:
+            self.output('\n')
+    
+def run():
+    return FSCat().run()
+    
+if __name__ == "__main__":
+    sys.exit(run())
+        
\ No newline at end of file

fs/commands/fscp

+#!/usr/bin/env python
+import sys
+from fs.commands.fscp import run
+sys.exit(run())
+
+
+

fs/commands/fscp.py

+from fs.opener import opener
+from fs.utils import copyfile, copystructure
+from fs.path import pathjoin
+from fs.errors import FSError
+from fs.commands.runner import Command
+import sys
+import Queue as queue
+import time
+import threading
+
+
+class FileOpThread(threading.Thread):        
+    
+    def __init__(self, action, name, dest_fs, queue, on_done, on_error):
+        self.action = action                  
+        self.dest_fs = dest_fs
+        self.queue = queue
+        self.on_done = on_done
+        self.on_error = on_error
+        self.finish_event = threading.Event()
+        super(FileOpThread, self).__init__()        
+    
+    def run(self):        
+        
+        try:
+            while not self.finish_event.isSet():            
+                try:
+                    path_type, fs, path, dest_path = self.queue.get(timeout=0.1)
+                except queue.Empty:
+                    continue
+                try:
+                    if path_type == FSCopy.DIR:
+                        self.dest_fs.makedir(path, recursive=True, allow_recreate=True)
+                    else:                                                                
+                        self.action(fs, path, self.dest_fs, dest_path, overwrite=True) 
+                except Exception, e:                    
+                    self.queue.task_done()                                  
+                    raise                   
+                else:
+                    self.queue.task_done() 
+                    self.on_done(path_type, fs, path, self.dest_fs, dest_path)
+                        
+        except Exception, e:            
+            self.on_error(e)            
+
+class FSCopy(Command):
+    
+    DIR, FILE = 0, 1
+    
+    def get_action(self):
+        return copyfile
+
+    def get_optparse(self):
+        optparse = super(FSCopy, self).get_optparse()
+        optparse.add_option('-p', '--progress', dest='progress', action="store_true", default=False,
+                            help="show progress", metavar="PROGRESS")
+        optparse.add_option('-t', '--threads', dest='threads', action="store", default=1,
+                            help="number of threads to use", type="int", metavar="THREAD_COUNT")        
+        return optparse
+        
+    def do_run(self, options, args):
+               
+        self.options = options 
+        if len(args) < 2:
+            self.error("at least two filesystems required\n")
+            return 1
+        
+        srcs = args[:-1]
+        dst = args[-1] 
+        
+        dst_fs, dst_path = self.open_fs(dst, writeable=True, create=True)
+        
+        if dst_path is not None and dst_fs.isfile(dst_path):
+            self.error('Destination must be a directory\n')
+            return 1
+        
+        if dst_path:
+            dst_fs = dst_fs.makeopendir(dst_path)
+            dst_path = None                      
+        
+        copy_fs_paths = []
+        
+        progress = options.progress
+        
+        self.root_dirs = [] 
+        for fs_url in srcs:
+            src_fs, src_path = self.open_fs(fs_url)            
+
+            if src_path is None:
+                src_path = '/'
+
+            if self.is_wildcard(src_path):
+                for file_path in src_fs.listdir(wildcard=src_path, full=True):
+                    copy_fs_paths.append((self.FILE, src_fs, file_path, file_path))
+                    
+            else:                
+                if src_fs.isdir(src_path): 
+                    self.root_dirs.append((src_fs, src_path))                                        
+                    src_sub_fs = src_fs.opendir(src_path)
+                    for dir_path, file_paths in src_sub_fs.walk():                                            
+                        copy_fs_paths.append((self.DIR, src_sub_fs, dir_path, dir_path))
+                        sub_fs = src_sub_fs.opendir(dir_path)
+                        for file_path in file_paths:                                                         
+                            copy_fs_paths.append((self.FILE, sub_fs, file_path, pathjoin(dir_path, file_path)))
+                else:
+                    if src_fs.exists(src_path):
+                        copy_fs_paths.append((self.FILE, src_fs, src_path, src_path))
+                    else:
+                        self.error('%s is not a file or directory\n' % src_path)
+                        return 1                    
+                                
+                
+        if self.options.threads > 1:            
+            copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
+            copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]            
+            for path_type, fs, path, dest_path in copy_fs_dirs:               
+                dst_fs.makedir(path, allow_recreate=True, recursive=True)                             
+        
+        self.lock = threading.RLock()
+            
+        self.total_files = len(copy_fs_paths)
+        self.done_files = 0
+                
+        file_queue = queue.Queue()
+        threads = [FileOpThread(self.get_action(),
+                                'T%i' % i,
+                                dst_fs,
+                                file_queue,
+                                self.on_done,
+                                self.on_error)
+                        for i in xrange(options.threads)]
+        for thread in threads:
+            thread.start()
+        
+        self.action_error = None
+        complete = False
+        try:        
+            enqueue = file_queue.put            
+            for resource in copy_fs_paths:
+                enqueue(resource)
+                        
+            while not file_queue.empty():
+                time.sleep(0)
+                if self.any_error():
+                    raise SystemExit
+            # Can't use queue.join here, or KeyboardInterrupt will not be
+            # caught until the queue is finished 
+            #file_queue.join()
+        
+        except KeyboardInterrupt:
+            print "!"
+            options.progress = False
+            if self.action_error:
+                self.error(self.wrap_error(unicode(self.action_error)) + '\n')  
+            else:          
+                self.output("\nCancelling...\n")
+                
+        except SystemExit:
+            options.progress = False
+            if self.action_error:
+                self.error(self.wrap_error(unicode(self.action_error)) + '\n')
+                               
+        finally:
+            sys.stdout.flush()                
+            for thread in threads:
+                thread.finish_event.set()                           
+            for thread in threads:
+                thread.join()
+            complete = True
+            self.post_actions()
+                                   
+        dst_fs.close()
+        
+        if complete and options.progress:
+            sys.stdout.write(self.progress_bar(self.total_files, self.done_files))
+            sys.stdout.write('\n')
+            sys.stdout.flush()        
+        
+    def post_actions(self):
+        pass
+        
+    def on_done(self, path_type, src_fs, src_path, dst_fs, dst_path):        
+        self.lock.acquire()        
+        try:
+            if self.options.verbose:
+                if path_type == self.DIR:
+                    print "mkdir %s" % dst_fs.desc(dst_path)
+                else:
+                    print "%s -> %s" % (src_fs.desc(src_path), dst_fs.desc(dst_path))
+            elif self.options.progress:
+                self.done_files += 1        
+                sys.stdout.write(self.progress_bar(self.total_files, self.done_files))
+                sys.stdout.flush()
+        finally:
+            self.lock.release()
+            
+    def on_error(self, e):
+        self.lock.acquire()
+        try:
+            self.action_error = e
+        finally:
+            self.lock.release()
+    
+    def any_error(self):                
+        self.lock.acquire()
+        try:
+            return bool(self.action_error)
+        finally:
+            self.lock.release()
+            
+    def progress_bar(self, total, remaining, msg=''):
+        bar_width = 20
+        throbber = '|/-\\'
+        throb = throbber[remaining % len(throbber)]
+        done = float(remaining) / total
+        
+        done_steps = int(done * bar_width)
+        bar_steps = ('#' * done_steps).ljust(bar_width) 
+        
+        msg = '%i%% ' % int(done * 100.0)
+        
+        if total == remaining:
+            throb = ''
+            
+        bar = '\r%s[%s] %s' % (throb, bar_steps, msg)
+        return bar
+        
+def run():
+    return FSCopy().run()
+
+if __name__ == "__main__":
+    sys.exit(run())
+     
\ No newline at end of file

fs/commands/fsinfo

+#!/usr/bin/env python
+import sys
+from fs.commands.fsinfo import run
+sys.exit(run())
+
+
+

fs/commands/fsinfo.py

+#!/usr/bin/env python
+
+from fs.errors import ResourceNotFoundError
+from fs.opener import opener
+from fs.commands.runner import Command
+import sys
+from datetime import datetime
+
+class FSInfo(Command):
+     
+    def get_optparse(self):
+        optparse = super(FSInfo, self).get_optparse()
+        optparse.add_option('-k', '--key', dest='keys', action='append', default=[],
+                            help='display KEYS only')
+        optparse.add_option('-s', '--simple', dest='simple', action='store_true', default=False,
+                            help='info displayed in simple format (no table)')
+        optparse.add_option('-o', '--omit', dest='omit', action='store_true', default=False,
+                            help='omit path name from output')
+        optparse.add_option('-d', '--dirsonly', dest='dirsonly', action="store_true", default=False,
+                            help="list directories only", metavar="DIRSONLY")
+        optparse.add_option('-f', '--filesonly', dest='filesonly', action="store_true", default=False,
+                            help="list files only", metavar="FILESONLY")
+        return optparse   
+        
+        
+    def do_run(self, options, args):
+        
+        def wrap_value(val):            
+            if val.rstrip() == '\0':
+                return self.wrap_error('... missing ...')
+            return val
+
+        def make_printable(text):
+            if not isinstance(text, basestring):
+                try:
+                    text = str(text)
+                except:
+                    try:
+                        text = unicode(text)
+                    except:
+                        text = repr(text)
+            return text
+                
+                
+        keys = options.keys or None
+        for fs, path, is_dir in self.get_resources(args,
+                                                   files_only=options.filesonly,
+                                                   dirs_only=options.dirsonly):                        
+            if not options.omit:
+                if options.simple:           
+                    file_line = u'%s\n' % self.wrap_filename(path)
+                else:
+                    file_line = u'[%s] %s\n' % (self.wrap_filename(path), self.wrap_faded(fs.desc(path)))
+                self.output(file_line)            
+            info = fs.getinfo(path)
+                                            
+            for k, v in info.items():
+                if k.startswith('_'):
+                    del info[k]
+                elif not isinstance(v, (basestring, int, float, bool, datetime)):
+                    del info[k]                
+                        
+            if keys:            
+                table = [(k, make_printable(info.get(k, '\0'))) for k in keys]
+            else:
+                keys = sorted(info.keys())
+                table = [(k, make_printable(info[k])) for k in sorted(info.keys())]
+                                 
+            if options.simple:
+                for row in table:
+                    self.output(row[-1] + '\n')
+            else:                                        
+                self.output_table(table, {0:self.wrap_table_header, 1:wrap_value})            
+
+        
+def run():
+    return FSInfo().run()         
+    
+if __name__ == "__main__":
+    sys.exit(run())    
+    
\ No newline at end of file

fs/commands/fsls

+#!/usr/bin/env python
+import sys
+from fs.commands.fsls import run
+sys.exit(run())
+
+
+

fs/commands/fsls.py

+#!/usr/bin/env python
+
+from fs.opener import opener
+from fs.path import pathsplit, abspath, isdotfile
+from fs.commands.runner import Command
+from collections import defaultdict
+import sys
+
+class FSList(Command):
+                
+    def get_optparse(self):
+        optparse = super(FSList, self).get_optparse()        
+        optparse.add_option('-u', '--full', dest='fullpath', action="store_true", default=False,
+                            help="output full path", metavar="FULL")
+        optparse.add_option('-s', '--syspath', dest='syspath', action="store_true", default=False,
+                            help="output system path", metavar="SYSPATH")         
+        optparse.add_option('-d', '--dirsonly', dest='dirsonly', action="store_true", default=False,
+                            help="list directories only", metavar="DIRSONLY")
+        optparse.add_option('-f', '--filesonly', dest='filesonly', action="store_true", default=False,
+                            help="list files only", metavar="FILESONLY")
+        optparse.add_option('-l', '--long', dest='long', action="store_true", default=False,
+                            help="use a long listing format", metavar="LONG")
+        optparse.add_option('-a', '--all', dest='all', action='store_true', default=False,
+                            help="do not hide dot files")
+        
+        return optparse
+        
+        
+    def do_run(self, options, args):            
+        output = self.output
+             
+        if not args:
+            args = [u'.']
+            
+        dir_paths = []   
+        file_paths = []             
+        for fs_url in args:                        
+            fs, path = self.open_fs(fs_url)                
+            path = path or '.'
+            wildcard = None
+            
+            if self.is_wildcard(path):
+                path, wildcard = pathsplit(path)
+                
+            if fs.isfile(path):                
+                if not options.dirsonly:                    
+                    file_paths.append(path)
+            else:                  
+                if not options.filesonly:                    
+                    dir_paths += fs.listdir(path,
+                                            wildcard=wildcard,
+                                            full=options.fullpath,
+                                            dirs_only=True)
+                
+                if not options.dirsonly:                                    
+                    file_paths += fs.listdir(path,
+                                             wildcard=wildcard,
+                                             full=options.fullpath,                                   
+                                             files_only=True)
+                        
+        if options.syspath:
+            dir_paths = [fs.getsyspath(path, allow_none=True) or path for path in dir_paths]
+            file_paths = [fs.getsyspath(path, allow_none=True) or path for path in file_paths]
+                        
+        dirs = frozenset(dir_paths)                
+        paths = sorted(dir_paths + file_paths, key=lambda p:p.lower())        
+        
+        if not options.all:            
+            paths = [path for path in paths if not isdotfile(path)]
+        
+        if not paths:
+            return                                                    
+        
+        def columnize(paths, num_columns):
+            
+            col_height = (len(paths) + num_columns - 1) / num_columns                
+            columns = [[] for _ in xrange(num_columns)]                
+            col_no = 0
+            col_pos = 0
+            for path in paths:
+                columns[col_no].append(path)
+                col_pos += 1
+                if col_pos >= col_height:
+                    col_no += 1
+                    col_pos = 0
+                            
+            padded_columns = []
+            
+            wrap_filename = self.wrap_filename
+            wrap_dirname = self.wrap_dirname
+            def wrap(path):                      
+                if path in dirs:
+                    return wrap_dirname(path.ljust(max_width))
+                else:
+                    return wrap_filename(path.ljust(max_width))
+            
+            for column in columns:
+                if column:
+                    max_width = max([len(path) for path in column])
+                else:
+                    max_width = 1
+                max_width = min(max_width, terminal_width)                         
+                padded_columns.append([wrap(path) for path in column])
+                
+            return padded_columns
+            
+
+        def condense_columns(columns):
+            max_column_height = max([len(col) for col in columns])
+            lines = [[] for _ in xrange(max_column_height)]
+            for column in columns:
+                for line, path in zip(lines, column):
+                    line.append(path)                                
+            return '\n'.join(u'  '.join(line) for line in lines)
+                                                        
+               
+        if options.long:            
+            for path in paths:
+                if path in dirs:
+                    output(self.wrap_dirname(path) + '\n')
+                else:
+                    output(self.wrap_filename(path) + '\n')            
+                                
+        else:            
+            terminal_width = self.terminal_width
+            path_widths = [len(path) for path in paths]
+            smallest_paths = min(path_widths)               
+            num_paths = len(paths) 
+                           
+            num_cols = min(terminal_width / (smallest_paths + 2), num_paths)                   
+            while num_cols:                                                                                                  
+                col_height = (num_paths + num_cols - 1) / num_cols                                   
+                line_width = 0                    
+                for col_no in xrange(num_cols):                          
+                    try:                                        
+                        col_width = max(path_widths[col_no*col_height:(col_no + 1) * col_height])
+                    except ValueError:
+                        continue
+                    line_width += col_width                    
+                    if line_width > terminal_width:                            
+                        break;
+                    line_width += 2
+                else:
+                    if line_width - 1 <= terminal_width:
+                        break
+                num_cols -= 1                                
+            num_cols = max(1, num_cols)                               
+            columns = columnize(paths, num_cols)                
+            output(condense_columns(columns))
+            output('\n')
+
+def run():
+    return FSList().run()            
+    
+if __name__ == "__main__":
+    sys.exit(run())
+        
\ No newline at end of file

fs/commands/fsmv

+#!/usr/bin/env python
+import sys
+from fs.commands.fsmv import run
+sys.exit(run())
+
+
+

fs/commands/fsmv.py

+from fs.utils import movefile, contains_files
+from fs.commands import fscp
+import sys
+
+class FSMove(fscp.FSCopy):
+    def get_action(self):        
+        return movefile
+    
+    def post_actions(self):
+        for fs, dirpath in self.root_dirs:
+            if not contains_files(fs, dirpath):                
+                fs.removedir(dirpath, force=True)
+    
+def run():
+    return FSMove().run()
+    
+if __name__ == "__main__":
+    sys.exit(run())

fs/commands/fsrm

+#!/usr/bin/env python
+import sys
+from fs.commands.fsrm import run
+sys.exit(run())
+
+
+

fs/commands/fsrm.py

+#!/usr/bin/env python
+
+from fs.errors import ResourceNotFoundError
+from fs.opener import opener
+from fs.commands.runner import Command
+import sys
+
+class FSrm(Command):
+    
+    def get_optparse(self):
+        optparse = super(FSrm, self).get_optparse()
+        optparse.add_option('-f', '--force', dest='force', action='store_true', default=False,
+                            help='ignore non-existent files, never prompt')
+        optparse.add_option('-i', '--interactive', dest='interactive', action='store_true', default=False,
+                            help='prompt before removing')
+        optparse.add_option('-r', '--recursive', dest='recursive', action='store_true', default=False,
+                            help='remove directories and their contents recursively')
+        return optparse
+    
+    def do_run(self, options, args):
+                        
+        interactive = options.interactive
+        verbose = options.verbose
+        
+        for fs, path, is_dir in self.get_resources(args):                     
+            if interactive:
+                if is_dir:
+                    msg = "remove directory '%s'?" % path
+                else:
+                    msg = "remove file '%s'?" % path
+                if not self.ask(msg) in 'yY':
+                    continue
+            try:
+                if is_dir:
+                    fs.removedir(path, force=options.recursive)
+                else:
+                    fs.remove(path)
+            except ResourceNotFoundError:
+                if not options.force:
+                    raise
+            else:
+                if verbose:
+                    self.output("removed '%s'\n" % path)
+            
+        
+def run():         
+    return FSrm().run()
+    
+if __name__ == "__main__":
+    sys.exit(run())    
+    
\ No newline at end of file

fs/commands/fsserve

+#!/usr/bin/env python
+import sys
+from fs.commands.fsserve import run
+sys.exit(run())
+
+
+

fs/commands/fsserve.py

+#!/usr/bin/env python
+import sys
+
+from fs.opener import opener
+from fs.commands.runner import Command
+from fs.utils import print_fs
+
+
+class FSServe(Command):
+        
+    def get_optparse(self):
+        optparse = super(FSServe, self).get_optparse()                
+        optparse.add_option('-t', '--type', dest='type', type="string", default="http",
+                            help="Server type to create (http, rpc, sftp)", metavar="TYPE")
+        optparse.add_option('-a', '--addr', dest='addr', type="string", default="",
+                            help="Server address", metavar="ADDR")
+        optparse.add_option('-p', '--port', dest='port', type="int",
+                            help="Port number", metavar="")        
+        return optparse
+        
+    def do_run(self, options, args):        
+    
+        try:
+            fs_url = args[0]
+        except IndexError:
+            self.error('FS required\n')
+            return 1
+            
+        fs, path = self.open_fs(fs_url)
+        if path and fs.isdir(path):
+            fs, path = fs.opendir(path), '/'        
+            
+        port = options.port
+                        
+        if options.type == 'http':            
+            from fs.expose.http import serve_fs            
+            if port is None:
+                port = 80
+            serve_fs(fs, options.addr, port)
+                
+        elif options.type == 'rpc':            
+            from fs.expose.xmlrpc import RPCFSServer
+            if port is None:
+                port = 80
+            s = RPCFSServer(fs, (options.addr, options.port))
+            s.serve_forever()
+        
+        elif options.type == 'sftp':            
+            from fs.expose.sftp import BaseSFTPServer
+            if port is None:
+                port = 22
+            server = BaseSFTPServer((options.addr, port), fs)
+            try:
+                server.serve_forever()
+            except Exception, e:
+                pass
+            finally:
+                server.server_close()
+        
+        else:
+            self.error("Server type '%s' not recognised\n" % options.type)
+            
+            
+def run():                               
+    return FSServe().run()        
+    
+if __name__ == "__main__":
+    sys.exit(run())
\ No newline at end of file

fs/commands/fstree

+#!/usr/bin/env python
+import sys
+from fs.commands.fstree import run
+sys.exit(run())
+
+
+

fs/commands/fstree.py

+#!/usr/bin/env python
+import sys
+
+from fs.opener import opener
+from fs.commands.runner import Command
+
+from fs.utils import print_fs
+
+class FSTree(Command):
+        
+    def get_optparse(self):
+        optparse = super(FSTree, self).get_optparse()        
+        optparse.add_option('-d', '--depth', dest='depth', type="int", default=5,
+                            help="Maximum depth to display", metavar="DEPTH")        
+        return optparse
+        
+    def do_run(self, options, args):        
+        
+        if not args:
+            args = ['.']
+    
+        for fs, path, is_dir in self.get_resources(args, single=True):
+            if path is not None:
+                fs.opendir(path)                
+            if not is_dir:
+                self.error(u"'%s' is not a dir\n" % path)
+                return 1
+            print_fs(fs, path or '',
+                     file_out=self.output_file,
+                     max_levels=options.depth,
+                     terminal_colors=self.is_terminal())        
+   
+def run():
+    return FSTree().run()          
+    
+if __name__ == "__main__":
+    sys.exit(run())
+   
\ No newline at end of file

fs/commands/runner.py

+import sys
+from optparse import OptionParser
+from fs.opener import opener, OpenerError
+from fs.errors import FSError
+from fs.path import splitext, pathsplit, isdotfile
+import platform
+from collections import defaultdict
+
+
+if platform.system() == 'Linux' :
+    def getTerminalSize():
+        def ioctl_GWINSZ(fd):
+            try:
+                import fcntl, termios, struct, os
+                cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
+            '1234'))
+            except:
+                return None
+            return cr
+        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
+        if not cr:
+            try:
+                fd = os.open(os.ctermid(), os.O_RDONLY)
+                cr = ioctl_GWINSZ(fd)
+                os.close(fd)
+            except:
+                pass
+        if not cr:
+            try:
+                cr = (env['LINES'], env['COLUMNS'])
+            except:
+                cr = (25, 80)
+        return int(cr[1]), int(cr[0])
+else:
+    def getTerminalSize():
+        return 80, 50
+
+def _unicode(text):
+    if not isinstance(text, unicode):
+        return text.decode('ascii', 'replace')
+    return text
+
+class Command(object):
+    
+    def __init__(self, usage='', version=''):
+        self.usage = usage
+        self.version = version
+        self.output_file = sys.stdout
+        self.error_file = sys.stderr
+        self.encoding = getattr(self.output_file, 'encoding', 'utf-8') or 'utf-8'
+        self.verbosity_level = 0
+        self.terminal_colors = not sys.platform.startswith('win') and self.is_terminal()
+        w, h = getTerminalSize()
+        self.terminal_width = w
+        self.name = self.__class__.__name__.lower()
+    
+    def is_wildcard(self, path):
+        if path is None:
+            return False
+        return '*' in path or '?' in path
+    
+    def is_terminal(self):
+        try:
+            return self.output_file.isatty()
+        except AttributeError:
+            return False        
+        
+    def wrap_dirname(self, dirname):
+        if not self.terminal_colors:
+            return dirname
+        return '\x1b[1;32m%s\x1b[0m' % dirname
+    
+    def wrap_error(self, msg):
+        if not self.terminal_colors:
+            return msg
+        return '\x1b[31m%s\x1b[0m' % msg
+    
+    def wrap_filename(self, fname):        
+        fname = _unicode(fname)
+        if not self.terminal_colors:
+            return fname            
+        if '.' in fname:
+            name, ext = splitext(fname)
+            fname = u'%s\x1b[36m%s\x1b[0m' % (name, ext)
+        if isdotfile(fname):
+            fname = u'\x1b[2m%s\x1b[0m' % fname
+        return fname
+    
+    def wrap_faded(self, text):
+        text = _unicode(text)
+        if not self.terminal_colors:
+            return text
+        return u'\x1b[2m%s\x1b[0m' % text
+    
+    def wrap_table_header(self, name):
+        if not self.terminal_colors:
+            return name
+        return '\x1b[1;32m%s\x1b[0m' % name
+        
+    def open_fs(self, fs_url, writeable=False, create=False):
+        try:
+            fs, path = opener.parse(fs_url, writeable=writeable, create=create)
+        except OpenerError, e:
+            self.error(str(e)+'\n')
+            sys.exit(1)
+        fs.cache_hint(True)        
+        return fs, path
+    
+    def expand_wildcard(self, fs, path):        
+        if path is None:
+            return [], []
+        pathname, resourcename = pathsplit(path)
+        if self.is_wildcard(resourcename):
+            dir_paths = fs.listdir(pathname,
+                                   wildcard=resourcename,
+                                   absolute=True,
+                                   dirs_only=True)
+            
+            file_paths = fs.listdir(pathname,
+                                    wildcard=resourcename,
+                                    absolute=True,
+                                    files_only=True)
+            return dir_paths, file_paths
+            
+        else:            
+            if fs.isdir(path):                
+                #file_paths = fs.listdir(path,
+                #                        absolute=True)
+                return [path], []
+            return [], [path]
+        
+    def get_resources(self, fs_urls, dirs_only=False, files_only=False, single=False):
+        
+        fs_paths = [self.open_fs(fs_url) for fs_url in fs_urls]        
+        
+        resources = []
+        
+        for fs, path in fs_paths:
+            if self.is_wildcard(path):
+                if not files_only:
+                    dir_paths = fs.listdir(wildcard=path, dirs_only=True)
+                    for path in dir_paths:
+                        resources.append([fs, path, True])                        
+                if not dirs_only:                    
+                    file_paths = fs.listdir(wildcard=path, files_only=True)
+                    for path in file_paths:
+                        resources.append([fs, path, False])                        
+            else:
+                path = path or '/'
+                is_dir = fs.isdir(path)
+                resource = [fs, path, is_dir]
+                if not files_only and not dirs_only:
+                    resources.append(resource)
+                elif files_only and not is_dir:
+                    resources.append(resource)
+                elif dirs_only and is_dir:
+                    resources.append(resource)                
+                    
+            if single:
+                break
+            
+        return resources                
+    
+    def ask(self, msg):
+        return raw_input('%s: %s ' % (self.name, msg))                     
+    
+    def text_encode(self, text):  
+
+        if not isinstance(text, unicode):
+            text = text.decode('ascii', 'replace')                
+        text = text.encode(self.encoding, 'replace')
+                    
+        return text
+    
+    def output(self, msg, verbose=False):
+        if verbose and not self.verbose:
+            return        
+        self.output_file.write(self.text_encode(msg))
+        
+    
+    def output_table(self, table, col_process=None, verbose=False):
+        
+        if verbose and not self.verbose:
+            return
+        
+        if col_process is None:
+            col_process = {}
+        
+        max_row_widths = defaultdict(int)
+        
+        for row in table:
+            for col_no, col in enumerate(row):
+                max_row_widths[col_no] = max(max_row_widths[col_no], len(col))
+        
+        lines = []        
+        for row in table:
+            out_col = []
+            for col_no, col in enumerate(row):
+                td = col.ljust(max_row_widths[col_no])
+                if col_no in col_process:
+                     td = col_process[col_no](td)
+                out_col.append(td)
+            lines.append(self.text_encode('%s\n' % '  '.join(out_col).rstrip()))
+        self.output(''.join(lines))        
+                                        
+    def error(self, msg):
+        self.error_file.write('%s: %s' % (self.name, self.text_encode(msg)))        
+        
+    def get_optparse(self):
+        optparse = OptionParser(usage=self.usage, version=self.version)
+        optparse.add_option('-v', '--verbose', dest='verbose', action="store_true", default=False,
+                            help="make output verbose", metavar="VERBOSE")        
+        return optparse
+        
+    def run(self):        
+        parser = self.get_optparse()
+        options, args = parser.parse_args()
+        args = [unicode(arg, sys.getfilesystemencoding()) for arg in args]
+        self.verbose = options.verbose        
+        try:
+            return self.do_run(options, args) or 0
+        except FSError, e:
+            self.error(self.wrap_error(unicode(e)) + '\n')
+            return 1        
+        except KeyboardInterrupt:
+            if self.is_terminal():
+                self.output("\n")
+            return 0
+        except SystemExit:
+            return 0
+        #except IOError:
+        #    return 1
+        except Exception, e:            
+            self.error(self.wrap_error('Internal Error - %s\n' % unicode(e)))
+            return 1
+        
+        
+        
+if __name__ == "__main__":
+    command = Command()
+    sys.exit(command.run())
\ No newline at end of file

fs/contrib/davfs/__init__.py

@@ -275,7 +275,7 @@ class DAVFS(FS):
                 msg = str(e)
             raise RemoteConnectionError("",msg=msg,details=e)
 
-    def setcontents(self,path,contents):
+    def setcontents(self,path, contents, chunk_size=1024*64):
         resp = self._request(path,"PUT",contents)
         resp.close()
         if resp.status == 405:

fs/contrib/tahoefs/__init__.py

@@ -315,9 +315,9 @@ class TahoeFS(CacheFS):
             self._uncache(dst,added=True)
 
     @_fix_path
-    def setcontents(self, path, file):
+    def setcontents(self, path, data, chunk_size=64*1024):
         try:
-            self.wrapped_fs.setcontents(path, file)
+            self.wrapped_fs.setcontents(path, data, chunk_size=chunk_size)
         finally:
             self._uncache(path, added=True)
 

fs/errors.py

@@ -28,6 +28,7 @@ __all__ = ['FSError',
            'DirectoryNotEmptyError',
            'ParentDirectoryMissingError',
            'ResourceLockedError',
+           'NoMMapError',
            'convert_fs_errors',
            'convert_os_errors'
            ]
@@ -174,6 +175,9 @@ class ResourceLockedError(ResourceError):
     """Exception raised when a resource can't be used because it is locked."""
     default_message = "Resource is locked: %(path)s"
 
+class NoMMapError(ResourceError):
+    """Exception raise when getmmap fails to create a mmap"""
+    default_message = "Can't get mmap for %(path)s"
 
 
 def convert_fs_errors(func):

fs/expose/xmlrpc.py

@@ -60,7 +60,7 @@ class RPCFSInterface(object):
 
     def set_contents(self, path, data):
         path = self.decode_path(path)
-        self.fs.createfile(path,data.data)
+        self.fs.setcontents(path,data.data)
 
     def exists(self, path):
         path = self.decode_path(path)

fs/filelike.py

@@ -102,6 +102,7 @@ class FileLikeBase(object):
         read at a time when looking for a newline character.  Setting this to
         a larger number when lines are long should improve efficiency.
         """
+        super(FileLikeBase, self).__init__()
         # File-like attributes
         self.closed = False
         self.softspace = 0
@@ -110,8 +111,8 @@ class FileLikeBase(object):
         self._rbuffer = None     # data that's been read but not returned
         self._wbuffer = None     # data that's been given but not written
         self._sbuffer = None     # data between real & apparent file pos
-        self._soffset = 0        # internal offset of file pointer
-
+        self._soffset = 0        # internal offset of file pointer        
+    
     #
     #  The following five methods are the ones that subclasses are expected
     #  to implement.  Carefully check their docstrings.

fs/ftpfs.py

@@ -741,7 +741,9 @@ class FTPFS(FS):
               'virtual': False,
               'read_only' : False,
               'unicode_paths' : True,
-              'case_insensitive_paths' : False,              
+              'case_insensitive_paths' : False,
+              'atomic.move' : True,
+              'atomic.copy' : True,              
               'atomic.makedir' : True,
               'atomic.rename' : True,
               'atomic.setcontents' : False,

fs/memoryfs.py

@@ -189,6 +189,8 @@ class MemoryFS(FS):
               'read_only' : False,
               'unicode_paths' : True,
               'case_insensitive_paths' : False,
+              'atomic.move' : False,
+              'atomic.copy' : False,
               'atomic.makedir' : True,
               'atomic.rename' : True,
               'atomic.setcontents' : False,              

fs/mountfs.py

@@ -244,14 +244,24 @@ class MountFS(FS):
         return fs.open(delegate_path, mode, **kwargs)
 
     @synchronize
-    def setcontents(self, path, contents):
+    def setcontents(self, path, data, chunk_size=64*1024):
         object = self.mount_tree.get(path, None)
         if type(object) is MountFS.FileMount:
-            return super(MountFS,self).setcontents(path,contents)
+            return super(MountFS,self).setcontents(path, data, chunk_size=chunk_size)
         fs, mount_path, delegate_path = self._delegate(path)
         if fs is self or fs is None:
             raise ParentDirectoryMissingError(path)
-        return fs.setcontents(delegate_path,contents)
+        return fs.setcontents(delegate_path, data, chunk_size)
+
+    @synchronize
+    def createfile(self, path):
+        object = self.mount_tree.get(path, None)
+        if type(object) is MountFS.FileMount:
+            return super(MountFS,self).setcontents(path, contents, chunk_size=chunk_size)
+        fs, mount_path, delegate_path = self._delegate(path)
+        if fs is self or fs is None:
+            raise ParentDirectoryMissingError(path)
+        return fs.createfile(delegate_path)
 
     @synchronize
     def remove(self, path):

fs/osfs/__init__.py

@@ -177,8 +177,6 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
                 return stat.f_bfree * stat.f_bsize
         
         return super(OSFS, self).getmeta(meta_name, default)
-            
-        
 
     @convert_os_errors
     def open(self, path, mode="r", **kwargs):
@@ -209,8 +207,8 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
         return os.path.isfile(path)
 
     @convert_os_errors
-    def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
-        paths = [self._decode_path(p) for p in os.listdir(self.getsyspath(path))]
+    def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):        
+        paths = [self._decode_path(p) for p in os.listdir(self.getsyspath(path))]        
         return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
 
     @convert_os_errors

fs/path.py

@@ -170,16 +170,57 @@ def pathsplit(path):
 
     >>> pathsplit("foo/bar/baz")
     ('foo/bar', 'baz')
+    
+    >>> pathsplit("/foo/bar/baz")
+    ('/foo/bar', 'baz')
 
     """
     split = normpath(path).rsplit('/', 1)
     if len(split) == 1:
         return (u'', split[0])
-    return tuple(split)
+    return split[0] or '/', split[1]
 
 # Allow pathsplit() to be used as fs.path.split()
 split = pathsplit
 
+def splitext(path):
+    """Splits the extension from the path, and returns the path (up to the last
+    '.' and the extension
+    
+    :param path: A path to split
+    
+    >>> splitext('baz.txt')
+    ('baz', 'txt')
+    
+    >>> splitext('foo/bar/baz.txt')
+    ('foo/bar/baz', 'txt')
+    
+    """
+    
+    parent_path, pathname = pathsplit(path)
+    if '.' not in pathname:
+        return path, ''
+    pathname, ext = pathname.rsplit('.', 1)
+    path = pathjoin(parent_path, pathname)
+    return path, '.' + ext
+
+def isdotfile(path):
+    """Detects if a path references a dot file, i.e. a resource who's name
+    starts with a '.'
+    
+    :param path: Path to check
+    
+    >>> isdotfile('.baz')
+    True
+    
+    >>> isdotfile('foo/bar/.baz')
+    True
+    
+    >>> isdotfile('foo/bar.baz')
+    False
+    
+    """
+    return pathsplit(path)[-1].startswith('.')
 
 def dirname(path):
     """Returns the parent directory of a path.

fs/remote.py

@@ -253,16 +253,30 @@ class RemoteFileBuffer(FileWrapper):
         if "w" in self.mode or "a" in self.mode or "+" in self.mode:
             pos = self.wrapped_file.tell()
             self.wrapped_file.seek(0)
-            self.fs.setcontents(self.path, self.wrapped_file)
+            
+#            chunk_size = 64*1024
+#            f = None
+#            try:
+#                f = self.fs.wrapped_fs.open(self.path, 'wb')                
+#                chunk = self.wrapped_file.read(chunk_size)
+#                while chunk:
+#                    f.write(chunk)
+#                    chunk = self.wrapped_file.read(chunk_size)
+#            finally:
+#                if f is not None:
+#                    f.close()
+            
+            self.fs.setcontents(self.path, self.wrapped_file)            
+            
             self.wrapped_file.seek(pos)
     
-    def close(self):
+    def close(self):        
         with self._lock:
-            if not self.closed:
+            if not self.closed:        
                 self._setcontents()
                 if self._rfile is not None:
                     self._rfile.close()
-                super(RemoteFileBuffer,self).close()
+                super(RemoteFileBuffer,self).close()                
 
 
 class ConnectionManagerFS(LazyFS):
@@ -309,8 +323,8 @@ class ConnectionManagerFS(LazyFS):
         self._poll_sleeper = threading.Event()
         self.connected = connected
 
-    def setcontents(self,path,data):
-        return self.wrapped_fs.setcontents(path,data)
+    def setcontents(self, path, data, chunk_size=64*1024):
+        return self.wrapped_fs.setcontents(path, data, chunk_size=chunk_size)
 
     def __getstate__(self):
         state = super(ConnectionManagerFS,self).__getstate__()
@@ -521,8 +535,8 @@ class CacheFS(WrapFS):
         self._uncache(path,unmoved=True)
         return f
 
-    def setcontents(self,path,contents):
-        res = super(CacheFS,self).setcontents(path,contents)
+    def setcontents(self, path, contents='', chunk_size=64*1024):
+        res = super(CacheFS,self).setcontents(path, contents, chunk_size=chunk_size)
         self._uncache(path,unmoved=True)
         return res
 

fs/s3fs.py

@@ -61,6 +61,8 @@ class S3FS(FS):
               'unicode_paths' : True,
               'case_insensitive_paths' : False,
               'network' : True,
+              'atomic.move' : True,
+              'atomic.copy' : True,
               'atomic.makedir' : True,
               'atomic.rename' : False,
               'atomic.setconetns' : True
@@ -241,9 +243,9 @@ class S3FS(FS):
         
         return url
 
-    def setcontents(self,path,contents):
+    def setcontents(self, path, data, chunk_size=64*1024):
         s3path = self._s3path(path)
-        self._sync_set_contents(s3path,contents)
+        self._sync_set_contents(s3path, data)
 
     def open(self,path,mode="r"):
         """Open the named file in the given mode.

fs/sftpfs.py

@@ -51,6 +51,8 @@ class SFTPFS(FS):
               'unicode_paths' : True,
               'case_insensitive_paths' : False,
               'network' : True,
+              'atomic.move' : True,
+              'atomic.copy' : True,
               'atomic.makedir' : True,
               'atomic.rename' : True,
               'atomic.setcontents' : False
@@ -135,6 +137,7 @@ class SFTPFS(FS):
                 self.client.close()
             if self._owns_transport and self._transport:
                 self._transport.close()
+            self.closed = True
 
     def _normpath(self,path):
         if not isinstance(path,unicode):
@@ -145,7 +148,7 @@ class SFTPFS(FS):
         return npath
 
     @convert_os_errors
-    def open(self,path,mode="r",bufsize=-1):
+    def open(self,path,mode="rb",bufsize=-1):
         npath = self._normpath(path)
         if self.isdir(path):
             msg = "that's a directory: %(path)s"
@@ -163,6 +166,11 @@ class SFTPFS(FS):
         f.truncate = new_truncate
         return f
 
+    def desc(self, path):
+        npath = self._normpath(path)
+        addr, port = self._transport.getpeername()
+        return u'%s on sftp://%s:%i' % (self.client.normalize(npath), addr, port)
+
     @convert_os_errors
     def exists(self,path):
         npath = self._normpath(path)
@@ -328,10 +336,10 @@ class SFTPFS(FS):
             raise
 
     @convert_os_errors
-    def getinfo(self, path):
+    def getinfo(self, path):        
         npath = self._normpath(path)
         stats = self.client.stat(npath)
-        info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') )
+        info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') )        
         info['size'] = info['st_size']
         ct = info.get('st_ctime', None)
         if ct is not None:

fs/tempfs.py

@@ -25,6 +25,8 @@ class TempFS(OSFS):
               'unicode_paths' : os.path.supports_unicode_filenames,
               'case_insensitive_paths' : os.path.normcase('Aa') == 'aa', 
               'network' : False,
+              'atomic.move' : True,
+              'atomic.copy' : True,
               'atomic.makedir' : True,
               'atomic.rename' : True,
               'atomic.setcontents' : False            

fs/tests/__init__.py

@@ -133,7 +133,7 @@ class FSTestCases(object):
         self.assertFalse(self.fs.exists("dir1"))
         self.assertFalse(self.fs.isdir("dir1"))
         self.assertFalse(self.fs.isfile("a.txt"))
-        self.fs.createfile("a.txt")
+        self.fs.setcontents("a.txt", '')
         self.assertFalse(self.fs.isdir("dir1"))
         self.assertTrue(self.fs.exists("a.txt"))
         self.assertTrue(self.fs.isfile("a.txt"))
@@ -149,10 +149,10 @@ class FSTestCases(object):
         def check_unicode(items):
             for item in items:
                 self.assertTrue(isinstance(item,unicode))
-        self.fs.createfile(u"a")
-        self.fs.createfile("b")
-        self.fs.createfile("foo")
-        self.fs.createfile("bar")
+        self.fs.setcontents(u"a", '')
+        self.fs.setcontents("b", '')
+        self.fs.setcontents("foo", '')
+        self.fs.setcontents("bar", '')
         # Test listing of the root directory
         d1 = self.fs.listdir()
         self.assertEqual(len(d1), 4)
@@ -173,10 +173,10 @@ class FSTestCases(object):
         # Create some deeper subdirectories, to make sure their
         # contents are not inadvertantly included
         self.fs.makedir("p/1/2/3",recursive=True)
-        self.fs.createfile("p/1/2/3/a")
-        self.fs.createfile("p/1/2/3/b")
-        self.fs.createfile("p/1/2/3/foo")
-        self.fs.createfile("p/1/2/3/bar")
+        self.fs.setcontents("p/1/2/3/a", '')
+        self.fs.setcontents("p/1/2/3/b", '')
+        self.fs.setcontents("p/1/2/3/foo", '')
+        self.fs.setcontents("p/1/2/3/bar", '')
         self.fs.makedir("q")
         # Test listing just files, just dirs, and wildcards
         dirs_only = self.fs.listdir(dirs_only=True)
@@ -213,10 +213,10 @@ class FSTestCases(object):
         def check_equal(items,target):
             names = [nm for (nm,info) in items] 
             self.assertEqual(sorted(names),sorted(target))
-        self.fs.createfile(u"a")
-        self.fs.createfile("b")
-        self.fs.createfile("foo")
-        self.fs.createfile("bar")
+        self.fs.setcontents(u"a", '')
+        self.fs.setcontents("b", '')
+        self.fs.setcontents("foo", '')
+        self.fs.setcontents("bar", '')
         # Test listing of the root directory
         d1 = self.fs.listdirinfo()
         self.assertEqual(len(d1), 4)
@@ -238,10 +238,10 @@ class FSTestCases(object):
         # Create some deeper subdirectories, to make sure their
         # contents are not inadvertantly included
         self.fs.makedir("p/1/2/3",recursive=True)
-        self.fs.createfile("p/1/2/3/a")
-        self.fs.createfile("p/1/2/3/b")
-        self.fs.createfile("p/1/2/3/foo")
-        self.fs.createfile("p/1/2/3/bar")
+        self.fs.setcontents("p/1/2/3/a", '')
+        self.fs.setcontents("p/1/2/3/b", '')
+        self.fs.setcontents("p/1/2/3/foo", '')
+        self.fs.setcontents("p/1/2/3/bar", '')
         self.fs.makedir("q")
         # Test listing just files, just dirs, and wildcards
         dirs_only = self.fs.listdirinfo(dirs_only=True)
@@ -275,8 +275,8 @@ class FSTestCases(object):
         alpha = u"\N{GREEK SMALL LETTER ALPHA}"
         beta = u"\N{GREEK SMALL LETTER BETA}"
         self.fs.makedir(alpha)
-        self.fs.createfile(alpha+"/a")
-        self.fs.createfile(alpha+"/"+beta)
+        self.fs.setcontents(alpha+"/a", '')
+        self.fs.setcontents(alpha+"/"+beta, '')
         self.assertTrue(self.check(alpha))
         self.assertEquals(sorted(self.fs.listdir(alpha)),["a",beta])
 
@@ -293,18 +293,18 @@ class FSTestCases(object):
         self.assert_(check("a/b/child"))
         self.assertRaises(DestinationExistsError,self.fs.makedir,"/a/b")
         self.fs.makedir("/a/b",allow_recreate=True)
-        self.fs.createfile("/a/file")
+        self.fs.setcontents("/a/file", '')
         self.assertRaises(ResourceInvalidError,self.fs.makedir,"a/file")
 
     def test_remove(self):
-        self.fs.createfile("a.txt")
+        self.fs.setcontents("a.txt", '')
         self.assertTrue(self.check("a.txt"))
         self.fs.remove("a.txt")
         self.assertFalse(self.check("a.txt"))
         self.assertRaises(ResourceNotFoundError,self.fs.remove,"a.txt")
         self.fs.makedir("dir1")
         self.assertRaises(ResourceInvalidError,self.fs.remove,"dir1")
-        self.fs.createfile("/dir1/a.txt")
+        self.fs.setcontents("/dir1/a.txt", '')
         self.assertTrue(self.check("dir1/a.txt"))
         self.fs.remove("dir1/a.txt")
         self.assertFalse(self.check("/dir1/a.txt"))
@@ -337,7 +337,7 @@ class FSTestCases(object):
         self.assert_(not check("foo/bar"))
         #  Ensure that force=True works as expected
         self.fs.makedir("frollic/waggle", recursive=True)
-        self.fs.createfile("frollic/waddle.txt","waddlewaddlewaddle")
+        self.fs.setcontents("frollic/waddle.txt","waddlewaddlewaddle")
         self.assertRaises(DirectoryNotEmptyError,self.fs.removedir,"frollic")
         self.assertRaises(ResourceInvalidError,self.fs.removedir,"frollic/waddle.txt")
         self.fs.removedir("frollic",force=True)
@@ -357,14 +357,14 @@ class FSTestCases(object):
     def test_rename(self):
         check = self.check
         # test renaming a file in the same directory
-        self.fs.createfile("foo.txt","Hello, World!")
+        self.fs.setcontents("foo.txt","Hello, World!")
         self.assert_(check("foo.txt"))
         self.fs.rename("foo.txt", "bar.txt")
         self.assert_(check("bar.txt"))
         self.assert_(not check("foo.txt"))
         # test renaming a directory in the same directory
         self.fs.makedir("dir_a")
-        self.fs.createfile("dir_a/test.txt","testerific")
+        self.fs.setcontents("dir_a/test.txt","testerific")
         self.assert_(check("dir_a"))
         self.fs.rename("dir_a","dir_b")
         self.assert_(check("dir_b"))
@@ -381,7 +381,7 @@ class FSTestCases(object):
 
     def test_info(self):
         test_str = "Hello, World!"
-        self.fs.createfile("info.txt",test_str)
+        self.fs.setcontents("info.txt",test_str)
         info = self.fs.getinfo("info.txt")
         self.assertEqual(info['size'], len(test_str))
         self.fs.desc("info.txt")
@@ -390,7 +390,7 @@ class FSTestCases(object):
 
     def test_getsize(self):
         test_str = "*"*23
-        self.fs.createfile("info.txt",test_str)
+        self.fs.setcontents("info.txt",test_str)
         size = self.fs.getsize("info.txt")
         self.assertEqual(size, len(test_str))
 
@@ -398,7 +398,7 @@ class FSTestCases(object):
         check = self.check
         contents = "If the implementation is hard to explain, it's a bad idea."
         def makefile(path):
-            self.fs.createfile(path,contents)
+            self.fs.setcontents(path,contents)
         def checkcontents(path):
             check_contents = self.fs.getcontents(path)
             self.assertEqual(check_contents,contents)
@@ -431,7 +431,7 @@ class FSTestCases(object):
         check = self.check
         contents = "If the implementation is hard to explain, it's a bad idea."
         def makefile(path):
-            self.fs.createfile(path, contents)
+            self.fs.setcontents(path, contents)
 
         self.assertRaises(ResourceNotFoundError,self.fs.movedir,"a","b")
         self.fs.makedir("a")
@@ -476,7 +476,7 @@ class FSTestCases(object):
         check = self.check
         contents = "If the implementation is hard to explain, it's a bad idea."
         def makefile(path,contents=contents):
-            self.fs.createfile(path,contents)
+            self.fs.setcontents(path,contents)
         def checkcontents(path,contents=contents):
             check_contents = self.fs.getcontents(path)
             self.assertEqual(check_contents,contents)
@@ -510,7 +510,7 @@ class FSTestCases(object):
         check = self.check
         contents = "If the implementation is hard to explain, it's a bad idea."
         def makefile(path):
-            self.fs.createfile(path,contents)
+            self.fs.setcontents(path,contents)
         def checkcontents(path):
             check_contents = self.fs.getcontents(path)
             self.assertEqual(check_contents,contents)
@@ -549,7 +549,7 @@ class FSTestCases(object):
         check = self.check
         contents = "If the implementation is hard to explain, it's a bad idea."
         def makefile(path):
-            self.fs.createfile(path,contents)
+            self.fs.setcontents(path,contents)
 
         self.fs.makedir("a")
         makefile("a/1.txt")
@@ -679,13 +679,14 @@ class FSTestCases(object):
             self.assertEquals(self.fs.getcontents('f.txt'),contents)
 
     def test_pickling(self):
-        self.fs.createfile("test1","hello world")
+        self.fs.setcontents("test1","hello world")
         fs2 = pickle.loads(pickle.dumps(self.fs))
         self.assert_(fs2.isfile("test1"))
         fs3 = pickle.loads(pickle.dumps(self.fs,-1))
         self.assert_(fs3.isfile("test1"))
 
     def test_big_file(self):
+        return
         chunk_size = 1024 * 256
         num_chunks = 4
         def chunk_stream():
@@ -721,7 +722,7 @@ class FSTestCases(object):
             return int(dts1) == int(dts2)       
         d1 = datetime.datetime(2010, 6, 20, 11, 0, 9, 987699)
         d2 = datetime.datetime(2010, 7, 5, 11, 0, 9, 500000)                        
-        self.fs.createfile('/dates.txt', 'check dates')        
+        self.fs.setcontents('/dates.txt', 'check dates')        
         # If the implementation supports settimes, check that the times
         # can be set and then retrieved
         try:
@@ -744,7 +745,9 @@ class ThreadingTestCases:
     __lock = threading.RLock()
 
     def _yield(self):
-        time.sleep(0.001)
+        #time.sleep(0.001)
+        # Yields without a delay
+        time.sleep(0)
 
     def _lock(self):
         self.__lock.acquire()

fs/tests/test_fs.py

@@ -25,6 +25,7 @@ class TestOSFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def check(self, p):
         return os.path.exists(os.path.join(self.temp_dir, relpath(p)))
@@ -40,6 +41,7 @@ class TestSubFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def check(self, p):
         p = os.path.join("foo/bar", relpath(p))
@@ -63,6 +65,9 @@ class TestMountFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
         self.mount_fs.mountdir("mounted/memfs", self.mem_fs)
         self.fs = self.mount_fs.opendir("mounted/memfs")
 
+    def tearDown(self):
+        self.fs.close()
+
     def check(self, p):
         return self.mount_fs.exists(os.path.join("mounted/memfs", relpath(p)))
 
@@ -73,6 +78,9 @@ class TestMountFS_atroot(unittest.TestCase,FSTestCases,ThreadingTestCases):
         self.fs = mountfs.MountFS()
         self.fs.mountdir("", self.mem_fs)
 
+    def tearDown(self):
+        self.fs.close()
+
     def check(self, p):
         return self.mem_fs.exists(p)
 
@@ -85,6 +93,9 @@ class TestMountFS_stacked(unittest.TestCase,FSTestCases,ThreadingTestCases):
         self.mount_fs.mountdir("mem", self.mem_fs1)
         self.mount_fs.mountdir("mem/two", self.mem_fs2)
         self.fs = self.mount_fs.opendir("/mem/two")
+        
+    def tearDown(self):
+        self.fs.close()
 
     def check(self, p):
         return self.mount_fs.exists(os.path.join("mem/two", relpath(p)))

fs/tests/test_ftpfs.py

@@ -43,6 +43,7 @@ class TestFTPFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
         else:
             os.system('kill '+str(self.ftp_server.pid))
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def check(self, p):
         return os.path.exists(os.path.join(self.temp_dir, relpath(p)))

fs/tests/test_path.py

@@ -87,7 +87,8 @@ class TestPathFunctions(unittest.TestCase):
                     ("a/b/c", ("a/b", "c")),
                     ("a", ("", "a")),
                     ("", ("", "")),
-                    ("/", ("", "")),
+                    ("/", ("/", "")),
+                    ("/foo", ("/", "foo")),
                     ("foo/bar", ("foo", "bar")),
                     ("foo/bar/baz", ("foo/bar", "baz")),
                 ]

fs/tests/test_remote.py

@@ -38,12 +38,12 @@ class RemoteTempFS(TempFS):
         return RemoteFileBuffer(self, path, mode, f,
                                       write_on_flush=write_on_flush)
         
-    def setcontents(self, path, content):
+    def setcontents(self, path, data, chunk_size=64*1024):
         f = super(RemoteTempFS, self).open(path, 'wb')
-        if getattr(content, 'read', False):
-            f.write(content.read())
+        if getattr(data, 'read', False):
+            f.write(data.read())
         else:
-            f.write(content)
+            f.write(data)
         f.close()
 
 
@@ -64,7 +64,7 @@ class TellAfterCloseFile(object):
             return self._finalpos
         return self.file.tell()
 
-    def __getattr__(self,attr):
+    def __getattr__(self,attr):        
         return getattr(self.file,attr)
 
 
@@ -75,10 +75,10 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
         self.fs = RemoteTempFS()
         self.original_setcontents = self.fs.setcontents
 
-    def tearDown(self):
+    def tearDown(self):        
         self.fs.close()
 
-    def fake_setcontents(self, path, content):
+    def fake_setcontents(self, path, content='', chunk_size=16*1024):
         ''' Fake replacement for RemoteTempFS setcontents() '''
         raise self.FakeException("setcontents should not be called here!")
     
@@ -245,7 +245,7 @@ class TestConnectionManagerFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
 class DisconnectingFS(WrapFS):
     """FS subclass that raises lots of RemoteConnectionErrors."""
 
-    def __init__(self,fs=None):
+    def __init__(self,fs=None):        
         if fs is None:
             fs = TempFS()
         self._connected = True
@@ -272,7 +272,7 @@ class DisconnectingFS(WrapFS):
             time.sleep(random.random()*0.1)
             self._connected = not self._connected
 
-    def setcontents(self, path, contents):
+    def setcontents(self, path, contents='', chunk_size=64*1024):
         return self.wrapped_fs.setcontents(path, contents)
 
     def close(self):

fs/tests/test_s3fs.py

@@ -26,6 +26,9 @@ class TestS3FS(unittest.TestCase,FSTestCases,ThreadingTestCases):
         for k in self.fs._s3bukt.list():
             self.fs._s3bukt.delete_key(k)
 
+    def tearDown(self):
+        self.fs.close()
+
     def test_concurrent_copydir(self):
         #  makedir() on S3FS is currently not atomic
         pass
@@ -46,3 +49,5 @@ class TestS3FS_prefix(TestS3FS):
         for k in self.fs._s3bukt.list():
             self.fs._s3bukt.delete_key(k)
 
+    def tearDown(self):
+        self.fs.close()

fs/tests/test_walk.py

@@ -10,10 +10,10 @@ class TestWalk(unittest.TestCase):
     
     def setUp(self):
         self.fs = MemoryFS()
-        self.fs.createfile('a.txt', 'hello')
-        self.fs.createfile('b.txt', 'world')
-        self.fs.makeopendir('foo').createfile('c', '123')
-        self.fs.makeopendir('.svn').createfile('ignored')
+        self.fs.setcontents('a.txt', 'hello')
+        self.fs.setcontents('b.txt', 'world')
+        self.fs.makeopendir('foo').setcontents('c', '123')
+        self.fs.makeopendir('.svn').setcontents('ignored', '')
     
     def test_wildcard(self):
         for dir_path, paths in self.fs.walk(wildcard='*.txt'):

fs/tests/test_wrapfs.py

@@ -26,6 +26,7 @@ class TestWrapFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def check(self, p):
         return os.path.exists(os.path.join(self.temp_dir, relpath(p)))
@@ -40,6 +41,7 @@ class TestLazyFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def check(self, p):
         return os.path.exists(os.path.join(self.temp_dir, relpath(p)))
@@ -58,6 +60,7 @@ class TestLimitSizeFS(TestWrapFS):
         self.fs.removedir("/",force=True)
         self.assertEquals(self.fs.cur_size,0)
         super(TestLimitSizeFS,self).tearDown()
+        self.fs.close()
 
     def test_storage_error(self):
         total_written = 0
@@ -86,6 +89,7 @@ class TestHideDotFilesFS(unittest.TestCase):
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
+        self.fs.close()
 
     def test_hidden(self):
         self.assertEquals(len(self.fs.listdir(hidden=False)), 2)

fs/tests/test_xattr.py

@@ -26,11 +26,11 @@ class XAttrTestCases:
             self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
             self.fs.delxattr(p,"xattr1")
             self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
-        self.fs.createfile("test.txt","hello")
+        self.fs.setcontents("test.txt","hello")
         do_getsetdel("test.txt")
         self.assertRaises(ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
         self.fs.makedir("mystuff")
-        self.fs.createfile("/mystuff/test.txt","")
+        self.fs.setcontents("/mystuff/test.txt","")
         do_getsetdel("mystuff")
         do_getsetdel("mystuff/test.txt")
 
@@ -49,15 +49,15 @@ class XAttrTestCases:
             self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2"])
             self.fs.delxattr(p,"attr2")
             self.assertEquals(sorted(self.fs.listxattrs(p)),[])
-        self.fs.createfile("test.txt","hello")
+        self.fs.setcontents("test.txt","hello")
         do_list("test.txt")
         self.fs.makedir("mystuff")
-        self.fs.createfile("/mystuff/test.txt","")
+        self.fs.setcontents("/mystuff/test.txt","")
         do_list("mystuff")
         do_list("mystuff/test.txt")
 
     def test_copy_xattrs(self):
-        self.fs.createfile("a.txt","content")
+        self.fs.setcontents("a.txt","content")
         self.fs.setxattr("a.txt","myattr","myvalue")
         self.fs.setxattr("a.txt","testattr","testvalue")
         self.fs.makedir("stuff")
@@ -75,7 +75,7 @@ class XAttrTestCases:
         self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
 
     def test_move_xattrs(self):
-        self.fs.createfile("a.txt","content")
+        self.fs.setcontents("a.txt","content")
         self.fs.setxattr("a.txt","myattr","myvalue")
         self.fs.setxattr("a.txt","testattr","testvalue")
         self.fs.makedir("stuff")

fs/utils.py

@@ -21,10 +21,11 @@ import stat
 from fs.mountfs import MountFS
 from fs.path import pathjoin, pathsplit
 from fs.errors import DestinationExistsError
+from fs.base import FS
 
 
 
-def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=16384):
+def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
     """Copy a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
     Otherwise file will be copied a chunk at a time.
 
@@ -49,30 +50,20 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=1638
 
     # System copy if there are two sys paths
     if src_syspath is not None and dst_syspath is not None:
-        shutil.copyfile(src_syspath, dst_syspath)
+        FS._shutil_copyfile(src_syspath, dst_syspath)
         return
 
-    src, dst = None, None        
-
+    src = None        
     try:
         # Chunk copy
-        src = src_fs.open(src_path, 'rb')
-        dst = dst_fs.open(dst_path, 'wb')
-
-        while True:
-            chunk = src.read(chunk_size)
-            if not chunk:
-                break
-            dst.write(chunk)
-
+        src = src_fs.open(src_path, 'rb')        
+        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
     finally:
         if src is not None:
             src.close()
-        if dst is not None:
-            dst.close()
 
 
-def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=16384):
+def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
     """Move a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
     Otherwise file will be copied a chunk at a time.
 
@@ -94,33 +85,21 @@ def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=1638
         return
 
     # System copy if there are two sys paths
-    if src_syspath is not None and dst_syspath is not None:
-        shutil.move(src_syspath, dst_syspath)
+    if src_syspath is not None and dst_syspath is not None:        
+        FS._shutil_movefile(src_syspath, dst_syspath)        
         return
 
-    src, dst = None, None
-
-    try:
-        # Chunk copy
+    src = None
+    try:        
         src = src_fs.open(src_path, 'rb')
-        dst = dst_fs.open(dst_path, 'wb')
-
-        while True:
-            chunk = src.read(chunk_size)
-            if not chunk:
-                break
-            dst.write(chunk)
-
+        dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)        
         src_fs.remove(src_path)
 
     finally:
         if src is not None:
             src.close()
-        if dst is not None:
-            dst.close()
-
 
-def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=16384):
+def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
     """Moves contents of a directory from one filesystem to another.
 
     :param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
@@ -142,12 +121,12 @@ def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=16384):
     mount_fs.mount('dst', fs2)
 
     mount_fs.movedir('src', 'dst',
-                     overwrite=True,
+                     overwrite=overwrite,
                      ignore_errors=ignore_errors,
                      chunk_size=chunk_size)
 
 
-def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=16384):
+def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
     """Copies contents of a directory from one filesystem to another.
 
     :param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
@@ -168,11 +147,24 @@ def copydir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=16384):
     mount_fs.mount('src', fs1)
     mount_fs.mount('dst', fs2)
     mount_fs.copydir('src', 'dst',
-                     overwrite=True,
+                     overwrite=overwrite,
                      ignore_errors=ignore_errors,
                      chunk_size=chunk_size)
 
 
+def copystructure(src_fs, dst_fs):
+    """Copies the directory structure from one filesystem to another, so that
+    all directories in `src_fs` will have a corresponding directory in `dst_fs`
+    
+    :param src_fs: Filesystem to copy structure from
+    :param dst_fs: Filesystem to copy structure to
+    
+    """
+    
+    for path in src_fs.walkdirs(wildcard="depth"):
+        dst_fs.makedir(path, allow_recreate=True)
+
+
 def countbytes(fs):
     """Returns the total number of bytes contained within files in a filesystem.
 
@@ -214,6 +206,13 @@ def isfile(fs,path,info=None):
                 return False
     return fs.isfile(path)
 
+def contains_files(fs, path='/'):
+    """Check if there are any files in the filesystem"""
+    try:
+        iter(fs.walkfiles(path)).next()
+    except StopIteration:
+        return False
+    return True
 
 def find_duplicates(fs,
                     compare_paths=None,
@@ -350,14 +349,16 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
     if file_out is None:
         file_out = sys.stdout
 
+    file_encoding = getattr(file_out, 'encoding', 'utf-8')
+
     if terminal_colors is None:
-        if sys.platform == 'win32':
+        if sys.platform.startswith('win'):
             terminal_colors = False
         else:
             terminal_colors = True
-
+    
     def write(line):
-        file_out.write(line.encode(file_out.encoding or 'utf-8')+'\n')
+        file_out.write(line.encode(file_encoding)+'\n')
         
     def wrap_prefix(prefix):
        if not terminal_colors:
@@ -407,7 +408,8 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
             if is_dir:                
                 write('%s %s' % (wrap_prefix(prefix + '--'), wrap_dirname(item)))
                 if max_levels is not None and len(levels) >= max_levels:
-                    write(wrap_prefix(prefix[:-1] + '       ') + wrap_error('max recursion levels reached'))
+                    pass
+                    #write(wrap_prefix(prefix[:-1] + '       ') + wrap_error('max recursion levels reached'))
                 else:
                     print_dir(fs, pathjoin(path, item), levels[:] + [is_last_item])                                            
             else:

fs/watch.py

@@ -312,6 +312,24 @@ class WatchableFS(WatchableFSMixin,WrapFS):
         self.notify_watchers(ACCESSED,path)
         return WatchedFile(f,self,path,mode)
 
+    def setcontents(self, path, data='', chunk_size=64*1024):
+        existed = self.wrapped_fs.isfile(path)
+        ret = super(WatchableFS, self).setcontents(path, data, chunk_size=chunk_size)
+        if not existed:
+            self.notify_watchers(CREATED,path)
+        self.notify_watchers(ACCESSED,path)
+        if data:
+            self.notify_watchers(MODIFIED,path,True)
+        return ret
+
+    def createfile(self, path):
+        existed = self.wrapped_fs.isfile(path)
+        ret = super(WatchableFS, self).createfile(path)
+        if not existed:
+            self.notify_watchers(CREATED,path)
+        self.notify_watchers(ACCESSED,path)
+        return retq
+
     def makedir(self,path,recursive=False,allow_recreate=False):
         existed = self.wrapped_fs.isdir(path)
         try:

fs/wrapfs/__init__.py

@@ -144,6 +144,14 @@ class WrapFS(FS):
         return self._file_wrap(f, mode)
 
     @rewrite_errors
+    def setcontents(self, path, data, chunk_size=64*1024):
+        return self.wrapped_fs.setcontents(self._encode(path), data, chunk_size=chunk_size)
+
+    @rewrite_errors
+    def createfile(self, path):
+        return self.wrapped_fs.createfile(self._encode(path))
+
+    @rewrite_errors
     def exists(self, path):
         return self.wrapped_fs.exists(self._encode(path))
 
@@ -156,7 +164,7 @@ class WrapFS(FS):
         return self.wrapped_fs.isfile(self._encode(path))
 
     @rewrite_errors
-    def listdir(self, path="", **kwds):
+    def listdir(self, path="", **kwds):        
         full = kwds.pop("full",False)
         absolute = kwds.pop("absolute",False)
         wildcard = kwds.pop("wildcard",None)
@@ -325,6 +333,6 @@ def wrap_fs_methods(decorator, cls=None, exclude=[]):
 wrap_fs_methods.method_names = ["open","exists","isdir","isfile","listdir",
     "makedir","remove","setcontents","removedir","rename","getinfo","copy",
     "move","copydir","movedir","close","getxattr","setxattr","delxattr",
-    "listxattrs","getsyspath","createfile"]
+    "listxattrs","getsyspath","createfile", "hasmeta", "getmeta"]
 
 

fs/wrapfs/debugfs.py

@@ -45,16 +45,17 @@ logger.setLevel(logging.DEBUG)
 logger.addHandler(logging.StreamHandler())
 
 class DebugFS(object):
-    def __init__(self, fs, identifier=None, skip=()):
+    def __init__(self, fs, identifier=None, skip=(), verbose=True):
         '''
             fs - Reference to object to debug
             identifier - Custom string-like object will be added
                 to each log line as identifier.
             skip - list of method names which DebugFS should not log
-        '''
+        '''        
         self.__wrapped_fs = fs
         self.__identifier = identifier
-        self.__skip = skip 
+        self.__skip = skip
+        self.__verbose = verbose 
         super(DebugFS, self).__init__()
         
     def __log(self, level, message):
@@ -65,8 +66,8 @@ class DebugFS(object):
         
     def __parse_param(self, value):
         if isinstance(value, basestring):
-            if len(value) > 20:
-                value = "%s (length %d)" % (repr(value[:20]), len(value))
+            if len(value) > 60:
+                value = "%s ... (length %d)" % (repr(value[:60]), len(value))
             else:
                 value = repr(value) 
         elif isinstance(value, list):
@@ -96,6 +97,7 @@ class DebugFS(object):
         self.__log(INFO, "%s %s%s -> %s" % (msg, str(key), args, value))
     
     def __getattr__(self, key):
+        
         if key.startswith('__'):
             # Internal calls, nothing interesting
             return object.__getattribute__(self, key)
@@ -117,7 +119,7 @@ class DebugFS(object):
         
         def _method(*args, **kwargs):
             try:
-                value = attr(*args, **kwargs)
+                value = attr(*args, **kwargs)                
                 self.__report("Call method", key, value, *args, **kwargs)
             except FSError, e:
                 self.__log(ERROR, "Call method %s%s -> Exception %s: %s" % \
@@ -134,6 +136,7 @@ class DebugFS(object):
                 raise e, None, tb
             return value
         
-        if key not in self.__skip:
-            self.__log(DEBUG, "Asking for method %s" % key)
+        if self.__verbose:
+            if key not in self.__skip:
+                self.__log(DEBUG, "Asking for method %s" % key)
         return _method

fs/wrapfs/lazyfs.py

@@ -88,8 +88,8 @@ class LazyFS(WrapFS):
 
     wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs)
 
-    def setcontents(self, path, data):
-        return self.wrapped_fs.setcontents(path, data)
+    def setcontents(self, path, data, chunk_size=64*1024):
+        return self.wrapped_fs.setcontents(path, data, chunk_size=chunk_size)
 
     def close(self):
         if not self.closed:

fs/wrapfs/limitsizefs.py

@@ -60,7 +60,22 @@ class LimitSizeFS(WrapFS):
                 size = 0
                 self._file_sizes[path] = 0
             return LimitSizeFile(self,path,f,mode,size)
-
+                
+    def setcontents(self, path, data, chunk_size=64*1024):            
+        f = None
+        try:                
+            f = self.open(path, 'w')
+            if hasattr(data, 'read'):
+                chunk = data.read(chunk_size)            
+                while chunk:
+                    f.write(chunk)
+                    chunk = data.read(chunk_size)
+            else:
+                f.write(data)
+        finally:
+            if f is not None:
+                f.close()
+    
     def _ensure_file_size(self, path, size, shrink=False):
         path = relpath(normpath(path))
         with self._size_lock:

fs/wrapfs/readonlyfs.py

@@ -59,3 +59,5 @@ class ReadOnlyFS(WrapFS):
     remove = _no_can_do
     removedir = _no_can_do
     settimes = _no_can_do
+    setcontents = _no_can_do
+    createfile = _no_can_do

fs/wrapfs/subfs.py

@@ -38,15 +38,13 @@ class SubFS(WrapFS):
     def __repr__(self):
         return str(self)
 
-    def desc(self, path):
-        if self.isdir(path):
-            return "Sub dir of %s" % str(self.wrapped_fs)
-        else:
-            return "File in sub dir of %s" % str(self.wrapped_fs)
-
-    def setcontents(self,path,contents):
+    def desc(self, path):        
+        desc = "%s in sub dir %s of %s" % (path, self.sub_dir, str(self.wrapped_fs))
+        return desc
+        
+    def setcontents(self, path, data, chunk_size=64*1024):
         path = self._encode(path)
-        return self.wrapped_fs.setcontents(path,contents)
+        return self.wrapped_fs.setcontents(path, data, chunk_size=chunk_size)
 
     def opendir(self, path):
         if not self.exists(path):

fs/xattrs.py

@@ -173,7 +173,7 @@ class SimulateXAttr(WrapFS):
             try:
                 self.wrapped_fs.removedir(path,recursive=recursive)
             except FSError:
-                self.wrapped_fs.createfile(attr_file,attr_file_contents)
+                self.wrapped_fs.setcontents(attr_file,attr_file_contents)
                 raise
 
     def copy(self,src,dst,**kwds):

fs/zipfs.py

@@ -77,7 +77,7 @@ class ZipFS(FS):
               'unicode_paths' : True,
               'case_insensitive_paths' : False,
               'network' : False,
-              'atomic.setcontents' : False             
+              'atomic.setcontents' : False
              }
 
     def __init__(self, zip_file, mode="r", compression="deflated", allow_zip_64=False, encoding="CP437", thread_synchronize=True):
@@ -106,19 +106,24 @@ class ZipFS(FS):
 
         self.zip_mode = mode
         self.encoding = encoding
+        
+        if isinstance(zip_file, basestring):
+            zip_file = os.path.expanduser(os.path.expandvars(zip_file))
+            zip_file = os.path.normpath(os.path.abspath(zip_file))
+        
         try:
             self.zf = ZipFile(zip_file, mode, compression_type, allow_zip_64)
         except BadZipfile, bzf:            
             raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                                details=bzf)
-        except IOError, ioe:
+        except IOError, ioe:            
             if str(ioe).startswith('[Errno 22] Invalid argument'):
                 raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                                    details=ioe)
             raise ZipNotFoundError("Zip file not found (%s)" % str(zip_file),
                                   details=ioe)
                 
-        self.zip_path = str(zip_file)        
+        self.zip_path = str(zip_file)
         self.temp_fs = None
         if mode in 'wa':
             self.temp_fs = tempfs.TempFS()
@@ -214,11 +219,9 @@ class ZipFS(FS):
         sys_path = self.temp_fs.getsyspath(filename)
         self.zf.write(sys_path, filename.encode(self.encoding))
 
-    def desc(self, path):
-        if self.isdir(path):
-            return "Dir in zip file: %s" % self.zip_path
-        else:
-            return "File in zip file: %s" % self.zip_path
+    def desc(self, path):        
+        return "%s in zip file %s" % (path, self.zip_path)
+        
 
     def isdir(self, path):
         return self._path_fs.isdir(path)
@@ -258,4 +261,5 @@ class ZipFS(FS):
         if 'date_time' in zinfo:
             info['created_time'] = datetime.datetime(*zinfo['date_time'])
         info.update(zinfo)
+        
         return info

setup.py

 #!/usr/bin/env python
 
 from distutils.core import setup
-
 from fs import __version__ as VERSION
 
+COMMANDS = ['fscat',
+            'fscp',
+            'fsinfo',
+            'fsls',
+            'fsmv',
+            'fscp',
+            'fsrm',
+            'fsserve',
+            'fstree']
+
+
 classifiers = [
     'Development Status :: 3 - Alpha',
     'Intended Audience :: Developers',
@@ -25,7 +35,8 @@ setup(name='fs',
       platforms = ['any'],
       packages=['fs','fs.expose','fs.expose.fuse','fs.tests','fs.wrapfs',
                 'fs.osfs','fs.contrib','fs.contrib.bigfs','fs.contrib.davfs',
-                'fs.expose.dokan',],
+                'fs.expose.dokan', 'fs.commands'],
+      scripts=['fs/commands/%s' % command for command in COMMANDS],
       classifiers=classifiers,
       )