add module "fs.filelike" with utils for building file-like objects.

This is a local copy of the guts of my "filelike" module, re-licensed under
the MIT license.

This commit also uses it to fix a few edge-cases in various filesystem
implementations (e.g. truncating StringIO objects).

ChangeLog

@@ -57,4 +57,6 @@
     * Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these
       can often determine whether a path is a file or directory by inspecting
       the info dict and avoid an additional query to the filesystem.
+    * Added utility module 'fs.filelike' with some helpers for building and
+      manipulating file-like objects.
 

fs/contrib/bigfs/__init__.py

@@ -12,15 +12,13 @@ Contributed under the terms of the BSD License:
 http://www.opensource.org/licenses/bsd-license.php
 """
 
+from struct import pack, unpack
+
 from fs.base import *
 from fs.memoryfs import MemoryFS
-from fs.contrib.bigfs.subrangefile import SubrangeFile
+from fs.filelike import StringIO
 
-from struct import pack, unpack
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
+from fs.contrib.bigfs.subrangefile import SubrangeFile
 
 class BIGEntry:
     def __init__(self, filename, offset, storedSize, isCompressed, realSize):

fs/expose/sftp.py

@@ -29,7 +29,6 @@ import stat as statinfo
 import time
 import SocketServer as sockserv
 import threading
-from StringIO import StringIO
 
 import paramiko
 
@@ -37,6 +36,7 @@ from fs.base import flags_to_mode
 from fs.path import *
 from fs.errors import *
 from fs.local_functools import wraps
+from fs.filelike import StringIO
 
 
 # Default host key used by BaseSFTPServer

fs/filelike.py

+"""
+fs.filelike
+===========
+
+This module takes care of the groundwork for implementing and manipulating
+objects that provide a rich file-like interface, including reading, writing,
+seeking and iteration.
+
+The main class is FileLikeBase, which implements the entire file-like interface
+on top of primitive _read(), _write(), _seek(), _tell() and _truncate() methods.
+Subclasses may implement any or all of these methods to obtain the related
+higher-level file behaviors.
+
+Other useful classes include:
+
+    * StringIO:   a version of the builtin StringIO class, patched to more
+                  closely preserve the semantics of a standard file.
+
+    * FileWrapper:  a generic base class for wrappers around a filelike object
+                    (think e.g. compression or decryption).
+
+    * SpooledTemporaryFile:  a version of the builtin SpooledTemporaryFile
+                             class, patched to more closely preserve the 
+                             semantics of a standard file.
+
+""" 
+# Copyright (C) 2006-2009, Ryan Kelly
+# All rights reserved; available under the terms of the MIT License.
+
+import tempfile as _tempfile
+
+try:
+    from cStringIO import StringIO as _StringIO
+except ImportError:
+    from StrimgIO import StringIO as _StringIO
+
+import fs
+
+
+class NotReadableError(IOError):
+    pass
+class NotWritableError(IOError):
+    pass
+class NotSeekableError(IOError):
+    pass
+class NotTruncatableError(IOError):
+    pass
+
+
+class FileLikeBase(object):
+    """Base class for implementing file-like objects.
+    
+    This class takes a lot of the legwork out of writing file-like objects
+    with a rich interface.  It implements the higher-level file-like methods
+    on top of five primitive methods: _read, _write, _seek, _tell and
+    _truncate. See their docstrings for precise details on how these methods
+    behave.
+    
+    Subclasses then need only implement some subset of these methods for
+    rich file-like interface compatability.  They may of course override
+    other methods as desired.
+
+    The class is missing the following attributes and methods, which dont
+    really make sense for anything but real files:
+        
+        * fileno()
+        * isatty()
+        * encoding
+        * mode
+        * name
+        * newlines
+
+    Unlike standard file objects, all read methods share the same buffer
+    and so can be freely mixed (e.g. read(), readline(), next(), ...).
+
+    This class understands and will accept the following mode strings,
+    with any additional characters being ignored:
+
+        * r    - open the file for reading only.
+        * r+   - open the file for reading and writing.
+        * r-   - open the file for streamed reading; do not allow seek/tell.
+        * w    - open the file for writing only; create the file if
+                 it doesn't exist; truncate it to zero length.
+        * w+   - open the file for reading and writing; create the file
+                 if it doesn't exist; truncate it to zero length.
+        * w-   - open the file for streamed writing; do not allow seek/tell.
+        * a    - open the file for writing only; create the file if it
+                 doesn't exist; place pointer at end of file.
+        * a+   - open the file for reading and writing; create the file
+                 if it doesn't exist; place pointer at end of file.
+
+    These are mostly standard except for the "-" indicator, which has
+    been added for efficiency purposes in cases where seeking can be
+    expensive to simulate (e.g. compressed files).  Note that any file
+    opened for both reading and writing must also support seeking.
+    """
+    
+    def __init__(self,bufsize=1024*64):
+        """FileLikeBase Constructor.
+
+        The optional argument 'bufsize' specifies the number of bytes to
+        read at a time when looking for a newline character.  Setting this to
+        a larger number when lines are long should improve efficiency.
+        """
+        # File-like attributes
+        self.closed = False
+        self.softspace = 0
+        # Our own attributes
+        self._bufsize = bufsize  # buffer size for chunked reading
+        self._rbuffer = None     # data that's been read but not returned
+        self._wbuffer = None     # data that's been given but not written
+        self._sbuffer = None     # data between real & apparent file pos
+        self._soffset = 0        # internal offset of file pointer
+
+    #
+    #  The following five methods are the ones that subclasses are expected
+    #  to implement.  Carefully check their docstrings.
+    #
+
+    def _read(self,sizehint=-1):
+        """Read approximately <sizehint> bytes from the file-like object.
+        
+        This method is to be implemented by subclasses that wish to be
+        readable.  It should read approximately <sizehint> bytes from the
+        file and return them as a string.  If <sizehint> is missing or
+        less than or equal to zero, try to read all the remaining contents.
+        
+        The method need not guarantee any particular number of bytes -
+        it may return more bytes than requested, or fewer.  If needed the
+        size hint may be completely ignored.  It may even return an empty
+        string if no data is yet available.
+        
+        Because of this, the method must return None to signify that EOF
+        has been reached.  The higher-level methods will never indicate EOF
+        until None has been read from _read().  Once EOF is reached, it
+        should be safe to call _read() again, immediately returning None.
+        """
+        raise NotReadableError("Object not readable")
+    
+    def _write(self,string,flushing=False):
+        """Write the given string to the file-like object.
+        
+        This method must be implemented by subclasses wishing to be writable.
+        It must attempt to write as much of the given data as possible to the
+        file, but need not guarantee that it is all written.  It may return
+        None to indicate that all data was written, or return as a string any
+        data that could not be written.
+        
+        If the keyword argument 'flushing' is true, it indicates that the
+        internal write buffers are being flushed, and *all* the given data
+        is expected to be written to the file. If unwritten data is returned
+        when 'flushing' is true, an IOError will be raised.
+        """
+        raise NotWritableError("Object not writable")
+
+    def _seek(self,offset,whence):
+        """Set the file's internal position pointer, approximately.
+ 
+        This method should set the file's position to approximately 'offset'
+        bytes relative to the position specified by 'whence'.  If it is
+        not possible to position the pointer exactly at the given offset,
+        it should be positioned at a convenient *smaller* offset and the
+        file data between the real and apparent position should be returned.
+
+        At minimum, this method must implement the ability to seek to
+        the start of the file, i.e. offset=0 and whence=0.  If more
+        complex seeks are difficult to implement then it may raise
+        NotImplementedError to have them simulated (inefficiently) by
+        the higher-level machinery of this class.
+        """
+        raise NotSeekableError("Object not seekable")
+
+    def _tell(self):
+        """Get the location of the file's internal position pointer.
+
+        This method must be implemented by subclasses that wish to be
+        seekable, and must return the position of the file's internal
+        pointer.
+
+        Due to buffering, the position seen by users of this class
+        (the "apparent position") may be different to the position
+        returned by this method (the "actual position").
+        """
+        raise NotSeekableError("Object not seekable")
+
+    def _truncate(self,size):
+        """Truncate the file's size to <size>.
+
+        This method must be implemented by subclasses that wish to be
+        truncatable.  It must truncate the file to exactly the given size
+        or fail with an IOError.
+
+        Note that <size> will never be None; if it was not specified by the
+        user then it is calculated as the file's apparent position (which may
+        be different to its actual position due to buffering).
+        """
+        raise NotTruncatableError("Object not truncatable")
+
+    #
+    #  The following methods provide the public API of the filelike object.
+    #  Subclasses shouldn't need to mess with these (except perhaps for
+    #  close() and flush())
+    #
+
+    def _check_mode(self,mode,mstr=None):
+        """Check whether the file may be accessed in the given mode.
+
+        'mode' must be one of "r" or "w", and this function returns False
+        if the file-like object has a 'mode' attribute, and it does not
+        permit access in that mode.  If there is no 'mode' attribute,
+        it defaults to "r+".
+
+        If seek support is not required, use "r-" or "w-" as the mode string.
+
+        To check a mode string other than self.mode, pass it in as the
+        second argument.
+        """
+        if mstr is None:
+            try:
+                mstr = self.mode
+            except AttributeError:
+                mstr = "r+"
+        if "+" in mstr:
+            return True
+        if "-" in mstr and "-" not in mode:
+            return False    
+        if "r" in mode:
+            if "r" not in mstr:
+                return False    
+        if "w" in mode:
+            if "w" not in mstr and "a" not in mstr:
+                return False
+        return True
+        
+    def _assert_mode(self,mode,mstr=None):
+        """Check whether the file may be accessed in the given mode.
+
+        This method is equivalent to _check_assert(), but raises IOError
+        instead of returning False.
+        """
+        if mstr is None:
+            try:
+                mstr = self.mode
+            except AttributeError:
+                mstr = "r+"
+        if "+" in mstr:
+            return True
+        if "-" in mstr and "-" not in mode:
+            raise NotSeekableError("File does not support seeking.")
+        if "r" in mode:
+            if "r" not in mstr:
+                raise NotReadableError("File not opened for reading")
+        if "w" in mode:
+            if "w" not in mstr and "a" not in mstr:
+                raise NotWritableError("File not opened for writing")
+        return True
+    
+    def flush(self):
+        """Flush internal write buffer, if necessary."""
+        if self.closed:
+            raise IOError("File has been closed")
+        if self._check_mode("w-") and self._wbuffer is not None:
+            buffered = ""
+            if self._sbuffer:
+                buffered = buffered + self._sbuffer
+                self._sbuffer = None
+            buffered = buffered + self._wbuffer
+            self._wbuffer = None
+            leftover = self._write(buffered,flushing=True)
+            if leftover:
+                raise IOError("Could not flush write buffer.")
+    
+    def close(self):
+        """Flush write buffers and close the file.
+
+        The file may not be accessed further once it is closed.
+        """
+        #  Errors in subclass constructors can cause this to be called without
+        #  having called FileLikeBase.__init__().  Since we need the attrs it
+        #  initialises in cleanup, ensure we call it here.
+        if not hasattr(self,"closed"):
+            FileLikeBase.__init__(self)
+        if not self.closed:
+            self.flush()
+            self.closed = True
+
+    def __del__(self):
+        self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self,exc_type,exc_val,exc_tb):
+        self.close()
+        return False
+    
+    def next(self):
+        """next() method complying with the iterator protocol.
+
+        File-like objects are their own iterators, with each call to
+        next() returning subsequent lines from the file.
+        """
+        ln = self.readline()
+        if ln == "":
+            raise StopIteration()
+        return ln
+    
+    def __iter__(self):
+        return self
+
+    def truncate(self,size=None):
+        """Truncate the file to the given size.
+
+        If <size> is not specified or is None, the current file position is
+        used.  Note that this method may fail at runtime if the underlying
+        filelike object is not truncatable.
+        """
+        if "-" in getattr(self,"mode",""):
+            raise NotTruncatableError("File is not seekable, can't truncate.")
+        if self._wbuffer:
+            self.flush()
+        if size is None:
+            size = self.tell()
+        self._truncate(size)
+
+    def seek(self,offset,whence=0):
+        """Move the internal file pointer to the given location."""
+        if whence > 2 or whence < 0:
+            raise ValueError("Invalid value for 'whence': " + str(whence))
+        if "-" in getattr(self,"mode",""):
+            raise NotSeekableError("File is not seekable.")
+        # Ensure that there's nothing left in the write buffer
+        if self._wbuffer:
+            self.flush()
+        # Adjust for any data left in the read buffer
+        if whence == 1 and self._rbuffer:
+            offset = offset - len(self._rbuffer)
+        self._rbuffer = None
+        # Adjust for any discrepancy in actual vs apparent seek position
+        if whence == 1:
+            if self._sbuffer:
+                offset = offset + len(self._sbuffer)
+            if self._soffset:
+                offset = offset + self._soffset
+        self._sbuffer = None
+        self._soffset = 0
+        # Shortcut the special case of staying put.
+        # As per posix, this has already cases the buffers to be flushed.
+        if offset == 0 and whence == 1:
+            return
+        # Catch any failed attempts to read while simulating seek
+        try:
+            # Try to do a whence-wise seek if it is implemented.
+            sbuf = None
+            try:
+                sbuf = self._seek(offset,whence)
+            except NotImplementedError:
+                # Try to simulate using an absolute seek.
+                try:
+                    if whence == 1:
+                        offset = self._tell() + offset
+                    elif whence == 2:
+                        if hasattr(self,"size"):
+                            offset = self.size + offset
+                        else:
+                            self._do_read_rest()
+                            offset = self.tell() + offset
+                    else:
+                        # absolute seek already failed, don't try again
+                        raise NotImplementedError
+                    sbuf = self._seek(offset,0)
+                except NotImplementedError:
+                    # Simulate by reseting to start
+                    self._seek(0,0)
+                    self._soffset = offset
+            finally:
+                self._sbuffer = sbuf
+        except NotReadableError:
+            raise NotSeekableError("File not readable, can't simulate seek")
+
+    def tell(self):
+        """Determine current position of internal file pointer."""
+        # Need to adjust for unread/unwritten data in buffers
+        pos = self._tell()
+        if self._rbuffer:
+            pos = pos - len(self._rbuffer)
+        if self._wbuffer:
+            pos = pos + len(self._wbuffer)
+        if self._sbuffer:
+            pos = pos + len(self._sbuffer)
+        if self._soffset:
+            pos = pos + self._soffset
+        return pos
+    
+    def read(self,size=-1):
+        """Read at most 'size' bytes from the file.
+
+        Bytes are returned as a string.  If 'size' is negative, zero or
+        missing, the remainder of the file is read.  If EOF is encountered
+        immediately, the empty string is returned.
+        """
+        if self.closed:
+            raise IOError("File has been closed")
+        self._assert_mode("r-")
+        return self._do_read(size)
+
+    def _do_read(self,size):
+        """Private method to read from the file.
+
+        This method behaves the same as self.read(), but skips some
+        permission and sanity checks.  It is intended for use in simulating
+        seek(), where we may want to read (and discard) information from
+        a file not opened in read mode.
+
+        Note that this may still fail if the file object actually can't
+        be read from - it just won't check whether the mode string gives
+        permission.
+        """
+        # If we were previously writing, ensure position is correct
+        if self._wbuffer is not None:
+            self.seek(0,1)
+        # Discard any data that should have been seeked over
+        if self._sbuffer:
+            s = len(self._sbuffer)
+            self._sbuffer = None
+            self.read(s)
+        elif self._soffset:
+            s = self._soffset
+            self._soffset = 0
+            while s > self._bufsize:
+                self._do_read(self._bufsize)
+                s -= self._bufsize
+            self._do_read(s)
+        # Should the entire file be read?
+        if size <= 0:
+            if self._rbuffer:
+                data = [self._rbuffer]
+            else:
+                data = []
+            self._rbuffer = ""
+            newData = self._read()
+            while newData is not None:
+                data.append(newData)
+                newData = self._read()
+            output = "".join(data)
+        # Otherwise, we need to return a specific amount of data
+        else:
+            if self._rbuffer:
+                newData = self._rbuffer
+                data = [newData]
+            else:
+                newData = ""
+                data = []
+            sizeSoFar = len(newData)
+            while sizeSoFar < size:
+                newData = self._read(size-sizeSoFar)
+                if newData is None:
+                    break
+                data.append(newData)
+                sizeSoFar += len(newData)
+            data = "".join(data)
+            if sizeSoFar > size:
+                # read too many bytes, store in the buffer
+                self._rbuffer = data[size:]
+                data = data[:size]
+            else:
+                self._rbuffer = ""
+            output = data
+        return output
+
+    def _do_read_rest(self):
+        """Private method to read the file through to EOF."""
+        data = self._do_read(self._bufsize)
+        while data != "":
+            data = self._do_read(self._bufsize)
+        
+    def readline(self,size=-1):
+        """Read a line from the file, or at most <size> bytes."""
+        bits = []
+        indx = -1
+        sizeSoFar = 0
+        while indx == -1:
+            nextBit = self.read(self._bufsize)
+            bits.append(nextBit)
+            sizeSoFar += len(nextBit)
+            if nextBit == "":
+                break
+            if size > 0 and sizeSoFar >= size:
+                break
+            indx = nextBit.find("\n")
+        # If not found, return whole string up to <size> length
+        # Any leftovers are pushed onto front of buffer
+        if indx == -1:
+            data = "".join(bits)
+            if size > 0 and sizeSoFar > size:
+                extra = data[size:]
+                data = data[:size]
+                self._rbuffer = extra + self._rbuffer
+            return data
+        # If found, push leftovers onto front of buffer
+        # Add one to preserve the newline in the return value
+        indx += 1
+        extra = bits[-1][indx:]
+        bits[-1] = bits[-1][:indx]
+        self._rbuffer = extra + self._rbuffer
+        return "".join(bits)
+    
+    def readlines(self,sizehint=-1):
+        """Return a list of all lines in the file."""
+        return [ln for ln in self]
+    
+    def xreadlines(self):
+        """Iterator over lines in the file - equivalent to iter(self)."""
+        return iter(self)
+
+    def write(self,string):
+        """Write the given string to the file."""
+        if self.closed:
+            raise IOError("File has been closed")
+        self._assert_mode("w-")
+        # If we were previously reading, ensure position is correct
+        if self._rbuffer is not None:
+            self.seek(0,1)
+        # If we're actually behind the apparent position, we must also
+        # write the data in the gap.
+        if self._sbuffer:
+            string = self._sbuffer + string
+            self._sbuffer = None
+        elif self._soffset:
+            s = self._soffset
+            self._soffset = 0
+            try:
+                string = self._do_read(s) + string
+            except NotReadableError:
+                raise NotSeekableError("File not readable, could not complete simulation of seek")
+            self.seek(0,0)
+        if self._wbuffer:
+            string = self._wbuffer + string
+        leftover = self._write(string)
+        if leftover is None:
+            self._wbuffer = ""
+        else:
+            self._wbuffer = leftover
+    
+    def writelines(self,seq):
+        """Write a sequence of lines to the file."""
+        for ln in seq:
+            self.write(ln)
+
+
+class FileWrapper(FileLikeBase):
+    """Base class for objects that wrap a file-like object.
+    
+    This class provides basic functionality for implementing file-like
+    objects that wrap another file-like object to alter its functionality
+    in some way.  It takes care of house-keeping duties such as flushing
+    and closing the wrapped file.
+
+    Access to the wrapped file is given by the attribute wrapped_file.
+    By convention, the subclass's constructor should accept this as its
+    first argument and pass it to its superclass's constructor in the
+    same position.
+    
+    This class provides a basic implementation of _read() and _write()
+    which just calls read() and write() on the wrapped object.  Subclasses
+    will probably want to override these.
+    """
+
+    _append_requires_overwrite = False
+
+    def __init__(self,wrapped_file,mode=None):
+        """FileWrapper constructor.
+        
+        'wrapped_file' must be a file-like object, which is to be wrapped
+        in another file-like object to provide additional functionality.
+        
+        If given, 'mode' must be the access mode string under which
+        the wrapped file is to be accessed.  If not given or None, it
+        is looked up on the wrapped file if possible.  Otherwise, it
+        is not set on the object.
+        """
+        # This is used for working around flush/close inefficiencies
+        self.__closing = False
+        super(FileWrapper,self).__init__()
+        self.wrapped_file = wrapped_file
+        if mode is None:
+            self.mode = getattr(wrapped_file,"mode","r+")
+        else:
+            self.mode = mode
+        self._validate_mode()
+        # Copy useful attributes of wrapped_file
+        if hasattr(wrapped_file,"name"):
+            self.name = wrapped_file.name
+        # Respect append-mode setting
+        if "a" in self.mode:
+            if self._check_mode("r"):
+                self.wrapped_file.seek(0)
+            self.seek(0,2)
+
+    def _validate_mode(self):
+        """Check that various file-mode conditions are satisfied."""
+        #  If append mode requires overwriting the underlying file,
+        #  if must not be opened in append mode.
+        if self._append_requires_overwrite:
+            if self._check_mode("w"):
+                if "a" in getattr(self.wrapped_file,"mode",""):
+                    raise ValueError("Underlying file can't be in append mode")
+
+    def __del__(self):
+        #  Errors in subclass constructors could result in this being called
+        #  without invoking FileWrapper.__init__.  Establish some simple
+        #  invariants to prevent errors in this case.
+        if not hasattr(self,"wrapped_file"):
+            self.wrapped_file = None
+        if not hasattr(self,"_FileWrapper__closing"):
+            self.__closing = False
+        #  Close the wrapper and the underlying file independently, so the
+        #  latter is still closed on cleanup even if the former errors out.
+        try:
+            super(FileWrapper,self).close()
+        except Exception:
+            if hasattr(getattr(self,"wrapped_file",None),"close"):
+                self.wrapped_file.close()
+            raise
+
+    def close(self):
+        """Close the object for reading/writing."""
+        #  The superclass implementation of this will call flush(),
+        #  which calls flush() on our wrapped object.  But we then call
+        #  close() on it, which will call its flush() again!  To avoid
+        #  this inefficiency, our flush() will not flush the wrapped
+        #  file when we're closing.
+        self.__closing = True
+        super(FileWrapper,self).close()
+        if hasattr(self.wrapped_file,"close"):
+            self.wrapped_file.close()
+
+    def flush(self):
+        """Flush the write buffers of the file."""
+        super(FileWrapper,self).flush()
+        if not self.__closing and hasattr(self.wrapped_file,"flush"):
+            self.wrapped_file.flush()
+
+    def _read(self,sizehint=-1):
+        data = self.wrapped_file.read(sizehint)
+        if data == "":
+            return None
+        return data
+
+    def _write(self,string,flushing=False):
+        return self.wrapped_file.write(string)
+
+    def _seek(self,offset,whence):
+        self.wrapped_file.seek(offset,whence)
+
+    def _tell(self):
+        return self.wrapped_file.tell()
+
+    def _truncate(self,size):
+        return self.wrapped_file.truncate(size)
+
+
+class StringIO(FileWrapper):
+    """StringIO wrapper that more closely matches standard file behaviour.
+
+    This is a simple compatability wrapper around the native StringIO class
+    which fixes some corner-cases of its behaviour.  Specifically:
+
+        * adding __enter__ and __exit__ methods
+        * having truncate(size) zero-fill when growing the file
+
+    """
+
+    def __init__(self,data=None,mode=None):
+        wrapped_file = _StringIO()
+        if data is not None:
+            wrapped_file.write(data)
+            wrapped_file.seek(0)
+        super(StringIO,self).__init__(wrapped_file,mode)
+
+    def getvalue(self):
+        return self.wrapped_file.getvalue()
+
+    def _truncate(self,size):
+        pos = self.wrapped_file.tell()
+        self.wrapped_file.truncate(size)
+        curlen = len(self.wrapped_file.getvalue())
+        if size > curlen:
+            self.wrapped_file.seek(curlen)
+            try:
+                self.wrapped_file.write("\x00"*(size-curlen))
+            finally:
+                self.wrapped_file.seek(pos)
+
+
+class SpooledTemporaryFile(FileWrapper):
+    """SpooledTemporaryFile wrapper with some compatability fixes.
+
+    This is a simple compatability wrapper around the native SpooledTempFile
+    class which fixes some corner-cases of its behaviour.  Specifically:
+
+        * have truncate() accept a size argument
+        * roll to disk is seeking past the max in-memory size
+        * use improved StringIO class from this module
+
+    """
+
+    def __init__(self,max_size=0,mode="w+b",bufsize=-1,*args,**kwds):
+        try:
+            stf_args = (max_size,mode,bufsize) + args
+            wrapped_file = _tempfile.SpooledTemporaryFile(*stf_args,**kwds)
+            wrapped_file._file = StringIO()
+            self.__is_spooled = True
+        except AttributeError:
+            ntf_args = (mode,bufsize) + args
+            wrapped_file = _tempfile.NamedTemporaryFile(*ntf_args,**kwds)
+            self.__is_spooled = False
+        super(SpooledTemporaryFile,self).__init__(wrapped_file)
+
+    def _seek(self,offset,whence):
+        if self.__is_spooled:
+            max_size = self.wrapped_file._max_size
+            if whence == fs.SEEK_SET:
+                if offset > max_size:
+                    self.wrapped_file.rollover()
+            elif whence == fs.SEEK_CUR:
+                if offset + self.wrapped_file.tell() > max_size:
+                    self.wrapped_file.rollover()
+            else:
+                if offset > 0:
+                    self.wrapped_file.rollover()
+        self.wrapped_file.seek(offset,whence)
+
+    def _truncate(self,size):
+        if self.__is_spooled:
+            self.wrapped_file._file.truncate(size)
+        else:
+            self.wrapped_file.truncate(size)
+
+    def fileno(self):
+        return self.wrapped_file.fileno()
+
+

fs/ftpfs.py

@@ -29,11 +29,6 @@ import re
 from socket import error as socket_error
 from fs.local_functools import wraps
 
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
 import time
 import sys
 

fs/memoryfs.py

@@ -16,11 +16,7 @@ from fs.path import iteratepath, pathsplit, normpath
 from fs.base import *
 from fs.errors import *
 from fs import _thread_synchronize_default
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
+from fs.filelike import StringIO
 
 
 def _check_mode(mode, mode_chars):

fs/multifs.py

@@ -3,9 +3,10 @@ fs.multifs
 ==========
 
 A MultiFS is a filesytem composed of a sequence of other filesystems, where
-the directory structure of each filesystem is overlaid over the previous filesystem.
-When you attempt to access a file from the MultiFS it will try each 'child'
-FS in order, until it either finds a path that exists or raises a ResourceNotFoundError.
+the directory structure of each filesystem is overlaid over the previous
+filesystem. When you attempt to access a file from the MultiFS it will try
+each 'child' FS in order, until it either finds a path that exists or raises a
+ResourceNotFoundError.
 
 One use for such a filesystem would be to selectively override a set of files,
 to customize behaviour. For example, to create a filesystem that could be used
@@ -61,11 +62,11 @@ from fs.errors import ResourceNotFoundError
 
 class MultiFS(FS):
 
-    """A MultiFS is a filesystem that delegates to a sequence of other filesystems.
-    Operations on the MultiFS will try each 'child' filesystem in order, until it
-    succeeds. In effect, creating a filesystem that combines the files and dirs of
-    its children.
+    """A filesystem that delegates to a sequence of other filesystems.
 
+    Operations on the MultiFS will try each 'child' filesystem in order, until
+    it succeeds. In effect, creating a filesystem that combines the files and
+    dirs of its children.
     """
 
     def __init__(self):

fs/remote.py

@@ -21,10 +21,12 @@ FS subclasses interfacing with a remote filesystem.  These include:
 
 """
 
+from __future__ import with_statement
+
 import sys
+import os
 import time
 import copy
-from StringIO import StringIO
 from errno import EINVAL
 
 from fs.base import FS, threading
@@ -33,22 +35,11 @@ from fs.wrapfs.lazyfs import LazyFS
 from fs.path import *
 from fs.errors import *
 from fs.local_functools import wraps
+from fs.filelike import StringIO, SpooledTemporaryFile, FileWrapper
 from fs import SEEK_SET, SEEK_CUR, SEEK_END
 
-try:
-   from tempfile import SpooledTemporaryFile
-   def _MakeSpooledTempFile(*args, **kwds):
-        return SpooledTemporaryFile(*args, **kwds)
-except ImportError:
-   from tempfile import NamedTemporaryFile
-   class SpooledTemporaryFile(object):
-        """Fake SpooledTemporaryFile, for faking out isinstance() checks."""
-        pass
-   def _MakeSpooledTempFile(max_size=0, *args, **kwds):
-       return NamedTemporaryFile(*args,**kwds)
-
 
-class RemoteFileBuffer(object):
+class RemoteFileBuffer(FileWrapper):
     """File-like object providing buffer for local file operations.
 
     Instances of this class manage a local tempfile buffer corresponding
@@ -74,19 +65,16 @@ class RemoteFileBuffer(object):
 
     max_size_in_memory = 1024 * 8
 
-    def __init__(self,fs,path,mode,rfile=None, 
-                 write_on_flush=True):
+    def __init__(self, fs, path, mode, rfile=None, write_on_flush=True):
         """RemoteFileBuffer constructor.
 
         The owning filesystem, path and mode must be provided.  If the
         optional argument 'rfile' is provided, it must be a read()-able
         object or a string containing the initial file contents.
         """
-        self.file = _MakeSpooledTempFile(max_size=self.max_size_in_memory)
+        wrapped_file = SpooledTemporaryFile(max_size=self.max_size_in_memory)
         self.fs = fs
         self.path = path
-        self.mode = mode
-        self.closed = False
         self.write_on_flush = write_on_flush
         self._changed = False
         self._readlen = 0 # How many bytes already loaded from rfile
@@ -107,17 +95,17 @@ class RemoteFileBuffer(object):
                 rfile = StringIO(unicode(rfile))
                 
             self._rfile = rfile
-            
-            # FIXME: What if mode with position on eof?
-            if "a" in mode:
-                # Not good enough...
-                self.seek(0, SEEK_END)
         else:
             # Do not use remote file object
             self._eof = True
             self._rfile = None
             if rfile is not None and hasattr(rfile,"close"):
                 rfile.close()
+        super(RemoteFileBuffer,self).__init__(wrapped_file,mode)
+        # FIXME: What if mode with position on eof?
+        if "a" in mode:
+            # Not good enough...
+            self.seek(0, SEEK_END)
 
     def __del__(self):
         #  Don't try to close a partially-constructed file
@@ -125,55 +113,19 @@ class RemoteFileBuffer(object):
             if not self.closed:
                 self.close()
 
-    def __getattr__(self,name):
-        if name in ("file","_lock","fs","path","mode","closed"):
-            raise AttributeError(name)
-        file = self.__dict__['file']
-        a = getattr(file, name)
-        if not callable(a):
-            return a
-        @wraps(a)
-        def call_with_lock(*args,**kwds):
-            self._lock.acquire()
-            try:
-                if "write" in name:
-                    self._changed = True
-                    #  Do we need to discard into from the buffer?
-                    toread = len(args[0]) - (self._readlen - self.file.tell())
+    def _write(self,data,flushing=False):
+        with self._lock:
+            #  Do we need to discard info from the buffer?
+            toread = len(data) - (self._readlen - self.wrapped_file.tell())
             if toread > 0:
                 if not self._eof:
                     self._fillbuffer(toread)
                 else:
                     self._readlen += toread
-                return a(*args,**kwds)
-            finally:
-                self._lock.release()
-        setattr(self, name, call_with_lock)
-        return call_with_lock
-
-    def __enter__(self):
-        self.file.__enter__()
-        return self
-
-    def __exit__(self,exc,value,tb):
-        self.close()
-        return False
-
-    def __iter__(self):          
-        # TODO: implement this with on-demand loading.
-        self._fillbuffer()
-        return self.file.__iter__()
-
-    def readline(self,size=None):
-        # TODO: implement this with on-demand loading.
-        if size is None:
-            self._fillbuffer()
-            return self.file.readline()
-        else:
-            self._fillbuffer(size)
-            return self.file.readline(size)
+            self._changed = True
+            self.wrapped_file.write(data)
 
-    def _read(self, length=None):
+    def _read_remote(self, length=None):
         """Read data from the remote file into the local buffer."""
         chunklen = 1024 * 256
         bytes_read = 0
@@ -191,7 +143,7 @@ class RemoteFileBuffer(object):
                 break
             
             bytes_read += datalen 
-            self.file.write(data)
+            self.wrapped_file.write(data)
             
             if datalen < toread:
                 # We reached EOF,
@@ -210,48 +162,43 @@ class RemoteFileBuffer(object):
         into the buffer.  It reads 'length' bytes from rfile and writes them
         into the buffer, seeking back to the original file position.
         """
-        curpos = self.file.tell()
+        curpos = self.wrapped_file.tell()
         if length == None:
             if not self._eof:
                 # Read all data and we didn't reached EOF
                 # Merge endpos - tell + bytes from rfile
-                self.file.seek(0, SEEK_END)
-                self._read()
+                self.wrapped_file.seek(0, SEEK_END)
+                self._read_remote()
                 self._eof = True
-                self.file.seek(curpos)
+                self.wrapped_file.seek(curpos)
                
         elif not self._eof:
             if curpos + length > self._readlen:
+                # Read all data and we didn't reached EOF
                 # Load endpos - tell() + len bytes from rfile
                 toload = length - (self._readlen - curpos)
-                self.file.seek(0, SEEK_END)
-                self._read(toload)
-                self.file.seek(curpos)
+                self.wrapped_file.seek(0, SEEK_END)
+                self._read_remote(toload)
+                self.wrapped_file.seek(curpos)
         
-    def read(self, length=None):
+    def _read(self, length=None):
+        if length < 0:
+            length = None
+        with self._lock:
             self._fillbuffer(length)
-        return self.file.read(length if length != None else -1)
-
-    def seek(self,offset,whence=SEEK_SET):           
-        if isinstance(self.file,SpooledTemporaryFile):
-            #  SpooledTemporaryFile.seek doesn't roll to disk if seeking
-            #  beyond the max in-memory size.
-            if whence == SEEK_SET:
-                if offset > self.file._max_size:
-                    self.file.rollover()
-            elif whence == SEEK_CUR:
-                if offset + self.file.tell() > self.file._max_size:
-                    self.file.rollover()
-            else:
-                if offset > 0:
-                    self.file.rollover()
+            data = self.wrapped_file.read(length if length != None else -1)
+            if not data:
+                data = None
+            return data
 
+    def _seek(self,offset,whence=SEEK_SET):
+        with self._lock:
             if not self._eof:
                 # Count absolute position of seeking
                 if whence == SEEK_SET:
                     abspos = offset
                 elif whence == SEEK_CUR:
-                abspos =  offset + self.file.tell()
+                    abspos =  offset + self.wrapped_file.tell()
                 elif whence == SEEK_END:
                     abspos = None
                 else:
@@ -260,30 +207,16 @@ class RemoteFileBuffer(object):
                 if abspos != None:
                     toread = abspos - self._readlen
                     if toread > 0:
-                    self.file.seek(self._readlen)
+                        self.wrapped_file.seek(self._readlen)
                         self._fillbuffer(toread)
                 else:
-                self.file.seek(self._readlen)
+                    self.wrapped_file.seek(self._readlen)
                     self._fillbuffer()
                 
-        self.file.seek(offset, whence)
-
-    def truncate(self,size=None):
-        self._lock.acquire()
-        try:
-            if isinstance(self.file,SpooledTemporaryFile):
-                #  SpooledTemporaryFile.truncate doesn't accept size argument.
-                if size is None:
-                    self.file._file.truncate()
-                else:
-                    self.file._file.truncate(size)
-            else:
-                if size is None:
-                    self.file.truncate()
-                else:
-                    self.file.truncate(size)
-            self._changed = True
+            self.wrapped_file.seek(offset, whence)
 
+    def _truncate(self,size):
+        with self._lock:
             if not self._eof and self._readlen < size:
                 # Read the rest of file
                 self._fillbuffer(size - self._readlen)
@@ -295,20 +228,18 @@ class RemoteFileBuffer(object):
                 # Lock rfile
                 self._eof = True
 
+            self.wrapped_file.truncate(size)
+            self._changed = True
+                
             self.flush()
             if self._rfile is not None:
                 self._rfile.close()
-        finally:
-            self._lock.release()
 
     def flush(self):
-        self._lock.acquire()
-        try:
-            self.file.flush()
+        with self._lock:
+            self.wrapped_file.flush()
             if self.write_on_flush:
                 self._setcontents()
-        finally:
-            self._lock.release()
 
     def _setcontents(self):
         if not self._changed:
@@ -320,22 +251,18 @@ class RemoteFileBuffer(object):
             self._fillbuffer()
             
         if "w" in self.mode or "a" in self.mode or "+" in self.mode:
-            pos = self.file.tell()
-            self.file.seek(0)
-            self.fs.setcontents(self.path, self.file)
-            self.file.seek(pos)
+            pos = self.wrapped_file.tell()
+            self.wrapped_file.seek(0)
+            self.fs.setcontents(self.path, self.wrapped_file)
+            self.wrapped_file.seek(pos)
     
     def close(self):
-        self._lock.acquire()
-        try:
+        with self._lock:
             if not self.closed:
                 self._setcontents()
-                self.file.close()
-                self.closed = True
                 if self._rfile is not None:
                     self._rfile.close()
-        finally:
-            self._lock.release()
+                super(RemoteFileBuffer,self).close()
 
 
 class ConnectionManagerFS(LazyFS):

fs/rpcfs.py

@@ -14,17 +14,7 @@ from fs.base import *
 from fs.errors import *
 from fs.path import *
 
-from StringIO import StringIO
-if hasattr(StringIO,"__exit__"):
-    class StringIO(StringIO):
-        pass
-else:
-    class StringIO(StringIO):
-        def __enter__(self):
-            return self
-        def __exit__(self,exc_type,exc_value,traceback):
-            self.close()
-            return False
+from fs.filelike import StringIO
 
 
 def re_raise_faults(func):
@@ -173,14 +163,19 @@ class RPCFS(FS):
             f.seek(0,2)
         oldflush = f.flush
         oldclose = f.close
+        oldtruncate = f.truncate
         def newflush():
             oldflush()
             self.proxy.set_contents(path,xmlrpclib.Binary(f.getvalue()))
         def newclose():
             f.flush()
             oldclose()
+        def newtruncate(size=None):
+            oldtruncate(size)
+            f.flush()
         f.flush = newflush
         f.close = newclose
+        f.truncate = newtruncate
         return f
 
     def exists(self, path):

fs/tests/test_remote.py

@@ -109,7 +109,7 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
         
         f = self.fs.open('test.txt', 'rb')
         self.assertEquals(f.read(10), contents[:10])
-        f.file.seek(0, SEEK_END)
+        f.wrapped_file.seek(0, SEEK_END)
         self.assertEquals(f._rfile.tell(), 10)
         f.seek(20)
         self.assertEquals(f.tell(), 20)
@@ -136,9 +136,9 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
         f._rfile.seek(len(contents[:-5]))
         # Write 10 new characters (will make contents longer for 5 chars)
         f.write(u'1234567890')
+        f.flush()
         # We are on the end of file (and buffer not serve anything anymore)
         self.assertEquals(f.read(), '')
-        f.close()
         
         self.fakeOn()
         

fs/zipfs.py

@@ -11,15 +11,11 @@ import datetime
 from fs.base import *
 from fs.path import *
 from fs.errors import *
+from fs.filelike import StringIO
 
 from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED, BadZipfile, LargeZipFile
 from memoryfs import MemoryFS
 
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
 import tempfs