Commit 8fcf4df8 by rfkelly0

add module "fs.filelike" with utils for building file-like objects.

This is a local copy of the guts of my "filelike" module, re-licensed under
the MIT license.

This commit also uses it to fix a few edge-cases in various filesystem
implementations (e.g. truncating StringIO objects).
parent 739272ca
...@@ -57,4 +57,6 @@ ...@@ -57,4 +57,6 @@
* Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these * Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these
can often determine whether a path is a file or directory by inspecting can often determine whether a path is a file or directory by inspecting
the info dict and avoid an additional query to the filesystem. the info dict and avoid an additional query to the filesystem.
* Added utility module 'fs.filelike' with some helpers for building and
manipulating file-like objects.
...@@ -12,15 +12,13 @@ Contributed under the terms of the BSD License: ...@@ -12,15 +12,13 @@ Contributed under the terms of the BSD License:
http://www.opensource.org/licenses/bsd-license.php http://www.opensource.org/licenses/bsd-license.php
""" """
from struct import pack, unpack
from fs.base import * from fs.base import *
from fs.memoryfs import MemoryFS from fs.memoryfs import MemoryFS
from fs.contrib.bigfs.subrangefile import SubrangeFile from fs.filelike import StringIO
from struct import pack, unpack from fs.contrib.bigfs.subrangefile import SubrangeFile
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
class BIGEntry: class BIGEntry:
def __init__(self, filename, offset, storedSize, isCompressed, realSize): def __init__(self, filename, offset, storedSize, isCompressed, realSize):
......
...@@ -29,7 +29,6 @@ import stat as statinfo ...@@ -29,7 +29,6 @@ import stat as statinfo
import time import time
import SocketServer as sockserv import SocketServer as sockserv
import threading import threading
from StringIO import StringIO
import paramiko import paramiko
...@@ -37,6 +36,7 @@ from fs.base import flags_to_mode ...@@ -37,6 +36,7 @@ from fs.base import flags_to_mode
from fs.path import * from fs.path import *
from fs.errors import * from fs.errors import *
from fs.local_functools import wraps from fs.local_functools import wraps
from fs.filelike import StringIO
# Default host key used by BaseSFTPServer # Default host key used by BaseSFTPServer
......
"""
fs.filelike
===========
This module takes care of the groundwork for implementing and manipulating
objects that provide a rich file-like interface, including reading, writing,
seeking and iteration.
The main class is FileLikeBase, which implements the entire file-like interface
on top of primitive _read(), _write(), _seek(), _tell() and _truncate() methods.
Subclasses may implement any or all of these methods to obtain the related
higher-level file behaviors.
Other useful classes include:
* StringIO: a version of the builtin StringIO class, patched to more
closely preserve the semantics of a standard file.
* FileWrapper: a generic base class for wrappers around a filelike object
(think e.g. compression or decryption).
* SpooledTemporaryFile: a version of the builtin SpooledTemporaryFile
class, patched to more closely preserve the
semantics of a standard file.
"""
# Copyright (C) 2006-2009, Ryan Kelly
# All rights reserved; available under the terms of the MIT License.
import tempfile as _tempfile
try:
from cStringIO import StringIO as _StringIO
except ImportError:
from StrimgIO import StringIO as _StringIO
import fs
class NotReadableError(IOError):
pass
class NotWritableError(IOError):
pass
class NotSeekableError(IOError):
pass
class NotTruncatableError(IOError):
pass
class FileLikeBase(object):
"""Base class for implementing file-like objects.
This class takes a lot of the legwork out of writing file-like objects
with a rich interface. It implements the higher-level file-like methods
on top of five primitive methods: _read, _write, _seek, _tell and
_truncate. See their docstrings for precise details on how these methods
behave.
Subclasses then need only implement some subset of these methods for
rich file-like interface compatability. They may of course override
other methods as desired.
The class is missing the following attributes and methods, which dont
really make sense for anything but real files:
* fileno()
* isatty()
* encoding
* mode
* name
* newlines
Unlike standard file objects, all read methods share the same buffer
and so can be freely mixed (e.g. read(), readline(), next(), ...).
This class understands and will accept the following mode strings,
with any additional characters being ignored:
* r - open the file for reading only.
* r+ - open the file for reading and writing.
* r- - open the file for streamed reading; do not allow seek/tell.
* w - open the file for writing only; create the file if
it doesn't exist; truncate it to zero length.
* w+ - open the file for reading and writing; create the file
if it doesn't exist; truncate it to zero length.
* w- - open the file for streamed writing; do not allow seek/tell.
* a - open the file for writing only; create the file if it
doesn't exist; place pointer at end of file.
* a+ - open the file for reading and writing; create the file
if it doesn't exist; place pointer at end of file.
These are mostly standard except for the "-" indicator, which has
been added for efficiency purposes in cases where seeking can be
expensive to simulate (e.g. compressed files). Note that any file
opened for both reading and writing must also support seeking.
"""
def __init__(self,bufsize=1024*64):
"""FileLikeBase Constructor.
The optional argument 'bufsize' specifies the number of bytes to
read at a time when looking for a newline character. Setting this to
a larger number when lines are long should improve efficiency.
"""
# File-like attributes
self.closed = False
self.softspace = 0
# Our own attributes
self._bufsize = bufsize # buffer size for chunked reading
self._rbuffer = None # data that's been read but not returned
self._wbuffer = None # data that's been given but not written
self._sbuffer = None # data between real & apparent file pos
self._soffset = 0 # internal offset of file pointer
#
# The following five methods are the ones that subclasses are expected
# to implement. Carefully check their docstrings.
#
def _read(self,sizehint=-1):
"""Read approximately <sizehint> bytes from the file-like object.
This method is to be implemented by subclasses that wish to be
readable. It should read approximately <sizehint> bytes from the
file and return them as a string. If <sizehint> is missing or
less than or equal to zero, try to read all the remaining contents.
The method need not guarantee any particular number of bytes -
it may return more bytes than requested, or fewer. If needed the
size hint may be completely ignored. It may even return an empty
string if no data is yet available.
Because of this, the method must return None to signify that EOF
has been reached. The higher-level methods will never indicate EOF
until None has been read from _read(). Once EOF is reached, it
should be safe to call _read() again, immediately returning None.
"""
raise NotReadableError("Object not readable")
def _write(self,string,flushing=False):
"""Write the given string to the file-like object.
This method must be implemented by subclasses wishing to be writable.
It must attempt to write as much of the given data as possible to the
file, but need not guarantee that it is all written. It may return
None to indicate that all data was written, or return as a string any
data that could not be written.
If the keyword argument 'flushing' is true, it indicates that the
internal write buffers are being flushed, and *all* the given data
is expected to be written to the file. If unwritten data is returned
when 'flushing' is true, an IOError will be raised.
"""
raise NotWritableError("Object not writable")
def _seek(self,offset,whence):
"""Set the file's internal position pointer, approximately.
This method should set the file's position to approximately 'offset'
bytes relative to the position specified by 'whence'. If it is
not possible to position the pointer exactly at the given offset,
it should be positioned at a convenient *smaller* offset and the
file data between the real and apparent position should be returned.
At minimum, this method must implement the ability to seek to
the start of the file, i.e. offset=0 and whence=0. If more
complex seeks are difficult to implement then it may raise
NotImplementedError to have them simulated (inefficiently) by
the higher-level machinery of this class.
"""
raise NotSeekableError("Object not seekable")
def _tell(self):
"""Get the location of the file's internal position pointer.
This method must be implemented by subclasses that wish to be
seekable, and must return the position of the file's internal
pointer.
Due to buffering, the position seen by users of this class
(the "apparent position") may be different to the position
returned by this method (the "actual position").
"""
raise NotSeekableError("Object not seekable")
def _truncate(self,size):
"""Truncate the file's size to <size>.
This method must be implemented by subclasses that wish to be
truncatable. It must truncate the file to exactly the given size
or fail with an IOError.
Note that <size> will never be None; if it was not specified by the
user then it is calculated as the file's apparent position (which may
be different to its actual position due to buffering).
"""
raise NotTruncatableError("Object not truncatable")
#
# The following methods provide the public API of the filelike object.
# Subclasses shouldn't need to mess with these (except perhaps for
# close() and flush())
#
def _check_mode(self,mode,mstr=None):
"""Check whether the file may be accessed in the given mode.
'mode' must be one of "r" or "w", and this function returns False
if the file-like object has a 'mode' attribute, and it does not
permit access in that mode. If there is no 'mode' attribute,
it defaults to "r+".
If seek support is not required, use "r-" or "w-" as the mode string.
To check a mode string other than self.mode, pass it in as the
second argument.
"""
if mstr is None:
try:
mstr = self.mode
except AttributeError:
mstr = "r+"
if "+" in mstr:
return True
if "-" in mstr and "-" not in mode:
return False
if "r" in mode:
if "r" not in mstr:
return False
if "w" in mode:
if "w" not in mstr and "a" not in mstr:
return False
return True
def _assert_mode(self,mode,mstr=None):
"""Check whether the file may be accessed in the given mode.
This method is equivalent to _check_assert(), but raises IOError
instead of returning False.
"""
if mstr is None:
try:
mstr = self.mode
except AttributeError:
mstr = "r+"
if "+" in mstr:
return True
if "-" in mstr and "-" not in mode:
raise NotSeekableError("File does not support seeking.")
if "r" in mode:
if "r" not in mstr:
raise NotReadableError("File not opened for reading")
if "w" in mode:
if "w" not in mstr and "a" not in mstr:
raise NotWritableError("File not opened for writing")
return True
def flush(self):
"""Flush internal write buffer, if necessary."""
if self.closed:
raise IOError("File has been closed")
if self._check_mode("w-") and self._wbuffer is not None:
buffered = ""
if self._sbuffer:
buffered = buffered + self._sbuffer
self._sbuffer = None
buffered = buffered + self._wbuffer
self._wbuffer = None
leftover = self._write(buffered,flushing=True)
if leftover:
raise IOError("Could not flush write buffer.")
def close(self):
"""Flush write buffers and close the file.
The file may not be accessed further once it is closed.
"""
# Errors in subclass constructors can cause this to be called without
# having called FileLikeBase.__init__(). Since we need the attrs it
# initialises in cleanup, ensure we call it here.
if not hasattr(self,"closed"):
FileLikeBase.__init__(self)
if not self.closed:
self.flush()
self.closed = True
def __del__(self):
self.close()
def __enter__(self):
return self
def __exit__(self,exc_type,exc_val,exc_tb):
self.close()
return False
def next(self):
"""next() method complying with the iterator protocol.
File-like objects are their own iterators, with each call to
next() returning subsequent lines from the file.
"""
ln = self.readline()
if ln == "":
raise StopIteration()
return ln
def __iter__(self):
return self
def truncate(self,size=None):
"""Truncate the file to the given size.
If <size> is not specified or is None, the current file position is
used. Note that this method may fail at runtime if the underlying
filelike object is not truncatable.
"""
if "-" in getattr(self,"mode",""):
raise NotTruncatableError("File is not seekable, can't truncate.")
if self._wbuffer:
self.flush()
if size is None:
size = self.tell()
self._truncate(size)
def seek(self,offset,whence=0):
"""Move the internal file pointer to the given location."""
if whence > 2 or whence < 0:
raise ValueError("Invalid value for 'whence': " + str(whence))
if "-" in getattr(self,"mode",""):
raise NotSeekableError("File is not seekable.")
# Ensure that there's nothing left in the write buffer
if self._wbuffer:
self.flush()
# Adjust for any data left in the read buffer
if whence == 1 and self._rbuffer:
offset = offset - len(self._rbuffer)
self._rbuffer = None
# Adjust for any discrepancy in actual vs apparent seek position
if whence == 1:
if self._sbuffer:
offset = offset + len(self._sbuffer)
if self._soffset:
offset = offset + self._soffset
self._sbuffer = None
self._soffset = 0
# Shortcut the special case of staying put.
# As per posix, this has already cases the buffers to be flushed.
if offset == 0 and whence == 1:
return
# Catch any failed attempts to read while simulating seek
try:
# Try to do a whence-wise seek if it is implemented.
sbuf = None
try:
sbuf = self._seek(offset,whence)
except NotImplementedError:
# Try to simulate using an absolute seek.
try:
if whence == 1:
offset = self._tell() + offset
elif whence == 2:
if hasattr(self,"size"):
offset = self.size + offset
else:
self._do_read_rest()
offset = self.tell() + offset
else:
# absolute seek already failed, don't try again
raise NotImplementedError
sbuf = self._seek(offset,0)
except NotImplementedError:
# Simulate by reseting to start
self._seek(0,0)
self._soffset = offset
finally:
self._sbuffer = sbuf
except NotReadableError:
raise NotSeekableError("File not readable, can't simulate seek")
def tell(self):
"""Determine current position of internal file pointer."""
# Need to adjust for unread/unwritten data in buffers
pos = self._tell()
if self._rbuffer:
pos = pos - len(self._rbuffer)
if self._wbuffer:
pos = pos + len(self._wbuffer)
if self._sbuffer:
pos = pos + len(self._sbuffer)
if self._soffset:
pos = pos + self._soffset
return pos
def read(self,size=-1):
"""Read at most 'size' bytes from the file.
Bytes are returned as a string. If 'size' is negative, zero or
missing, the remainder of the file is read. If EOF is encountered
immediately, the empty string is returned.
"""
if self.closed:
raise IOError("File has been closed")
self._assert_mode("r-")
return self._do_read(size)
def _do_read(self,size):
"""Private method to read from the file.
This method behaves the same as self.read(), but skips some
permission and sanity checks. It is intended for use in simulating
seek(), where we may want to read (and discard) information from
a file not opened in read mode.
Note that this may still fail if the file object actually can't
be read from - it just won't check whether the mode string gives
permission.
"""
# If we were previously writing, ensure position is correct
if self._wbuffer is not None:
self.seek(0,1)
# Discard any data that should have been seeked over
if self._sbuffer:
s = len(self._sbuffer)
self._sbuffer = None
self.read(s)
elif self._soffset:
s = self._soffset
self._soffset = 0
while s > self._bufsize:
self._do_read(self._bufsize)
s -= self._bufsize
self._do_read(s)
# Should the entire file be read?
if size <= 0:
if self._rbuffer:
data = [self._rbuffer]
else:
data = []
self._rbuffer = ""
newData = self._read()
while newData is not None:
data.append(newData)
newData = self._read()
output = "".join(data)
# Otherwise, we need to return a specific amount of data
else:
if self._rbuffer:
newData = self._rbuffer
data = [newData]
else:
newData = ""
data = []
sizeSoFar = len(newData)
while sizeSoFar < size:
newData = self._read(size-sizeSoFar)
if newData is None:
break
data.append(newData)
sizeSoFar += len(newData)
data = "".join(data)
if sizeSoFar > size:
# read too many bytes, store in the buffer
self._rbuffer = data[size:]
data = data[:size]
else:
self._rbuffer = ""
output = data
return output
def _do_read_rest(self):
"""Private method to read the file through to EOF."""
data = self._do_read(self._bufsize)
while data != "":
data = self._do_read(self._bufsize)
def readline(self,size=-1):
"""Read a line from the file, or at most <size> bytes."""
bits = []
indx = -1
sizeSoFar = 0
while indx == -1:
nextBit = self.read(self._bufsize)
bits.append(nextBit)
sizeSoFar += len(nextBit)
if nextBit == "":
break
if size > 0 and sizeSoFar >= size:
break
indx = nextBit.find("\n")
# If not found, return whole string up to <size> length
# Any leftovers are pushed onto front of buffer
if indx == -1:
data = "".join(bits)
if size > 0 and sizeSoFar > size:
extra = data[size:]
data = data[:size]
self._rbuffer = extra + self._rbuffer
return data
# If found, push leftovers onto front of buffer
# Add one to preserve the newline in the return value
indx += 1
extra = bits[-1][indx:]
bits[-1] = bits[-1][:indx]
self._rbuffer = extra + self._rbuffer
return "".join(bits)
def readlines(self,sizehint=-1):
"""Return a list of all lines in the file."""
return [ln for ln in self]
def xreadlines(self):
"""Iterator over lines in the file - equivalent to iter(self)."""
return iter(self)
def write(self,string):
"""Write the given string to the file."""
if self.closed:
raise IOError("File has been closed")
self._assert_mode("w-")
# If we were previously reading, ensure position is correct
if self._rbuffer is not None:
self.seek(0,1)
# If we're actually behind the apparent position, we must also
# write the data in the gap.
if self._sbuffer:
string = self._sbuffer + string
self._sbuffer = None
elif self._soffset:
s = self._soffset
self._soffset = 0
try:
string = self._do_read(s) + string
except NotReadableError:
raise NotSeekableError("File not readable, could not complete simulation of seek")
self.seek(0,0)
if self._wbuffer:
string = self._wbuffer + string
leftover = self._write(string)
if leftover is None:
self._wbuffer = ""
else:
self._wbuffer = leftover
def writelines(self,seq):
"""Write a sequence of lines to the file."""
for ln in seq:
self.write(ln)
class FileWrapper(FileLikeBase):
"""Base class for objects that wrap a file-like object.
This class provides basic functionality for implementing file-like
objects that wrap another file-like object to alter its functionality
in some way. It takes care of house-keeping duties such as flushing
and closing the wrapped file.
Access to the wrapped file is given by the attribute wrapped_file.
By convention, the subclass's constructor should accept this as its
first argument and pass it to its superclass's constructor in the
same position.
This class provides a basic implementation of _read() and _write()
which just calls read() and write() on the wrapped object. Subclasses
will probably want to override these.
"""
_append_requires_overwrite = False
def __init__(self,wrapped_file,mode=None):
"""FileWrapper constructor.
'wrapped_file' must be a file-like object, which is to be wrapped
in another file-like object to provide additional functionality.
If given, 'mode' must be the access mode string under which
the wrapped file is to be accessed. If not given or None, it
is looked up on the wrapped file if possible. Otherwise, it
is not set on the object.
"""
# This is used for working around flush/close inefficiencies
self.__closing = False
super(FileWrapper,self).__init__()
self.wrapped_file = wrapped_file
if mode is None:
self.mode = getattr(wrapped_file,"mode","r+")
else:
self.mode = mode
self._validate_mode()
# Copy useful attributes of wrapped_file
if hasattr(wrapped_file,"name"):
self.name = wrapped_file.name
# Respect append-mode setting
if "a" in self.mode:
if self._check_mode("r"):
self.wrapped_file.seek(0)
self.seek(0,2)
def _validate_mode(self):
"""Check that various file-mode conditions are satisfied."""
# If append mode requires overwriting the underlying file,
# if must not be opened in append mode.
if self._append_requires_overwrite:
if self._check_mode("w"):
if "a" in getattr(self.wrapped_file,"mode",""):
raise ValueError("Underlying file can't be in append mode")
def __del__(self):
# Errors in subclass constructors could result in this being called
# without invoking FileWrapper.__init__. Establish some simple
# invariants to prevent errors in this case.
if not hasattr(self,"wrapped_file"):
self.wrapped_file = None
if not hasattr(self,"_FileWrapper__closing"):
self.__closing = False
# Close the wrapper and the underlying file independently, so the
# latter is still closed on cleanup even if the former errors out.
try:
super(FileWrapper,self).close()
except Exception:
if hasattr(getattr(self,"wrapped_file",None),"close"):
self.wrapped_file.close()
raise
def close(self):
"""Close the object for reading/writing."""
# The superclass implementation of this will call flush(),
# which calls flush() on our wrapped object. But we then call
# close() on it, which will call its flush() again! To avoid
# this inefficiency, our flush() will not flush the wrapped
# file when we're closing.
self.__closing = True
super(FileWrapper,self).close()
if hasattr(self.wrapped_file,"close"):
self.wrapped_file.close()
def flush(self):
"""Flush the write buffers of the file."""
super(FileWrapper,self).flush()
if not self.__closing and hasattr(self.wrapped_file,"flush"):
self.wrapped_file.flush()
def _read(self,sizehint=-1):
data = self.wrapped_file.read(sizehint)
if data == "":
return None
return data
def _write(self,string,flushing=False):
return self.wrapped_file.write(string)
def _seek(self,offset,whence):
self.wrapped_file.seek(offset,whence)
def _tell(self):
return self.wrapped_file.tell()
def _truncate(self,size):
return self.wrapped_file.truncate(size)
class StringIO(FileWrapper):
"""StringIO wrapper that more closely matches standard file behaviour.
This is a simple compatability wrapper around the native StringIO class
which fixes some corner-cases of its behaviour. Specifically:
* adding __enter__ and __exit__ methods
* having truncate(size) zero-fill when growing the file
"""
def __init__(self,data=None,mode=None):
wrapped_file = _StringIO()
if data is not None:
wrapped_file.write(data)
wrapped_file.seek(0)
super(StringIO,self).__init__(wrapped_file,mode)
def getvalue(self):
return self.wrapped_file.getvalue()
def _truncate(self,size):
pos = self.wrapped_file.tell()
self.wrapped_file.truncate(size)
curlen = len(self.wrapped_file.getvalue())
if size > curlen:
self.wrapped_file.seek(curlen)
try:
self.wrapped_file.write("\x00"*(size-curlen))
finally:
self.wrapped_file.seek(pos)
class SpooledTemporaryFile(FileWrapper):
"""SpooledTemporaryFile wrapper with some compatability fixes.
This is a simple compatability wrapper around the native SpooledTempFile
class which fixes some corner-cases of its behaviour. Specifically:
* have truncate() accept a size argument
* roll to disk is seeking past the max in-memory size
* use improved StringIO class from this module
"""
def __init__(self,max_size=0,mode="w+b",bufsize=-1,*args,**kwds):
try:
stf_args = (max_size,mode,bufsize) + args
wrapped_file = _tempfile.SpooledTemporaryFile(*stf_args,**kwds)
wrapped_file._file = StringIO()
self.__is_spooled = True
except AttributeError:
ntf_args = (mode,bufsize) + args
wrapped_file = _tempfile.NamedTemporaryFile(*ntf_args,**kwds)
self.__is_spooled = False
super(SpooledTemporaryFile,self).__init__(wrapped_file)
def _seek(self,offset,whence):
if self.__is_spooled:
max_size = self.wrapped_file._max_size
if whence == fs.SEEK_SET:
if offset > max_size:
self.wrapped_file.rollover()
elif whence == fs.SEEK_CUR:
if offset + self.wrapped_file.tell() > max_size:
self.wrapped_file.rollover()
else:
if offset > 0:
self.wrapped_file.rollover()
self.wrapped_file.seek(offset,whence)
def _truncate(self,size):
if self.__is_spooled:
self.wrapped_file._file.truncate(size)
else:
self.wrapped_file.truncate(size)
def fileno(self):
return self.wrapped_file.fileno()
...@@ -29,11 +29,6 @@ import re ...@@ -29,11 +29,6 @@ import re
from socket import error as socket_error from socket import error as socket_error
from fs.local_functools import wraps from fs.local_functools import wraps
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import time import time
import sys import sys
......
...@@ -16,11 +16,7 @@ from fs.path import iteratepath, pathsplit, normpath ...@@ -16,11 +16,7 @@ from fs.path import iteratepath, pathsplit, normpath
from fs.base import * from fs.base import *
from fs.errors import * from fs.errors import *
from fs import _thread_synchronize_default from fs import _thread_synchronize_default
from fs.filelike import StringIO
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
def _check_mode(mode, mode_chars): def _check_mode(mode, mode_chars):
......
...@@ -3,9 +3,10 @@ fs.multifs ...@@ -3,9 +3,10 @@ fs.multifs
========== ==========
A MultiFS is a filesytem composed of a sequence of other filesystems, where A MultiFS is a filesytem composed of a sequence of other filesystems, where
the directory structure of each filesystem is overlaid over the previous filesystem. the directory structure of each filesystem is overlaid over the previous
When you attempt to access a file from the MultiFS it will try each 'child' filesystem. When you attempt to access a file from the MultiFS it will try
FS in order, until it either finds a path that exists or raises a ResourceNotFoundError. each 'child' FS in order, until it either finds a path that exists or raises a
ResourceNotFoundError.
One use for such a filesystem would be to selectively override a set of files, One use for such a filesystem would be to selectively override a set of files,
to customize behaviour. For example, to create a filesystem that could be used to customize behaviour. For example, to create a filesystem that could be used
...@@ -61,11 +62,11 @@ from fs.errors import ResourceNotFoundError ...@@ -61,11 +62,11 @@ from fs.errors import ResourceNotFoundError
class MultiFS(FS): class MultiFS(FS):
"""A MultiFS is a filesystem that delegates to a sequence of other filesystems. """A filesystem that delegates to a sequence of other filesystems.
Operations on the MultiFS will try each 'child' filesystem in order, until it
succeeds. In effect, creating a filesystem that combines the files and dirs of
its children.
Operations on the MultiFS will try each 'child' filesystem in order, until
it succeeds. In effect, creating a filesystem that combines the files and
dirs of its children.
""" """
def __init__(self): def __init__(self):
......
...@@ -21,10 +21,12 @@ FS subclasses interfacing with a remote filesystem. These include: ...@@ -21,10 +21,12 @@ FS subclasses interfacing with a remote filesystem. These include:
""" """
from __future__ import with_statement
import sys import sys
import os
import time import time
import copy import copy
from StringIO import StringIO
from errno import EINVAL from errno import EINVAL
from fs.base import FS, threading from fs.base import FS, threading
...@@ -33,22 +35,11 @@ from fs.wrapfs.lazyfs import LazyFS ...@@ -33,22 +35,11 @@ from fs.wrapfs.lazyfs import LazyFS
from fs.path import * from fs.path import *
from fs.errors import * from fs.errors import *
from fs.local_functools import wraps from fs.local_functools import wraps
from fs.filelike import StringIO, SpooledTemporaryFile, FileWrapper
from fs import SEEK_SET, SEEK_CUR, SEEK_END from fs import SEEK_SET, SEEK_CUR, SEEK_END
try:
from tempfile import SpooledTemporaryFile
def _MakeSpooledTempFile(*args, **kwds):
return SpooledTemporaryFile(*args, **kwds)
except ImportError:
from tempfile import NamedTemporaryFile
class SpooledTemporaryFile(object):
"""Fake SpooledTemporaryFile, for faking out isinstance() checks."""
pass
def _MakeSpooledTempFile(max_size=0, *args, **kwds):
return NamedTemporaryFile(*args,**kwds)
class RemoteFileBuffer(object): class RemoteFileBuffer(FileWrapper):
"""File-like object providing buffer for local file operations. """File-like object providing buffer for local file operations.
Instances of this class manage a local tempfile buffer corresponding Instances of this class manage a local tempfile buffer corresponding
...@@ -74,19 +65,16 @@ class RemoteFileBuffer(object): ...@@ -74,19 +65,16 @@ class RemoteFileBuffer(object):
max_size_in_memory = 1024 * 8 max_size_in_memory = 1024 * 8
def __init__(self,fs,path,mode,rfile=None, def __init__(self, fs, path, mode, rfile=None, write_on_flush=True):
write_on_flush=True):
"""RemoteFileBuffer constructor. """RemoteFileBuffer constructor.
The owning filesystem, path and mode must be provided. If the The owning filesystem, path and mode must be provided. If the
optional argument 'rfile' is provided, it must be a read()-able optional argument 'rfile' is provided, it must be a read()-able
object or a string containing the initial file contents. object or a string containing the initial file contents.
""" """
self.file = _MakeSpooledTempFile(max_size=self.max_size_in_memory) wrapped_file = SpooledTemporaryFile(max_size=self.max_size_in_memory)
self.fs = fs self.fs = fs
self.path = path self.path = path
self.mode = mode
self.closed = False
self.write_on_flush = write_on_flush self.write_on_flush = write_on_flush
self._changed = False self._changed = False
self._readlen = 0 # How many bytes already loaded from rfile self._readlen = 0 # How many bytes already loaded from rfile
...@@ -107,17 +95,17 @@ class RemoteFileBuffer(object): ...@@ -107,17 +95,17 @@ class RemoteFileBuffer(object):
rfile = StringIO(unicode(rfile)) rfile = StringIO(unicode(rfile))
self._rfile = rfile self._rfile = rfile
# FIXME: What if mode with position on eof?
if "a" in mode:
# Not good enough...
self.seek(0, SEEK_END)
else: else:
# Do not use remote file object # Do not use remote file object
self._eof = True self._eof = True
self._rfile = None self._rfile = None
if rfile is not None and hasattr(rfile,"close"): if rfile is not None and hasattr(rfile,"close"):
rfile.close() rfile.close()
super(RemoteFileBuffer,self).__init__(wrapped_file,mode)
# FIXME: What if mode with position on eof?
if "a" in mode:
# Not good enough...
self.seek(0, SEEK_END)
def __del__(self): def __del__(self):
# Don't try to close a partially-constructed file # Don't try to close a partially-constructed file
...@@ -125,55 +113,19 @@ class RemoteFileBuffer(object): ...@@ -125,55 +113,19 @@ class RemoteFileBuffer(object):
if not self.closed: if not self.closed:
self.close() self.close()
def __getattr__(self,name): def _write(self,data,flushing=False):
if name in ("file","_lock","fs","path","mode","closed"): with self._lock:
raise AttributeError(name) # Do we need to discard info from the buffer?
file = self.__dict__['file'] toread = len(data) - (self._readlen - self.wrapped_file.tell())
a = getattr(file, name) if toread > 0:
if not callable(a): if not self._eof:
return a self._fillbuffer(toread)
@wraps(a) else:
def call_with_lock(*args,**kwds): self._readlen += toread
self._lock.acquire() self._changed = True
try: self.wrapped_file.write(data)
if "write" in name:
self._changed = True def _read_remote(self, length=None):
# Do we need to discard into from the buffer?
toread = len(args[0]) - (self._readlen - self.file.tell())
if toread > 0:
if not self._eof:
self._fillbuffer(toread)
else:
self._readlen += toread
return a(*args,**kwds)
finally:
self._lock.release()
setattr(self, name, call_with_lock)
return call_with_lock
def __enter__(self):
self.file.__enter__()
return self
def __exit__(self,exc,value,tb):
self.close()
return False
def __iter__(self):
# TODO: implement this with on-demand loading.
self._fillbuffer()
return self.file.__iter__()
def readline(self,size=None):
# TODO: implement this with on-demand loading.
if size is None:
self._fillbuffer()
return self.file.readline()
else:
self._fillbuffer(size)
return self.file.readline(size)
def _read(self, length=None):
"""Read data from the remote file into the local buffer.""" """Read data from the remote file into the local buffer."""
chunklen = 1024 * 256 chunklen = 1024 * 256
bytes_read = 0 bytes_read = 0
...@@ -191,7 +143,7 @@ class RemoteFileBuffer(object): ...@@ -191,7 +143,7 @@ class RemoteFileBuffer(object):
break break
bytes_read += datalen bytes_read += datalen
self.file.write(data) self.wrapped_file.write(data)
if datalen < toread: if datalen < toread:
# We reached EOF, # We reached EOF,
...@@ -210,80 +162,61 @@ class RemoteFileBuffer(object): ...@@ -210,80 +162,61 @@ class RemoteFileBuffer(object):
into the buffer. It reads 'length' bytes from rfile and writes them into the buffer. It reads 'length' bytes from rfile and writes them
into the buffer, seeking back to the original file position. into the buffer, seeking back to the original file position.
""" """
curpos = self.file.tell() curpos = self.wrapped_file.tell()
if length == None: if length == None:
if not self._eof: if not self._eof:
# Read all data and we didn't reached EOF # Read all data and we didn't reached EOF
# Merge endpos - tell + bytes from rfile # Merge endpos - tell + bytes from rfile
self.file.seek(0, SEEK_END) self.wrapped_file.seek(0, SEEK_END)
self._read() self._read_remote()
self._eof = True self._eof = True
self.file.seek(curpos) self.wrapped_file.seek(curpos)
elif not self._eof: elif not self._eof:
if curpos + length > self._readlen: if curpos + length > self._readlen:
# Read all data and we didn't reached EOF
# Load endpos - tell() + len bytes from rfile # Load endpos - tell() + len bytes from rfile
toload = length - (self._readlen - curpos) toload = length - (self._readlen - curpos)
self.file.seek(0, SEEK_END) self.wrapped_file.seek(0, SEEK_END)
self._read(toload) self._read_remote(toload)
self.file.seek(curpos) self.wrapped_file.seek(curpos)
def read(self, length=None): def _read(self, length=None):
self._fillbuffer(length) if length < 0:
return self.file.read(length if length != None else -1) length = None
with self._lock:
def seek(self,offset,whence=SEEK_SET): self._fillbuffer(length)
if isinstance(self.file,SpooledTemporaryFile): data = self.wrapped_file.read(length if length != None else -1)
# SpooledTemporaryFile.seek doesn't roll to disk if seeking if not data:
# beyond the max in-memory size. data = None
if whence == SEEK_SET: return data
if offset > self.file._max_size:
self.file.rollover() def _seek(self,offset,whence=SEEK_SET):
elif whence == SEEK_CUR: with self._lock:
if offset + self.file.tell() > self.file._max_size: if not self._eof:
self.file.rollover() # Count absolute position of seeking
else: if whence == SEEK_SET:
if offset > 0: abspos = offset
self.file.rollover() elif whence == SEEK_CUR:
abspos = offset + self.wrapped_file.tell()
if not self._eof: elif whence == SEEK_END:
# Count absolute position of seeking abspos = None
if whence == SEEK_SET:
abspos = offset
elif whence == SEEK_CUR:
abspos = offset + self.file.tell()
elif whence == SEEK_END:
abspos = None
else:
raise IOError(EINVAL, 'Invalid whence')
if abspos != None:
toread = abspos - self._readlen
if toread > 0:
self.file.seek(self._readlen)
self._fillbuffer(toread)
else:
self.file.seek(self._readlen)
self._fillbuffer()
self.file.seek(offset, whence)
def truncate(self,size=None):
self._lock.acquire()
try:
if isinstance(self.file,SpooledTemporaryFile):
# SpooledTemporaryFile.truncate doesn't accept size argument.
if size is None:
self.file._file.truncate()
else: else:
self.file._file.truncate(size) raise IOError(EINVAL, 'Invalid whence')
else:
if size is None: if abspos != None:
self.file.truncate() toread = abspos - self._readlen
if toread > 0:
self.wrapped_file.seek(self._readlen)
self._fillbuffer(toread)
else: else:
self.file.truncate(size) self.wrapped_file.seek(self._readlen)
self._changed = True self._fillbuffer()
self.wrapped_file.seek(offset, whence)
def _truncate(self,size):
with self._lock:
if not self._eof and self._readlen < size: if not self._eof and self._readlen < size:
# Read the rest of file # Read the rest of file
self._fillbuffer(size - self._readlen) self._fillbuffer(size - self._readlen)
...@@ -294,21 +227,19 @@ class RemoteFileBuffer(object): ...@@ -294,21 +227,19 @@ class RemoteFileBuffer(object):
self._readlen = size if size != None else 0 self._readlen = size if size != None else 0
# Lock rfile # Lock rfile
self._eof = True self._eof = True
self.wrapped_file.truncate(size)
self._changed = True
self.flush() self.flush()
if self._rfile is not None: if self._rfile is not None:
self._rfile.close() self._rfile.close()
finally:
self._lock.release()
def flush(self): def flush(self):
self._lock.acquire() with self._lock:
try: self.wrapped_file.flush()
self.file.flush()
if self.write_on_flush: if self.write_on_flush:
self._setcontents() self._setcontents()
finally:
self._lock.release()
def _setcontents(self): def _setcontents(self):
if not self._changed: if not self._changed:
...@@ -320,22 +251,18 @@ class RemoteFileBuffer(object): ...@@ -320,22 +251,18 @@ class RemoteFileBuffer(object):
self._fillbuffer() self._fillbuffer()
if "w" in self.mode or "a" in self.mode or "+" in self.mode: if "w" in self.mode or "a" in self.mode or "+" in self.mode:
pos = self.file.tell() pos = self.wrapped_file.tell()
self.file.seek(0) self.wrapped_file.seek(0)
self.fs.setcontents(self.path, self.file) self.fs.setcontents(self.path, self.wrapped_file)
self.file.seek(pos) self.wrapped_file.seek(pos)
def close(self): def close(self):
self._lock.acquire() with self._lock:
try:
if not self.closed: if not self.closed:
self._setcontents() self._setcontents()
self.file.close()
self.closed = True
if self._rfile is not None: if self._rfile is not None:
self._rfile.close() self._rfile.close()
finally: super(RemoteFileBuffer,self).close()
self._lock.release()
class ConnectionManagerFS(LazyFS): class ConnectionManagerFS(LazyFS):
......
...@@ -14,17 +14,7 @@ from fs.base import * ...@@ -14,17 +14,7 @@ from fs.base import *
from fs.errors import * from fs.errors import *
from fs.path import * from fs.path import *
from StringIO import StringIO from fs.filelike import StringIO
if hasattr(StringIO,"__exit__"):
class StringIO(StringIO):
pass
else:
class StringIO(StringIO):
def __enter__(self):
return self
def __exit__(self,exc_type,exc_value,traceback):
self.close()
return False
def re_raise_faults(func): def re_raise_faults(func):
...@@ -173,14 +163,19 @@ class RPCFS(FS): ...@@ -173,14 +163,19 @@ class RPCFS(FS):
f.seek(0,2) f.seek(0,2)
oldflush = f.flush oldflush = f.flush
oldclose = f.close oldclose = f.close
oldtruncate = f.truncate
def newflush(): def newflush():
oldflush() oldflush()
self.proxy.set_contents(path,xmlrpclib.Binary(f.getvalue())) self.proxy.set_contents(path,xmlrpclib.Binary(f.getvalue()))
def newclose(): def newclose():
f.flush() f.flush()
oldclose() oldclose()
def newtruncate(size=None):
oldtruncate(size)
f.flush()
f.flush = newflush f.flush = newflush
f.close = newclose f.close = newclose
f.truncate = newtruncate
return f return f
def exists(self, path): def exists(self, path):
......
...@@ -36,7 +36,7 @@ class RemoteTempFS(TempFS): ...@@ -36,7 +36,7 @@ class RemoteTempFS(TempFS):
f = None f = None
return RemoteFileBuffer(self, path, mode, f, return RemoteFileBuffer(self, path, mode, f,
write_on_flush=write_on_flush) write_on_flush=write_on_flush)
def setcontents(self, path, content): def setcontents(self, path, content):
f = super(RemoteTempFS, self).open(path, 'wb') f = super(RemoteTempFS, self).open(path, 'wb')
...@@ -109,7 +109,7 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases): ...@@ -109,7 +109,7 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
f = self.fs.open('test.txt', 'rb') f = self.fs.open('test.txt', 'rb')
self.assertEquals(f.read(10), contents[:10]) self.assertEquals(f.read(10), contents[:10])
f.file.seek(0, SEEK_END) f.wrapped_file.seek(0, SEEK_END)
self.assertEquals(f._rfile.tell(), 10) self.assertEquals(f._rfile.tell(), 10)
f.seek(20) f.seek(20)
self.assertEquals(f.tell(), 20) self.assertEquals(f.tell(), 20)
...@@ -136,9 +136,9 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases): ...@@ -136,9 +136,9 @@ class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
f._rfile.seek(len(contents[:-5])) f._rfile.seek(len(contents[:-5]))
# Write 10 new characters (will make contents longer for 5 chars) # Write 10 new characters (will make contents longer for 5 chars)
f.write(u'1234567890') f.write(u'1234567890')
f.flush()
# We are on the end of file (and buffer not serve anything anymore) # We are on the end of file (and buffer not serve anything anymore)
self.assertEquals(f.read(), '') self.assertEquals(f.read(), '')
f.close()
self.fakeOn() self.fakeOn()
......
...@@ -11,15 +11,11 @@ import datetime ...@@ -11,15 +11,11 @@ import datetime
from fs.base import * from fs.base import *
from fs.path import * from fs.path import *
from fs.errors import * from fs.errors import *
from fs.filelike import StringIO
from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED, BadZipfile, LargeZipFile from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED, BadZipfile, LargeZipFile
from memoryfs import MemoryFS from memoryfs import MemoryFS
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import tempfs import tempfs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment