Skip to content

P
pyfs
Commit c56e8ae6 by willmcgugan
Options

Fixes and documentation

parent 462723b6
Showing with 296 additions and 68 deletions
docs/concepts.rst
Concepts Concepts
======== ========
It is generally quite easy to get in to the mind-set of using PyFilesystem interface over lower level interfaces (since the code tends to be simpler) but there are a few concepts which you will need to keep in mind. Working with PyFilesystem is generally easier than working with lower level interfaces, as long as you are aware these simple concepts.
Sandboxing Sandboxing
---------- ----------
...@@ -25,7 +25,9 @@ We can open the `foo` directory with the following code:: ...@@ -25,7 +25,9 @@ We can open the `foo` directory with the following code::
from fs.osfs import OSFS from fs.osfs import OSFS
foo_fs = OSFS('foo') foo_fs = OSFS('foo')
The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable, especially if we are passing `foo_fs` to an untrusted function or one that could potentially delete files. Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method:: The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable,
especially if we are passing `foo_fs` to an untrusted function or to a function that has the potential to delete files.
Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method::
bar_fs = foo_fs.opendir('bar') bar_fs = foo_fs.opendir('bar')
...@@ -54,7 +56,7 @@ When working with paths in FS objects, keep in mind the following: ...@@ -54,7 +56,7 @@ When working with paths in FS objects, keep in mind the following:
* A double dot means 'previous directory' * A double dot means 'previous directory'
Note that paths used by the FS interface will use this format, but the constructor or additional methods may not. Note that paths used by the FS interface will use this format, but the constructor or additional methods may not.
Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which can be platform-dependent. Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which is platform-dependent.
There are many helpful functions for working with paths in the :mod:`fs.path` module. There are many helpful functions for working with paths in the :mod:`fs.path` module.
......
docs/getting_started.rst
...@@ -14,13 +14,21 @@ Add the -U switch if you want to upgrade a previous installation:: ...@@ -14,13 +14,21 @@ Add the -U switch if you want to upgrade a previous installation::
easy_install -U fs easy_install -U fs
This will install the latest stable release. If you would prefer to install the cutting edge release then you can get the latest copy of the source via SVN:: If you prefer to use Pip (http://pypi.python.org/pypi/pip) to install Python packages, the procedure is much the same::
pip install fs
Or to upgrade::
pip install fs --upgrade
You can also install the cutting edge release by checking out the source via SVN::
svn checkout http://pyfilesystem.googlecode.com/svn/trunk/ pyfilesystem-read-only svn checkout http://pyfilesystem.googlecode.com/svn/trunk/ pyfilesystem-read-only
cd pyfilesystem-read-only cd pyfilesystem-read-only
python setup.py install python setup.py install
You should now have the `fs` module on your path: Whichever method you use, you should now have the `fs` module on your path (version number may vary)::
>>> import fs >>> import fs
>>> fs.__version__ >>> fs.__version__
......
fs/base.py
...@@ -205,15 +205,15 @@ class FS(object): ...@@ -205,15 +205,15 @@ class FS(object):
# type of lock that should be there. None == no lock, # type of lock that should be there. None == no lock,
# True == a proper lock, False == a dummy lock. # True == a proper lock, False == a dummy lock.
state = self.__dict__.copy() state = self.__dict__.copy()
lock = state.get("_lock",None) lock = state.get("_lock", None)
if lock is not None: if lock is not None:
if isinstance(lock,threading._RLock): if isinstance(lock, threading._RLock):
state["_lock"] = True state["_lock"] = True
else: else:
state["_lock"] = False state["_lock"] = False
return state return state
def __setstate__(self,state): def __setstate__(self, state):
self.__dict__.update(state) self.__dict__.update(state)
lock = state.get("_lock") lock = state.get("_lock")
if lock is not None: if lock is not None:
...@@ -336,26 +336,39 @@ class FS(object): ...@@ -336,26 +336,39 @@ class FS(object):
"""Open a the given path as a file-like object. """Open a the given path as a file-like object.
:param path: a path to file that should be opened :param path: a path to file that should be opened
:type path: string
:param mode: mode of file to open, identical to the mode string used :param mode: mode of file to open, identical to the mode string used
in 'file' and 'open' builtins in 'file' and 'open' builtins
:type mode: string
:param kwargs: additional (optional) keyword parameters that may :param kwargs: additional (optional) keyword parameters that may
be required to open the file be required to open the file
:type kwargs: dict
:rtype: a file-like object :rtype: a file-like object
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if an intermediate directory is an file
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
""" """
raise UnsupportedError("open file") raise UnsupportedError("open file")
def safeopen(self, path, mode="r", **kwargs): def safeopen(self, path, mode="r", **kwargs):
"""Like :py:meth:`~fs.base.FS.open`, but returns a :py:class:`~fs.base.NullFile` if the file could not be opened. """Like :py:meth:`~fs.base.FS.open`, but returns a
:py:class:`~fs.base.NullFile` if the file could not be opened.
A ``NullFile`` is a dummy file which has all the methods of a file-like object, A ``NullFile`` is a dummy file which has all the methods of a file-like object,
but contains no data. but contains no data.
:param path: a path to file that should be opened :param path: a path to file that should be opened
:type path: string
:param mode: mode of file to open, identical to the mode string used :param mode: mode of file to open, identical to the mode string used
in 'file' and 'open' builtins in 'file' and 'open' builtins
:type mode: string
:param kwargs: additional (optional) keyword parameters that may :param kwargs: additional (optional) keyword parameters that may
be required to open the file be required to open the file
:type kwargs: dict
:rtype: a file-like object :rtype: a file-like object
""" """
...@@ -369,6 +382,8 @@ class FS(object): ...@@ -369,6 +382,8 @@ class FS(object):
"""Check if a path references a valid resource. """Check if a path references a valid resource.
:param path: A path in the filesystem :param path: A path in the filesystem
:type path: string
:rtype: bool :rtype: bool
""" """
...@@ -378,6 +393,8 @@ class FS(object): ...@@ -378,6 +393,8 @@ class FS(object):
"""Check if a path references a directory. """Check if a path references a directory.
:param path: a path in the filesystem :param path: a path in the filesystem
:type path: string
:rtype: bool :rtype: bool
""" """
...@@ -387,6 +404,8 @@ class FS(object): ...@@ -387,6 +404,8 @@ class FS(object):
"""Check if a path references a file. """Check if a path references a file.
:param path: a path in the filesystem :param path: a path in the filesystem
:type path: string
:rtype: bool :rtype: bool
""" """
...@@ -419,10 +438,12 @@ class FS(object): ...@@ -419,10 +438,12 @@ class FS(object):
:type dirs_only: bool :type dirs_only: bool
:param files_only: if True, only return files :param files_only: if True, only return files
:type files_only: bool :type files_only: bool
:rtype: iterable of paths :rtype: iterable of paths
:raises `fs.errors.ResourceNotFoundError`: if the path is not found :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path exists, but is not a directory :raises `fs.errors.ResourceInvalidError`: if the path exists, but is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
""" """
raise UnsupportedError("list directory") raise UnsupportedError("list directory")
...@@ -460,7 +481,7 @@ class FS(object): ...@@ -460,7 +481,7 @@ class FS(object):
if full or absolute: if full or absolute:
return self.getinfo(p) return self.getinfo(p)
else: else:
return self.getinfo(pathjoin(path,p)) return self.getinfo(pathjoin(path, p))
except FSError: except FSError:
return {} return {}
...@@ -553,6 +574,7 @@ class FS(object): ...@@ -553,6 +574,7 @@ class FS(object):
"""Make a directory on the filesystem. """Make a directory on the filesystem.
:param path: path of directory :param path: path of directory
:type path: string
:param recursive: if True, any intermediate directories will also be created :param recursive: if True, any intermediate directories will also be created
:type recursive: bool :type recursive: bool
:param allow_recreate: if True, re-creating a directory wont be an error :param allow_recreate: if True, re-creating a directory wont be an error
...@@ -561,6 +583,7 @@ class FS(object): ...@@ -561,6 +583,7 @@ class FS(object):
:raises `fs.errors.DestinationExistsError`: if the path is already a directory, and allow_recreate is False :raises `fs.errors.DestinationExistsError`: if the path is already a directory, and allow_recreate is False
:raises `fs.errors.ParentDirectoryMissingError`: if a containing directory is missing and recursive is False :raises `fs.errors.ParentDirectoryMissingError`: if a containing directory is missing and recursive is False
:raises `fs.errors.ResourceInvalidError`: if a path is an existing file :raises `fs.errors.ResourceInvalidError`: if a path is an existing file
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
""" """
raise UnsupportedError("make directory") raise UnsupportedError("make directory")
...@@ -569,9 +592,11 @@ class FS(object): ...@@ -569,9 +592,11 @@ class FS(object):
"""Remove a file from the filesystem. """Remove a file from the filesystem.
:param path: Path of the resource to remove :param path: Path of the resource to remove
:type path: string
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is a directory :raises `fs.errors.ResourceInvalidError`: if the path is a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
""" """
raise UnsupportedError("remove resource") raise UnsupportedError("remove resource")
...@@ -580,14 +605,16 @@ class FS(object): ...@@ -580,14 +605,16 @@ class FS(object):
"""Remove a directory from the filesystem """Remove a directory from the filesystem
:param path: path of the directory to remove :param path: path of the directory to remove
:type path: string
:param recursive: if True, empty parent directories will be removed :param recursive: if True, empty parent directories will be removed
:type recursive: bool :type recursive: bool
:param force: if True, any directory contents will be removed :param force: if True, any directory contents will be removed
:type force: bool :type force: bool
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.DirectoryNotEmptyError`: if the directory is not empty and force is False :raises `fs.errors.DirectoryNotEmptyError`: if the directory is not empty and force is False
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
""" """
raise UnsupportedError("remove directory") raise UnsupportedError("remove directory")
...@@ -596,7 +623,15 @@ class FS(object): ...@@ -596,7 +623,15 @@ class FS(object):
"""Renames a file or directory """Renames a file or directory
:param src: path to rename :param src: path to rename
:type src: string
:param dst: new name :param dst: new name
:type dst: string
:raises ParentDirectoryMissingError: if a containing directory is missing
:raises ResourceInvalidError: if the path or a parent path is not a
directory or src is a parent of dst or one of src or dst is a dir
and the other don't
:raises ResourceNotFoundError: if the src path does not exist
""" """
raise UnsupportedError("rename resource") raise UnsupportedError("rename resource")
...@@ -606,8 +641,11 @@ class FS(object): ...@@ -606,8 +641,11 @@ class FS(object):
"""Set the accessed time and modified time of a file """Set the accessed time and modified time of a file
:param path: path to a file :param path: path to a file
:param accessed_time: a datetime object the file was accessed (defaults to current time) :type path: string
:param modified_time: a datetime object the file was modified (defaults to current time) :param accessed_time: the datetime the file was accessed (defaults to current time)
:type accessed_time: datetime
:param modified_time: the datetime the file was modified (defaults to current time)
:type modified_time: datetime
""" """
...@@ -637,8 +675,14 @@ class FS(object): ...@@ -637,8 +675,14 @@ class FS(object):
* "modified_time" - A datetime object containing the time the resource was modified * "modified_time" - A datetime object containing the time the resource was modified
:param path: a path to retrieve information for :param path: a path to retrieve information for
:type path: string
:rtype: dict :rtype: dict
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
""" """
raise UnsupportedError("get resource info") raise UnsupportedError("get resource info")
...@@ -675,7 +719,7 @@ class FS(object): ...@@ -675,7 +719,7 @@ class FS(object):
if f is not None: if f is not None:
f.close() f.close()
def setcontents(self, path, data, chunk_size=1024*64): def setcontents(self, path, data, chunk_size=1024 * 64):
"""A convenience method to create a new file from a string or file-like object """A convenience method to create a new file from a string or file-like object
:param path: a path of the file to create :param path: a path of the file to create
...@@ -708,7 +752,7 @@ class FS(object): ...@@ -708,7 +752,7 @@ class FS(object):
def setcontents_async(self, def setcontents_async(self,
path, path,
data, data,
chunk_size=1024*64, chunk_size=1024 * 64,
progress_callback=None, progress_callback=None,
finished_callback=None, finished_callback=None,
error_callback=None): error_callback=None):
...@@ -795,6 +839,9 @@ class FS(object): ...@@ -795,6 +839,9 @@ class FS(object):
"""Opens a directory and returns a FS object representing its contents. """Opens a directory and returns a FS object representing its contents.
:param path: path to directory to open :param path: path to directory to open
:type path: string
:return: the opened dir
:rtype: an FS object :rtype: an FS object
""" """
...@@ -815,6 +862,7 @@ class FS(object): ...@@ -815,6 +862,7 @@ class FS(object):
contents. contents.
:param path: root path to start walking :param path: root path to start walking
:type path: string
:param wildcard: if given, only return files that match this wildcard :param wildcard: if given, only return files that match this wildcard
:type wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean :type wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard :param dir_wildcard: if given, only walk directories that match the wildcard
...@@ -825,6 +873,9 @@ class FS(object): ...@@ -825,6 +873,9 @@ class FS(object):
* ``"depth"`` yields the deepest paths first * ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory :param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of (current_path, paths)
""" """
...@@ -893,16 +944,24 @@ class FS(object): ...@@ -893,16 +944,24 @@ class FS(object):
wildcard=None, wildcard=None,
dir_wildcard=None, dir_wildcard=None,
search="breadth", search="breadth",
ignore_errors=False ): ignore_errors=False):
"""Like the 'walk' method, but just yields file paths. """Like the 'walk' method, but just yields file paths.
:param path: root path to start walking :param path: root path to start walking
:type path: string
:param wildcard: if given, only return files that match this wildcard :param wildcard: if given, only return files that match this wildcard
:type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean :type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard :param dir_wildcard: if given, only walk directories that match the wildcard
:type dir_wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean :type dir_wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean
:param search: same as walk method :param search: a string identifying the method used to walk the directories. There are two such methods:
* ``"breadth"`` yields paths in the top directories first
* ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory :param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of file paths
""" """
for path, files in self.walk(path, wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors): for path, files in self.walk(path, wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors):
...@@ -917,10 +976,18 @@ class FS(object): ...@@ -917,10 +976,18 @@ class FS(object):
"""Like the 'walk' method but yields directories. """Like the 'walk' method but yields directories.
:param path: root path to start walking :param path: root path to start walking
:type path: string
:param wildcard: if given, only return directories that match this wildcard :param wildcard: if given, only return directories that match this wildcard
:type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean :type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean
:param search: same as the walk method :param search: a string identifying the method used to walk the directories. There are two such methods:
* ``"breadth"`` yields paths in the top directories first
* ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory :param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of dir paths
""" """
for p, _files in self.walk(path, dir_wildcard=wildcard, search=search, ignore_errors=ignore_errors): for p, _files in self.walk(path, dir_wildcard=wildcard, search=search, ignore_errors=ignore_errors):
...@@ -931,8 +998,10 @@ class FS(object): ...@@ -931,8 +998,10 @@ class FS(object):
"""Returns the size (in bytes) of a resource. """Returns the size (in bytes) of a resource.
:param path: a path to the resource :param path: a path to the resource
:rtype: integer :type path: string
:returns: the size of the file :returns: the size of the file
:rtype: integer
""" """
info = self.getinfo(path) info = self.getinfo(path)
...@@ -941,22 +1010,26 @@ class FS(object): ...@@ -941,22 +1010,26 @@ class FS(object):
raise OperationFailedError("get size of resource", path) raise OperationFailedError("get size of resource", path)
return size return size
def copy(self, src, dst, overwrite=False, chunk_size=1024*64): def copy(self, src, dst, overwrite=False, chunk_size=1024 * 64):
"""Copies a file from src to dst. """Copies a file from src to dst.
:param src: the source path :param src: the source path
:type src: string
:param dst: the destination path :param dst: the destination path
:type dst: string
:param overwrite: if True, then an existing file at the destination may :param overwrite: if True, then an existing file at the destination may
be overwritten; If False then DestinationExistsError be overwritten; If False then DestinationExistsError
will be raised. will be raised.
:type overwrite: bool
:param chunk_size: size of chunks to use if a simple copy is required :param chunk_size: size of chunks to use if a simple copy is required
(defaults to 64K). (defaults to 64K).
:type chunk_size: bool
""" """
if not self.isfile(src): if not self.isfile(src):
if self.isdir(src): if self.isdir(src):
raise ResourceInvalidError(src,msg="Source is not a file: %(path)s") raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
raise ResourceNotFoundError(src) raise ResourceNotFoundError(src)
if not overwrite and self.exists(dst): if not overwrite and self.exists(dst):
raise DestinationExistsError(dst) raise DestinationExistsError(dst)
...@@ -985,7 +1058,7 @@ class FS(object): ...@@ -985,7 +1058,7 @@ class FS(object):
shutil.copyfile(src_syspath, dst_syspath) shutil.copyfile(src_syspath, dst_syspath)
except IOError, e: except IOError, e:
# shutil reports ENOENT when a parent directory is missing # shutil reports ENOENT when a parent directory is missing
if getattr(e,"errno",None) == 2: if getattr(e, "errno", None) == 2:
if not os.path.exists(dirname(dst_syspath)): if not os.path.exists(dirname(dst_syspath)):
raise ParentDirectoryMissingError(dst_syspath) raise ParentDirectoryMissingError(dst_syspath)
raise raise
...@@ -1000,10 +1073,9 @@ class FS(object): ...@@ -1000,10 +1073,9 @@ class FS(object):
"""moves a file from one location to another. """moves a file from one location to another.
:param src: source path :param src: source path
:type src: string
:param dst: destination path :param dst: destination path
:param overwrite: if True, then an existing file at the destination path :type dst: string
will be silently overwritten; if False then an exception
will be raised in this case.
:param overwrite: When True the destination will be overwritten (if it exists), :param overwrite: When True the destination will be overwritten (if it exists),
otherwise a DestinationExistsError will be thrown otherwise a DestinationExistsError will be thrown
:type overwrite: bool :type overwrite: bool
...@@ -1011,6 +1083,8 @@ class FS(object): ...@@ -1011,6 +1083,8 @@ class FS(object):
is required is required
:type chunk_size: integer :type chunk_size: integer
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
""" """
src_syspath = self.getsyspath(src, allow_none=True) src_syspath = self.getsyspath(src, allow_none=True)
...@@ -1037,13 +1111,20 @@ class FS(object): ...@@ -1037,13 +1111,20 @@ class FS(object):
"""moves a directory from one location to another. """moves a directory from one location to another.
:param src: source directory path :param src: source directory path
:type src: string
:param dst: destination directory path :param dst: destination directory path
:type dst: string
:param overwrite: if True then any existing files in the destination :param overwrite: if True then any existing files in the destination
directory will be overwritten directory will be overwritten
:type overwrite: bool
:param ignore_errors: if True then this method will ignore FSError :param ignore_errors: if True then this method will ignore FSError
exceptions when moving files exceptions when moving files
:type ignore_errors: bool
:param chunk_size: size of chunks to use when copying, if a simple copy :param chunk_size: size of chunks to use when copying, if a simple copy
is required is required
:type chunk_size: integer
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
""" """
if not self.isdir(src): if not self.isdir(src):
...@@ -1058,7 +1139,7 @@ class FS(object): ...@@ -1058,7 +1139,7 @@ class FS(object):
if src_syspath is not None and dst_syspath is not None: if src_syspath is not None and dst_syspath is not None:
try: try:
os.rename(src_syspath,dst_syspath) os.rename(src_syspath, dst_syspath)
return return
except OSError: except OSError:
pass pass
...@@ -1097,7 +1178,9 @@ class FS(object): ...@@ -1097,7 +1178,9 @@ class FS(object):
"""copies a directory from one location to another. """copies a directory from one location to another.
:param src: source directory path :param src: source directory path
:type src: string
:param dst: destination directory path :param dst: destination directory path
:type dst: string
:param overwrite: if True then any existing files in the destination :param overwrite: if True then any existing files in the destination
directory will be overwritten directory will be overwritten
:type overwrite: bool :type overwrite: bool
...@@ -1144,6 +1227,7 @@ class FS(object): ...@@ -1144,6 +1227,7 @@ class FS(object):
"""Check if a directory is empty (contains no files or sub-directories) """Check if a directory is empty (contains no files or sub-directories)
:param path: a directory path :param path: a directory path
:rtype: bool :rtype: bool
""" """
...@@ -1162,6 +1246,9 @@ class FS(object): ...@@ -1162,6 +1246,9 @@ class FS(object):
:param path: path to the new directory :param path: path to the new directory
:param recursive: if True any intermediate directories will be created :param recursive: if True any intermediate directories will be created
:return: the opened dir
:rtype: an FS object
""" """
self.makedir(path, allow_recreate=True, recursive=recursive) self.makedir(path, allow_recreate=True, recursive=recursive)
......
fs/expose/serve/server.py
...@@ -19,10 +19,28 @@ class _SocketFile(object): ...@@ -19,10 +19,28 @@ class _SocketFile(object):
def write(self, data): def write(self, data):
self.socket.sendall(data) self.socket.sendall(data)
class ConnectionThread(threading.Thread):
def remote_call(method_name=None):
method = method_name
def deco(f):
if not hasattr(f, '_remote_call_names'):
f._remote_call_names = []
f._remote_call_names.append(method or f.__name__)
return f
return deco
class RemoteResponse(Exception):
def __init__(self, header, payload):
self.header = header
self.payload = payload
class ConnectionHandlerBase(threading.Thread):
_methods = {}
def __init__(self, server, connection_id, socket, address): def __init__(self, server, connection_id, socket, address):
super(ConnectionThread, self).__init__() super(ConnectionHandlerBase, self).__init__()
self.server = server self.server = server
self.connection_id = connection_id self.connection_id = connection_id
self.socket = socket self.socket = socket
...@@ -34,6 +52,16 @@ class ConnectionThread(threading.Thread): ...@@ -34,6 +52,16 @@ class ConnectionThread(threading.Thread):
self._lock = threading.RLock() self._lock = threading.RLock()
self.socket_error = None self.socket_error = None
if not self._methods:
for method_name in dir(self):
method = getattr(self, method_name)
if callable(method) and hasattr(method, '_remote_call_names'):
for name in method._remote_call_names:
self._methods[name] = method
print self._methods
self.fs = None self.fs = None
def run(self): def run(self):
...@@ -71,16 +99,36 @@ class ConnectionThread(threading.Thread): ...@@ -71,16 +99,36 @@ class ConnectionThread(threading.Thread):
print '-' * 30 print '-' * 30
print repr(header) print repr(header)
print repr(payload) print repr(payload)
if header['type'] == 'rpc':
if header['method'] == 'ping': method = header['method']
self.encoder.write({'client_ref':header['client_ref']}, payload) args = header['args']
kwargs = header['kwargs']
method_callable = self._methods[method]
remote = dict(type='rpcresult',
client_ref = header['client_ref'])
try:
response = method_callable(*args, **kwargs)
remote['response'] = response
self.encoder.write(remote, '')
except RemoteResponse, response:
self.encoder.write(response.header, response.payload)
class RemoteFSConnection(ConnectionHandlerBase):
@remote_call()
def auth(self, username, password, resource):
self.username = username
self.password = password
self.resource = resource
from fs.memoryfs import MemoryFS
self.fs = MemoryFS()
class Server(object): class Server(object):
def __init__(self, addr='', port=3000): def __init__(self, addr='', port=3000, connection_factory=RemoteFSConnection):
self.addr = addr self.addr = addr
self.port = port self.port = port
self.connection_factory = connection_factory
self.socket = None self.socket = None
self.connection_id = 0 self.connection_id = 0
self.threads = {} self.threads = {}
...@@ -124,7 +172,7 @@ class Server(object): ...@@ -124,7 +172,7 @@ class Server(object):
print "Connection from", address print "Connection from", address
with self._lock: with self._lock:
self.connection_id += 1 self.connection_id += 1
thread = ConnectionThread(self, thread = self.connection_factory(self,
self.connection_id, self.connection_id,
clientsocket, clientsocket,
address) address)
......
fs/expose/sftp.py
...@@ -245,7 +245,10 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler): ...@@ -245,7 +245,10 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler):
t.start_server(server=self.server) t.start_server(server=self.server)
class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface): class ThreadedTCPServer(sockserv.TCPServer, sockserv.ThreadingMixIn):
pass
class BaseSFTPServer(ThreadedTCPServer, paramiko.ServerInterface):
"""SocketServer.TCPServer subclass exposing an FS via SFTP. """SocketServer.TCPServer subclass exposing an FS via SFTP.
BaseSFTPServer combines a simple SocketServer.TCPServer subclass with an BaseSFTPServer combines a simple SocketServer.TCPServer subclass with an
...@@ -318,6 +321,7 @@ if __name__ == "__main__": ...@@ -318,6 +321,7 @@ if __name__ == "__main__":
from fs.tempfs import TempFS from fs.tempfs import TempFS
server = BaseSFTPServer(("localhost",8022),TempFS()) server = BaseSFTPServer(("localhost",8022),TempFS())
try: try:
#import rpdb2; rpdb2.start_embedded_debugger('password')
server.serve_forever() server.serve_forever()
except (SystemExit,KeyboardInterrupt): except (SystemExit,KeyboardInterrupt):
server.server_close() server.server_close()
......
fs/ftpfs.py
...@@ -1209,7 +1209,7 @@ class FTPFS(FS): ...@@ -1209,7 +1209,7 @@ class FTPFS(FS):
@ftperrors @ftperrors
def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = normpath(path) path = normpath(path)
self.clear_dircache(path) #self.clear_dircache(path)
if not self.exists(path): if not self.exists(path):
raise ResourceNotFoundError(path) raise ResourceNotFoundError(path)
if not self.isdir(path): if not self.isdir(path):
......
fs/opener.py
...@@ -174,7 +174,7 @@ class OpenerRegistry(object): ...@@ -174,7 +174,7 @@ class OpenerRegistry(object):
for name in opener.names: for name in opener.names:
self.registry[name] = index self.registry[name] = index
def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False): def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False, cache_hint=True):
"""Parses a FS url and returns an fs object a path within that FS object """Parses a FS url and returns an fs object a path within that FS object
(if indicated in the path). A tuple of (<FS instance>, <path>) is returned. (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
...@@ -217,6 +217,7 @@ class OpenerRegistry(object): ...@@ -217,6 +217,7 @@ class OpenerRegistry(object):
raise OpenerError("Unable to parse '%s'" % orig_url) raise OpenerError("Unable to parse '%s'" % orig_url)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir) fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
fs.cache_hint(cache_hint)
if fs_path and iswildcard(fs_path): if fs_path and iswildcard(fs_path):
pathname, resourcename = pathsplit(fs_path or '') pathname, resourcename = pathsplit(fs_path or '')
......
fs/path.py
...@@ -237,6 +237,7 @@ def splitext(path): ...@@ -237,6 +237,7 @@ def splitext(path):
path = pathjoin(parent_path, pathname) path = pathjoin(parent_path, pathname)
return path, '.' + ext return path, '.' + ext
def isdotfile(path): def isdotfile(path):
"""Detects if a path references a dot file, i.e. a resource who's name """Detects if a path references a dot file, i.e. a resource who's name
starts with a '.' starts with a '.'
...@@ -255,6 +256,7 @@ def isdotfile(path): ...@@ -255,6 +256,7 @@ def isdotfile(path):
""" """
return basename(path).startswith('.') return basename(path).startswith('.')
def dirname(path): def dirname(path):
"""Returns the parent directory of a path. """Returns the parent directory of a path.
...@@ -266,10 +268,14 @@ def dirname(path): ...@@ -266,10 +268,14 @@ def dirname(path):
>>> dirname('foo/bar/baz') >>> dirname('foo/bar/baz')
'foo/bar' 'foo/bar'
>>> dirname('/foo/bar')
'/foo'
>>> dirname('/foo')
'/'
""" """
if '/' not in path: return pathsplit(path)[0]
return ''
return path.rsplit('/', 1)[0]
def basename(path): def basename(path):
...@@ -290,9 +296,7 @@ def basename(path): ...@@ -290,9 +296,7 @@ def basename(path):
'' ''
""" """
if '/' not in path: return pathsplit(path)[1]
return path
return path.rsplit('/', 1)[-1]
def issamedir(path1, path2): def issamedir(path1, path2):
...@@ -309,11 +313,13 @@ def issamedir(path1, path2): ...@@ -309,11 +313,13 @@ def issamedir(path1, path2):
""" """
return dirname(normpath(path1)) == dirname(normpath(path2)) return dirname(normpath(path1)) == dirname(normpath(path2))
def isbase(path1, path2): def isbase(path1, path2):
p1 = forcedir(abspath(path1)) p1 = forcedir(abspath(path1))
p2 = forcedir(abspath(path2)) p2 = forcedir(abspath(path2))
return p1 == p2 or p1.startswith(p2) return p1 == p2 or p1.startswith(p2)
def isprefix(path1, path2): def isprefix(path1, path2):
"""Return true is path1 is a prefix of path2. """Return true is path1 is a prefix of path2.
...@@ -341,6 +347,7 @@ def isprefix(path1, path2): ...@@ -341,6 +347,7 @@ def isprefix(path1, path2):
return False return False
return True return True
def forcedir(path): def forcedir(path):
"""Ensure the path ends with a trailing / """Ensure the path ends with a trailing /
......
fs/remotefs.py
# Work in Progress - Do not use # Work in Progress - Do not use
from __future__ import with_statement
from fs.base import FS from fs.base import FS
from fs.expose.serve import packetstream from fs.expose.serve import packetstream
...@@ -109,17 +109,38 @@ class _SocketFile(object): ...@@ -109,17 +109,38 @@ class _SocketFile(object):
self.socket.close() self.socket.close()
class _RemoteFile(object):
def __init__(self, path, connection):
self.path = path
self.connection = connection
class RemoteFS(FS): class RemoteFS(FS):
def __init__(self, addr='', port=3000, transport=None): _meta = { 'thead_safe' : True,
'network' : True,
'virtual' : False,
'read_only' : False,
'unicode_paths' : True,
}
def __init__(self, addr='', port=3000, username=None, password=None, resource=None, transport=None):
self.addr = addr self.addr = addr
self.port = port self.port = port
self.username = None
self.password = None
self.resource = None
self.transport = transport self.transport = transport
if self.transport is None: if self.transport is None:
self.transport = self._open_connection() self.transport = self._open_connection()
self.packet_handler = PacketHandler(self.transport) self.packet_handler = PacketHandler(self.transport)
self.packet_handler.start() self.packet_handler.start()
self._remote_call('auth',
username=username,
password=password,
resource=resource)
def _open_connection(self): def _open_connection(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.addr, self.port)) sock.connect((self.addr, self.port))
...@@ -127,6 +148,19 @@ class RemoteFS(FS): ...@@ -127,6 +148,19 @@ class RemoteFS(FS):
socket_file.write('pyfs/0.1\n') socket_file.write('pyfs/0.1\n')
return socket_file return socket_file
def _make_call(self, method_name, *args, **kwargs):
call = dict(type='rpc',
method=method_name,
args=args,
kwargs=kwargs)
return call
def _remote_call(self, method_name, *args, **kwargs):
call = self._make_call(method_name, *args, **kwargs)
call_id = self.packet_handler.send_packet(call)
header, payload = self.packet_handler.get_packet(call_id)
return header, payload
def ping(self, msg): def ping(self, msg):
call_id = self.packet_handler.send_packet({'type':'rpc', 'method':'ping'}, msg) call_id = self.packet_handler.send_packet({'type':'rpc', 'method':'ping'}, msg)
header, payload = self.packet_handler.get_packet(call_id) header, payload = self.packet_handler.get_packet(call_id)
...@@ -138,10 +172,17 @@ class RemoteFS(FS): ...@@ -138,10 +172,17 @@ class RemoteFS(FS):
self.transport.close() self.transport.close()
self.packet_handler.join() self.packet_handler.join()
def open(self, path, mode="r", **kwargs):
pass
def exists(self, path):
remote = self._remote_call('exists', path)
return remote.get('response')
if __name__ == "__main__": if __name__ == "__main__":
rfs = RemoteFS() rfs = RemoteFS()
rfs.ping("Hello, World!")
rfs.close() rfs.close()
\ No newline at end of file
fs/sftpfs.py
...@@ -222,6 +222,8 @@ class SFTPFS(FS): ...@@ -222,6 +222,8 @@ class SFTPFS(FS):
@property @property
@synchronize @synchronize
def client(self): def client(self):
if self.closed:
return None
client = getattr(self._tlocal, 'client', None) client = getattr(self._tlocal, 'client', None)
if client is None: if client is None:
if self._transport is None: if self._transport is None:
...@@ -242,9 +244,10 @@ class SFTPFS(FS): ...@@ -242,9 +244,10 @@ class SFTPFS(FS):
def close(self): def close(self):
"""Close the connection to the remote server.""" """Close the connection to the remote server."""
if not self.closed: if not self.closed:
if self.client: self._tlocal = None
self.client.close() #if self.client:
if self._owns_transport and self._transport: # self.client.close()
if self._owns_transport and self._transport and self._transport.is_active:
self._transport.close() self._transport.close()
self.closed = True self.closed = True
......
fs/tests/test_expose.py
...@@ -29,6 +29,22 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases): ...@@ -29,6 +29,22 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
port = 3000 port = 3000
self.temp_fs = TempFS() self.temp_fs = TempFS()
self.server = None self.server = None
self.serve_more_requests = True
self.server_thread = threading.Thread(target=self.runServer)
self.server_thread.setDaemon(True)
self.start_event = threading.Event()
self.end_event = threading.Event()
self.server_thread.start()
self.start_event.wait()
def runServer(self):
"""Run the server, swallowing shutdown-related execptions."""
port = 3000
while not self.server: while not self.server:
try: try:
self.server = self.makeServer(self.temp_fs,("127.0.0.1",port)) self.server = self.makeServer(self.temp_fs,("127.0.0.1",port))
...@@ -37,37 +53,48 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases): ...@@ -37,37 +53,48 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
port += 1 port += 1
else: else:
raise raise
self.server_addr = ("127.0.0.1",port) self.server_addr = ("127.0.0.1", port)
self.serve_more_requests = True
self.server_thread = threading.Thread(target=self.runServer) self.server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_thread.daemon = True
self.server_thread.start() # if sys.platform != "win32":
# try:
# self.server.socket.settimeout(1)
# except socket.error:
# pass
#
self.start_event.set()
def runServer(self):
"""Run the server, swallowing shutdown-related execptions."""
if sys.platform != "win32":
try:
self.server.socket.settimeout(0.1)
except socket.error:
pass
try: try:
#self.server.serve_forever()
while self.serve_more_requests: while self.serve_more_requests:
self.server.handle_request() self.server.handle_request()
except Exception, e: except Exception, e:
pass pass
self.end_event.set()
def setUp(self): def setUp(self):
self.startServer() self.startServer()
self.fs = rpcfs.RPCFS("http://%s:%d" % self.server_addr) self.fs = rpcfs.RPCFS("http://%s:%d" % self.server_addr)
def tearDown(self): def tearDown(self):
self.serve_more_requests = False self.serve_more_requests = False
#self.server.socket.close()
# self.server.socket.shutdown(socket.SHUT_RDWR)
# self.server.socket.close()
# self.temp_fs.close()
#self.server_thread.join()
#self.end_event.wait()
#return
try: try:
self.bump() self.bump()
self.server.server_close() self.server.server_close()
except Exception: except Exception:
pass pass
self.server_thread.join() #self.server_thread.join()
self.temp_fs.close() self.temp_fs.close()
def bump(self): def bump(self):
......
fs/tests/test_path.py
...@@ -105,7 +105,6 @@ class TestPathFunctions(unittest.TestCase): ...@@ -105,7 +105,6 @@ class TestPathFunctions(unittest.TestCase):
self.assertEquals(recursepath("hello",reverse=True),["/hello","/"]) self.assertEquals(recursepath("hello",reverse=True),["/hello","/"])
self.assertEquals(recursepath("",reverse=True),["/"]) self.assertEquals(recursepath("",reverse=True),["/"])
def test_isdotfile(self): def test_isdotfile(self):
for path in ['.foo', for path in ['.foo',
'.svn', '.svn',
...@@ -125,7 +124,9 @@ class TestPathFunctions(unittest.TestCase): ...@@ -125,7 +124,9 @@ class TestPathFunctions(unittest.TestCase):
tests = [('foo', ''), tests = [('foo', ''),
('foo/bar', 'foo'), ('foo/bar', 'foo'),
('foo/bar/baz', 'foo/bar'), ('foo/bar/baz', 'foo/bar'),
('/', '')] ('/foo/bar', '/foo'),
('/foo', '/'),
('/', '/')]
for path, test_dirname in tests: for path, test_dirname in tests:
self.assertEqual(dirname(path), test_dirname) self.assertEqual(dirname(path), test_dirname)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment