Skip to content

P
pyfs
Commit c56e8ae6 by willmcgugan
Options

Fixes and documentation

parent 462723b6
Showing with 416 additions and 186 deletions
docs/concepts.rst
Concepts
========
It is generally quite easy to get in to the mind-set of using PyFilesystem interface over lower level interfaces (since the code tends to be simpler) but there are a few concepts which you will need to keep in mind.
Working with PyFilesystem is generally easier than working with lower level interfaces, as long as you are aware these simple concepts.
Sandboxing
----------
......@@ -25,7 +25,9 @@ We can open the `foo` directory with the following code::
from fs.osfs import OSFS
foo_fs = OSFS('foo')
The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable, especially if we are passing `foo_fs` to an untrusted function or one that could potentially delete files. Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method::
The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable,
especially if we are passing `foo_fs` to an untrusted function or to a function that has the potential to delete files.
Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method::
bar_fs = foo_fs.opendir('bar')
......@@ -54,7 +56,7 @@ When working with paths in FS objects, keep in mind the following:
* A double dot means 'previous directory'
Note that paths used by the FS interface will use this format, but the constructor or additional methods may not.
Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which can be platform-dependent.
Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which is platform-dependent.
There are many helpful functions for working with paths in the :mod:`fs.path` module.
......
docs/getting_started.rst
......@@ -13,14 +13,22 @@ The easiest way to install PyFilesystem is with `easy_install <http://peak.telec
Add the -U switch if you want to upgrade a previous installation::
easy_install -U fs
If you prefer to use Pip (http://pypi.python.org/pypi/pip) to install Python packages, the procedure is much the same::
This will install the latest stable release. If you would prefer to install the cutting edge release then you can get the latest copy of the source via SVN::
pip install fs
Or to upgrade::
pip install fs --upgrade
You can also install the cutting edge release by checking out the source via SVN::
svn checkout http://pyfilesystem.googlecode.com/svn/trunk/ pyfilesystem-read-only
cd pyfilesystem-read-only
python setup.py install
You should now have the `fs` module on your path:
Whichever method you use, you should now have the `fs` module on your path (version number may vary)::
>>> import fs
>>> fs.__version__
......
fs/base.py
......@@ -52,10 +52,10 @@ class DummyLock(object):
def release(self):
"""Releasing a DummyLock always succeeds."""
pass
def __enter__(self):
pass
def __exit__(self, *args):
pass
......@@ -95,7 +95,7 @@ class NullFile(object):
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.closed = True
......@@ -147,7 +147,7 @@ class FS(object):
An instance of a class derived from FS is an abstraction on some kind of filesystem, such as the OS filesystem or a zip file.
"""
_meta = {}
def __init__(self, thread_synchronize=False):
......@@ -157,7 +157,7 @@ class FS(object):
:type thread_synchronize: bool
"""
super(FS, self).__init__()
self.closed = False
self.thread_synchronize = thread_synchronize
......@@ -169,11 +169,11 @@ class FS(object):
def __del__(self):
if not getattr(self, 'closed', True):
self.close()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
def __exit__(self, type, value, traceback):
self.close()
def cachehint(self, enabled):
......@@ -188,7 +188,7 @@ class FS(object):
"""
pass
# Deprecating cache_hint in favour of no underscore version, for consistency
cache_hint = cachehint
cache_hint = cachehint
def close(self):
"""Close the filesystem. This will perform any shutdown related
......@@ -205,16 +205,16 @@ class FS(object):
# type of lock that should be there. None == no lock,
# True == a proper lock, False == a dummy lock.
state = self.__dict__.copy()
lock = state.get("_lock",None)
lock = state.get("_lock", None)
if lock is not None:
if isinstance(lock,threading._RLock):
if isinstance(lock, threading._RLock):
state["_lock"] = True
else:
state["_lock"] = False
return state
def __setstate__(self,state):
self.__dict__.update(state)
def __setstate__(self, state):
self.__dict__.update(state)
lock = state.get("_lock")
if lock is not None:
if lock:
......@@ -254,13 +254,13 @@ class FS(object):
:param default: An option default to return, if the meta value isn't present
:raises `fs.errors.NoMetaError`: If specified meta value is not present, and there is no default
"""
"""
if meta_name not in self._meta:
if default is not NoDefaultMeta:
return default
raise NoMetaError(meta_name=meta_name)
raise NoMetaError(meta_name=meta_name)
return self._meta[meta_name]
def hasmeta(self, meta_name):
"""Check that a meta value is supported
......@@ -272,7 +272,7 @@ class FS(object):
self.getmeta(meta_name)
except NoMetaError:
return False
return True
return True
def getsyspath(self, path, allow_none=False):
"""Returns the system path (a path recognized by the OS) if one is present.
......@@ -302,7 +302,7 @@ class FS(object):
"""
return self.getsyspath(path, allow_none=True) is not None
def getpathurl(self, path, allow_none=False):
"""Returns a url that corresponds to the given path, if one exists.
......@@ -321,7 +321,7 @@ class FS(object):
if not allow_none:
raise NoPathURLError(path=path)
return None
def haspathurl(self, path):
"""Check if the path has an equivalent URL form
......@@ -336,26 +336,39 @@ class FS(object):
"""Open a the given path as a file-like object.
:param path: a path to file that should be opened
:type path: string
:param mode: mode of file to open, identical to the mode string used
in 'file' and 'open' builtins
:type mode: string
:param kwargs: additional (optional) keyword parameters that may
be required to open the file
be required to open the file
:type kwargs: dict
:rtype: a file-like object
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if an intermediate directory is an file
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
"""
raise UnsupportedError("open file")
def safeopen(self, path, mode="r", **kwargs):
"""Like :py:meth:`~fs.base.FS.open`, but returns a :py:class:`~fs.base.NullFile` if the file could not be opened.
"""Like :py:meth:`~fs.base.FS.open`, but returns a
:py:class:`~fs.base.NullFile` if the file could not be opened.
A ``NullFile`` is a dummy file which has all the methods of a file-like object,
but contains no data.
:param path: a path to file that should be opened
:type path: string
:param mode: mode of file to open, identical to the mode string used
in 'file' and 'open' builtins
:type mode: string
:param kwargs: additional (optional) keyword parameters that may
be required to open the file
be required to open the file
:type kwargs: dict
:rtype: a file-like object
"""
......@@ -369,8 +382,10 @@ class FS(object):
"""Check if a path references a valid resource.
:param path: A path in the filesystem
:type path: string
:rtype: bool
"""
return self.isfile(path) or self.isdir(path)
......@@ -378,6 +393,8 @@ class FS(object):
"""Check if a path references a directory.
:param path: a path in the filesystem
:type path: string
:rtype: bool
"""
......@@ -387,6 +404,8 @@ class FS(object):
"""Check if a path references a file.
:param path: a path in the filesystem
:type path: string
:rtype: bool
"""
......@@ -419,10 +438,12 @@ class FS(object):
:type dirs_only: bool
:param files_only: if True, only return files
:type files_only: bool
:rtype: iterable of paths
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path exists, but is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
"""
raise UnsupportedError("list directory")
......@@ -460,7 +481,7 @@ class FS(object):
if full or absolute:
return self.getinfo(p)
else:
return self.getinfo(pathjoin(path,p))
return self.getinfo(pathjoin(path, p))
except FSError:
return {}
......@@ -493,7 +514,7 @@ class FS(object):
if wildcard is not None:
if not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
wildcard = lambda fn:bool (wildcard_re.match(fn))
entries = [p for p in entries if wildcard(p)]
if dirs_only:
......@@ -553,14 +574,16 @@ class FS(object):
"""Make a directory on the filesystem.
:param path: path of directory
:type path: string
:param recursive: if True, any intermediate directories will also be created
:type recursive: bool
:param allow_recreate: if True, re-creating a directory wont be an error
:type allow_create: bool
:raises `fs.errors.DestinationExistsError`: if the path is already a directory, and allow_recreate is False
:raises `fs.errors.ParentDirectoryMissingError`: if a containing directory is missing and recursive is False
:raises `fs.errors.ResourceInvalidError`: if a path is an existing file
:raises `fs.errors.ResourceNotFoundError`: if the path is not found
"""
raise UnsupportedError("make directory")
......@@ -569,9 +592,11 @@ class FS(object):
"""Remove a file from the filesystem.
:param path: Path of the resource to remove
:type path: string
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
"""
raise UnsupportedError("remove resource")
......@@ -580,15 +605,17 @@ class FS(object):
"""Remove a directory from the filesystem
:param path: path of the directory to remove
:type path: string
:param recursive: if True, empty parent directories will be removed
:type recursive: bool
:param force: if True, any directory contents will be removed
:type force: bool
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.DirectoryNotEmptyError`: if the directory is not empty and force is False
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
"""
raise UnsupportedError("remove directory")
......@@ -596,49 +623,66 @@ class FS(object):
"""Renames a file or directory
:param src: path to rename
:type src: string
:param dst: new name
:type dst: string
:raises ParentDirectoryMissingError: if a containing directory is missing
:raises ResourceInvalidError: if the path or a parent path is not a
directory or src is a parent of dst or one of src or dst is a dir
and the other don't
:raises ResourceNotFoundError: if the src path does not exist
"""
raise UnsupportedError("rename resource")
@convert_os_errors
def settimes(self, path, accessed_time=None, modified_time=None):
"""Set the accessed time and modified time of a file
:param path: path to a file
:param accessed_time: a datetime object the file was accessed (defaults to current time)
:param modified_time: a datetime object the file was modified (defaults to current time)
:type path: string
:param accessed_time: the datetime the file was accessed (defaults to current time)
:type accessed_time: datetime
:param modified_time: the datetime the file was modified (defaults to current time)
:type modified_time: datetime
"""
sys_path = self.getsyspath(path, allow_none=True)
if sys_path is not None:
if sys_path is not None:
now = datetime.datetime.now()
if accessed_time is None:
accessed_time = now
if modified_time is None:
modified_time = now
modified_time = now
accessed_time = int(time.mktime(accessed_time.timetuple()))
modified_time = int(time.mktime(modified_time.timetuple()))
os.utime(sys_path, (accessed_time, modified_time))
return True
else:
raise UnsupportedError("settimes")
def getinfo(self, path):
"""Returns information for a path as a dictionary. The exact content of
this dictionary will vary depending on the implementation, but will
likely include a few common values. The following values will be found
in info dictionaries for most implementations:
* "size" - Number of bytes used to store the file or directory
* "created_time" - A datetime object containing the time the resource was created
* "accessed_time" - A datetime object containing the time the resource was last accessed
* "modified_time" - A datetime object containing the time the resource was modified
:param path: a path to retrieve information for
:type path: string
:rtype: dict
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is not a directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
"""
raise UnsupportedError("get resource info")
......@@ -655,9 +699,9 @@ class FS(object):
try:
sys_path = self.getsyspath(path)
except NoSysPathError:
return "No description available"
return "No description available"
return sys_path
def getcontents(self, path):
"""Returns the contents of a file as a string.
......@@ -675,7 +719,7 @@ class FS(object):
if f is not None:
f.close()
def setcontents(self, path, data, chunk_size=1024*64):
def setcontents(self, path, data, chunk_size=1024 * 64):
"""A convenience method to create a new file from a string or file-like object
:param path: a path of the file to create
......@@ -683,7 +727,7 @@ class FS(object):
:param chunk_size: Number of bytes to read in a chunk, if the implementation has to resort to a read / copy loop
"""
if not data:
self.createfile(path)
else:
......@@ -704,11 +748,11 @@ class FS(object):
finally:
if f is not None:
f.close()
def setcontents_async(self,
path,
data,
chunk_size=1024*64,
chunk_size=1024 * 64,
progress_callback=None,
finished_callback=None,
error_callback=None):
......@@ -728,18 +772,18 @@ class FS(object):
:returns: An event object that is set when the copy is complete, call
the `wait` method of this object to block until the data is written
"""
"""
if progress_callback is None:
progress_callback = lambda bytes_written:None
def do_setcontents():
try:
progress_callback = lambda bytes_written:None
def do_setcontents():
try:
f = None
try:
f = self.open(path, 'wb')
progress_callback(0)
progress_callback(0)
if hasattr(data, "read"):
bytes_written = 0
read = data.read
......@@ -749,30 +793,30 @@ class FS(object):
write(chunk)
bytes_written += len(chunk)
progress_callback(bytes_written)
chunk = read(chunk_size)
else:
f.write(data)
chunk = read(chunk_size)
else:
f.write(data)
progress_callback(len(data))
if finished_callback is not None:
finished_callback()
finally:
if f is not None:
f.close()
except Exception, e:
if error_callback is not None:
error_callback(e)
finally:
finished_event.set()
finished_event = threading.Event()
finished_event = threading.Event()
threading.Thread(target=do_setcontents).start()
return finished_event
def createfile(self, path, wipe=False):
"""Creates an empty file if it doesn't exist
......@@ -782,23 +826,26 @@ class FS(object):
"""
if not wipe and self.isfile(path):
return
f = None
try:
f = self.open(path, 'w')
finally:
if f is not None:
f.close()
def opendir(self, path):
"""Opens a directory and returns a FS object representing its contents.
:param path: path to directory to open
:type path: string
:return: the opened dir
:rtype: an FS object
"""
from fs.wrapfs.subfs import SubFS
if not self.exists(path):
raise ResourceNotFoundError(path)
......@@ -815,19 +862,23 @@ class FS(object):
contents.
:param path: root path to start walking
:type path: string
:param wildcard: if given, only return files that match this wildcard
:type wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard
:type dir_wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean
:param search: a string identifying the method used to walk the directories. There are two such methods:
* ``"breadth"`` yields paths in the top directories first
* ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of (current_path, paths)
"""
"""
def listdir(path, *args, **kwargs):
if ignore_errors:
try:
......@@ -836,21 +887,21 @@ class FS(object):
return []
else:
return self.listdir(path, *args, **kwargs)
if wildcard is None:
wildcard = lambda f:True
elif not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
if dir_wildcard is None:
dir_wildcard = lambda f:True
elif not callable(dir_wildcard):
dir_wildcard_re = re.compile(fnmatch.translate(dir_wildcard))
dir_wildcard = lambda fn:bool (dir_wildcard_re.match(fn))
dir_wildcard = lambda fn:bool (dir_wildcard_re.match(fn))
if search == "breadth":
dirs = [path]
while dirs:
current_path = dirs.pop()
......@@ -858,16 +909,16 @@ class FS(object):
try:
for filename in listdir(current_path):
path = pathjoin(current_path, filename)
if self.isdir(path):
if self.isdir(path):
if dir_wildcard(path):
dirs.append(path)
else:
dirs.append(path)
else:
if wildcard(filename):
paths.append(filename)
except ResourceNotFoundError:
# Could happen if another thread / process deletes something whilst we are walking
pass
yield (current_path, paths)
elif search == "depth":
......@@ -884,7 +935,7 @@ class FS(object):
for p in recurse(path):
yield p
else:
raise ValueError("Search should be 'breadth' or 'depth'")
......@@ -893,17 +944,25 @@ class FS(object):
wildcard=None,
dir_wildcard=None,
search="breadth",
ignore_errors=False ):
ignore_errors=False):
"""Like the 'walk' method, but just yields file paths.
:param path: root path to start walking
:type path: string
:param wildcard: if given, only return files that match this wildcard
:type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard
:type dir_wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean
:param search: same as walk method
:param search: a string identifying the method used to walk the directories. There are two such methods:
* ``"breadth"`` yields paths in the top directories first
* ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of file paths
"""
for path, files in self.walk(path, wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors):
for f in files:
......@@ -917,11 +976,19 @@ class FS(object):
"""Like the 'walk' method but yields directories.
:param path: root path to start walking
:type path: string
:param wildcard: if given, only return directories that match this wildcard
:type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean
:param search: same as the walk method
:param search: a string identifying the method used to walk the directories. There are two such methods:
* ``"breadth"`` yields paths in the top directories first
* ``"depth"`` yields the deepest paths first
:param ignore_errors: ignore any errors reading the directory
:type ignore_errors: bool
:rtype: iterator of dir paths
"""
for p, _files in self.walk(path, dir_wildcard=wildcard, search=search, ignore_errors=ignore_errors):
yield p
......@@ -931,32 +998,38 @@ class FS(object):
"""Returns the size (in bytes) of a resource.
:param path: a path to the resource
:rtype: integer
:type path: string
:returns: the size of the file
:rtype: integer
"""
info = self.getinfo(path)
size = info.get('size', None)
if size is None:
raise OperationFailedError("get size of resource", path)
return size
def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
def copy(self, src, dst, overwrite=False, chunk_size=1024 * 64):
"""Copies a file from src to dst.
:param src: the source path
:type src: string
:param dst: the destination path
:type dst: string
:param overwrite: if True, then an existing file at the destination may
be overwritten; If False then DestinationExistsError
will be raised.
:type overwrite: bool
:param chunk_size: size of chunks to use if a simple copy is required
(defaults to 64K).
:type chunk_size: bool
"""
if not self.isfile(src):
if self.isdir(src):
raise ResourceInvalidError(src,msg="Source is not a file: %(path)s")
raise ResourceInvalidError(src, msg="Source is not a file: %(path)s")
raise ResourceNotFoundError(src)
if not overwrite and self.exists(dst):
raise DestinationExistsError(dst)
......@@ -967,50 +1040,51 @@ class FS(object):
if src_syspath is not None and dst_syspath is not None:
self._shutil_copyfile(src_syspath, dst_syspath)
else:
src_file = None
src_file = None
try:
src_file = self.open(src, "rb")
self.setcontents(dst, src_file, chunk_size=chunk_size)
src_file = self.open(src, "rb")
self.setcontents(dst, src_file, chunk_size=chunk_size)
except ResourceNotFoundError:
if self.exists(src) and not self.exists(dirname(dst)):
raise ParentDirectoryMissingError(dst)
finally:
if src_file is not None:
src_file.close()
src_file.close()
@classmethod
@convert_os_errors
@convert_os_errors
def _shutil_copyfile(cls, src_syspath, dst_syspath):
try:
shutil.copyfile(src_syspath, dst_syspath)
except IOError, e:
# shutil reports ENOENT when a parent directory is missing
if getattr(e,"errno",None) == 2:
if getattr(e, "errno", None) == 2:
if not os.path.exists(dirname(dst_syspath)):
raise ParentDirectoryMissingError(dst_syspath)
raise
@classmethod
@convert_os_errors
def _shutil_movefile(cls, src_syspath, dst_syspath):
shutil.move(src_syspath, dst_syspath)
def move(self, src, dst, overwrite=False, chunk_size=16384):
"""moves a file from one location to another.
:param src: source path
:type src: string
:param dst: destination path
:param overwrite: if True, then an existing file at the destination path
will be silently overwritten; if False then an exception
will be raised in this case.
:type dst: string
:param overwrite: When True the destination will be overwritten (if it exists),
otherwise a DestinationExistsError will be thrown
:type overwrite: bool
:param chunk_size: Size of chunks to use when copying, if a simple copy
is required
:type chunk_size: integer
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
"""
src_syspath = self.getsyspath(src, allow_none=True)
......@@ -1032,19 +1106,26 @@ class FS(object):
pass
self.copy(src, dst, overwrite=overwrite, chunk_size=chunk_size)
self.remove(src)
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
"""moves a directory from one location to another.
:param src: source directory path
:type src: string
:param dst: destination directory path
:type dst: string
:param overwrite: if True then any existing files in the destination
directory will be overwritten
:type overwrite: bool
:param ignore_errors: if True then this method will ignore FSError
exceptions when moving files
:type ignore_errors: bool
:param chunk_size: size of chunks to use when copying, if a simple copy
is required
:type chunk_size: integer
:raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False
"""
if not self.isdir(src):
if self.isfile(src):
......@@ -1058,7 +1139,7 @@ class FS(object):
if src_syspath is not None and dst_syspath is not None:
try:
os.rename(src_syspath,dst_syspath)
os.rename(src_syspath, dst_syspath)
return
except OSError:
pass
......@@ -1092,12 +1173,14 @@ class FS(object):
movefile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size)
self.removedir(dirname)
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
"""copies a directory from one location to another.
:param src: source directory path
:type src: string
:param dst: destination directory path
:type dst: string
:param overwrite: if True then any existing files in the destination
directory will be overwritten
:type overwrite: bool
......@@ -1105,7 +1188,7 @@ class FS(object):
:type ignore_errors: bool
:param chunk_size: size of chunks to use when copying, if a simple copy
is required (defaults to 16K)
"""
if not self.isdir(src):
raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s")
......@@ -1144,8 +1227,9 @@ class FS(object):
"""Check if a directory is empty (contains no files or sub-directories)
:param path: a directory path
:rtype: bool
"""
path = normpath(path)
iter_dir = iter(self.listdir(path))
......@@ -1162,6 +1246,9 @@ class FS(object):
:param path: path to the new directory
:param recursive: if True any intermediate directories will be created
:return: the opened dir
:rtype: an FS object
"""
self.makedir(path, allow_recreate=True, recursive=recursive)
......@@ -1175,10 +1262,10 @@ class FS(object):
5. Set to None for no limit
"""
from fs.utils import print_fs
from fs.utils import print_fs
print_fs(self, max_levels=max_levels)
tree = printtree
def browse(self, hide_dotfiles=False):
"""Displays the FS tree in a graphical window (requires wxPython)
......@@ -1198,16 +1285,16 @@ class FS(object):
:param copy: If False then changes wont be written back to the file
:raises `fs.errors.NoMMapError`: Only paths that have a syspath can be opened as a mmap
"""
"""
syspath = self.getsyspath(path, allow_none=True)
if syspath is None:
raise NoMMapError(path)
try:
import mmap
import mmap
except ImportError:
raise NoMMapError(msg="mmap not supported")
if read_only:
f = open(syspath, 'rb')
access = mmap.ACCESS_READ
......@@ -1217,14 +1304,14 @@ class FS(object):
access = mmap.ACCESS_COPY
else:
f = open(syspath, 'r+b')
access = mmap.ACCESS_WRITE
access = mmap.ACCESS_WRITE
m = mmap.mmap(f.fileno(), 0, access=access)
return m
def flags_to_mode(flags):
"""Convert an os.O_* flag bitmask into an FS mode string."""
"""Convert an os.O_* flag bitmask into an FS mode string."""
if flags & os.O_WRONLY:
if flags & os.O_TRUNC:
mode = "w"
......@@ -1240,9 +1327,9 @@ def flags_to_mode(flags):
else:
mode = "r+"
else:
mode = "r"
mode = "r"
if flags & os.O_EXCL:
mode += "x"
return mode
fs/commands/fsls.py
......@@ -42,8 +42,8 @@ List contents of [PATH]"""
dir_paths = []
file_paths = []
fs_used = set()
for fs_url in args:
fs, path = self.open_fs(fs_url)
for fs_url in args:
fs, path = self.open_fs(fs_url)
fs_used.add(fs)
path = path or '.'
wildcard = None
......
fs/commands/runner.py
......@@ -245,7 +245,7 @@ class Command(object):
for col_no, col in enumerate(row):
td = col.ljust(max_row_widths[col_no])
if col_no in col_process:
td = col_process[col_no](td)
td = col_process[col_no](td)
out_col.append(td)
lines.append(self.text_encode('%s\n' % ' '.join(out_col).rstrip()))
self.output(''.join(lines))
......
fs/expose/serve/server.py
......@@ -19,10 +19,28 @@ class _SocketFile(object):
def write(self, data):
self.socket.sendall(data)
class ConnectionThread(threading.Thread):
def remote_call(method_name=None):
method = method_name
def deco(f):
if not hasattr(f, '_remote_call_names'):
f._remote_call_names = []
f._remote_call_names.append(method or f.__name__)
return f
return deco
class RemoteResponse(Exception):
def __init__(self, header, payload):
self.header = header
self.payload = payload
class ConnectionHandlerBase(threading.Thread):
_methods = {}
def __init__(self, server, connection_id, socket, address):
super(ConnectionThread, self).__init__()
super(ConnectionHandlerBase, self).__init__()
self.server = server
self.connection_id = connection_id
self.socket = socket
......@@ -33,6 +51,16 @@ class ConnectionThread(threading.Thread):
self._lock = threading.RLock()
self.socket_error = None
if not self._methods:
for method_name in dir(self):
method = getattr(self, method_name)
if callable(method) and hasattr(method, '_remote_call_names'):
for name in method._remote_call_names:
self._methods[name] = method
print self._methods
self.fs = None
......@@ -70,17 +98,37 @@ class ConnectionThread(threading.Thread):
def on_packet(self, header, payload):
print '-' * 30
print repr(header)
print repr(payload)
if header['method'] == 'ping':
self.encoder.write({'client_ref':header['client_ref']}, payload)
print repr(payload)
if header['type'] == 'rpc':
method = header['method']
args = header['args']
kwargs = header['kwargs']
method_callable = self._methods[method]
remote = dict(type='rpcresult',
client_ref = header['client_ref'])
try:
response = method_callable(*args, **kwargs)
remote['response'] = response
self.encoder.write(remote, '')
except RemoteResponse, response:
self.encoder.write(response.header, response.payload)
class RemoteFSConnection(ConnectionHandlerBase):
@remote_call()
def auth(self, username, password, resource):
self.username = username
self.password = password
self.resource = resource
from fs.memoryfs import MemoryFS
self.fs = MemoryFS()
class Server(object):
def __init__(self, addr='', port=3000):
def __init__(self, addr='', port=3000, connection_factory=RemoteFSConnection):
self.addr = addr
self.port = port
self.connection_factory = connection_factory
self.socket = None
self.connection_id = 0
self.threads = {}
......@@ -124,10 +172,10 @@ class Server(object):
print "Connection from", address
with self._lock:
self.connection_id += 1
thread = ConnectionThread(self,
self.connection_id,
clientsocket,
address)
thread = self.connection_factory(self,
self.connection_id,
clientsocket,
address)
self.threads[self.connection_id] = thread
thread.start()
......
fs/expose/sftp.py
......@@ -245,7 +245,10 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler):
t.start_server(server=self.server)
class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
class ThreadedTCPServer(sockserv.TCPServer, sockserv.ThreadingMixIn):
pass
class BaseSFTPServer(ThreadedTCPServer, paramiko.ServerInterface):
"""SocketServer.TCPServer subclass exposing an FS via SFTP.
BaseSFTPServer combines a simple SocketServer.TCPServer subclass with an
......@@ -318,6 +321,7 @@ if __name__ == "__main__":
from fs.tempfs import TempFS
server = BaseSFTPServer(("localhost",8022),TempFS())
try:
#import rpdb2; rpdb2.start_embedded_debugger('password')
server.serve_forever()
except (SystemExit,KeyboardInterrupt):
server.server_close()
......
fs/ftpfs.py
......@@ -1209,7 +1209,7 @@ class FTPFS(FS):
@ftperrors
def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = normpath(path)
self.clear_dircache(path)
#self.clear_dircache(path)
if not self.exists(path):
raise ResourceNotFoundError(path)
if not self.isdir(path):
......
fs/opener.py
......@@ -174,7 +174,7 @@ class OpenerRegistry(object):
for name in opener.names:
self.registry[name] = index
def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False, cache_hint=True):
"""Parses a FS url and returns an fs object a path within that FS object
(if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
......@@ -216,7 +216,8 @@ class OpenerRegistry(object):
if fs_url is None:
raise OpenerError("Unable to parse '%s'" % orig_url)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
fs.cache_hint(cache_hint)
if fs_path and iswildcard(fs_path):
pathname, resourcename = pathsplit(fs_path or '')
......@@ -432,7 +433,7 @@ examples:
dirpath, resourcepath = pathsplit(path)
url = netloc
ftpfs = FTPFS(url, user=username or '', passwd=password or '')
ftpfs.cache_hint(True)
......
fs/path.py
......@@ -237,6 +237,7 @@ def splitext(path):
path = pathjoin(parent_path, pathname)
return path, '.' + ext
def isdotfile(path):
"""Detects if a path references a dot file, i.e. a resource who's name
starts with a '.'
......@@ -255,6 +256,7 @@ def isdotfile(path):
"""
return basename(path).startswith('.')
def dirname(path):
"""Returns the parent directory of a path.
......@@ -265,12 +267,16 @@ def dirname(path):
>>> dirname('foo/bar/baz')
'foo/bar'
>>> dirname('/foo/bar')
'/foo'
>>> dirname('/foo')
'/'
"""
if '/' not in path:
return ''
return path.rsplit('/', 1)[0]
return pathsplit(path)[0]
def basename(path):
"""Returns the basename of the resource referenced by a path.
......@@ -290,9 +296,7 @@ def basename(path):
''
"""
if '/' not in path:
return path
return path.rsplit('/', 1)[-1]
return pathsplit(path)[1]
def issamedir(path1, path2):
......@@ -309,11 +313,13 @@ def issamedir(path1, path2):
"""
return dirname(normpath(path1)) == dirname(normpath(path2))
def isbase(path1, path2):
p1 = forcedir(abspath(path1))
p2 = forcedir(abspath(path2))
return p1 == p2 or p1.startswith(p2)
def isprefix(path1, path2):
"""Return true is path1 is a prefix of path2.
......@@ -341,6 +347,7 @@ def isprefix(path1, path2):
return False
return True
def forcedir(path):
"""Ensure the path ends with a trailing /
......
fs/remotefs.py
# Work in Progress - Do not use
from __future__ import with_statement
from fs.base import FS
from fs.expose.serve import packetstream
......@@ -107,26 +107,60 @@ class _SocketFile(object):
def close(self):
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
class _RemoteFile(object):
def __init__(self, path, connection):
self.path = path
self.connection = connection
class RemoteFS(FS):
def __init__(self, addr='', port=3000, transport=None):
_meta = { 'thead_safe' : True,
'network' : True,
'virtual' : False,
'read_only' : False,
'unicode_paths' : True,
}
def __init__(self, addr='', port=3000, username=None, password=None, resource=None, transport=None):
self.addr = addr
self.port = port
self.username = None
self.password = None
self.resource = None
self.transport = transport
if self.transport is None:
self.transport = self._open_connection()
self.packet_handler = PacketHandler(self.transport)
self.packet_handler.start()
self._remote_call('auth',
username=username,
password=password,
resource=resource)
def _open_connection(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.addr, self.port))
socket_file = _SocketFile(sock)
socket_file.write('pyfs/0.1\n')
return socket_file
def _make_call(self, method_name, *args, **kwargs):
call = dict(type='rpc',
method=method_name,
args=args,
kwargs=kwargs)
return call
def _remote_call(self, method_name, *args, **kwargs):
call = self._make_call(method_name, *args, **kwargs)
call_id = self.packet_handler.send_packet(call)
header, payload = self.packet_handler.get_packet(call_id)
return header, payload
def ping(self, msg):
call_id = self.packet_handler.send_packet({'type':'rpc', 'method':'ping'}, msg)
header, payload = self.packet_handler.get_packet(call_id)
......@@ -137,11 +171,18 @@ class RemoteFS(FS):
def close(self):
self.transport.close()
self.packet_handler.join()
def open(self, path, mode="r", **kwargs):
pass
def exists(self, path):
remote = self._remote_call('exists', path)
return remote.get('response')
if __name__ == "__main__":
rfs = RemoteFS()
rfs.ping("Hello, World!")
rfs = RemoteFS()
rfs.close()
\ No newline at end of file
fs/sftpfs.py
......@@ -139,10 +139,10 @@ class SFTPFS(FS):
if not connection.is_active():
raise RemoteConnectionError(msg='Unable to connect')
if no_auth:
if no_auth:
try:
connection.auth_none('')
except paramiko.SSHException:
except paramiko.SSHException:
pass
elif not connection.is_authenticated():
......@@ -222,6 +222,8 @@ class SFTPFS(FS):
@property
@synchronize
def client(self):
if self.closed:
return None
client = getattr(self._tlocal, 'client', None)
if client is None:
if self._transport is None:
......@@ -242,9 +244,10 @@ class SFTPFS(FS):
def close(self):
"""Close the connection to the remote server."""
if not self.closed:
if self.client:
self.client.close()
if self._owns_transport and self._transport:
self._tlocal = None
#if self.client:
# self.client.close()
if self._owns_transport and self._transport and self._transport.is_active:
self._transport.close()
self.closed = True
......
fs/tests/test_expose.py
......@@ -29,6 +29,22 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
port = 3000
self.temp_fs = TempFS()
self.server = None
self.serve_more_requests = True
self.server_thread = threading.Thread(target=self.runServer)
self.server_thread.setDaemon(True)
self.start_event = threading.Event()
self.end_event = threading.Event()
self.server_thread.start()
self.start_event.wait()
def runServer(self):
"""Run the server, swallowing shutdown-related execptions."""
port = 3000
while not self.server:
try:
self.server = self.makeServer(self.temp_fs,("127.0.0.1",port))
......@@ -37,24 +53,26 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
port += 1
else:
raise
self.server_addr = ("127.0.0.1",port)
self.serve_more_requests = True
self.server_thread = threading.Thread(target=self.runServer)
self.server_thread.daemon = True
self.server_thread.start()
def runServer(self):
"""Run the server, swallowing shutdown-related execptions."""
if sys.platform != "win32":
try:
self.server.socket.settimeout(0.1)
except socket.error:
pass
self.server_addr = ("127.0.0.1", port)
self.server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# if sys.platform != "win32":
# try:
# self.server.socket.settimeout(1)
# except socket.error:
# pass
#
self.start_event.set()
try:
#self.server.serve_forever()
while self.serve_more_requests:
self.server.handle_request()
except Exception, e:
pass
self.end_event.set()
def setUp(self):
self.startServer()
......@@ -62,12 +80,21 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
def tearDown(self):
self.serve_more_requests = False
#self.server.socket.close()
# self.server.socket.shutdown(socket.SHUT_RDWR)
# self.server.socket.close()
# self.temp_fs.close()
#self.server_thread.join()
#self.end_event.wait()
#return
try:
self.bump()
self.server.server_close()
except Exception:
pass
self.server_thread.join()
#self.server_thread.join()
self.temp_fs.close()
def bump(self):
......
fs/tests/test_path.py
......@@ -105,7 +105,6 @@ class TestPathFunctions(unittest.TestCase):
self.assertEquals(recursepath("hello",reverse=True),["/hello","/"])
self.assertEquals(recursepath("",reverse=True),["/"])
def test_isdotfile(self):
for path in ['.foo',
'.svn',
......@@ -125,7 +124,9 @@ class TestPathFunctions(unittest.TestCase):
tests = [('foo', ''),
('foo/bar', 'foo'),
('foo/bar/baz', 'foo/bar'),
('/', '')]
('/foo/bar', '/foo'),
('/foo', '/'),
('/', '/')]
for path, test_dirname in tests:
self.assertEqual(dirname(path), test_dirname)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment