Skip to content

P
pyfs
Commit 75fe116f by willmcgugan
Options

Mostly doc changes, and some pedantic pep-8 tweaks

parent 4ded92bc
Showing with 364 additions and 249 deletions
fs/__init__.py
......@@ -28,13 +28,13 @@ import path
_thread_synchronize_default = True
def set_thread_synchronize_default(sync):
"""Sets the default thread synctonisation flag.
"""Sets the default thread synchronisation flag.
FS objects are made thread-safe through the use of a per-FS threading Lock
object. Since this can introduce an small overhead it can be disabled with
this function if the code is single-threaded.
:param sync: Set wether to use thread synchronization for new FS objects
:param sync: Set whether to use thread synchronisation for new FS objects
"""
global _thread_synchronization_default
......
fs/base.py
......@@ -997,8 +997,7 @@ class SubFS(FS):
def flags_to_mode(flags):
"""Convert an os.O_* flag bitmask into an FS mode string."""
print flags
"""Convert an os.O_* flag bitmask into an FS mode string."""
if flags & os.O_EXCL:
raise UnsupportedError("open",msg="O_EXCL is not supported")
if flags & os.O_WRONLY:
......@@ -1016,6 +1015,5 @@ def flags_to_mode(flags):
else:
mode = "r+"
else:
mode = "r"
print mode
mode = "r"
return mode
fs/expose/django_storage.py
......@@ -8,6 +8,7 @@ Use an FS object for Django File Storage
from django.conf import settings
from django.core.files.storage import Storage
from django.core.files import File
from fs.path import abspath, dirname
from fs.errors import convert_fs_errors
......@@ -15,45 +16,49 @@ from fs.errors import convert_fs_errors
class FSStorage(Storage):
"""Expose an FS object as a Django File Storage object."""
def __init__(self,fs=None,base_url=None):
def __init__(self, fs=None, base_url=None):
"""
:param fs: an FS object
:param base_url: The url to prepend to the path
"""
if fs is None:
fs = settings.DEFAULT_FILE_STORAGE_FS
if base_url is None:
base_url = settings.MEDIA_URL
while base_url.endswith("/"):
base_url = base_url[:-1]
base_url = base_url.rstrip('/')
self.fs = fs
self.base_url = base_url
def exists(self,name):
def exists(self, name):
return self.fs.isfile(name)
def path(self,name):
def path(self, name):
path = self.fs.getsyspath(name)
if path is None:
raise NotImplementedError
return path
@convert_fs_errors
def size(self,name):
def size(self, name):
return self.fs.getsize(name)
@convert_fs_errors
def url(self,name):
def url(self, name):
return self.base_url + abspath(name)
@convert_fs_errors
def _open(self,name,mode):
return self.fs.open(name,mode)
def _open(self, name, mode):
return File(self.fs.open(name, mode))
@convert_fs_errors
def _save(self,name,content):
self.fs.makedir(dirname(name),allow_recreate=True,recursive=True)
self.fs.setcontents(name,content)
def _save(self, name, content):
self.fs.makedir(dirname(name), allow_recreate=True, recursive=True)
self.fs.setcontents(name, content)
return name
@convert_fs_errors
def delete(self,name):
def delete(self, name):
try:
self.fs.remove(name)
except ResourceNotFoundError:
......
fs/expose/sftp.py
......@@ -70,7 +70,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
This SFTPServerInterface subclass expects a single additional argument,
the fs object to be exposed. Use it to set up a transport subsystem
handler like so:
handler like so::
t.set_subsystem_handler("sftp",SFTPServer,SFTPServerInterface,fs)
......@@ -79,7 +79,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
paramiko server infrastructure.
"""
def __init__(self,server,fs,encoding=None,*args,**kwds):
def __init__(self, server, fs, encoding=None, *args, **kwds):
self.fs = fs
if encoding is None:
encoding = "utf8"
......@@ -87,12 +87,12 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
super(SFTPServerInterface,self).__init__(server,*args,**kwds)
@report_sftp_errors
def open(self,path,flags,attr):
return SFTPHandle(self,path,flags)
def open(self, path, flags, attr):
return SFTPHandle(self, path, flags)
@report_sftp_errors
def list_folder(self,path):
if not isinstance(path,unicode):
def list_folder(self, path):
if not isinstance(path, unicode):
path = path.decode(self.encoding)
stats = []
for entry in self.fs.listdir(path,absolute=True):
......@@ -100,8 +100,8 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
return stats
@report_sftp_errors
def stat(self,path):
if not isinstance(path,unicode):
def stat(self, path):
if not isinstance(path, unicode):
path = path.decode(self.encoding)
info = self.fs.getinfo(path)
stat = paramiko.SFTPAttributes()
......@@ -115,52 +115,52 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
stat.st_mode = 0777 | statinfo.S_IFREG
return stat
def lstat(self,path):
def lstat(self, path):
return self.stat(path)
@report_sftp_errors
def remove(self,path):
def remove(self, path):
if not isinstance(path,unicode):
path = path.decode(self.encoding)
self.fs.remove(path)
return paramiko.SFTP_OK
@report_sftp_errors
def rename(self,oldpath,newpath):
if not isinstance(oldpath,unicode):
def rename(self, oldpath, newpath):
if not isinstance(oldpath, unicode):
oldpath = oldpath.decode(self.encoding)
if not isinstance(newpath,unicode):
if not isinstance(newpath, unicode):
newpath = newpath.decode(self.encoding)
if self.fs.isfile(oldpath):
self.fs.move(oldpath,newpath)
self.fs.move(oldpath, newpath)
else:
self.fs.movedir(oldpath,newpath)
self.fs.movedir(oldpath, newpath)
return paramiko.SFTP_OK
@report_sftp_errors
def mkdir(self,path,attr):
def mkdir(self, path, attr):
if not isinstance(path,unicode):
path = path.decode(self.encoding)
self.fs.makedir(path)
return paramiko.SFTP_OK
@report_sftp_errors
def rmdir(self,path):
def rmdir(self, path):
if not isinstance(path,unicode):
path = path.decode(self.encoding)
self.fs.removedir(path)
return paramiko.SFTP_OK
def canonicalize(self,path):
def canonicalize(self, path):
return abspath(normpath(path))
def chattr(self,path,attr):
def chattr(self, path, attr):
return paramiko.SFTP_OP_UNSUPPORTED
def readlink(self,path):
def readlink(self, path):
return paramiko.SFTP_OP_UNSUPPORTED
def symlink(self,path):
def symlink(self, path):
return paramiko.SFTP_OP_UNSUPPORTED
......@@ -171,7 +171,7 @@ class SFTPHandle(paramiko.SFTPHandle):
and write requests directly through the to underlying file from the FS.
"""
def __init__(self,owner,path,flags):
def __init__(self, owner, path, flags):
super(SFTPHandle,self).__init__(flags)
mode = flags_to_mode(flags) + "b"
self.owner = owner
......@@ -186,12 +186,12 @@ class SFTPHandle(paramiko.SFTPHandle):
return paramiko.SFTP_OK
@report_sftp_errors
def read(self,offset,length):
def read(self, offset, length):
self._file.seek(offset)
return self._file.read(length)
@report_sftp_errors
def write(self,offset,data):
def write(self, offset, data):
self._file.seek(offset)
self._file.write(data)
return paramiko.SFTP_OK
......@@ -215,7 +215,7 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler):
def handle(self):
t = paramiko.Transport(self.request)
t.add_server_key(self.server.host_key)
t.set_subsystem_handler("sftp",paramiko.SFTPServer,SFTPServerInterface,self.server.fs,getattr(self.server,"encoding",None))
t.set_subsystem_handler("sftp", paramiko.SFTPServer, SFTPServerInterface, self.server.fs, getattr(self.server,"encoding",None))
# Note that this actually spawns a new thread to handle the requests.
# (Actually, paramiko.Transport is a subclass of Thread)
t.start_server(server=self.server)
......@@ -249,7 +249,7 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
"""
def __init__(self,address,fs=None,encoding=None,host_key=None,RequestHandlerClass=None):
def __init__(self, address, fs=None, encoding=None, host_key=None, RequestHandlerClass=None):
self.fs = fs
self.encoding = encoding
if host_key is None:
......@@ -259,25 +259,25 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
RequestHandlerClass = SFTPRequestHandler
sockserv.TCPServer.__init__(self,address,RequestHandlerClass)
def close_request(self,request):
def close_request(self, request):
# paramiko.Transport closes itself when finished.
# If we close it here, we'll break the Transport thread.
pass
def check_channel_request(self,kind,chanid):
def check_channel_request(self, kind, chanid):
if kind == 'session':
return paramiko.OPEN_SUCCEEDED
return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
def check_auth_none(self,username):
def check_auth_none(self, username):
"""Check whether the user can proceed without authentication."""
return paramiko.AUTH_SUCCESSFUL
def check_auth_publickey(self,username,key):
def check_auth_publickey(self, username,key):
"""Check whether the given public key is valid for authentication."""
return paramiko.AUTH_FAILED
def check_auth_password(self,username,password):
def check_auth_password(self, username, password):
"""Check whether the given password is valid for authentication."""
return paramiko.AUTH_FAILED
......
fs/expose/xmlrpc.py
......@@ -26,10 +26,10 @@ class RPCFSInterface(object):
the contents of files.
"""
def __init__(self,fs):
def __init__(self, fs):
self.fs = fs
def encode_path(self,path):
def encode_path(self, path):
"""Encode a filesystem path for sending over the wire.
Unfortunately XMLRPC only supports ASCII strings, so this method
......@@ -38,99 +38,99 @@ class RPCFSInterface(object):
"""
return path.encode("utf8").encode("base64")
def decode_path(self,path):
def decode_path(self, path):
"""Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8")
def get_contents(self,path):
def get_contents(self, path):
path = self.decode_path(path)
data = self.fs.getcontents(path)
return xmlrpclib.Binary(data)
def set_contents(self,path,data):
def set_contents(self, path, data):
path = self.decode_path(path)
self.fs.createfile(path,data.data)
def exists(self,path):
def exists(self, path):
path = self.decode_path(path)
return self.fs.exists(path)
def isdir(self,path):
def isdir(self, path):
path = self.decode_path(path)
return self.fs.isdir(path)
def isfile(self,path):
def isfile(self, path):
path = self.decode_path(path)
return self.fs.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = self.decode_path(path)
entries = self.fs.listdir(path,wildcard,full,absolute,dirs_only,files_only)
return [self.encode_path(e) for e in entries]
def makedir(self,path,recursive=False,allow_recreate=False):
def makedir(self, path, recursive=False, allow_recreate=False):
path = self.decode_path(path)
return self.fs.makedir(path,recursive,allow_recreate)
return self.fs.makedir(path, recursive, allow_recreate)
def remove(self,path):
def remove(self, path):
path = self.decode_path(path)
return self.fs.remove(path)
def removedir(self,path,recursive=False,force=False):
def removedir(self, path, recursive=False, force=False):
path = self.decode_path(path)
return self.fs.removedir(path,recursive,force)
return self.fs.removedir(path, recursive, force)
def rename(self,src,dst):
def rename(self, src, dst):
src = self.decode_path(src)
dst = self.decode_path(dst)
return self.fs.rename(src,dst)
return self.fs.rename(src, dst)
def getinfo(self,path):
def getinfo(self, path):
path = self.decode_path(path)
return self.fs.getinfo(path)
def desc(self,path):
def desc(self, path):
path = self.decode_path(path)
return self.fs.desc(path)
def getxattr(self,path,attr,default=None):
def getxattr(self, path, attr, default=None):
path = self.decode_path(path)
attr = self.decode_path(attr)
return self.fs.getxattr(path,attr,default)
return self.fs.getxattr(path, attr, default)
def setxattr(self,path,attr,value):
def setxattr(self, path, attr, value):
path = self.decode_path(path)
attr = self.decode_path(attr)
return self.fs.setxattr(path,attr,value)
return self.fs.setxattr(path, attr, value)
def delxattr(self,path,attr):
def delxattr(self, path, attr):
path = self.decode_path(path)
attr = self.decode_path(attr)
return self.fs.delxattr(path,attr)
return self.fs.delxattr(path, attr)
def listxattrs(self,path):
def listxattrs(self, path):
path = self.decode_path(path)
return [self.encode_path(a) for a in self.fs.listxattrs(path)]
def copy(self,src,dst,overwrite=False,chunk_size=16384):
def copy(self, src, dst, overwrite=False, chunk_size=16384):
src = self.decode_path(src)
dst = self.decode_path(dst)
return self.fs.copy(src,dst,overwrite,chunk_size)
return self.fs.copy(src, dst, overwrite, chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384):
src = self.decode_path(src)
dst = self.decode_path(dst)
return self.fs.move(src,dst,overwrite,chunk_size)
return self.fs.move(src, dst, overwrite, chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.decode_path(src)
dst = self.decode_path(dst)
return self.fs.movedir(src,dst,overwrite,ignore_errors,chunk_size)
return self.fs.movedir(src, dst, overwrite, ignore_errors, chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.decode_path(src)
dst = self.decode_path(dst)
return self.fs.copydir(src,dst,overwrite,ignore_errors,chunk_size)
return self.fs.copydir(src, dst, overwrite, ignore_errors, chunk_size)
class RPCFSServer(SimpleXMLRPCServer):
......@@ -148,7 +148,7 @@ class RPCFSServer(SimpleXMLRPCServer):
attribute "serve_more_requests" to False.
"""
def __init__(self,fs,addr,requestHandler=None,logRequests=None):
def __init__(self, fs, addr, requestHandler=None, logRequests=None):
kwds = dict(allow_none=True)
if requestHandler is not None:
kwds['requestHandler'] = requestHandler
......
fs/memoryfs.py
......@@ -305,8 +305,7 @@ class MemoryFS(FS):
if dir_item is None:
parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
return self
def _orphan_files(self, file_dir_entry):
for f in file_dir_entry.open_files:
......@@ -505,11 +504,11 @@ class MemoryFS(FS):
info['modified_time'] = dir_entry.modified_time
info['accessed_time'] = dir_entry.accessed_time
if dir_entry.isfile():
if dir_entry.isdir():
info['st_mode'] = 0755
else:
info['size'] = len(dir_entry.data or '')
info['st_mode'] = 0666
else:
info['st_mode'] = 0700
return info
......
fs/mountfs.py
......@@ -2,8 +2,44 @@
fs.mountfs
==========
Contains MountFS class which is a virtual Filesystem which can have other Filesystems linked as branched directories, much like a symlink in Linux
Contains MountFS class which is a virtual filesystem which can have other filesystems linked as branched directories.
For example, lets say we have two filesystems containing config files and resource respectively::
[config_fs]
|-- config.cfg
`-- defaults.cfg
[resources_fs]
|-- images
| |-- logo.jpg
| `-- photo.jpg
`-- data.dat
We can combine these filesystems in to a single filesystem with the following code::
from fs.mountfs import MountFS
combined_fs = MountFS
combined_fs.mountdir('config', config_fs)
combined_fs.mountdir('resources', resources_fs)
This will create a single filesystem where paths under `config` map to `config_fs`, and paths under `resources` map to `resources_fs`::
[combined_fs]
|-- config
| |-- config.cfg
| `-- defaults.cfg
`-- resources
|-- images
| |-- logo.jpg
| `-- photo.jpg
`-- data.dat
Now both filesystems can be accessed with the same path structure::
print combined_fs.getcontents('/config/defaults.cfg')
read_jpg(combined_fs.open('/resources/images/logo.jpg')
"""
from fs.base import *
......@@ -267,7 +303,7 @@ class MountFS(FS):
@synchronize
def mountdir(self, path, fs):
"""Mounts a host FS object on a given path.
:param path: A path within the MountFS
:param fs: A filesystem object to mount
......@@ -278,7 +314,13 @@ class MountFS(FS):
@synchronize
def mountfile(self, path, open_callable=None, info_callable=None):
"""Mounts a single file path. """
"""Mounts a single file path.
:param path: A path within the MountFS
:param open_Callable: A callable that returns a file-like object
:param info_callable: A callable that returns a dictionary with information regarding the file-like object
"""
path = normpath(path)
self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable)
......
fs/multifs.py
......@@ -9,18 +9,46 @@ FS in order, until it either finds a path that exists or raises a ResourceNotFou
One use for such a filesystem would be to selectively override a set of files,
to customize behaviour. For example, to create a filesystem that could be used
to *theme* a web application::
to *theme* a web application. We start with the following directories::
`-- templates
|-- snippets
| `-- panel.html
|-- index.html
|-- profile.html
`-- base.html
`-- theme
|-- snippets
| |-- widget.html
| `-- extra.html
|-- index.html
`-- theme.html
And we want to create a single filesystem that looks for files in `templates` if
they don't exist in `theme`. We can do this with the following code::
from fs.osfs import OSFS
from fs.multifs import MultiFS
themed_template_fs.addfs('templates', OSFS('templates'))
themed_template_fs.addfs('theme', OSFS('themes'))
Now we have a `themed_template_fs` FS object presents a single view of both
directories::
|-- snippets
| |-- panel.html
| |-- widget.html
| `-- extra.html
|-- index.html
|-- profile.html
|-- base.html
`-- theme.html
index_template = themed_template_fs.getcontent('index.html')
This will read the contents of *themes/index.html*, if it exists, otherwise
it will look for it in *templates/index.html*.
"""
......
fs/osfs/__init__.py
......@@ -31,7 +31,7 @@ def _os_stat(path):
return os.stat(path)
class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
"""Expose the underlying operating-system filesystem as an FS object.
This is the most basic of filesystems, which simply shadows the underlaying
......@@ -39,15 +39,15 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
methods in the os and os.path modules.
"""
def __init__(self, root_path, dir_mode=0700, thread_synchronize=_thread_synchronize_default, encoding=None, create=False):
def __init__(self, root_path, thread_synchronize=_thread_synchronize_default, encoding=None, create=False, dir_mode=0700):
"""
Creates an FS object that represents the OS Filesystem under a given root path
:param root_path: The root OS path
:param dir_mode: srt
:param root_path: The root OS path
:param thread_synchronize: If True, this object will be thread-safe by use of a threading.Lock object
:param encoding: The encoding method for path strings
:param create: Of True, then root_path will be created (if necessary)
:param create: If True, then root_path will be created if it doesn't already exist
:param dir_mode: The mode to use when creating the directory
"""
......@@ -90,11 +90,16 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
path = self._decode_path(path)
return path
def unsyspath(self,path):
def unsyspath(self, path):
"""Convert a system-level path into an FS-level path.
This basically the reverse of getsyspath(). If the path does not
refer to a location within this filesystem, ValueError is raised.
:param path: a system path
:returns: a path within this FS object
:rtype: string
"""
path = os.path.normpath(os.path.abspath(path))
if not path.startswith(self.root_path + os.path.sep):
......@@ -206,9 +211,8 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
if e.errno == errno.ENOENT:
if not os.path.exists(dirname(path_dst)):
raise ParentDirectoryMissingError(dst)
raise
raise
def _stat(self,path):
"""Stat the given path, normalising error codes."""
sys_path = self.getsyspath(path)
......
fs/path.py
......@@ -4,8 +4,8 @@ fs.path
Useful functions for FS path manipulation.
This is broadly similar to the standard 'os.path' module but works with
paths in the canonical format expected by all FS objects (backslash-separated,
This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (forwardslash-separated,
optional leading slash).
"""
......@@ -19,7 +19,8 @@ def normpath(path):
tries very hard to return a new path string the canonical FS format.
If the path is invalid, ValueError will be raised.
:param path: Path to normalize
:param path: path to normalize
:returns: a valid FS path
>>> normpath(r"foo\\bar\\baz")
'foo/bar/baz'
......@@ -50,7 +51,7 @@ def normpath(path):
if path[0] in "\\/":
if not components:
components = [""]
components.insert(0,"")
components.insert(0, "")
if isinstance(path, unicode):
return u"/".join(components)
else:
......@@ -73,7 +74,14 @@ def iteratepath(path, numsplits=None):
return map(None, path.split('/', numsplits))
def recursepath(path, reverse=False):
"""Iterate from root to path, returning intermediate paths"""
"""Returns intermediate paths from the root to the given path
:param reverse: reverses the order of the paths
>>> recursepath('a/b/c')
['/', u'/a', u'/a/b', u'/a/b/c']
"""
if reverse:
paths = []
path = abspath(path).rstrip("/")
......@@ -106,6 +114,9 @@ def relpath(path):
path if it is present.
:param path: Path to adjust
>>> relpath('/a/b')
'a/b'
"""
while path and path[0] == "/":
......@@ -147,9 +158,9 @@ join = pathjoin
def pathsplit(path):
"""Splits a path into (head,tail) pair.
"""Splits a path into (head, tail) pair.
This function splits a path into a pair (head,tail) where 'tail' is the
This function splits a path into a pair (head, tail) where 'tail' is the
last pathname component and 'head' is all preceeding components.
:param path: Path to split
......@@ -269,7 +280,7 @@ class PathMap(object):
A PathMap is like a dictionary where the keys are all FS paths. It allows
various dictionary operations (e.g. listing values, clearing values) to
be performed on a subset of the keys sharing some common prefix, e.g.:
be performed on a subset of the keys sharing some common prefix, e.g.::
# list all values in the map
pm.values()
......
fs/rpcfs.py
"""
fs.rpcfs: client to access an FS via XML-RPC
fs.rpcfs
========
This module provides the class 'RPCFS' to access a remote FS object over
XML-RPC. You probably want to use this in conjunction with the 'RPCFSServer'
class from the fs.expose.xmlrpc module.
class from the :mod:`fs.expose.xmlrpc` module.
"""
......@@ -90,7 +90,7 @@ class RPCFS(FS):
This class provides the client-side logic for accessing a remote FS
object, and is dual to the RPCFSServer class defined in fs.expose.xmlrpc.
Example:
Example::
fs = RPCFS("http://my.server.com/filesystem/location/")
......@@ -102,6 +102,9 @@ class RPCFS(FS):
The only required argument is the uri of the server to connect
to. This will be passed to the underlying XML-RPC server proxy
object, along with the 'transport' argument if it is provided.
:param uri: address of the server
"""
self.uri = uri
self._transport = transport
......@@ -127,12 +130,12 @@ class RPCFS(FS):
pass
return state
def __setstate__(self,state):
def __setstate__(self, state):
for (k,v) in state.iteritems():
self.__dict__[k] = v
self.proxy = self._make_proxy()
def encode_path(self,path):
def encode_path(self, path):
"""Encode a filesystem path for sending over the wire.
Unfortunately XMLRPC only supports ASCII strings, so this method
......@@ -141,11 +144,11 @@ class RPCFS(FS):
"""
return path.encode("utf8").encode("base64")
def decode_path(self,path):
def decode_path(self, path):
"""Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8")
def open(self,path,mode="r"):
def open(self, path, mode="r"):
# TODO: chunked transport of large files
path = self.encode_path(path)
if "w" in mode:
......@@ -178,83 +181,83 @@ class RPCFS(FS):
f.close = newclose
return f
def exists(self,path):
def exists(self, path):
path = self.encode_path(path)
return self.proxy.exists(path)
def isdir(self,path):
def isdir(self, path):
path = self.encode_path(path)
return self.proxy.isdir(path)
def isfile(self,path):
def isfile(self, path):
path = self.encode_path(path)
return self.proxy.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = self.encode_path(path)
entries = self.proxy.listdir(path,wildcard,full,absolute,dirs_only,files_only)
return [self.decode_path(e) for e in entries]
def makedir(self,path,recursive=False,allow_recreate=False):
def makedir(self, path, recursive=False, allow_recreate=False):
path = self.encode_path(path)
return self.proxy.makedir(path,recursive,allow_recreate)
def remove(self,path):
def remove(self, path):
path = self.encode_path(path)
return self.proxy.remove(path)
def removedir(self,path,recursive=False,force=False):
def removedir(self, path, recursive=False, force=False):
path = self.encode_path(path)
return self.proxy.removedir(path,recursive,force)
def rename(self,src,dst):
def rename(self, src, dst):
src = self.encode_path(src)
dst = self.encode_path(dst)
return self.proxy.rename(src,dst)
def getinfo(self,path):
def getinfo(self, path):
path = self.encode_path(path)
return self.proxy.getinfo(path)
def desc(self,path):
def desc(self, path):
path = self.encode_path(path)
return self.proxy.desc(path)
def getxattr(self,path,attr,default=None):
def getxattr(self, path, attr, default=None):
path = self.encode_path(path)
attr = self.encode_path(attr)
return self.fs.getxattr(path,attr,default)
def setxattr(self,path,attr,value):
def setxattr(self, path, attr, value):
path = self.encode_path(path)
attr = self.encode_path(attr)
return self.fs.setxattr(path,attr,value)
def delxattr(self,path,attr):
def delxattr(self, path, attr):
path = self.encode_path(path)
attr = self.encode_path(attr)
return self.fs.delxattr(path,attr)
def listxattrs(self,path):
def listxattrs(self, path):
path = self.encode_path(path)
return [self.decode_path(a) for a in self.fs.listxattrs(path)]
def copy(self,src,dst,overwrite=False,chunk_size=16384):
def copy(self, src, dst, overwrite=False, chunk_size=16384):
src = self.encode_path(src)
dst = self.encode_path(dst)
return self.proxy.copy(src,dst,overwrite,chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384):
def move(self, src, dst, overwrite=False, chunk_size=16384):
src = self.encode_path(src)
dst = self.encode_path(dst)
return self.proxy.move(src,dst,overwrite,chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.encode_path(src)
dst = self.encode_path(dst)
return self.proxy.movedir(src,dst,overwrite,ignore_errors,chunk_size)
return self.proxy.movedir(src, dst, overwrite, ignore_errors, chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.encode_path(src)
dst = self.encode_path(dst)
return self.proxy.copydir(src,dst,overwrite,ignore_errors,chunk_size)
......
fs/sftpfs.py
......@@ -42,7 +42,7 @@ class SFTPFS(FS):
class in the paramiko module.
"""
def __init__(self,connection,root_path="/",encoding=None,**credentials):
def __init__(self, connection, root_path="/", encoding=None, **credentials):
"""SFTPFS constructor.
The only required argument is 'connection', which must be something
......@@ -58,6 +58,10 @@ class SFTPFS(FS):
machine - access to files outsite this root wil be prevented. Any
other keyword arguments are assumed to be credentials to be used when
connecting the transport.
:param connection: a connection string
:param root_path: The root path to open
"""
if encoding is None:
encoding = "utf8"
......
fs/utils.py
......@@ -181,7 +181,7 @@ def find_duplicates(fs,
other attributes not take in to account).
:param fs: A filesystem object
:param compare_paths: An iterable of paths within the FS object, or all files if omited
:param compare_paths: An iterable of paths within the FS object, or all files if omitted
:param quick: If set to True, the quick method of finding duplicates will be used, which can potentially return false positives if the files have the same size and start with the same data. Do not use when deleting files!
:param signature_chunk_size: The number of bytes to read before generating a signature checksum value
:param signature_size: The total number of bytes read to generate a signature
......@@ -281,7 +281,7 @@ def find_duplicates(fs,
paths = list(set(paths).difference(dups))
def print_fs(fs, path='/', max_levels=5, file_out=sys.stdout, terminal_colors=None):
def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
"""Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid.
Be careful about printing a OSFS, or any other large filesystem.
Without max_levels set, this function will traverse the entire directory tree.
......@@ -303,6 +303,9 @@ def print_fs(fs, path='/', max_levels=5, file_out=sys.stdout, terminal_colors=No
"""
if file_out is None:
file_out = sys.stdout
if terminal_colors is None:
if sys.platform == 'win32':
terminal_colors = False
......
fs/wrapfs/__init__.py
......@@ -47,20 +47,20 @@ class WrapFS(FS):
The following methods can be overridden to control how files are
accessed in the underlying FS object:
_file_wrap(file,mode): called for each file that is opened from
* _file_wrap(file, mode): called for each file that is opened from
the underlying FS; may return a modified
file-like object.
_encode(path): encode a path for access in the underlying FS
* _encode(path): encode a path for access in the underlying FS
_decode(path): decode a path from the underlying FS
* _decode(path): decode a path from the underlying FS
If the required path translation proceeds one component at a time,
it may be simpler to override the _encode_name() and _decode_name()
methods.
"""
def __init__(self,fs):
def __init__(self, fs):
super(WrapFS,self).__init__()
try:
self._lock = fs._lock
......@@ -68,19 +68,19 @@ class WrapFS(FS):
self._lock = None
self.wrapped_fs = fs
def _file_wrap(self,f,mode):
def _file_wrap(self, f, mode):
"""Apply wrapping to an opened file."""
return f
def _encode_name(self,name):
def _encode_name(self, name):
"""Encode path component for the underlying FS."""
return name
def _decode_name(self,name):
def _decode_name(self, name):
"""Decode path component from the underlying FS."""
return name
def _encode(self,path):
def _encode(self, path):
"""Encode path for the underlying FS."""
names = path.split("/")
e_names = []
......@@ -91,7 +91,7 @@ class WrapFS(FS):
e_names.append(self._encode_name(name))
return "/".join(e_names)
def _decode(self,path):
def _decode(self, path):
"""Decode path from the underlying FS."""
names = path.split("/")
d_names = []
......@@ -102,7 +102,7 @@ class WrapFS(FS):
d_names.append(self._decode_name(name))
return "/".join(d_names)
def _adjust_mode(self,mode):
def _adjust_mode(self, mode):
"""Adjust the mode used to open a file in the underlying FS.
This method takes the mode given when opening a file, and should
......@@ -116,11 +116,11 @@ class WrapFS(FS):
return (mode,mode)
@rewrite_errors
def getsyspath(self,path,allow_none=False):
def getsyspath(self, path, allow_none=False):
return self.wrapped_fs.getsyspath(self._encode(path),allow_none)
@rewrite_errors
def hassyspath(self,path):
def hassyspath(self, path):
return self.wrapped_fs.hassyspath(self._encode(path))
@rewrite_errors
......@@ -130,19 +130,19 @@ class WrapFS(FS):
return self._file_wrap(f, mode)
@rewrite_errors
def exists(self,path):
def exists(self, path):
return self.wrapped_fs.exists(self._encode(path))
@rewrite_errors
def isdir(self,path):
def isdir(self, path):
return self.wrapped_fs.isdir(self._encode(path))
@rewrite_errors
def isfile(self,path):
def isfile(self, path):
return self.wrapped_fs.isfile(self._encode(path))
@rewrite_errors
def listdir(self,path="",**kwds):
def listdir(self, path="", **kwds):
wildcard = kwds.pop("wildcard","*")
info = kwds.get("info",False)
entries = []
......@@ -160,74 +160,74 @@ class WrapFS(FS):
return entries
@rewrite_errors
def makedir(self,path,*args,**kwds):
def makedir(self, path, *args, **kwds):
return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)
@rewrite_errors
def remove(self,path):
def remove(self, path):
return self.wrapped_fs.remove(self._encode(path))
@rewrite_errors
def removedir(self,path,*args,**kwds):
def removedir(self, path, *args, **kwds):
return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)
@rewrite_errors
def rename(self,src,dst):
def rename(self, src, dst):
return self.wrapped_fs.rename(self._encode(src),self._encode(dst))
@rewrite_errors
def getinfo(self,path):
def getinfo(self, path):
return self.wrapped_fs.getinfo(self._encode(path))
@rewrite_errors
def desc(self,path):
def desc(self, path):
return self.wrapped_fs.desc(self._encode(path))
@rewrite_errors
def copy(self,src,dst,**kwds):
def copy(self, src, dst, **kwds):
return self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors
def move(self,src,dst,**kwds):
def move(self, src, dst, **kwds):
return self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors
def movedir(self,src,dst,**kwds):
def movedir(self, src, dst, **kwds):
return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors
def copydir(self,src,dst,**kwds):
def copydir(self, src, dst, **kwds):
return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors
def getxattr(self,path,name,default=None):
def getxattr(self, path, name, default=None):
try:
return self.wrapped_fs.getxattr(self._encode(path),name,default)
except AttributeError:
raise UnsupportedError("getxattr")
@rewrite_errors
def setxattr(self,path,name,value):
def setxattr(self, path, name, value):
try:
return self.wrapped_fs.setxattr(self._encode(path),name,value)
except AttributeError:
raise UnsupportedError("setxattr")
@rewrite_errors
def delxattr(self,path,name):
def delxattr(self, path, name):
try:
return self.wrapped_fs.delxattr(self._encode(path),name)
except AttributeError:
raise UnsupportedError("delxattr")
@rewrite_errors
def listxattrs(self,path):
def listxattrs(self, path):
try:
return self.wrapped_fs.listxattrs(self._encode(path))
except AttributeError:
raise UnsupportedError("listxattrs")
def __getattr__(self,attr):
def __getattr__(self, attr):
# These attributes can be used by the destructor, but may not be
# defined if there are errors in the constructor.
if attr == "closed":
......@@ -243,18 +243,18 @@ class WrapFS(FS):
super(WrapFS,self).close()
def wrap_fs_methods(decorator,cls=None,exclude=[]):
def wrap_fs_methods(decorator, cls=None, exclude=[]):
"""Apply the given decorator to all FS methods on the given class.
This function can be used in two ways. When called with two arguments it
applies the given function 'decorator' to each FS method of the given
class. When called with just a single argument, it creates and returns
a class decorator which will do the same thing when applied. So you can
use it like this:
use it like this::
wrap_fs_methods(mydecorator,MyFSClass)
Or on more recent Python versions, like this:
Or on more recent Python versions, like this::
@wrap_fs_methods(mydecorator)
class MyFSClass(FS):
......
fs/wrapfs/hidedotfilesfs.py
"""
fs.wrapfs.hidedotfiles
======================
fs.wrapfs.hidedotfilesfs
========================
An FS wrapper class for hiding dot-files in directory listings.
......@@ -17,17 +17,17 @@ class HideDotFilesFS(WrapFS):
It is False by default.
"""
def is_hidden(self,path):
def is_hidden(self, path):
"""Check whether the given path should be hidden."""
return path and basename(path)[0] == "."
def _encode(self,path):
def _encode(self, path):
return path
def _decode(self,path):
def _decode(self, path):
return path
def listdir(self,path="",**kwds):
def listdir(self, path="", **kwds):
hidden = kwds.pop("hidden",True)
entries = self.wrapped_fs.listdir(path,**kwds)
if not hidden:
......
fs/wrapfs/lazyfs.py
......@@ -26,7 +26,7 @@ class LazyFS(WrapFS):
the first time it is accessed.
"""
def __init__(self,fs):
def __init__(self, fs):
super(LazyFS,self).__init__(fs)
self._lazy_creation_lock = Lock()
......@@ -35,7 +35,7 @@ class LazyFS(WrapFS):
del state["_lazy_creation_lock"]
return state
def __setstate__(self,state):
def __setstate__(self, state):
self.__dict__.update(state)
self._lazy_creation_lock = Lock()
......@@ -55,7 +55,7 @@ class LazyFS(WrapFS):
finally:
self._lazy_creation_lock.release()
def _set_wrapped_fs(self,fs):
def _set_wrapped_fs(self, fs):
if isinstance(fs,FS):
self.__dict__["wrapped_fs"] = fs
elif isinstance(fs,type):
......@@ -75,7 +75,7 @@ class LazyFS(WrapFS):
wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs)
def setcontents(self,path,data):
def setcontents(self, path, data):
return self.wrapped_fs.setcontents(path,data)
def close(self):
......
fs/wrapfs/limitsizefs.py
......@@ -18,14 +18,14 @@ from fs.wrapfs import WrapFS
class LimitSizeFS(WrapFS):
"""FS wrapper class to limit total size of files stored."""
def __init__(self,fs,max_size):
def __init__(self, fs, max_size):
super(LimitSizeFS,self).__init__(fs)
self.max_size = max_size
self.cur_size = sum(self.getsize(f) for f in self.walkfiles())
self._size_lock = threading.Lock()
self._file_sizes = {}
def _decr_size(self,decr):
def _decr_size(self, decr):
with self._size_lock:
self.cur_size -= decr
......@@ -35,16 +35,16 @@ class LimitSizeFS(WrapFS):
del state["_file_sizes"]
return state
def __setstate__(self,state):
def __setstate__(self, state):
super(LimitSizeFS,self).__setstate__(state)
self._size_lock = threading.Lock()
def getsyspath(self,path,allow_none=False):
def getsyspath(self, path, allow_none=False):
if not allow_none:
raise NoSysPathError(path)
return None
def open(self,path,mode="r"):
def open(self, path, mode="r"):
path = relpath(normpath(path))
with self._size_lock:
try:
......@@ -60,7 +60,7 @@ class LimitSizeFS(WrapFS):
self._file_sizes[path] = 0
return LimitSizeFile(self,path,f,mode,size)
def _ensure_file_size(self,path,size):
def _ensure_file_size(self, path, size):
path = relpath(normpath(path))
with self._size_lock:
if path not in self._file_sizes:
......@@ -73,22 +73,22 @@ class LimitSizeFS(WrapFS):
self.cur_size += diff
self._file_sizes[path] = size
def copy(self,src,dst,**kwds):
def copy(self, src, dst, **kwds):
FS.copy(self,src,dst,**kwds)
def copydir(self,src,dst,**kwds):
def copydir(self, src, dst, **kwds):
FS.copydir(self,src,dst,**kwds)
def move(self,src,dst,**kwds):
def move(self, src, dst, **kwds):
FS.move(self,src,dst,**kwds)
path = relpath(normpath(src))
with self._size_lock:
self._file_sizes.pop(path,None)
def movedir(self,src,dst,**kwds):
def movedir(self, src, dst, **kwds):
FS.movedir(self,src,dst,**kwds)
def remove(self,path):
def remove(self, path):
size = self.getsize(path)
super(LimitSizeFS,self).remove(path)
self._decr_size(size)
......@@ -96,12 +96,12 @@ class LimitSizeFS(WrapFS):
with self._size_lock:
self._file_sizes.pop(path,None)
def removedir(self,path,recursive=False,force=False):
def removedir(self, path, recursive=False, force=False):
size = sum(self.getsize(f) for f in self.walkfiles(path))
super(LimitSizeFS,self).removedir(path,recursive=recursive,force=force)
self._decr_size(size)
def rename(self,src,dst):
def rename(self, src, dst):
try:
size = self.getsize(dst)
except ResourceNotFoundError:
......@@ -116,7 +116,7 @@ class LimitSizeFS(WrapFS):
class LimitSizeFile(object):
"""Filelike wrapper class for use by LimitSizeFS."""
def __init__(self,fs,path,file,mode,size):
def __init__(self, fs, path, file, mode, size):
self._lock = fs._lock
self.fs = fs
self.path = path
......@@ -126,17 +126,17 @@ class LimitSizeFile(object):
self.closed = False
@synchronize
def write(self,data):
def write(self, data):
pos = self.file.tell()
self.size = self.fs._ensure_file_size(self.path,pos+len(data))
self.file.write(data)
def writelines(self,lines):
def writelines(self, lines):
for line in lines:
self.write(line)
@synchronize
def truncate(self,size=None):
def truncate(self, size=None):
pos = self.file.tell()
if size is None:
size = pos
......@@ -145,7 +145,7 @@ class LimitSizeFile(object):
self.size = size
# This is lifted straight from the stdlib's tempfile.py
def __getattr__(self,name):
def __getattr__(self, name):
file = self.__dict__['file']
a = getattr(file, name)
if not issubclass(type(a), type(0)):
......@@ -156,7 +156,7 @@ class LimitSizeFile(object):
self.file.__enter__()
return self
def __exit__(self,exc,value,tb):
def __exit__(self, exc, value, tb):
self.close()
return False
......
fs/wrapfs/readonlyfs.py
......@@ -15,7 +15,7 @@ class ReadOnlyFS(WrapFS):
Note that this isn't a secure sandbox, untrusted code could work around the
read-only restrictions by getting the base class. Its main purpose is to
provide a degree of safety if you want to protect an FS object from
modification.
accidental modification.
"""
......
fs/xattrs.py
......@@ -128,6 +128,7 @@ class SimulateXAttr(WrapFS):
pass
self._set_attr_dict(path, attrs)
@synchronize
def listxattrs(self,path):
"""List all the extended attribute keys set on the given path."""
if not self.exists(path):
......
fs/zipfs.py
......@@ -61,7 +61,7 @@ class ZipFS(FS):
:param zip_file: A (system) path, or a file-like object
:param mode: Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
:param compression: Can be 'deflated' (default) to compress data or 'stored' to just store date
:param allow_zip_64: -- Set to True to use zip files greater than 2 MB, default is False
:param allow_zip_64: -- Set to True to use zip files greater than 2 GB, default is False
:param encoding: -- The encoding to use for unicode filenames
:param thread_synchronize: -- Set to True (default) to enable thread-safety
......@@ -203,6 +203,9 @@ class ZipFS(FS):
try:
zi = self.zf.getinfo(path.encode(self.encoding))
zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
for k, v in zinfo.iteritems():
if callable(v):
zinfo[k] = v()
except KeyError:
zinfo = {'file_size':0}
info = {'size' : zinfo['file_size'] }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment