Skip to content

P
pyfs
Commit f68f9863 by rfkelly0
Options

merge branch "rfk-ideas" into trunk

parent 29a1ca1c
Showing with 2027 additions and 877 deletions
AUTHORS 0 → 100644
Will McGugan (will@willmcgugan.com)
Ryan Kelly (ryan@rfk.id.au)
MANIFEST.in 0 → 100644
include AUTHORS
NOTES.txt 0 → 100644
Rename 'helpers' module to 'path' since it's basically an imitation of os.path.
Minify the contained functions:
- make normpath() do more e.g. collapse backrefs
- remove resolvepath() as it just confuses the issue
- resourcename() -> basename() in line with os.path notation
- remove isabsolutepath(), it wasn't used
Put error class definitions in separate submodule 'errors'
- less redundancy (e.g. no more `raise UnsupportedError("UNSUPPORTED")`)
- deeper exception heirachy (e.g. `ParentDirectoryMissingError`)
This is designed to allow me to be more optimistic; rather than checking all
preconditions before trying an action, I can just let it fail and branch on
the exception. Important for reducing the number of accesses to a remote FS.
Remove the notion of hidden files from the base FS class.
- has lead to several bugs with copying/moving directories
- it's not the filesystem's business to decide what files I want to see
- moved the logic into a separate wrapper class "HideDotFiles"
Remove xattr support from base FS class, making it a separate interface.
- has lead to several bugs with copying/moving files and directories
- now defined in fs.xattrs module
- SimulateXAttr wrapper class contains the same logic
- removexattr() -> delxattr() in line with python's get/set/del tradition
Operational changes to the base methods:
- don't require makedir() to support the "mode" argument, since this only
makes sense for OS-level files.
- when copy() is given an existing directory as destination, raise an error
rather than copying into the directory (explicit is better than implicit)
Split up the test definitions a little:
- use a separate FSTestCases mixin rather than subclassing TestOSFS
- path helpers testcases in their own module
- zipfs in its own module since it's different to all the others
- s3fs in its own module since it's very slow and costs money to test
fs/__init__.py
""" """
A filesystem abstraction.
""" fs: a filesystem abstraction.
This module provides an abstract base class 'FS' that defines a consistent
interface to different kinds of filesystem, along with a range of concrete
implementations of this interface such as:
__version__ = "0.1.1dev" OSFS: access the local filesystem, through the 'os' module
TempFS: a temporary filesystem that's automatically cleared on exit
MemoryFS: a filesystem that exists only in memory
ZipFS: access a zipfile like a filesystem
S3FS: access files stored in Amazon S3
"""
__version__ = "0.2.0"
__author__ = "Will McGugan (will@willmcgugan.com)" __author__ = "Will McGugan (will@willmcgugan.com)"
# 'base' imports * from 'path' and 'errors', so their contents
# will be available here as well.
from base import * from base import *
from helpers import *
__all__ = ['memoryfs', # provide these by default so people cna be 'fs.path.basename' etc.
'mountfs', import errors
'multifs', import path
'osfs',
'utils',
'zipfs',
'helpers',
'tempfs']
\ No newline at end of file
fs/browsewin.py
...@@ -151,15 +151,16 @@ class BrowseFrame(wx.Frame): ...@@ -151,15 +151,16 @@ class BrowseFrame(wx.Frame):
info_frame = InfoFrame(path, self.fs.desc(path), info) info_frame = InfoFrame(path, self.fs.desc(path), info)
info_frame.Show() info_frame.Show()
def browse(fs):
def browse(fs):
app = wx.PySimpleApp() app = wx.PySimpleApp()
frame = BrowseFrame(fs) frame = BrowseFrame(fs)
frame.Show() frame.Show()
app.MainLoop() app.MainLoop()
if __name__ == "__main__":
if __name__ == "__main__":
from osfs import OSFS from osfs import OSFS
home_fs = OSFS("~/") home_fs = OSFS("~/")
browse(home_fs) browse(home_fs)
fs/errors.py 0 → 100644
"""
fs.errors: error class definitions for FS
"""
import sys
import errno
try:
from functools import wraps
except ImportError:
def wraps(func):
def decorator(wfunc):
wfunc.__name__ == func.__name__
wfunc.__doc__ == func.__doc__
wfunc.__module__ == func.__module__
return decorator
class FSError(Exception):
"""Base exception class for the FS module."""
default_message = "Unspecified error"
def __init__(self,msg=None,details=None):
if msg is None:
msg = self.default_message
self.msg = msg
self.details = details
def __str__(self):
keys = dict((k,str(v)) for k,v in self.__dict__.iteritems())
return self.msg % keys
def __unicode__(self):
return unicode(str(self))
class PathError(FSError):
"""Exception for errors to do with a path string."""
default_message = "Path is invalid: %(path)s"
def __init__(self,path,**kwds):
self.path = path
super(PathError,self).__init__(**kwds)
class OperationFailedError(FSError):
"""Base exception class for errors associated with a specific operation."""
default_message = "Unable to %(opname)s: unspecified error [%(errno)s - %(details)s]"
def __init__(self,opname,path=None,**kwds):
self.opname = opname
self.path = path
self.errno = getattr(kwds.get("details",None),"errno",None)
super(OperationFailedError,self).__init__(**kwds)
class UnsupportedError(OperationFailedError):
"""Exception raised for operations that are not supported by the FS."""
default_message = "Unable to %(opname)s: not supported by this filesystem"
class RemoteConnectionError(OperationFailedError):
"""Exception raised when operations encounter remote connection trouble."""
default_message = "Unable to %(opname)s: remote connection errror"
class StorageSpaceError(OperationFailedError):
"""Exception raised when operations encounter storage space trouble."""
default_message = "Unable to %(opname)s: insufficient storage space"
class PermissionDeniedError(OperationFailedError):
default_message = "Unable to %(opname)s: permission denied"
class ResourceError(FSError):
"""Base exception class for error associated with a specific resource."""
default_message = "Unspecified resource error: %(path)s"
def __init__(self,path,**kwds):
self.path = path
self.opname = kwds.pop("opname",None)
super(ResourceError,self).__init__(**kwds)
class NoSysPathError(ResourceError):
"""Exception raised when there is no syspath for a given path."""
default_message = "No mapping to OS filesystem: %(path)s"
class ResourceNotFoundError(ResourceError):
"""Exception raised when a required resource is not found."""
default_message = "Resource not found: %(path)s"
class ResourceInvalidError(ResourceError):
"""Exception raised when a resource is the wrong type."""
default_message = "Resource is invalid: %(path)s"
class DestinationExistsError(ResourceError):
"""Exception raised when a target destination already exists."""
default_message = "Destination exists: %(path)s"
class DirectoryNotEmptyError(ResourceError):
"""Exception raised when a directory to be removed is not empty."""
default_message = "Directory is not empty: %(path)s"
class ParentDirectoryMissingError(ResourceError):
"""Exception raised when a parent directory is missing."""
default_message = "Parent directory is missing: %(path)s"
class ResourceLockedError(ResourceError):
"""Exception raised when a resource can't be used because it is locked."""
default_message = "Resource is locked: %(path)s"
def convert_fs_errors(func):
"""Function wrapper to convert FSError instances into OSErrors."""
@wraps(func)
def wrapper(*args,**kwds):
try:
return func(*args,**kwds)
except ResourceNotFoundError, e:
raise OSError(errno.ENOENT,str(e))
except ResourceInvalidError, e:
raise OSError(errno.EINVAL,str(e))
except PermissionDeniedError, e:
raise OSError(errno.EACCESS,str(e))
except DirectoryNotEmptyError, e:
raise OSError(errno.ENOTEMPTY,str(e))
except DestinationExistsError, e:
raise OSError(errno.EEXIST,str(e))
except StorageSpaceError, e:
raise OSError(errno.ENOSPC,str(e))
except RemoteConnectionError, e:
raise OSError(errno.ENONET,str(e))
except UnsupportedError, e:
raise OSError(errno.ENOSYS,str(e))
except FSError, e:
raise OSError(errno.EFAULT,str(e))
return wrapper
def convert_os_errors(func):
"""Function wrapper to convert OSError/IOError instances into FSErrors."""
opname = func.__name__
@wraps(func)
def wrapper(*args,**kwds):
try:
return func(*args,**kwds)
except (OSError,IOError), e:
if not hasattr(e,"errno") or not e.errno:
raise OperationFailedError(opname,details=e)
if e.errno == errno.ENOENT:
raise ResourceNotFoundError(e.filename,opname=opname,details=e)
if e.errno == errno.ENOTEMPTY:
raise DirectoryNotEmptyError(e.filename,opname=opname,details=e)
if e.errno == errno.EEXIST:
raise DestinationExistsError(e.filename,opname=opname,details=e)
if e.errno == 183: # some sort of win32 equivalent to EEXIST
raise DestinationExistsError(e.filename,opname=opname,details=e)
if e.errno == errno.ENOTDIR:
raise ResourceInvalidError(e.filename,opname=opname,details=e)
if e.errno == errno.EISDIR:
raise ResourceInvalidError(e.filename,opname=opname,details=e)
if e.errno == errno.EINVAL:
raise ResourceInvalidError(e.filename,opname=opname,details=e)
raise OperationFailedError(opname,details=e)
return wrapper
fs/expose/xmlrpc.py 0 → 100644
"""
fs.expose.xmlrpc: server to expose an FS via XML-RPC
This module provides the necessary infrastructure to expose an FS object
over XML-RPC. The main class is 'RPCFSServer', a SimpleXMLRPCServer subclass
designed to expose an underlying FS.
If you need to use a more powerful server than SimpleXMLRPCServer, you can
use the RPCFSInterface class to provide an XML-RPC-compatible wrapper around
an FS object, which can then be exposed using whatever server you choose
(e.g. Twisted's XML-RPC server).
"""
import xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer
class RPCFSInterface(object):
"""Wrapper to expose an FS via a XML-RPC compatible interface.
The only real trick is using xmlrpclib.Binary objects to transport
the contents of files.
"""
def __init__(self,fs):
self.fs = fs
def get_contents(self,path):
data = self.fs.getcontents(path)
return xmlrpclib.Binary(data)
def set_contents(self,path,data):
self.fs.createfile(path,data.data)
def exists(self,path):
return self.fs.exists(path)
def isdir(self,path):
return self.fs.isdir(path)
def isfile(self,path):
return self.fs.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
return list(self.fs.listdir(path,wildcard,full,absolute,dirs_only,files_only))
def makedir(self,path,recursive=False,allow_recreate=False):
return self.fs.makedir(path,recursive,allow_recreate)
def remove(self,path):
return self.fs.remove(path)
def removedir(self,path,recursive=False,force=False):
return self.fs.removedir(path,recursive,force)
def rename(self,src,dst):
return self.fs.rename(src,dst)
def getinfo(self,path):
return self.fs.getinfo(path)
def desc(self,path):
return self.fs.desc(path)
def getattr(self,path,attr):
return self.fs.getattr(path,attr)
def setattr(self,path,attr,value):
return self.fs.setattr(path,attr,value)
def copy(self,src,dst,overwrite=False,chunk_size=16384):
return self.fs.copy(src,dst,overwrite,chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384):
return self.fs.move(src,dst,overwrite,chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.fs.movedir(src,dst,overwrite,ignore_errors,chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.fs.copydir(src,dst,overwrite,ignore_errors,chunk_size)
class RPCFSServer(SimpleXMLRPCServer):
"""Server to expose an FS object via XML-RPC.
This class takes as its first argument an FS instance, and as its second
argument a (hostname,port) tuple on which to listen for XML-RPC requests.
Example:
fs = OSFS('/var/srv/myfiles')
s = RPCFSServer(fs,("",8080))
s.serve_forever()
To cleanly shut down the server after calling serve_forever, set the
attribute "serve_more_requests" to False.
"""
def __init__(self,fs,addr,requestHandler=None,logRequests=None):
kwds = dict(allow_none=True)
if requestHandler is not None:
kwds['requestHandler'] = requestHandler
if logRequests is not None:
kwds['logRequests'] = logRequests
self.serve_more_requests = True
SimpleXMLRPCServer.__init__(self,addr,**kwds)
self.register_instance(RPCFSInterface(fs))
def serve_forever(self):
"""Override serve_forever to allow graceful shutdown."""
while self.serve_more_requests:
self.handle_request()
fs/fuseserver.py deleted 100755 → 0
#!/usr/bin/env python
import base
import fuse
fuse.fuse_python_api = (0, 2)
from datetime import datetime
import time
from os import errno
import sys
from stat import *
def showtb(f):
def run(*args, **kwargs):
print
print "-"*80
print f, args, kwargs
try:
ret = f(*args, **kwargs)
print "\tReturned:", repr(ret)
return ret
except Exception, e:
print e
raise
print "-"*80
print
return run
"""
::*'''<code>open(path, flags)</code>'''
::*'''<code>create(path, flags, mode)</code>'''
::*'''<code>read(path, length, offset, fh=None)</code>'''
::*'''<code>write(path, buf, offset, fh=None)</code>'''
::*'''<code>fgetattr(path, fh=None)</code>'''
::*'''<code>ftruncate(path, len, fh=None)</code>'''
::*'''<code>flush(path, fh=None)</code>'''
::*'''<code>release(path, fh=None)</code>'''
::*'''<code>fsync(path, fdatasync, fh=None)</code>'''
"""
class FuseFile(object):
def __init__(self, f):
self.f = f
_run_t = time.time()
class FSFUSE(fuse.Fuse):
def __init__(self, fs, *args, **kwargs):
fuse.Fuse.__init__(self, *args, **kwargs)
self._fs = fs
@showtb
def fsinit(self):
return 0
def __getattr__(self, name):
print name
raise AttributeError
#@showtb
def getattr(self, path):
if not self._fs.exists(path):
return -errno.ENOENT
class Stat(fuse.Stat):
def __init__(self, context, fs, path):
fuse.Stat.__init__(self)
info = fs.getinfo(path)
isdir = fs.isdir(path)
fsize = fs.getsize(path) or 1024
self.st_ino = 0
self.st_dev = 0
self.st_nlink = 2 if isdir else 1
self.st_blksize = fsize
self.st_mode = info.get('st_mode', S_IFDIR | 0755 if isdir else S_IFREG | 0666)
print self.st_mode
self.st_uid = context['uid']
self.st_gid = context['gid']
self.st_rdev = 0
self.st_size = fsize
self.st_blocks = 1
for key, value in info.iteritems():
if not key.startswith('_'):
setattr(self, key, value)
def do_time(attr, key):
if not hasattr(self, attr):
if key in info:
info_t = info[key]
setattr(self, attr, time.mktime(info_t.timetuple()))
else:
setattr(self, attr, _run_t)
do_time('st_atime', 'accessed_time')
do_time('st_mtime', 'modified_time')
do_time('st_ctime', 'created_time')
#for v in dir(self):
# if not v.startswith('_'):
# print v, getattr(self, v)
return Stat(self.GetContext(), self._fs, path)
@showtb
def chmod(self, path, mode):
return 0
@showtb
def chown(self, path, user, group):
return 0
@showtb
def utime(self, path, times):
return 0
@showtb
def utimens(self, path, times):
return 0
@showtb
def fsyncdir(self):
pass
@showtb
def bmap(self):
return 0
@showtb
def ftruncate(self, path, flags, fh):
if fh is not None:
fh.truncate()
fh.flush()
return 0
def fsdestroy(self):
return 0
@showtb
def statfs(self):
return (0, 0, 0, 0, 0, 0, 0)
#def setattr
#
#
#@showtb
#def getdir(self, path, offset):
# paths = ['.', '..']
# paths += self._fs.listdir(path)
# print repr(paths)
#
# for p in paths:
# yield fuse.Direntry(p)
@showtb
def opendir(self, path):
return 0
@showtb
def getxattr(self, path, name, default):
return self._fs.getattr(path, name, default)
@showtb
def setxattr(self, path, name, value):
self._fs.setattr(path, name)
return 0
@showtb
def removeattr(self, path, name):
self._fs.removeattr(path, name)
return 0
@showtb
def listxattr(self, path, something):
return self._fs.listattrs(path)
@showtb
def open(self, path, flags):
return self._fs.open(path, flags=flags)
@showtb
def create(self, path, flags, mode):
return self._fs.open(path, "w")
@showtb
def read(self, path, length, offset, fh=None):
if fh:
fh.seek(offset)
return fh.read(length)
@showtb
def write(self, path, buf, offset, fh=None):
if fh:
fh.seek(offset)
# FUSE seems to expect a return value of the number of bytes written,
# but Python file objects don't return that information,
# so we will assume all bytes are written...
bytes_written = fh.write(buf) or len(buf)
return bytes_written
@showtb
def release(self, path, flags, fh=None):
if fh:
fh.close()
return 0
@showtb
def flush(self, path, fh=None):
if fh:
try:
fh.flush()
except base.FSError:
return 0
return 0
@showtb
def access(self, path, *args, **kwargs):
return 0
#@showtb
def readdir(self, path, offset):
paths = ['.', '..']
paths += self._fs.listdir(path)
return [fuse.Direntry(p) for p in paths]
#@showtb
#def fgetattr(self, path, fh=None):
# fh.flush()
# return self.getattr(path)
@showtb
def readlink(self, path):
return path
@showtb
def symlink(self, path, path1):
return 0
@showtb
def mknod(self, path, mode, rdev):
f = None
try:
f = self._fs.open(path, mode)
finally:
f.close()
return 0
@showtb
def mkdir(self, path, mode):
self._fs.mkdir(path, mode)
return 0
@showtb
def rmdir(self, path):
self._fs.removedir(path, True)
return 0
@showtb
def unlink(self, path):
try:
self._fs.remove(path)
except base.FSError:
return 0
return 0
#symlink(target, name)
@showtb
def rename(self, old, new):
self._fs.rename(old, new)
return 0
#@showtb
#def read(self, path, size, offset):
# pass
def main(fs):
usage="""
FSFS: Exposes an FS
""" + fuse.Fuse.fusage
server = FSFUSE(fs, version="%prog 0.1",
usage=usage, dash_s_do='setsingle')
#server.readdir('.', 0)
server.parse(errex=1)
server.main()
if __name__ == "__main__":
import memoryfs
import osfs
mem_fs = memoryfs.MemoryFS()
mem_fs.makedir("test")
mem_fs.createfile("a.txt", "This is a test")
mem_fs.createfile("test/b.txt", "This is in a sub-dir")
#fs = osfs.OSFS('/home/will/fusetest/')
#main(fs)
main(mem_fs)
# To run do ./fuserserver.py -d -f testfs
# This will map a fs.memoryfs to testfs/ on the local filesystem under tests/fs
# To unmouont, do fusermount -u testfs
\ No newline at end of file
fs/helpers.py deleted 100644 → 0
"""Contains a number of standalone functions for path manipulation."""
from itertools import chain
def _iteratepath(path, numsplits=None):
path = resolvepath(path)
if not path:
return []
if numsplits == None:
return filter(lambda p:bool(p), path.split('/'))
else:
return filter(lambda p:bool(p), path.split('/', numsplits))
def isabsolutepath(path):
"""Returns True if a given path is absolute.
>>> isabsolutepath("a/b/c")
False
>>> isabsolutepath("/foo/bar")
True
"""
if path:
return path[0] in '\\/'
return False
def normpath(path):
"""Normalizes a path to be in the formated expected by FS objects.
Returns a new path string.
>>> normpath(r"foo\\bar\\baz")
'foo/bar/baz'
"""
return path.replace('\\', '/')
def pathjoin(*paths):
"""Joins any number of paths together. Returns a new path string.
paths -- An iterable of path strings
>>> pathjoin('foo', 'bar', 'baz')
'foo/bar/baz'
>>> pathjoin('foo/bar', '../baz')
'foo/baz'
"""
absolute = False
relpaths = []
for p in paths:
if p:
if p[0] in '\\/':
del relpaths[:]
absolute = True
relpaths.append(p)
pathstack = []
for component in chain(*(normpath(path).split('/') for path in relpaths)):
if component == "..":
if not pathstack:
raise ValueError("Relative path is invalid")
sub = pathstack.pop()
elif component == ".":
pass
elif component:
pathstack.append(component)
if absolute:
return "/" + "/".join(pathstack)
else:
return "/".join(pathstack)
def pathsplit(path):
"""Splits a path on a path separator. Returns a tuple containing the path up
to that last separator and the remaining path component.
>>> pathsplit("foo/bar")
('foo', 'bar')
>>> pathsplit("foo/bar/baz")
('foo/bar', 'baz')
"""
split = normpath(path).rsplit('/', 1)
if len(split) == 1:
return ('', split[0])
return tuple(split)
def dirname(path):
"""Returns the parent directory of a path.
path -- A FS path
>>> dirname('foo/bar/baz')
'foo/bar'
"""
return pathsplit(path)[0]
def resourcename(path):
"""Returns the resource references by a path.
path -- A FS path
>>> resourcename('foo/bar/baz')
'baz'
"""
return pathsplit(path)[1]
def resolvepath(path):
"""Normalises the path and removes any relative path components.
path -- A path string
>>> resolvepath(r"foo\\bar\\..\\baz")
'foo/baz'
"""
return pathjoin(path)
def makerelative(path):
"""Makes a path relative by removing initial separator.
path -- A path
>>> makerelative("/foo/bar")
'foo/bar'
"""
path = normpath(path)
if path.startswith('/'):
return path[1:]
return path
def makeabsolute(path):
"""Makes a path absolute by adding a separater at the beginning of the path.
path -- A path
>>> makeabsolute("foo/bar/baz")
'/foo/bar/baz'
"""
path = normpath(path)
if not path.startswith('/'):
return '/'+path
return path
def issamedir(path1, path2):
"""Return true if two paths reference a resource in the same directory.
path1 -- First path
path2 -- Second path
>>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
True
>>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
False
"""
return pathsplit(resolvepath(path1))[0] == pathsplit(resolvepath(path2))[0]
fs/mountfs.py
...@@ -4,27 +4,32 @@ from base import * ...@@ -4,27 +4,32 @@ from base import *
from objecttree import ObjectTree from objecttree import ObjectTree
from memoryfs import MemoryFS from memoryfs import MemoryFS
class MountFS(FS):
"""A filesystem that delegates to other filesystems.""" class DirMount(object):
def __init__(self, path, fs):
self.path = path
self.fs = fs
class DirMount(object): def __str__(self):
def __init__(self, path, fs): return "Mount point: %s"%self.path
self.path = path
self.fs = fs
def __str__(self):
return "Mount point: %s"%self.path
class FileMount(object): class FileMount(object):
def __init__(self, path, open_callable, info_callable=None): def __init__(self, path, open_callable, info_callable=None):
self.open_callable = open_callable self.open_callable = open_callable
def no_info_callable(path): def no_info_callable(path):
return {} return {}
self.info_callable = info_callable or no_info_callable self.info_callable = info_callable or no_info_callable
def __init__(self, thread_syncronize=True):
FS.__init__(self, thread_syncronize=thread_syncronize) class MountFS(FS):
"""A filesystem that delegates to other filesystems."""
DirMount = DirMount
FileMount = FileMount
def __init__(self, thread_synchronize=True):
FS.__init__(self, thread_synchronize=thread_synchronize)
self.mount_tree = ObjectTree() self.mount_tree = ObjectTree()
def __str__(self): def __str__(self):
...@@ -66,7 +71,7 @@ class MountFS(FS): ...@@ -66,7 +71,7 @@ class MountFS(FS):
try: try:
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError(path)
if fs is self: if fs is self:
object = self.mount_tree.get(path, None) object = self.mount_tree.get(path, None)
...@@ -82,7 +87,7 @@ class MountFS(FS): ...@@ -82,7 +87,7 @@ class MountFS(FS):
try: try:
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
return ResourceNotFoundError("NO_RESOURCE", path) return ResourceNotFoundError(path)
if fs is self: if fs is self:
object = self.mount_tree.get(path, None) object = self.mount_tree.get(path, None)
...@@ -92,7 +97,7 @@ class MountFS(FS): ...@@ -92,7 +97,7 @@ class MountFS(FS):
finally: finally:
self._lock.release() self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=True, dirs_only=False, files_only=False): def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
self._lock.acquire() self._lock.acquire()
try: try:
...@@ -100,7 +105,7 @@ class MountFS(FS): ...@@ -100,7 +105,7 @@ class MountFS(FS):
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_DIR", path) raise ResourceNotFoundError(path)
if fs is self: if fs is self:
if files_only: if files_only:
...@@ -112,7 +117,6 @@ class MountFS(FS): ...@@ -112,7 +117,6 @@ class MountFS(FS):
wildcard, wildcard,
full, full,
absolute, absolute,
hidden,
dirs_only, dirs_only,
files_only) files_only)
else: else:
...@@ -120,28 +124,27 @@ class MountFS(FS): ...@@ -120,28 +124,27 @@ class MountFS(FS):
wildcard=wildcard, wildcard=wildcard,
full=False, full=False,
absolute=False, absolute=False,
hidden=hidden,
dirs_only=dirs_only, dirs_only=dirs_only,
files_only=files_only) files_only=files_only)
if full or absolute: if full or absolute:
if full: if full:
path = makeabsolute(path) path = abspath(normpath(path))
else: else:
path = makerelative(path) path = relpath(normpath(path))
paths = [pathjoin(path, p) for p in paths] paths = [pathjoin(path, p) for p in paths]
return paths return paths
finally: finally:
self._lock.release() self._lock.release()
def makedir(self, path, mode=0777, recursive=False, allow_recreate=False): def makedir(self, path, recursive=False, allow_recreate=False):
path = normpath(path) path = normpath(path)
self._lock.acquire() self._lock.acquire()
try: try:
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is self: if fs is self:
raise UnsupportedError("UNSUPPORTED", msg="Can only makedir for mounted paths" ) raise UnsupportedError("make directory", msg="Can only makedir for mounted paths" )
return fs.makedir(delegate_path, mode, recursive=recursive, allow_recreate=allow_recreate) return fs.makedir(delegate_path, recursive=recursive, allow_recreate=allow_recreate)
finally: finally:
self._lock.release() self._lock.release()
...@@ -158,7 +161,7 @@ class MountFS(FS): ...@@ -158,7 +161,7 @@ class MountFS(FS):
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
return fs.open(delegate_path, mode, **kwargs) return fs.open(delegate_path, mode, **kwargs)
...@@ -190,9 +193,9 @@ class MountFS(FS): ...@@ -190,9 +193,9 @@ class MountFS(FS):
path = normpath(path) path = normpath(path)
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
if fs is self: if fs is self:
raise UnsupportedError("UNSUPPORTED", msg="Can only remove paths within a mounted dir" ) raise UnsupportedError("remove file", msg="Can only remove paths within a mounted dir")
return fs.remove(delegate_path) return fs.remove(delegate_path)
finally: finally:
...@@ -207,10 +210,10 @@ class MountFS(FS): ...@@ -207,10 +210,10 @@ class MountFS(FS):
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None or fs is self: if fs is None or fs is self:
raise OperationFailedError("REMOVEDIR_FAILED", path, msg="Can not removedir for an un-mounted path") raise ResourceInvalidError(path, msg="Can not removedir for an un-mounted path")
if not force and not fs.isdirempty(delegate_path): if not force and not fs.isdirempty(delegate_path):
raise OperationFailedError("REMOVEDIR_FAILED", "Directory is not empty: %(path)s") raise DirectoryNotEmptyError("Directory is not empty: %(path)s")
return fs.removedir(delegate_path, recursive, force) return fs.removedir(delegate_path, recursive, force)
...@@ -220,7 +223,7 @@ class MountFS(FS): ...@@ -220,7 +223,7 @@ class MountFS(FS):
def rename(self, src, dst): def rename(self, src, dst):
if not issamedir(src, dst): if not issamedir(src, dst):
raise ValueError("Destination path must the same directory (user the move method for moving to a different directory)") raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)")
self._lock.acquire() self._lock.acquire()
try: try:
...@@ -228,7 +231,7 @@ class MountFS(FS): ...@@ -228,7 +231,7 @@ class MountFS(FS):
fs2, mount_path2, delegate_path2 = self._delegate(dst) fs2, mount_path2, delegate_path2 = self._delegate(dst)
if fs1 is not fs2: if fs1 is not fs2:
raise OperationFailedError("RENAME_FAILED", src) raise OperationFailedError("rename resource", path=src)
if fs1 is not self: if fs1 is not self:
return fs1.rename(delegate_path1, delegate_path2) return fs1.rename(delegate_path1, delegate_path2)
...@@ -240,11 +243,11 @@ class MountFS(FS): ...@@ -240,11 +243,11 @@ class MountFS(FS):
object2 = self.mount_tree.get(path_dst, None) object2 = self.mount_tree.get(path_dst, None)
if object1 is None: if object1 is None:
raise NoResourceError("NO_RESOURCE", src) raise ResourceNotFoundError(src)
# TODO! # TODO!
raise UnsupportedError("UNSUPPORTED", src) raise UnsupportedError("rename resource", path=src)
finally: finally:
self._lock.release() self._lock.release()
...@@ -280,7 +283,7 @@ class MountFS(FS): ...@@ -280,7 +283,7 @@ class MountFS(FS):
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError(path)
if fs is self: if fs is self:
if self.isfile(path): if self.isfile(path):
...@@ -297,13 +300,13 @@ class MountFS(FS): ...@@ -297,13 +300,13 @@ class MountFS(FS):
fs, mount_path, delegate_path = self._delegate(path) fs, mount_path, delegate_path = self._delegate(path)
if fs is None: if fs is None:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
if fs is self: if fs is self:
object = self.mount_tree.get(path, None) object = self.mount_tree.get(path, None)
if object is None or isinstance(object, dict): if object is None or isinstance(object, dict):
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
size = self.mount_tree[path].info_callable(path).get("size", None) size = self.mount_tree[path].info_callable(path).get("size", None)
return size return size
...@@ -312,43 +315,3 @@ class MountFS(FS): ...@@ -312,43 +315,3 @@ class MountFS(FS):
except: except:
self._lock.release() self._lock.release()
if __name__ == "__main__":
help(MountFS)
fs1 = MemoryFS()
fs1.makedir("Memroot/B/C/D", recursive=True)
fs1.open("test.txt", 'w').write("Hello, World!")
#print_fs(fs1)
mountfs = MountFS()
mountfs.mountdir('1/2', fs1)
mountfs.mountdir('1/another', fs1)
def testfile(*args, **kwargs):
print args, kwargs
def testfile_info(*args, **kwargs):
print "testfile_info", args, kwargs
return {'size':100}
mountfs.mountfile('filedir/file.txt', testfile, testfile_info)
print mountfs.getinfo("filedir/file.txt")
#print mountfs.listdir('1/2/Memroot/B/C')
print mountfs.isdir("1")
print mountfs.desc('1/2/Memroot/B')
print_fs(mountfs)
import browsewin
browsewin.browse(mountfs)
print mountfs.getinfo("1/2")
#print mountfs._delegate('1/2/Memroot/B')
fs/multifs.py
#!/usr/in/env python #!/usr/in/env python
from base import FS, FSError from fs.base import FS, FSError
from helpers import * from fs.path import *
class MultiFS(FS): class MultiFS(FS):
...@@ -13,7 +14,7 @@ class MultiFS(FS): ...@@ -13,7 +14,7 @@ class MultiFS(FS):
""" """
def __init__(self): def __init__(self):
FS.__init__(self, thread_syncronize=True) FS.__init__(self, thread_synchronize=True)
self.fs_sequence = [] self.fs_sequence = []
self.fs_lookup = {} self.fs_lookup = {}
...@@ -99,7 +100,7 @@ class MultiFS(FS): ...@@ -99,7 +100,7 @@ class MultiFS(FS):
for fs_name, fs_object in self.fs_lookup.iteritems(): for fs_name, fs_object in self.fs_lookup.iteritems():
if fs is fs_object: if fs is fs_object:
return fs_name, fs return fs_name, fs
raise ResourceNotFoundError("NO_RESOURCE", path, msg="Path does not map to any filesystem: %(path)s") raise ResourceNotFoundError(path, msg="Path does not map to any filesystem: %(path)s")
finally: finally:
self._lock.release() self._lock.release()
...@@ -109,7 +110,7 @@ class MultiFS(FS): ...@@ -109,7 +110,7 @@ class MultiFS(FS):
fs = self._delegate_search(path) fs = self._delegate_search(path)
if fs is not None: if fs is not None:
return fs.getsyspath(path, allow_none=allow_none) return fs.getsyspath(path, allow_none=allow_none)
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
...@@ -117,7 +118,7 @@ class MultiFS(FS): ...@@ -117,7 +118,7 @@ class MultiFS(FS):
self._lock.acquire() self._lock.acquire()
try: try:
if not self.exists(path): if not self.exists(path):
raise ResourceNotFoundError("NO_RESOURCE", path) raise ResourceNotFoundError(path)
name, fs = self.which(path) name, fs = self.which(path)
if name is None: if name is None:
...@@ -135,7 +136,7 @@ class MultiFS(FS): ...@@ -135,7 +136,7 @@ class MultiFS(FS):
fs_file = fs.open(path, mode, **kwargs) fs_file = fs.open(path, mode, **kwargs)
return fs_file return fs_file
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
...@@ -166,16 +167,6 @@ class MultiFS(FS): ...@@ -166,16 +167,6 @@ class MultiFS(FS):
finally: finally:
self._lock.release() self._lock.release()
def ishidden(self, path):
self._lock.acquire()
try:
fs = self._delegate_search(path)
if fs is not None:
return fs.isfile(path)
return False
finally:
self._lock.release()
def listdir(self, path="./", *args, **kwargs): def listdir(self, path="./", *args, **kwargs):
self._lock.acquire() self._lock.acquire()
try: try:
...@@ -197,7 +188,7 @@ class MultiFS(FS): ...@@ -197,7 +188,7 @@ class MultiFS(FS):
if fs.exists(path): if fs.exists(path):
fs.remove(path) fs.remove(path)
return return
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
...@@ -208,20 +199,20 @@ class MultiFS(FS): ...@@ -208,20 +199,20 @@ class MultiFS(FS):
if fs.isdir(path): if fs.isdir(path):
fs.removedir(path, recursive) fs.removedir(path, recursive)
return return
raise ResourceNotFoundError("NO_DIR", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
def rename(self, src, dst): def rename(self, src, dst):
if not issamedir(src, dst): if not issamedir(src, dst):
raise ValueError("Destination path must the same directory (user the move method for moving to a different directory)") raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)")
self._lock.acquire() self._lock.acquire()
try: try:
for fs in self: for fs in self:
if fs.exists(src): if fs.exists(src):
fs.rename(src, dst) fs.rename(src, dst)
return return
raise FSError("NO_RESOURCE", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
...@@ -232,31 +223,7 @@ class MultiFS(FS): ...@@ -232,31 +223,7 @@ class MultiFS(FS):
if fs.exists(path): if fs.exists(path):
return fs.getinfo(path) return fs.getinfo(path)
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
finally: finally:
self._lock.release() self._lock.release()
if __name__ == "__main__":
import fs
import osfs
osfs = osfs.OSFS('~/')
import memoryfs
mem_fs = memoryfs.MemoryFS()
mem_fs.makedir('projects/test2', recursive=True)
mem_fs.makedir('projects/A', recursive=True)
mem_fs.makedir('projects/A/B', recursive=True)
mem_fs.open("projects/test2/readme.txt", 'w').write("Hello, World!")
mem_fs.open("projects/A/readme.txt", 'w').write("\nSecond Line")
multifs = MultiFS()
multifs.addfs("osfs", osfs)
multifs.addfs("mem_fs", mem_fs)
import browsewin
browsewin.browse(multifs)
\ No newline at end of file
fs/objecttree.py
#!/usr/bin/env python
from helpers import _iteratepath, pathsplit
class _ObjectDict(dict): class _ObjectDict(dict):
pass pass
class ObjectTree(object):
class ObjectTree(object):
"""A class to facilitate the creation of tree structures.""" """A class to facilitate the creation of tree structures."""
def __init__(self): def __init__(self):
...@@ -105,15 +103,3 @@ class ObjectTree(object): ...@@ -105,15 +103,3 @@ class ObjectTree(object):
return self.root.iteritems() return self.root.iteritems()
if __name__ == "__main__":
ot = ObjectTree()
ot['a/b/c'] = "Hai!"
print ot['a/b/c']
print ot.partialget("/a/b/c/d/e/f")
ot['a/b/c/d'] = "?"
print ot['a/b/c'].keys()
\ No newline at end of file
fs/path.py 0 → 100644
"""
fs.path: useful functions for FS path manipulation.
This is broadly similar to the standard 'os.path' module but works with
paths in the canonical format expected by all FS objects (backslash-separated,
optional leading slash).
"""
def normpath(path):
"""Normalizes a path to be in the format expected by FS objects.
This function remove any leading or trailing slashes, collapses
duplicate slashes, replaces forward with backward slashes, and generally
tries very hard to return a new path string the canonical FS format.
If the path is invalid, ValueError will be raised.
>>> normpath(r"foo\\bar\\baz")
'foo/bar/baz'
>>> normpath("/foo//bar/frob/../baz")
'/foo/bar/baz'
>>> normpath("foo/../../bar")
Traceback (most recent call last)
...
ValueError: too many backrefs in path 'foo/../../bar'
"""
if not path:
return path
components = []
for comp in path.replace('\\','/').split("/"):
if not comp or comp == ".":
pass
elif comp == "..":
try:
components.pop()
except IndexError:
err = "too many backrefs in path '%s'" % (path,)
raise ValueError(err)
else:
components.append(comp)
if path[0] in "\\/":
if not components:
components = [""]
components.insert(0,"")
return "/".join(components)
def iteratepath(path, numsplits=None):
"""Iterate over the individual components of a path."""
path = relpath(normpath(path))
if not path:
return []
if numsplits == None:
return path.split('/')
else:
return path.split('/', numsplits)
def abspath(path):
"""Convert the given path to an absolute path.
Since FS objects have no concept of a 'current directory' this simply
adds a leading '/' character if the path doesn't already have one.
"""
if not path:
return "/"
if path[0] != "/":
return "/" + path
return path
def relpath(path):
"""Convert the given path to a relative path.
This is the inverse of abspath(), stripping a leading '/' from the
path if it is present.
"""
while path and path[0] == "/":
path = path[1:]
return path
def pathjoin(*paths):
"""Joins any number of paths together, returning a new path string.
>>> pathjoin('foo', 'bar', 'baz')
'foo/bar/baz'
>>> pathjoin('foo/bar', '../baz')
'foo/baz'
>>> pathjoin('foo/bar', '/baz')
'/baz'
"""
absolute = False
relpaths = []
for p in paths:
if p:
if p[0] in '\\/':
del relpaths[:]
absolute = True
relpaths.append(p)
path = normpath("/".join(relpaths))
if absolute and not path.startswith("/"):
path = "/" + path
return path
# Allow pathjoin() to be used as fs.path.join()
join = pathjoin
def pathsplit(path):
"""Splits a path into (head,tail) pair.
This function splits a path into a pair (head,tail) where 'tail' is the
last pathname component and 'head' is all preceeding components.
>>> pathsplit("foo/bar")
('foo', 'bar')
>>> pathsplit("foo/bar/baz")
('foo/bar', 'baz')
"""
split = normpath(path).rsplit('/', 1)
if len(split) == 1:
return ('', split[0])
return tuple(split)
# Allow pathsplit() to be used as fs.path.split()
split = pathsplit
def dirname(path):
"""Returns the parent directory of a path.
This is always equivalent to the 'head' component of the value returned
by pathsplit(path).
>>> dirname('foo/bar/baz')
'foo/bar'
"""
return pathsplit(path)[0]
def basename(path):
"""Returns the basename of the resource referenced by a path.
This is always equivalent to the 'head' component of the value returned
by pathsplit(path).
>>> basename('foo/bar/baz')
'baz'
"""
return pathsplit(path)[1]
def issamedir(path1, path2):
"""Return true if two paths reference a resource in the same directory.
>>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
True
>>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
False
"""
return pathsplit(normpath(path1))[0] == pathsplit(normpath(path2))[0]
def isprefix(path1,path2):
"""Return true is path1 is a prefix of path2.
>>> isprefix("foo/bar", "foo/bar/spam.txt")
True
>>> isprefix("foo/bar/", "foo/bar")
True
>>> isprefix("foo/barry", "foo/baz/bar")
False
>>> isprefix("foo/bar/baz/", "foo/baz/bar")
False
"""
bits1 = path1.split("/")
bits2 = path2.split("/")
while bits1 and bits1[-1] == "":
bits1.pop()
if len(bits1) > len(bits2):
return False
for (bit1,bit2) in zip(bits1,bits2):
if bit1 != bit2:
return False
return True
fs/rpcfs.py
""" """
fs.rpcfs: Client and Server to expose an FS via XML-RPC fs.rpcfs: client to access an FS via XML-RPC
This module provides the following pair of classes that can be used to expose This module provides the class 'RPCFS' to access a remote FS object over
a remote filesystem using XML-RPC: XML-RPC. You probably want to use this in conjunction with the 'RPCFSServer'
class from the fs.expose.xmlrpc module.
RPCFSServer: a subclass of SimpleXMLRPCServer that exposes the methods
of an FS instance via XML-RPC
RPCFS: a subclass of FS that delegates all filesystem operations to
a remote server using XML-RPC.
If you need to use a more powerful server than SimpleXMLRPCServer, you can
use the RPCFSInterface class to provide an XML-RPC-compatible wrapper around
an FS object, which can then be exposed using whatever server you choose
(e.g. Twisted's XML-RPC server).
""" """
import xmlrpclib import xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer
from fs.base import * from fs.base import *
from StringIO import StringIO from StringIO import StringIO
if hasattr(StringIO,"__exit__"):
class StringIO(StringIO):
class ObjProxy: pass
"""Simple object proxy allowing us to replace read-only attributes. else:
class StringIO(StringIO):
This is used to put a modified 'close' method on files returned by def __enter__(self):
open(), such that they will be uploaded to the server when closed. return self
""" def __exit__(self,exc_type,exc_value,traceback):
self.close()
def __init__(self,obj): return False
self._obj = obj
def __getattr__(self,attr):
return getattr(self._obj,attr)
def re_raise_faults(func): def re_raise_faults(func):
...@@ -103,7 +88,7 @@ class RPCFS(FS): ...@@ -103,7 +88,7 @@ class RPCFS(FS):
"""Access a filesystem exposed via XML-RPC. """Access a filesystem exposed via XML-RPC.
This class provides the client-side logic for accessing a remote FS This class provides the client-side logic for accessing a remote FS
object, and is dual to the RPCFSServer class also defined in this module. object, and is dual to the RPCFSServer class defined in fs.expose.xmlrpc.
Example: Example:
...@@ -116,21 +101,37 @@ class RPCFS(FS): ...@@ -116,21 +101,37 @@ class RPCFS(FS):
The only required argument is the uri of the server to connect The only required argument is the uri of the server to connect
to. This will be passed to the underlying XML-RPC server proxy to. This will be passed to the underlying XML-RPC server proxy
object along with the 'transport' argument if it is provided. object, along with the 'transport' argument if it is provided.
""" """
self.uri = uri self.uri = uri
if transport is not None: self._transport = transport
proxy = xmlrpclib.ServerProxy(uri,transport,allow_none=True) self.proxy = self._make_proxy()
def _make_proxy(self):
kwds = dict(allow_none=True)
if self._transport is not None:
proxy = xmlrpclib.ServerProxy(self.uri,self._transport,**kwds)
else: else:
proxy = xmlrpclib.ServerProxy(uri,allow_none=True) proxy = xmlrpclib.ServerProxy(self.uri,**kwds)
self.proxy = ReRaiseFaults(proxy) return ReRaiseFaults(proxy)
def __str__(self): def __str__(self):
return '<RPCFS: %s>' % (self.uri,) return '<RPCFS: %s>' % (self.uri,)
__repr__ = __str__ def __getstate__(self):
state = super(RPCFS,self).__getstate__()
try:
del state['proxy']
except KeyError:
pass
return state
def open(self,path,mode): def __setstate__(self,state):
for (k,v) in state.iteritems():
self.__dict__[k] = v
self.proxy = self._make_proxy()
def open(self,path,mode="r"):
# TODO: chunked transport of large files # TODO: chunked transport of large files
if "w" in mode: if "w" in mode:
self.proxy.set_contents(path,xmlrpclib.Binary("")) self.proxy.set_contents(path,xmlrpclib.Binary(""))
...@@ -139,13 +140,13 @@ class RPCFS(FS): ...@@ -139,13 +140,13 @@ class RPCFS(FS):
data = self.proxy.get_contents(path).data data = self.proxy.get_contents(path).data
except IOError: except IOError:
if "w" not in mode and "a" not in mode: if "w" not in mode and "a" not in mode:
raise ResourceNotFoundError("NO_FILE",path) raise ResourceNotFoundError(path)
if not self.isdir(dirname(path)): if not self.isdir(dirname(path)):
raise OperationFailedError("OPEN_FAILED", path,msg="Parent directory does not exist") raise ParentDirectoryMissingError(path)
self.proxy.set_contents(path,xmlrpclib.Binary("")) self.proxy.set_contents(path,xmlrpclib.Binary(""))
else: else:
data = "" data = ""
f = ObjProxy(StringIO(data)) f = StringIO(data)
if "a" not in mode: if "a" not in mode:
f.seek(0,0) f.seek(0,0)
else: else:
...@@ -171,11 +172,11 @@ class RPCFS(FS): ...@@ -171,11 +172,11 @@ class RPCFS(FS):
def isfile(self,path): def isfile(self,path):
return self.proxy.isfile(path) return self.proxy.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,hidden=True,dirs_only=False,files_only=False): def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
return self.proxy.listdir(path,wildcard,full,absolute,hidden,dirs_only,files_only) return self.proxy.listdir(path,wildcard,full,absolute,dirs_only,files_only)
def makedir(self,path,mode=0777,recursive=False,allow_recreate=False): def makedir(self,path,recursive=False,allow_recreate=False):
return self.proxy.makedir(path,mode,recursive,allow_recreate) return self.proxy.makedir(path,recursive,allow_recreate)
def remove(self,path): def remove(self,path):
return self.proxy.remove(path) return self.proxy.remove(path)
...@@ -211,99 +212,3 @@ class RPCFS(FS): ...@@ -211,99 +212,3 @@ class RPCFS(FS):
return self.proxy.copydir(src,dst,overwrite,ignore_errors,chunk_size) return self.proxy.copydir(src,dst,overwrite,ignore_errors,chunk_size)
class RPCFSInterface(object):
"""Wrapper to expose an FS via a XML-RPC compatible interface.
The only real trick is using xmlrpclib.Binary objects to trasnport
the contents of files.
"""
def __init__(self,fs):
self.fs = fs
def get_contents(self,path):
data = self.fs.getcontents(path)
return xmlrpclib.Binary(data)
def set_contents(self,path,data):
self.fs.createfile(path,data.data)
def exists(self,path):
return self.fs.exists(path)
def isdir(self,path):
return self.fs.isdir(path)
def isfile(self,path):
return self.fs.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,hidden=True,dirs_only=False,files_only=False):
return list(self.fs.listdir(path,wildcard,full,absolute,hidden,dirs_only,files_only))
def makedir(self,path,mode=0777,recursive=False,allow_recreate=False):
return self.fs.makedir(path,mode,recursive,allow_recreate)
def remove(self,path):
return self.fs.remove(path)
def removedir(self,path,recursive=False,force=False):
return self.fs.removedir(path,recursive,force)
def rename(self,src,dst):
return self.fs.rename(src,dst)
def getinfo(self,path):
return self.fs.getinfo(path)
def desc(self,path):
return self.fs.desc(path)
def getattr(self,path,attr):
return self.fs.getattr(path,attr)
def setattr(self,path,attr,value):
return self.fs.setattr(path,attr,value)
def copy(self,src,dst,overwrite=False,chunk_size=16384):
return self.fs.copy(src,dst,overwrite,chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384):
return self.fs.move(src,dst,overwrite,chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.fs.movedir(src,dst,overwrite,ignore_errors,chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.fs.copydir(src,dst,overwrite,ignore_errors,chunk_size)
class RPCFSServer(SimpleXMLRPCServer):
"""Server to expose an FS object via XML-RPC.
This class takes as its first argument an FS instance, and as its second
argument a (hostname,port) tuple on which to listen for XML-RPC requests.
Example:
fs = OSFS('/var/srv/myfiles')
s = RPCFSServer(fs,("",8080))
s.serve_forever()
To cleanly shut down the server after calling serve_forever, set the
attribute "serve_more_requests" to False.
"""
def __init__(self,fs,addr,requestHandler=None,logRequests=None):
kwds = dict(allow_none=True)
if requestHandler is not None:
kwds['requestHandler'] = requestHandler
if logRequests is not None:
kwds['logRequests'] = logRequests
self.serve_more_requests = True
SimpleXMLRPCServer.__init__(self,addr,**kwds)
self.register_instance(RPCFSInterface(fs))
def serve_forever(self):
"""Override serve_forever to allow graceful shutdown."""
while self.serve_more_requests:
self.handle_request()
fs/sftpfs.py 0 → 100644
This diff is collapsed. Click to expand it.
fs/tempfs.py
...@@ -9,16 +9,16 @@ class TempFS(OSFS): ...@@ -9,16 +9,16 @@ class TempFS(OSFS):
"""Create a Filesystem in a tempory directory (with tempfile.mkdtemp), """Create a Filesystem in a tempory directory (with tempfile.mkdtemp),
and removes it when the TempFS object is cleaned up.""" and removes it when the TempFS object is cleaned up."""
def __init__(self, identifier=None, thread_syncronize=True): def __init__(self, identifier=None, temp_dir=None, dir_mode=0700, thread_synchronize=True):
"""Creates a temporary Filesystem """Creates a temporary Filesystem
identifier -- A string that is included in the name of the temporary directory, identifier -- A string that is included in the name of the temporary directory,
default uses "TempFS" default uses "TempFS"
""" """
self._temp_dir = tempfile.mkdtemp(identifier or "TempFS") self._temp_dir = tempfile.mkdtemp(identifier or "TempFS",dir=temp_dir)
self._cleaned = False self._cleaned = False
OSFS.__init__(self, self._temp_dir, thread_syncronize=thread_syncronize) OSFS.__init__(self, self._temp_dir, dir_mode=dir_mode, thread_synchronize=thread_synchronize)
def __str__(self): def __str__(self):
return '<TempFS: %s>' % self._temp_dir return '<TempFS: %s>' % self._temp_dir
...@@ -44,7 +44,4 @@ class TempFS(OSFS): ...@@ -44,7 +44,4 @@ class TempFS(OSFS):
def __del__(self): def __del__(self):
self.close() self.close()
if __name__ == "__main__":
tfs = TempFS()
print tfs
fs/tests/test_expose.py 0 → 100644
"""
fs.tests.test_expose: testcases for fs.expose and associated FS classes
"""
import unittest
import sys
import os, os.path
import socket
import threading
import time
from fs.tests import FSTestCases
from fs.tempfs import TempFS
from fs.osfs import OSFS
from fs.path import *
from fs import rpcfs
from fs.expose.xmlrpc import RPCFSServer
class TestRPCFS(unittest.TestCase,FSTestCases):
def makeServer(self,fs,addr):
return RPCFSServer(fs,addr,logRequests=False)
def startServer(self):
port = 8000
self.temp_fs = TempFS()
self.server = None
while not self.server:
try:
self.server = self.makeServer(self.temp_fs,("localhost",port))
except socket.error, e:
if e.args[1] == "Address already in use":
port += 1
else:
raise
self.server_addr = ("localhost",port)
self.serve_more_requests = True
self.server_thread = threading.Thread(target=self.runServer)
self.server_thread.start()
def runServer(self):
"""Run the server, swallowing shutdown-related execptions."""
self.server.socket.settimeout(0.1)
try:
while self.serve_more_requests:
self.server.handle_request()
except Exception, e:
pass
def setUp(self):
self.startServer()
self.fs = rpcfs.RPCFS("http://%s:%d" % self.server_addr)
def tearDown(self):
self.serve_more_requests = False
try:
self.bump()
self.server.server_close()
except Exception:
pass
self.server_thread.join()
self.temp_fs.close()
def bump(self):
host, port = self.server_addr
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
af, socktype, proto, cn, sa = res
sock = None
try:
sock = socket.socket(af, socktype, proto)
sock.settimeout(1)
sock.connect(sa)
sock.send("\n")
except socket.error, e:
pass
finally:
if sock is not None:
sock.close()
from fs import sftpfs
from fs.expose.sftp import BaseSFTPServer
class TestSFTPFS(TestRPCFS):
def makeServer(self,fs,addr):
return BaseSFTPServer(addr,fs)
def setUp(self):
self.startServer()
self.fs = sftpfs.SFTPFS(self.server_addr)
def bump(self):
# paramiko doesn't like being bumped, just wait for it to timeout.
# TODO: do this using a paramiko.Transport() connection
pass
from fs.expose import fuse
from fs.osfs import OSFS
class TestFUSE(unittest.TestCase,FSTestCases):
def setUp(self):
self.temp_fs = TempFS()
self.temp_fs.makedir("root")
self.temp_fs.makedir("mount")
self.mounted_fs = self.temp_fs.opendir("root")
self.mount_point = self.temp_fs.getsyspath("mount")
self.fs = OSFS(self.temp_fs.getsyspath("mount"))
self.mount_proc = fuse.mount(self.mounted_fs,self.mount_point)
def tearDown(self):
self.mount_proc.unmount()
self.temp_fs.close()
def check(self,p):
return self.mounted_fs.exists(p)
fs/tests/test_fs.py 0 → 100644
"""
fs.tests.test_fs: testcases for basic FS implementations
"""
from fs.tests import FSTestCases
import unittest
import os
import shutil
import tempfile
from fs.path import *
from fs import osfs
class TestOSFS(unittest.TestCase,FSTestCases):
def setUp(self):
self.temp_dir = tempfile.mkdtemp("fstest")
self.fs = osfs.OSFS(self.temp_dir)
def tearDown(self):
shutil.rmtree(self.temp_dir)
def check(self, p):
return os.path.exists(os.path.join(self.temp_dir, relpath(p)))
class TestSubFS(unittest.TestCase,FSTestCases):
def setUp(self):
self.temp_dir = tempfile.mkdtemp("fstest")
self.parent_fs = osfs.OSFS(self.temp_dir)
self.parent_fs.makedir("foo/bar", recursive=True)
self.fs = self.parent_fs.opendir("foo/bar")
def tearDown(self):
shutil.rmtree(self.temp_dir)
def check(self, p):
p = os.path.join("foo/bar", relpath(p))
full_p = os.path.join(self.temp_dir, p)
return os.path.exists(full_p)
from fs import memoryfs
class TestMemoryFS(unittest.TestCase,FSTestCases):
def setUp(self):
self.fs = memoryfs.MemoryFS()
from fs import mountfs
class TestMountFS(unittest.TestCase,FSTestCases):
def setUp(self):
self.mount_fs = mountfs.MountFS()
self.mem_fs = memoryfs.MemoryFS()
self.mount_fs.mountdir("mounted/memfs", self.mem_fs)
self.fs = self.mount_fs.opendir("mounted/memfs")
def tearDown(self):
pass
def check(self, p):
return self.mount_fs.exists(os.path.join("mounted/memfs", relpath(p)))
from fs import tempfs
class TestTempFS(unittest.TestCase,FSTestCases):
def setUp(self):
self.fs = tempfs.TempFS()
def tearDown(self):
td = self.fs._temp_dir
self.fs.close()
self.assert_(not os.path.exists(td))
def check(self, p):
td = self.fs._temp_dir
return os.path.exists(os.path.join(td, relpath(p)))
fs/tests/test_objecttree.py 0 → 100644
"""
fs.tests.test_objectree: testcases for the fs objecttree module
"""
import unittest
import fs.tests
from fs import objecttree
class TestObjectTree(unittest.TestCase):
"""Testcases for the ObjectTree class."""
def test_getset(self):
ot = objecttree.ObjectTree()
ot['foo'] = "bar"
self.assertEqual(ot['foo'], 'bar')
ot = objecttree.ObjectTree()
ot['foo/bar'] = "baz"
self.assertEqual(ot['foo'], {'bar':'baz'})
self.assertEqual(ot['foo/bar'], 'baz')
del ot['foo/bar']
self.assertEqual(ot['foo'], {})
ot = objecttree.ObjectTree()
ot['a/b/c'] = "A"
ot['a/b/d'] = "B"
ot['a/b/e'] = "C"
ot['a/b/f'] = "D"
self.assertEqual(sorted(ot['a/b'].values()), ['A', 'B', 'C', 'D'])
self.assert_(ot.get('a/b/x', -1) == -1)
self.assert_('a/b/c' in ot)
self.assert_('a/b/x' not in ot)
self.assert_(ot.isobject('a/b/c'))
self.assert_(ot.isobject('a/b/d'))
self.assert_(not ot.isobject('a/b'))
left, object, right = ot.partialget('a/b/e/f/g')
self.assertEqual(left, "a/b/e")
self.assertEqual(object, "C")
self.assertEqual(right, "f/g")
fs/tests/test_path.py 0 → 100644
"""
fs.tests.test_path: testcases for the fs path functions
"""
import unittest
import fs.tests
from fs.path import *
class TestPathFunctions(unittest.TestCase):
"""Testcases for FS path functions."""
def test_normpath(self):
tests = [ ("\\a\\b\\c", "/a/b/c"),
("", ""),
("/a/b/c", "/a/b/c"),
("a/b/c", "a/b/c"),
("a/b/../c/", "a/c"),
("/","/"),
]
for path, result in tests:
self.assertEqual(normpath(path), result)
def test_pathjoin(self):
tests = [ ("", "a", "a"),
("a", "a", "a/a"),
("a/b", "../c", "a/c"),
("a/b/../c", "d", "a/c/d"),
("/a/b/c", "d", "/a/b/c/d"),
("/a/b/c", "../../../d", "/d"),
("a", "b", "c", "a/b/c"),
("a/b/c", "../d", "c", "a/b/d/c"),
("a/b/c", "../d", "/a", "/a"),
("aaa", "bbb/ccc", "aaa/bbb/ccc"),
("aaa", "bbb\ccc", "aaa/bbb/ccc"),
("aaa", "bbb", "ccc", "/aaa", "eee", "/aaa/eee"),
("a/b", "./d", "e", "a/b/d/e"),
("/", "/", "/"),
("/", "", "/"),
]
for testpaths in tests:
paths = testpaths[:-1]
result = testpaths[-1]
self.assertEqual(fs.pathjoin(*paths), result)
self.assertRaises(ValueError, fs.pathjoin, "../")
self.assertRaises(ValueError, fs.pathjoin, "./../")
self.assertRaises(ValueError, fs.pathjoin, "a/b", "../../..")
self.assertRaises(ValueError, fs.pathjoin, "a/b/../../../d")
def test_relpath(self):
tests = [ ("/a/b", "a/b"),
("a/b", "a/b"),
("/", "") ]
for path, result in tests:
self.assertEqual(fs.relpath(path), result)
def test_abspath(self):
tests = [ ("/a/b", "/a/b"),
("a/b", "/a/b"),
("/", "/") ]
for path, result in tests:
self.assertEqual(fs.abspath(path), result)
def test_iteratepath(self):
tests = [ ("a/b", ["a", "b"]),
("", [] ),
("aaa/bbb/ccc", ["aaa", "bbb", "ccc"]),
("a/b/c/../d", ["a", "b", "d"]) ]
for path, results in tests:
for path_component, expected in zip(iteratepath(path), results):
self.assertEqual(path_component, expected)
self.assertEqual(list(iteratepath("a/b/c/d", 1)), ["a", "b/c/d"])
self.assertEqual(list(iteratepath("a/b/c/d", 2)), ["a", "b", "c/d"])
def test_pathsplit(self):
tests = [ ("a/b", ("a", "b")),
("a/b/c", ("a/b", "c")),
("a", ("", "a")),
("", ("", "")),
("/", ("", "")),
("foo/bar", ("foo", "bar")),
("foo/bar/baz", ("foo/bar", "baz")),
]
for path, result in tests:
self.assertEqual(fs.pathsplit(path), result)
fs/tests/test_s3fs.py 0 → 100644
#!/usr/bin/env python
"""
fs.tests.test_s3fs: testcases for the S3FS module
These tests are set up to be skipped by default, since they're very slow,
require a valid AWS account, and cost money. You'll have to set the '__test__'
attribute the True on te TestS3FS class to get them running.
"""
import unittest
from fs.tests import FSTestCases
from fs.path import *
from fs import s3fs
class TestS3FS(unittest.TestCase,FSTestCases):
# Disable the tests by default
__test__ = False
bucket = "test-s3fs.rfk.id.au"
def setUp(self):
self.fs = s3fs.S3FS(self.bucket)
self._clear()
def _clear(self):
for (path,files) in self.fs.walk(search="depth"):
for fn in files:
self.fs.remove(pathjoin(path,fn))
if path and path != "/":
self.fs.removedir(path)
def tearDown(self):
self._clear()
for k in self.fs._s3bukt.list():
self.fs._s3bukt.delete_key(k)
self.fs._s3conn.delete_bucket(self.bucket)
class TestS3FS_prefix(TestS3FS):
def setUp(self):
self.fs = s3fs.S3FS(self.bucket,"/unittest/files")
self._clear()
fs/tests/test_xattr.py 0 → 100644
"""
fs.tests.test_xattr: testcases for extended attribute support
"""
import unittest
import os
from fs.path import *
from fs.errors import *
from fs.tests import FSTestCases
class XAttrTestCases:
"""Testcases for filesystems providing extended attribute support.
This class should be used as a mixin to the unittest.TestCase class
for filesystems that provide extended attribute support.
"""
def test_getsetdel(self):
def do_getsetdel(p):
self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
self.fs.setxattr(p,"xattr1","value1")
self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
self.fs.delxattr(p,"xattr1")
self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
self.fs.createfile("test.txt","hello")
do_getsetdel("test.txt")
self.assertRaises(ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
self.fs.makedir("mystuff")
self.fs.createfile("/mystuff/test.txt","")
do_getsetdel("mystuff")
do_getsetdel("mystuff/test.txt")
def test_list_xattrs(self):
def do_list(p):
self.assertEquals(sorted(self.fs.listxattrs(p)),[])
self.fs.setxattr(p,"xattr1","value1")
self.assertEquals(sorted(self.fs.listxattrs(p)),["xattr1"])
self.fs.setxattr(p,"attr2","value2")
self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2","xattr1"])
self.fs.delxattr(p,"xattr1")
self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2"])
self.fs.delxattr(p,"attr2")
self.assertEquals(sorted(self.fs.listxattrs(p)),[])
self.fs.createfile("test.txt","hello")
do_list("test.txt")
self.fs.makedir("mystuff")
self.fs.createfile("/mystuff/test.txt","")
do_list("mystuff")
do_list("mystuff/test.txt")
def test_copy_xattrs(self):
self.fs.createfile("a.txt","content")
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
self.fs.copy("a.txt","stuff/a.txt")
self.assertTrue(self.fs.exists("stuff/a.txt"))
self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue")
self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue")
self.assertEquals(self.fs.getxattr("a.txt","myattr"),"myvalue")
self.assertEquals(self.fs.getxattr("a.txt","testattr"),"testvalue")
self.fs.setxattr("stuff","dirattr","a directory")
self.fs.copydir("stuff","stuff2")
self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue")
self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue")
self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory")
self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
def test_move_xattrs(self):
self.fs.createfile("a.txt","content")
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
self.fs.move("a.txt","stuff/a.txt")
self.assertTrue(self.fs.exists("stuff/a.txt"))
self.assertEquals(self.fs.getxattr("stuff/a.txt","myattr"),"myvalue")
self.assertEquals(self.fs.getxattr("stuff/a.txt","testattr"),"testvalue")
self.fs.setxattr("stuff","dirattr","a directory")
self.fs.movedir("stuff","stuff2")
self.assertEquals(self.fs.getxattr("stuff2/a.txt","myattr"),"myvalue")
self.assertEquals(self.fs.getxattr("stuff2/a.txt","testattr"),"testvalue")
self.assertEquals(self.fs.getxattr("stuff2","dirattr"),"a directory")
from fs.xattrs import ensure_xattrs
from fs import tempfs
class TestXAttr_TempFS(unittest.TestCase,FSTestCases,XAttrTestCases):
def setUp(self):
self.fs = ensure_xattrs(tempfs.TempFS())
def tearDown(self):
td = self.fs._temp_dir
self.fs.close()
self.assert_(not os.path.exists(td))
def check(self, p):
td = self.fs._temp_dir
return os.path.exists(os.path.join(td, relpath(p)))
from fs import memoryfs
class TestXAttr_MemoryFS(unittest.TestCase,FSTestCases,XAttrTestCases):
def setUp(self):
self.fs = ensure_xattrs(memoryfs.MemoryFS())
def check(self, p):
return self.fs.exists(p)
fs/tests/test_zipfs.py 0 → 100644
"""
fs.tests.test_zipfs: testcases for the ZipFS class
"""
import unittest
import os
import random
import zipfile
import tempfile
import fs.tests
from fs.path import *
from fs import zipfs
class TestReadZipFS(unittest.TestCase):
def setUp(self):
self.temp_filename = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(6))+".zip"
self.temp_filename = os.path.join(tempfile.gettempdir(), self.temp_filename)
self.zf = zipfile.ZipFile(self.temp_filename, "w")
zf = self.zf
zf.writestr("a.txt", "Hello, World!")
zf.writestr("b.txt", "b")
zf.writestr("1.txt", "1")
zf.writestr("foo/bar/baz.txt", "baz")
zf.writestr("foo/second.txt", "hai")
zf.close()
self.fs = zipfs.ZipFS(self.temp_filename, "r")
def tearDown(self):
self.fs.close()
os.remove(self.temp_filename)
def check(self, p):
try:
self.zipfile.getinfo(p)
return True
except:
return False
def test_reads(self):
def read_contents(path):
f = self.fs.open(path)
contents = f.read()
return contents
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
def test_getcontents(self):
def read_contents(path):
return self.fs.getcontents(path)
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
def test_is(self):
self.assert_(self.fs.isfile('a.txt'))
self.assert_(self.fs.isfile('1.txt'))
self.assert_(self.fs.isfile('foo/bar/baz.txt'))
self.assert_(self.fs.isdir('foo'))
self.assert_(self.fs.isdir('foo/bar'))
self.assert_(self.fs.exists('a.txt'))
self.assert_(self.fs.exists('1.txt'))
self.assert_(self.fs.exists('foo/bar/baz.txt'))
self.assert_(self.fs.exists('foo'))
self.assert_(self.fs.exists('foo/bar'))
def test_listdir(self):
def check_listing(path, expected):
dir_list = self.fs.listdir(path)
self.assert_(sorted(dir_list) == sorted(expected))
check_listing('/', ['a.txt', '1.txt', 'foo', 'b.txt'])
check_listing('foo', ['second.txt', 'bar'])
check_listing('foo/bar', ['baz.txt'])
class TestWriteZipFS(unittest.TestCase):
def setUp(self):
self.temp_filename = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(6))+".zip"
self.temp_filename = os.path.join(tempfile.gettempdir(), self.temp_filename)
zip_fs = zipfs.ZipFS(self.temp_filename, 'w')
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
makefile("foo/bar/baz.txt", "baz")
makefile("foo/second.txt", "hai")
zip_fs.close()
def tearDown(self):
os.remove(self.temp_filename)
def test_valid(self):
zf = zipfile.ZipFile(self.temp_filename, "r")
self.assert_(zf.testzip() is None)
zf.close()
def test_creation(self):
zf = zipfile.ZipFile(self.temp_filename, "r")
def check_contents(filename, contents):
zcontents = zf.read(filename)
self.assertEqual(contents, zcontents)
check_contents("a.txt", "Hello, World!")
check_contents("b.txt", "b")
check_contents("foo/bar/baz.txt", "baz")
check_contents("foo/second.txt", "hai")
class TestAppendZipFS(TestWriteZipFS):
def setUp(self):
self.temp_filename = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(6))+".zip"
self.temp_filename = os.path.join(tempfile.gettempdir(), self.temp_filename)
zip_fs = zipfs.ZipFS(self.temp_filename, 'w')
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
zip_fs.close()
zip_fs = zipfs.ZipFS(self.temp_filename, 'a')
makefile("foo/bar/baz.txt", "baz")
makefile("foo/second.txt", "hai")
zip_fs.close()
fs/utils.py
"""Contains a number of high level utility functions for working with FS objects.""" """
fs.utils: high-level utility functions for working with FS objects.
"""
import shutil import shutil
from mountfs import MountFS from mountfs import MountFS
...@@ -82,6 +86,7 @@ def movefile(src_fs, src_path, dst_fs, dst_path, chunk_size=16384): ...@@ -82,6 +86,7 @@ def movefile(src_fs, src_path, dst_fs, dst_path, chunk_size=16384):
if dst is not None: if dst is not None:
dst.close() dst.close()
def movedir(fs1, fs2, ignore_errors=False, chunk_size=16384): def movedir(fs1, fs2, ignore_errors=False, chunk_size=16384):
"""Moves contents of a directory from one filesystem to another. """Moves contents of a directory from one filesystem to another.
...@@ -103,6 +108,7 @@ def movedir(fs1, fs2, ignore_errors=False, chunk_size=16384): ...@@ -103,6 +108,7 @@ def movedir(fs1, fs2, ignore_errors=False, chunk_size=16384):
mount_fs.mount('dir2', fs2) mount_fs.mount('dir2', fs2)
mount_fs.movedir('dir1', 'dir2', ignore_errors=ignore_errors, chunk_size=chunk_size) mount_fs.movedir('dir1', 'dir2', ignore_errors=ignore_errors, chunk_size=chunk_size)
def copydir(fs1, fs2, ignore_errors=False, chunk_size=16384): def copydir(fs1, fs2, ignore_errors=False, chunk_size=16384):
"""Copies contents of a directory from one filesystem to another. """Copies contents of a directory from one filesystem to another.
...@@ -124,11 +130,13 @@ def copydir(fs1, fs2, ignore_errors=False, chunk_size=16384): ...@@ -124,11 +130,13 @@ def copydir(fs1, fs2, ignore_errors=False, chunk_size=16384):
mount_fs.mount('dir2', fs2) mount_fs.mount('dir2', fs2)
mount_fs.copydir('dir1', 'dir2', ignore_errors=ignore_errors, chunk_size=chunk_size) mount_fs.copydir('dir1', 'dir2', ignore_errors=ignore_errors, chunk_size=chunk_size)
def countbytes(count_fs):
def countbytes(fs):
"""Returns the total number of bytes contained within files in a filesystem. """Returns the total number of bytes contained within files in a filesystem.
count_fs -- A filesystem object fs -- A filesystem object
""" """
total = sum(count_fs.getsize(f) for f in count_fs.walkfiles()) total = sum(fs.getsize(f) for f in fs.walkfiles())
return total return total
fs/wrapfs.py 0 → 100644
"""
fs.wrapfs: class for wrapping an existing FS object with added functionality
This module provides the class WrapFS, a base class for objects that wrap
another FS object and provide some transformation of its contents. It could
be very useful for implementing e.g. transparent encryption or compression
services.
As a simple example of how this class could be used, the 'HideDotFiles' class
implements the standard unix shell functionality of hiding dot files in
directory listings.
"""
from fs.base import FS
class WrapFS(FS):
"""FS that wraps another FS, providing translation etc.
This class allows simple transforms to be applied to the names
and/or contents of files in an FS. It could be used to implement
e.g. compression or encryption in a relatively painless manner.
The following methods can be overridden to control how files are
accessed in the underlying FS object:
_file_wrap(file,mode): called for each file that is opened from
the underlying FS; may return a modified
file-like object.
_encode(path): encode a path for access in the underlying FS
_decode(path): decode a path from the underlying FS
If the required path translation proceeds one component at a time,
it may be simpler to override the _encode_name() and _decode_name()
methods.
"""
def __init__(self,fs):
super(WrapFS,self).__init__()
self.wrapped_fs = fs
def _file_wrap(self,f,mode):
"""Apply wrapping to an opened file."""
return f
def _encode_name(self,name):
"""Encode path component for the underlying FS."""
return name
def _decode_name(self,name):
"""Decode path component from the underlying FS."""
return name
def _encode(self,path):
"""Encode path for the underlying FS."""
names = path.split("/")
e_names = []
for name in names:
if name == "":
e_names.append("")
else:
e_names.append(self._encode_name(name))
return "/".join(e_names)
def _decode(self,path):
"""Decode path from the underlying FS."""
names = path.split("/")
d_names = []
for name in names:
if name == "":
d_names.append("")
else:
d_names.append(self._decode_name(name))
return "/".join(d_names)
def _adjust_mode(self,mode):
"""Adjust the mode used to open a file in the underlying FS.
This method takes the mode given when opening a file, and should
return a two-tuple giving the mode to be used in this FS as first
item, and the mode to be used in the underlying FS as the second.
An example of why this is needed is a WrapFS subclass that does
transparent file compression - in this case files from the wrapped
FS cannot be opened in append mode.
"""
return (mode,mode)
def getsyspath(self,path,allow_none=False):
return self.wrapped_fs.getsyspath(self._encode(path),allow_none)
def hassyspath(self,path):
return self.wrapped_fs.hassyspath(self._encode(path))
def open(self,path,mode="r"):
(mode,wmode) = self._adjust_mode(mode)
f = self.wrapped_fs.open(self._encode(path),wmode)
return self._file_wrap(f,mode)
def exists(self,path):
return self.wrapped_fs.exists(self._encode(path))
def isdir(self,path):
return self.wrapped_fs.isdir(self._encode(path))
def isfile(self,path):
return self.wrapped_fs.isfile(self._encode(path))
def listdir(self,path="",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
entries = []
for name in self.wrapped_fs.listdir(self._encode(path),wildcard=None,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only):
entries.append(self._decode(name))
return self._listdir_helper(path,entries,wildcard=wildcard,full=False,absolute=False,dirs_only=False,files_only=False)
def makedir(self,path,*args,**kwds):
return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)
def remove(self,path):
return self.wrapped_fs.remove(self._encode(path))
def removedir(self,path,*args,**kwds):
return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)
def rename(self,src,dst):
return self.wrapped_fs.rename(self._encode(src),self._encode(dst))
def getinfo(self,path):
return self.wrapped_fs.getinfo(self._encode(path))
def desc(self,path):
return self.wrapped_fs.desc(self._encode(path))
def copy(self,src,dst,overwrite=False,chunk_size=16384):
return self.wrapped_fs.copy(self._encode(src),self._encode(dst),overwrite,chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384):
return self.wrapped_fs.move(self._encode(src),self._encode(dst),overwrite,chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384):
return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),overwrite,ignore_errors,chunk_size)
def __getattr__(self,attr):
return getattr(self.wrapped_fs,attr)
def close(self):
if hasattr(self.wrapped_fs,"close"):
self.wrapped_fs.close()
class HideDotFiles(WrapFS):
"""FS wrapper class that hides dot-files in directory listings.
The listdir() function takes an extra keyword argument 'hidden'
indicating whether hidden dotfiles shoud be included in the output.
It is False by default.
"""
def is_hidden(self,path):
"""Check whether the given path should be hidden."""
return path and basename(path)[0] == "."
def _encode(self,path):
return path
def _decode(self,path):
return path
def listdir(self,path="",**kwds):
hidden = kwds.pop("hidden",True)
entries = self.wrapped_fs.listdir(path,**kwds)
if not hidden:
entries = [e for e in entries if not self.is_hidden(e)]
return entries
def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",hidden=False):
if search == "breadth":
dirs = [path]
while dirs:
current_path = dirs.pop()
paths = []
for filename in self.listdir(current_path,hidden=hidden):
path = pathjoin(current_path, filename)
if self.isdir(path):
if dir_wildcard is not None:
if fnmatch.fnmatch(path, dir_wilcard):
dirs.append(path)
else:
dirs.append(path)
else:
if wildcard is not None:
if fnmatch.fnmatch(path, wildcard):
paths.append(filename)
else:
paths.append(filename)
yield (current_path, paths)
elif search == "depth":
def recurse(recurse_path):
for path in self.listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True,hidden=hidden):
for p in recurse(path):
yield p
yield (recurse_path, self.listdir(recurse_path, wildcard=wildcard, files_only=True,hidden=hidden))
for p in recurse(path):
yield p
else:
raise ValueError("Search should be 'breadth' or 'depth'")
def isdirempty(self, path):
path = normpath(path)
iter_dir = iter(self.listdir(path,hidden=True))
try:
iter_dir.next()
except StopIteration:
return True
return False
fs/xattrs.py 0 → 100644
"""
fs.xattrs: extended attribute support for FS
This module defines a standard interface for FS subclasses that want to
support extended file attributes, and a WrapFS subclass that can simulate
extended attributes on top of an ordinery FS.
FS instances offering extended attribute support must provide the following
methods:
getxattr(path,name) - get the named attribute for the given path,
or None if it does not exist
setxattr(path,name,value) - set the named attribute for the given path
to the given value
delxattr(path,name) - delete the named attribute for the given path,
raising KeyError if it does not exist
listxattrs(path) - iterator over all stored attribute names for
the given path
If extended attributes are required by FS-consuming code, it should use the
function 'ensure_xattrs'. This will interrogate an FS object to determine
if it has native xattr support, and return a wrapped version if it does not.
"""
try:
import cPickle as pickle
except ImportError:
import pickle
from fs.path import *
from fs.errors import *
from fs.wrapfs import WrapFS
def ensure_xattrs(fs):
"""Ensure that the given FS supports xattrs, simulating them if required.
Given an FS object, this function returns an equivalent FS that has support
for extended attributes. This may be the original object if they are
supported natively, or a wrapper class is they must be simulated.
"""
try:
# This attr doesn't have to exist, None should be returned by default
fs.getxattr("/","testingx-xattr")
return fs
except Exception:
return SimulateXAttr(fs)
class SimulateXAttr(WrapFS):
"""FS wrapper class that simulates xattr support.
The following methods are supplied for manipulating extended attributes:
* xattrs: list all extended attribute names for a path
* getxattr: get an xattr of a path by name
* setxattr: set an xattr of a path by name
* delxattr: delete an xattr of a path by name
For each file in the underlying FS, this class maintains a corresponding
'.xattrs.FILENAME' file containing its extended attributes. Extended
attributes of a directory are stored in the file '.xattrs' within the
directory itself.
"""
def _get_attr_path(self, path):
"""Get the path of the file containing xattrs for the given path."""
if self.wrapped_fs.isdir(path):
attr_path = pathjoin(path, '.xattrs')
else:
dir_path, file_name = pathsplit(path)
attr_path = pathjoin(dir_path, '.xattrs.'+file_name)
return attr_path
def _is_attr_path(self, path):
"""Check whether the given path references an xattrs file."""
_,name = pathsplit(path)
if name.startswith(".xattrs"):
return True
return False
def _get_attr_dict(self, path):
"""Retrieve the xattr dictionary for the given path."""
attr_path = self._get_attr_path(path)
if self.wrapped_fs.exists(attr_path):
return pickle.loads(self.wrapped_fs.getcontents(attr_path))
else:
return {}
def _set_attr_dict(self, path, attrs):
"""Store the xattr dictionary for the given path."""
attr_path = self._get_attr_path(path)
self.wrapped_fs.setcontents(attr_path, pickle.dumps(attrs))
def setxattr(self, path, key, value):
"""Set an extended attribute on the given path."""
if not self.exists(path):
raise ResourceNotFoundError(path)
attrs = self._get_attr_dict(path)
attrs[key] = str(value)
self._set_attr_dict(path, attrs)
def getxattr(self, path, key, default=None):
"""Retrieve an extended attribute for the given path."""
if not self.exists(path):
raise ResourceNotFoundError(path)
attrs = self._get_attr_dict(path)
return attrs.get(key, default)
def delxattr(self, path, key):
if not self.exists(path):
raise ResourceNotFoundError(path)
attrs = self._get_attr_dict(path)
try:
del attrs[key]
except KeyError:
pass
self._set_attr_dict(path, attrs)
def listxattrs(self,path):
"""List all the extended attribute keys set on the given path."""
if not self.exists(path):
raise ResourceNotFoundError(path)
return self._get_attr_dict(path).keys()
def _encode(self,path):
"""Prevent requests for operations on .xattr files."""
if self._is_attr_path(path):
raise PathError(path,msg="Paths cannot contain '.xattrs': %(path)s")
return path
def _decode(self,path):
return path
def listdir(self,path="",**kwds):
"""Prevent .xattr from appearing in listings."""
entries = self.wrapped_fs.listdir(path,**kwds)
return [e for e in entries if not self._is_attr_path(e)]
def copy(self,src,dst,**kwds):
"""Ensure xattrs are copied when copying a file."""
self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)
s_attr_file = self._get_attr_path(src)
d_attr_file = self._get_attr_path(dst)
try:
self.wrapped_fs.copy(s_attr_file,d_attr_file,overwrite=True)
except ResourceNotFoundError,e:
pass
def move(self,src,dst,**kwds):
"""Ensure xattrs are preserved when moving a file."""
self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)
s_attr_file = self._get_attr_path(src)
d_attr_file = self._get_attr_path(dst)
try:
self.wrapped_fs.move(s_attr_file,d_attr_file,overwrite=True)
except ResourceNotFoundError:
pass
fs/zipfs.py
#!/usr/bin/env python #!/usr/bin/env python
from base import * from fs.base import *
from helpers import *
from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
from memoryfs import MemoryFS from memoryfs import MemoryFS
...@@ -50,17 +49,17 @@ class ZipFS(FS): ...@@ -50,17 +49,17 @@ class ZipFS(FS):
"""A FileSystem that represents a zip file.""" """A FileSystem that represents a zip file."""
def __init__(self, zip_file, mode="r", compression="deflated", allowZip64=False, thread_syncronize=True): def __init__(self, zip_file, mode="r", compression="deflated", allowZip64=False, thread_synchronize=True):
"""Create a FS that maps on to a zip file. """Create a FS that maps on to a zip file.
zip_file -- A (system) path, or a file-like object zip_file -- A (system) path, or a file-like object
mode -- Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending mode -- Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
compression -- Can be 'deflated' (default) to compress data or 'stored' to just store date compression -- Can be 'deflated' (default) to compress data or 'stored' to just store date
allowZip64 -- Set to True to use zip files greater than 2 MB, default is False allowZip64 -- Set to True to use zip files greater than 2 MB, default is False
thread_syncronize -- Set to True (default) to enable thread-safety thread_synchronize -- Set to True (default) to enable thread-safety
""" """
FS.__init__(self, thread_syncronize=thread_syncronize) FS.__init__(self, thread_synchronize=thread_synchronize)
if compression == "deflated": if compression == "deflated":
compression_type = ZIP_DEFLATED compression_type = ZIP_DEFLATED
elif compression == "stored": elif compression == "stored":
...@@ -75,7 +74,7 @@ class ZipFS(FS): ...@@ -75,7 +74,7 @@ class ZipFS(FS):
try: try:
self.zf = ZipFile(zip_file, mode, compression_type, allowZip64) self.zf = ZipFile(zip_file, mode, compression_type, allowZip64)
except IOError: except IOError:
raise ResourceNotFoundError("NO_FILE", str(zip_file), msg="Zip file does not exist: %(path)s") raise ResourceNotFoundError(str(zip_file), msg="Zip file does not exist: %(path)s")
self.zip_path = str(zip_file) self.zip_path = str(zip_file)
self.temp_fs = None self.temp_fs = None
...@@ -129,11 +128,11 @@ class ZipFS(FS): ...@@ -129,11 +128,11 @@ class ZipFS(FS):
if 'r' in mode: if 'r' in mode:
if self.zip_mode not in 'ra': if self.zip_mode not in 'ra':
raise OperationFailedError("OPEN_FAILED", path, msg="Zip file must be opened for reading ('r') or appending ('a')") raise OperationFailedError("open file", path=path, msg="Zip file must be opened for reading ('r') or appending ('a')")
try: try:
contents = self.zf.read(path) contents = self.zf.read(path)
except KeyError: except KeyError:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
return StringIO(contents) return StringIO(contents)
if 'w' in mode: if 'w' in mode:
...@@ -154,14 +153,14 @@ class ZipFS(FS): ...@@ -154,14 +153,14 @@ class ZipFS(FS):
self._lock.acquire() self._lock.acquire()
try: try:
if not self.exists(path): if not self.exists(path):
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
path = normpath(path) path = normpath(path)
try: try:
contents = self.zf.read(path) contents = self.zf.read(path)
except KeyError: except KeyError:
raise ResourceNotFoundError("NO_FILE", path) raise ResourceNotFoundError(path)
except RuntimeError: except RuntimeError:
raise OperationFailedError("READ_FAILED", path, "Zip file must be oppened with 'r' or 'a' to read") raise OperationFailedError("read file", path=path, msg="Zip file must be oppened with 'r' or 'a' to read")
return contents return contents
finally: finally:
self._lock.release() self._lock.release()
...@@ -194,23 +193,23 @@ class ZipFS(FS): ...@@ -194,23 +193,23 @@ class ZipFS(FS):
try: try:
dirname = normpath(dirname) dirname = normpath(dirname)
if self.zip_mode not in "wa": if self.zip_mode not in "wa":
raise OperationFailedError("MAKEDIR_FAILED", dirname, "Zip file must be opened for writing ('w') or appending ('a')") raise OperationFailedError("create directory", path=dirname, msg="Zip file must be opened for writing ('w') or appending ('a')")
if not dirname.endswith('/'): if not dirname.endswith('/'):
dirname += '/' dirname += '/'
self._add_resource(dirname) self._add_resource(dirname)
finally: finally:
self._lock.release() self._lock.release()
def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=True, dirs_only=False, files_only=False): def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
return self._path_fs.listdir(path, wildcard, full, absolute, hidden, dirs_only, files_only) return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)
def getinfo(self, path): def getinfo(self, path):
self._lock.acquire() self._lock.acquire()
try: try:
if not self.exists(path): if not self.exists(path):
return ResourceNotFoundError("NO_RESOURCE", path) return ResourceNotFoundError(path)
path = normpath(path).lstrip('/') path = normpath(path).lstrip('/')
try: try:
zi = self.zf.getinfo(path) zi = self.zf.getinfo(path)
...@@ -225,46 +224,4 @@ class ZipFS(FS): ...@@ -225,46 +224,4 @@ class ZipFS(FS):
finally: finally:
self._lock.release() self._lock.release()
if __name__ == "__main__":
def test():
zfs = ZipFS("t.zip", "w")
zfs.createfile("t.txt", "Hello, World!")
zfs.close()
rfs = ZipFS("t.zip", 'r')
print rfs.getcontents("t.txt")
print rfs.getcontents("w.txt")
def test2():
zfs = ZipFS("t2.zip", "r")
print zfs.listdir("/tagging-trunk")
print zfs.listdir("/")
import browsewin
browsewin.browse(zfs)
zfs.close()
#zfs.open("t.txt")
#print zfs.listdir("/")
test2()
zfs = ZipFS("t3.zip", "w")
zfs.createfile("t.txt", "Hello, World!")
zfs.createfile("foo/bar/baz/t.txt", "Hello, World!")
print zfs.getcontents('t.txt')
#print zfs.isdir("t.txt")
#print zfs.isfile("t.txt")
#print zfs.isfile("foo/bar")
zfs.close()
zfs = ZipFS("t3.zip", "r")
print "--"
print zfs.listdir("foo")
print zfs.isdir("foo/bar")
print zfs.listdir("foo/bar")
print zfs.listdir("foo/bar/baz")
print_fs(zfs)
#zfs = ZipFS("t3.zip", "r")
#print zfs.zf.getinfo("asd.txt")
#zfs.close()
setup.py
...@@ -23,6 +23,6 @@ setup(name='fs', ...@@ -23,6 +23,6 @@ setup(name='fs',
url="http://code.google.com/p/pyfilesystem/", url="http://code.google.com/p/pyfilesystem/",
download_url="http://code.google.com/p/pyfilesystem/downloads/list", download_url="http://code.google.com/p/pyfilesystem/downloads/list",
platforms = ['any'], platforms = ['any'],
packages=['fs'], packages=['fs','fs.expose','fs.expose.fuse','fs.tests'],
classifiers=classifiers, classifiers=classifiers,
) )
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment