Skip to content

P
pyfs
Commit 3b900b1c by willmcgugan
Options

Python3 compatibility

parent 754fcc98
Showing with 1460 additions and 93 deletions
fs/expose/fuse/__init__.py
......@@ -66,8 +66,13 @@ from fs.errors import *
from fs.path import *
from fs.local_functools import wraps
from six import PY3
try:
import fuse_ctypes as fuse
if PY3:
import fuse3 as fuse
else:
import fuse
except NotImplementedError:
raise ImportError("FUSE found but not usable")
try:
......@@ -178,7 +183,7 @@ class FSOperations(Operations):
# FUSE doesn't seem to pass correct mode information here - at least,
# I haven't figured out how to distinguish between "w" and "w+".
# Go with the most permissive option.
fh = self._reg_file(self.fs.open(path,"w+"),path)
fh = self._reg_file(self.fs.open(path,"wb+"),path)
fi.fh = fh
fi.keep_cache = 0
......@@ -318,10 +323,10 @@ class FSOperations(Operations):
def truncate(self, path, length, fh=None):
path = path.decode(NATIVE_ENCODING)
if fh is None and length == 0:
self.fs.open(path,"w").close()
self.fs.open(path,"wb").close()
else:
if fh is None:
f = self.fs.open(path,"r+")
f = self.fs.open(path,"rb+")
if not hasattr(f,"truncate"):
raise UnsupportedError("truncate")
f.truncate(length)
......@@ -544,7 +549,7 @@ class MountProcess(subprocess.Popen):
os.close(w)
if os.read(r,1) != "S":
self.terminate()
raise RuntimeError("FUSE error: " + os.read(r,20)).decode(NATIVE_ENCODING)
raise RuntimeError("FUSE error: " + os.read(r,20).decode(NATIVE_ENCODING))
def unmount(self):
"""Cleanly unmount the FUSE filesystem, terminating this subprocess."""
......
fs/expose/fuse/fuse.py 0 → 100644
# Copyright (c) 2008 Giorgos Verigakis <verigak@gmail.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
from __future__ import division
from ctypes import *
from ctypes.util import find_library
from errno import *
from functools import partial
from os import strerror
from platform import machine, system
from stat import S_IFDIR
from traceback import print_exc
_system = system()
_machine = machine()
if _system == 'Darwin':
_libfuse_path = find_library('fuse4x') or find_library('fuse')
else:
_libfuse_path = find_library('fuse')
if not _libfuse_path:
raise EnvironmentError('Unable to find libfuse')
if _system == 'Darwin':
_libiconv = CDLL(find_library('iconv'), RTLD_GLOBAL) # libfuse dependency
_libfuse = CDLL(_libfuse_path)
if _system == 'Darwin' and hasattr(_libfuse, 'macfuse_version'):
_system = 'Darwin-MacFuse'
class c_timespec(Structure):
_fields_ = [('tv_sec', c_long), ('tv_nsec', c_long)]
class c_utimbuf(Structure):
_fields_ = [('actime', c_timespec), ('modtime', c_timespec)]
class c_stat(Structure):
pass # Platform dependent
if _system in ('Darwin', 'Darwin-MacFuse', 'FreeBSD'):
ENOTSUP = 45
c_dev_t = c_int32
c_fsblkcnt_t = c_ulong
c_fsfilcnt_t = c_ulong
c_gid_t = c_uint32
c_mode_t = c_uint16
c_off_t = c_int64
c_pid_t = c_int32
c_uid_t = c_uint32
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte),
c_size_t, c_int, c_uint32)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte),
c_size_t, c_uint32)
if _system == 'Darwin':
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_mode', c_mode_t),
('st_nlink', c_uint16),
('st_ino', c_uint64),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec),
('st_birthtimespec', c_timespec),
('st_size', c_off_t),
('st_blocks', c_int64),
('st_blksize', c_int32),
('st_flags', c_int32),
('st_gen', c_int32),
('st_lspare', c_int32),
('st_qspare', c_int64)]
else:
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_uint32),
('st_mode', c_mode_t),
('st_nlink', c_uint16),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec),
('st_size', c_off_t),
('st_blocks', c_int64),
('st_blksize', c_int32)]
elif _system == 'Linux':
ENOTSUP = 95
c_dev_t = c_ulonglong
c_fsblkcnt_t = c_ulonglong
c_fsfilcnt_t = c_ulonglong
c_gid_t = c_uint
c_mode_t = c_uint
c_off_t = c_longlong
c_pid_t = c_int
c_uid_t = c_uint
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t)
if _machine == 'x86_64':
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_ulong),
('st_nlink', c_ulong),
('st_mode', c_mode_t),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('__pad0', c_int),
('st_rdev', c_dev_t),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_long),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec)]
elif _machine == 'ppc':
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_ulonglong),
('st_mode', c_mode_t),
('st_nlink', c_uint),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('__pad2', c_ushort),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_longlong),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec)]
else:
# i686, use as fallback for everything else
c_stat._fields_ = [
('st_dev', c_dev_t),
('__pad1', c_ushort),
('__st_ino', c_ulong),
('st_mode', c_mode_t),
('st_nlink', c_uint),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('__pad2', c_ushort),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_longlong),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec),
('st_ino', c_ulonglong)]
else:
raise NotImplementedError('%s is not supported.' % _system)
class c_statvfs(Structure):
_fields_ = [
('f_bsize', c_ulong),
('f_frsize', c_ulong),
('f_blocks', c_fsblkcnt_t),
('f_bfree', c_fsblkcnt_t),
('f_bavail', c_fsblkcnt_t),
('f_files', c_fsfilcnt_t),
('f_ffree', c_fsfilcnt_t),
('f_favail', c_fsfilcnt_t)]
if _system == 'FreeBSD':
c_fsblkcnt_t = c_uint64
c_fsfilcnt_t = c_uint64
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t)
class c_statvfs(Structure):
_fields_ = [
('f_bavail', c_fsblkcnt_t),
('f_bfree', c_fsblkcnt_t),
('f_blocks', c_fsblkcnt_t),
('f_favail', c_fsfilcnt_t),
('f_ffree', c_fsfilcnt_t),
('f_files', c_fsfilcnt_t),
('f_bsize', c_ulong),
('f_flag', c_ulong),
('f_frsize', c_ulong)]
class fuse_file_info(Structure):
_fields_ = [
('flags', c_int),
('fh_old', c_ulong),
('writepage', c_int),
('direct_io', c_uint, 1),
('keep_cache', c_uint, 1),
('flush', c_uint, 1),
('padding', c_uint, 29),
('fh', c_uint64),
('lock_owner', c_uint64)]
class fuse_context(Structure):
_fields_ = [
('fuse', c_voidp),
('uid', c_uid_t),
('gid', c_gid_t),
('pid', c_pid_t),
('private_data', c_voidp)]
_libfuse.fuse_get_context.restype = POINTER(fuse_context)
class fuse_operations(Structure):
_fields_ = [
('getattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat))),
('readlink', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)),
('getdir', c_voidp), # Deprecated, use readdir
('mknod', CFUNCTYPE(c_int, c_char_p, c_mode_t, c_dev_t)),
('mkdir', CFUNCTYPE(c_int, c_char_p, c_mode_t)),
('unlink', CFUNCTYPE(c_int, c_char_p)),
('rmdir', CFUNCTYPE(c_int, c_char_p)),
('symlink', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('rename', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('link', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('chmod', CFUNCTYPE(c_int, c_char_p, c_mode_t)),
('chown', CFUNCTYPE(c_int, c_char_p, c_uid_t, c_gid_t)),
('truncate', CFUNCTYPE(c_int, c_char_p, c_off_t)),
('utime', c_voidp), # Deprecated, use utimens
('open', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('read', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, c_off_t,
POINTER(fuse_file_info))),
('write', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, c_off_t,
POINTER(fuse_file_info))),
('statfs', CFUNCTYPE(c_int, c_char_p, POINTER(c_statvfs))),
('flush', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('release', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('fsync', CFUNCTYPE(c_int, c_char_p, c_int, POINTER(fuse_file_info))),
('setxattr', setxattr_t),
('getxattr', getxattr_t),
('listxattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)),
('removexattr', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('opendir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('readdir', CFUNCTYPE(c_int, c_char_p, c_voidp, CFUNCTYPE(c_int, c_voidp,
c_char_p, POINTER(c_stat), c_off_t), c_off_t, POINTER(fuse_file_info))),
('releasedir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('fsyncdir', CFUNCTYPE(c_int, c_char_p, c_int, POINTER(fuse_file_info))),
('init', CFUNCTYPE(c_voidp, c_voidp)),
('destroy', CFUNCTYPE(c_voidp, c_voidp)),
('access', CFUNCTYPE(c_int, c_char_p, c_int)),
('create', CFUNCTYPE(c_int, c_char_p, c_mode_t, POINTER(fuse_file_info))),
('ftruncate', CFUNCTYPE(c_int, c_char_p, c_off_t, POINTER(fuse_file_info))),
('fgetattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat),
POINTER(fuse_file_info))),
('lock', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info), c_int, c_voidp)),
('utimens', CFUNCTYPE(c_int, c_char_p, POINTER(c_utimbuf))),
('bmap', CFUNCTYPE(c_int, c_char_p, c_size_t, POINTER(c_ulonglong)))]
def time_of_timespec(ts):
return ts.tv_sec + ts.tv_nsec / 10 ** 9
def set_st_attrs(st, attrs):
for key, val in attrs.items():
if key in ('st_atime', 'st_mtime', 'st_ctime'):
timespec = getattr(st, key + 'spec')
timespec.tv_sec = int(val)
timespec.tv_nsec = int((val - timespec.tv_sec) * 10 ** 9)
elif hasattr(st, key):
setattr(st, key, val)
def fuse_get_context():
"""Returns a (uid, gid, pid) tuple"""
ctxp = _libfuse.fuse_get_context()
ctx = ctxp.contents
return ctx.uid, ctx.gid, ctx.pid
class FuseOSError(OSError):
def __init__(self, errno):
super(FuseOSError, self).__init__(errno, strerror(errno))
class FUSE(object):
"""This class is the lower level interface and should not be subclassed
under normal use. Its methods are called by fuse.
Assumes API version 2.6 or later."""
def __init__(self, operations, mountpoint, raw_fi=False, **kwargs):
"""Setting raw_fi to True will cause FUSE to pass the fuse_file_info
class as is to Operations, instead of just the fh field.
This gives you access to direct_io, keep_cache, etc."""
self.operations = operations
self.raw_fi = raw_fi
args = ['fuse']
if kwargs.pop('foreground', False):
args.append('-f')
if kwargs.pop('debug', False):
args.append('-d')
if kwargs.pop('nothreads', False):
args.append('-s')
kwargs.setdefault('fsname', operations.__class__.__name__)
args.append('-o')
args.append(','.join(key if val == True else '%s=%s' % (key, val)
for key, val in kwargs.items()))
args.append(mountpoint)
argv = (c_char_p * len(args))(*args)
fuse_ops = fuse_operations()
for name, prototype in fuse_operations._fields_:
if prototype != c_voidp and getattr(operations, name, None):
op = partial(self._wrapper_, getattr(self, name))
setattr(fuse_ops, name, prototype(op))
err = _libfuse.fuse_main_real(len(args), argv, pointer(fuse_ops),
sizeof(fuse_ops), None)
del self.operations # Invoke the destructor
if err:
raise RuntimeError(err)
def _wrapper_(self, func, *args, **kwargs):
"""Decorator for the methods that follow"""
try:
return func(*args, **kwargs) or 0
except OSError, e:
return -(e.errno or EFAULT)
except:
print_exc()
return -EFAULT
def getattr(self, path, buf):
return self.fgetattr(path, buf, None)
def readlink(self, path, buf, bufsize):
ret = self.operations('readlink', path)
data = create_string_buffer(ret[:bufsize - 1])
memmove(buf, data, len(data))
return 0
def mknod(self, path, mode, dev):
return self.operations('mknod', path, mode, dev)
def mkdir(self, path, mode):
return self.operations('mkdir', path, mode)
def unlink(self, path):
return self.operations('unlink', path)
def rmdir(self, path):
return self.operations('rmdir', path)
def symlink(self, source, target):
return self.operations('symlink', target, source)
def rename(self, old, new):
return self.operations('rename', old, new)
def link(self, source, target):
return self.operations('link', target, source)
def chmod(self, path, mode):
return self.operations('chmod', path, mode)
def chown(self, path, uid, gid):
# Check if any of the arguments is a -1 that has overflowed
if c_uid_t(uid + 1).value == 0:
uid = -1
if c_gid_t(gid + 1).value == 0:
gid = -1
return self.operations('chown', path, uid, gid)
def truncate(self, path, length):
return self.operations('truncate', path, length)
def open(self, path, fip):
fi = fip.contents
if self.raw_fi:
return self.operations('open', path, fi)
else:
fi.fh = self.operations('open', path, fi.flags)
return 0
def read(self, path, buf, size, offset, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
ret = self.operations('read', path, size, offset, fh)
if not ret:
return 0
data = create_string_buffer(ret[:size], size)
memmove(buf, data, size)
return size
def write(self, path, buf, size, offset, fip):
data = string_at(buf, size)
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('write', path, data, offset, fh)
def statfs(self, path, buf):
stv = buf.contents
attrs = self.operations('statfs', path)
for key, val in attrs.items():
if hasattr(stv, key):
setattr(stv, key, val)
return 0
def flush(self, path, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('flush', path, fh)
def release(self, path, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('release', path, fh)
def fsync(self, path, datasync, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('fsync', path, datasync, fh)
def setxattr(self, path, name, value, size, options, *args):
data = string_at(value, size)
return self.operations('setxattr', path, name, data, options, *args)
def getxattr(self, path, name, value, size, *args):
ret = self.operations('getxattr', path, name, *args)
retsize = len(ret)
buf = create_string_buffer(ret, retsize) # Does not add trailing 0
if bool(value):
if retsize > size:
return -ERANGE
memmove(value, buf, retsize)
return retsize
def listxattr(self, path, namebuf, size):
ret = self.operations('listxattr', path)
buf = create_string_buffer('\x00'.join(ret)) if ret else ''
bufsize = len(buf)
if bool(namebuf):
if bufsize > size:
return -ERANGE
memmove(namebuf, buf, bufsize)
return bufsize
def removexattr(self, path, name):
return self.operations('removexattr', path, name)
def opendir(self, path, fip):
# Ignore raw_fi
fip.contents.fh = self.operations('opendir', path)
return 0
def readdir(self, path, buf, filler, offset, fip):
# Ignore raw_fi
for item in self.operations('readdir', path, fip.contents.fh):
if isinstance(item, str):
name, st, offset = item, None, 0
else:
name, attrs, offset = item
if attrs:
st = c_stat()
set_st_attrs(st, attrs)
else:
st = None
if filler(buf, name, st, offset) != 0:
break
return 0
def releasedir(self, path, fip):
# Ignore raw_fi
return self.operations('releasedir', path, fip.contents.fh)
def fsyncdir(self, path, datasync, fip):
# Ignore raw_fi
return self.operations('fsyncdir', path, datasync, fip.contents.fh)
def init(self, conn):
return self.operations('init', '/')
def destroy(self, private_data):
return self.operations('destroy', '/')
def access(self, path, amode):
return self.operations('access', path, amode)
def create(self, path, mode, fip):
fi = fip.contents
if self.raw_fi:
return self.operations('create', path, mode, fi)
else:
fi.fh = self.operations('create', path, mode)
return 0
def ftruncate(self, path, length, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('truncate', path, length, fh)
def fgetattr(self, path, buf, fip):
memset(buf, 0, sizeof(c_stat))
st = buf.contents
fh = fip and (fip.contents if self.raw_fi else fip.contents.fh)
attrs = self.operations('getattr', path, fh)
set_st_attrs(st, attrs)
return 0
def lock(self, path, fip, cmd, lock):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('lock', path, fh, cmd, lock)
def utimens(self, path, buf):
if buf:
atime = time_of_timespec(buf.contents.actime)
mtime = time_of_timespec(buf.contents.modtime)
times = (atime, mtime)
else:
times = None
return self.operations('utimens', path, times)
def bmap(self, path, blocksize, idx):
return self.operations('bmap', path, blocksize, idx)
class Operations(object):
"""This class should be subclassed and passed as an argument to FUSE on
initialization. All operations should raise a FuseOSError exception
on error.
When in doubt of what an operation should do, check the FUSE header
file or the corresponding system call man page."""
def __call__(self, op, *args):
if not hasattr(self, op):
raise FuseOSError(EFAULT)
return getattr(self, op)(*args)
def access(self, path, amode):
return 0
bmap = None
def chmod(self, path, mode):
raise FuseOSError(EROFS)
def chown(self, path, uid, gid):
raise FuseOSError(EROFS)
def create(self, path, mode, fi=None):
"""When raw_fi is False (default case), fi is None and create should
return a numerical file handle.
When raw_fi is True the file handle should be set directly by create
and return 0."""
raise FuseOSError(EROFS)
def destroy(self, path):
"""Called on filesystem destruction. Path is always /"""
pass
def flush(self, path, fh):
return 0
def fsync(self, path, datasync, fh):
return 0
def fsyncdir(self, path, datasync, fh):
return 0
def getattr(self, path, fh=None):
"""Returns a dictionary with keys identical to the stat C structure
of stat(2).
st_atime, st_mtime and st_ctime should be floats.
NOTE: There is an incombatibility between Linux and Mac OS X concerning
st_nlink of directories. Mac OS X counts all files inside the directory,
while Linux counts only the subdirectories."""
if path != '/':
raise FuseOSError(ENOENT)
return dict(st_mode=(S_IFDIR | 0755), st_nlink=2)
def getxattr(self, path, name, position=0):
raise FuseOSError(ENOTSUP)
def init(self, path):
"""Called on filesystem initialization. Path is always /
Use it instead of __init__ if you start threads on initialization."""
pass
def link(self, target, source):
raise FuseOSError(EROFS)
def listxattr(self, path):
return []
lock = None
def mkdir(self, path, mode):
raise FuseOSError(EROFS)
def mknod(self, path, mode, dev):
raise FuseOSError(EROFS)
def open(self, path, flags):
"""When raw_fi is False (default case), open should return a numerical
file handle.
When raw_fi is True the signature of open becomes:
open(self, path, fi)
and the file handle should be set directly."""
return 0
def opendir(self, path):
"""Returns a numerical file handle."""
return 0
def read(self, path, size, offset, fh):
"""Returns a string containing the data requested."""
raise FuseOSError(EIO)
def readdir(self, path, fh):
"""Can return either a list of names, or a list of (name, attrs, offset)
tuples. attrs is a dict as in getattr."""
return ['.', '..']
def readlink(self, path):
raise FuseOSError(ENOENT)
def release(self, path, fh):
return 0
def releasedir(self, path, fh):
return 0
def removexattr(self, path, name):
raise FuseOSError(ENOTSUP)
def rename(self, old, new):
raise FuseOSError(EROFS)
def rmdir(self, path):
raise FuseOSError(EROFS)
def setxattr(self, path, name, value, options, position=0):
raise FuseOSError(ENOTSUP)
def statfs(self, path):
"""Returns a dictionary with keys identical to the statvfs C structure
of statvfs(3).
On Mac OS X f_bsize and f_frsize must be a power of 2 (minimum 512)."""
return {}
def symlink(self, target, source):
raise FuseOSError(EROFS)
def truncate(self, path, length, fh=None):
raise FuseOSError(EROFS)
def unlink(self, path):
raise FuseOSError(EROFS)
def utimens(self, path, times=None):
"""Times is a (atime, mtime) tuple. If None use current time."""
return 0
def write(self, path, data, offset, fh):
raise FuseOSError(EROFS)
class LoggingMixIn:
def __call__(self, op, path, *args):
print '->', op, path, repr(args)
ret = '[Unhandled Exception]'
try:
ret = getattr(self, op)(path, *args)
return ret
except OSError, e:
ret = str(e)
raise
finally:
print '<-', op, repr(ret)
fs/expose/fuse/fuse3.py 0 → 100644
# Copyright (c) 2008 Giorgos Verigakis <verigak@gmail.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
from ctypes import *
from ctypes.util import find_library
from errno import *
from functools import partial
from platform import machine, system
from stat import S_IFDIR
from traceback import print_exc
import logging
class c_timespec(Structure):
_fields_ = [('tv_sec', c_long), ('tv_nsec', c_long)]
class c_utimbuf(Structure):
_fields_ = [('actime', c_timespec), ('modtime', c_timespec)]
class c_stat(Structure):
pass # Platform dependent
_system = system()
if _system in ('Darwin', 'FreeBSD'):
_libiconv = CDLL(find_library("iconv"), RTLD_GLOBAL) # libfuse dependency
ENOTSUP = 45
c_dev_t = c_int32
c_fsblkcnt_t = c_ulong
c_fsfilcnt_t = c_ulong
c_gid_t = c_uint32
c_mode_t = c_uint16
c_off_t = c_int64
c_pid_t = c_int32
c_uid_t = c_uint32
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte),
c_size_t, c_int, c_uint32)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte),
c_size_t, c_uint32)
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_uint32),
('st_mode', c_mode_t),
('st_nlink', c_uint16),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec),
('st_size', c_off_t),
('st_blocks', c_int64),
('st_blksize', c_int32)]
elif _system == 'Linux':
ENOTSUP = 95
c_dev_t = c_ulonglong
c_fsblkcnt_t = c_ulonglong
c_fsfilcnt_t = c_ulonglong
c_gid_t = c_uint
c_mode_t = c_uint
c_off_t = c_longlong
c_pid_t = c_int
c_uid_t = c_uint
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t)
_machine = machine()
if _machine == 'x86_64':
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_ulong),
('st_nlink', c_ulong),
('st_mode', c_mode_t),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('__pad0', c_int),
('st_rdev', c_dev_t),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_long),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec)]
elif _machine == 'ppc':
c_stat._fields_ = [
('st_dev', c_dev_t),
('st_ino', c_ulonglong),
('st_mode', c_mode_t),
('st_nlink', c_uint),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('__pad2', c_ushort),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_longlong),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec)]
else:
# i686, use as fallback for everything else
c_stat._fields_ = [
('st_dev', c_dev_t),
('__pad1', c_ushort),
('__st_ino', c_ulong),
('st_mode', c_mode_t),
('st_nlink', c_uint),
('st_uid', c_uid_t),
('st_gid', c_gid_t),
('st_rdev', c_dev_t),
('__pad2', c_ushort),
('st_size', c_off_t),
('st_blksize', c_long),
('st_blocks', c_longlong),
('st_atimespec', c_timespec),
('st_mtimespec', c_timespec),
('st_ctimespec', c_timespec),
('st_ino', c_ulonglong)]
else:
raise NotImplementedError('%s is not supported.' % _system)
class c_statvfs(Structure):
_fields_ = [
('f_bsize', c_ulong),
('f_frsize', c_ulong),
('f_blocks', c_fsblkcnt_t),
('f_bfree', c_fsblkcnt_t),
('f_bavail', c_fsblkcnt_t),
('f_files', c_fsfilcnt_t),
('f_ffree', c_fsfilcnt_t),
('f_favail', c_fsfilcnt_t)]
if _system == 'FreeBSD':
c_fsblkcnt_t = c_uint64
c_fsfilcnt_t = c_uint64
setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int)
getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t)
class c_statvfs(Structure):
_fields_ = [
('f_bavail', c_fsblkcnt_t),
('f_bfree', c_fsblkcnt_t),
('f_blocks', c_fsblkcnt_t),
('f_favail', c_fsfilcnt_t),
('f_ffree', c_fsfilcnt_t),
('f_files', c_fsfilcnt_t),
('f_bsize', c_ulong),
('f_flag', c_ulong),
('f_frsize', c_ulong)]
class fuse_file_info(Structure):
_fields_ = [
('flags', c_int),
('fh_old', c_ulong),
('writepage', c_int),
('direct_io', c_uint, 1),
('keep_cache', c_uint, 1),
('flush', c_uint, 1),
('padding', c_uint, 29),
('fh', c_uint64),
('lock_owner', c_uint64)]
class fuse_context(Structure):
_fields_ = [
('fuse', c_voidp),
('uid', c_uid_t),
('gid', c_gid_t),
('pid', c_pid_t),
('private_data', c_voidp)]
class fuse_operations(Structure):
_fields_ = [
('getattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat))),
('readlink', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)),
('getdir', c_voidp), # Deprecated, use readdir
('mknod', CFUNCTYPE(c_int, c_char_p, c_mode_t, c_dev_t)),
('mkdir', CFUNCTYPE(c_int, c_char_p, c_mode_t)),
('unlink', CFUNCTYPE(c_int, c_char_p)),
('rmdir', CFUNCTYPE(c_int, c_char_p)),
('symlink', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('rename', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('link', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('chmod', CFUNCTYPE(c_int, c_char_p, c_mode_t)),
('chown', CFUNCTYPE(c_int, c_char_p, c_uid_t, c_gid_t)),
('truncate', CFUNCTYPE(c_int, c_char_p, c_off_t)),
('utime', c_voidp), # Deprecated, use utimens
('open', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('read', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, c_off_t,
POINTER(fuse_file_info))),
('write', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, c_off_t,
POINTER(fuse_file_info))),
('statfs', CFUNCTYPE(c_int, c_char_p, POINTER(c_statvfs))),
('flush', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('release', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('fsync', CFUNCTYPE(c_int, c_char_p, c_int, POINTER(fuse_file_info))),
('setxattr', setxattr_t),
('getxattr', getxattr_t),
('listxattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)),
('removexattr', CFUNCTYPE(c_int, c_char_p, c_char_p)),
('opendir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('readdir', CFUNCTYPE(c_int, c_char_p, c_voidp, CFUNCTYPE(c_int, c_voidp,
c_char_p, POINTER(c_stat), c_off_t), c_off_t, POINTER(fuse_file_info))),
('releasedir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))),
('fsyncdir', CFUNCTYPE(c_int, c_char_p, c_int, POINTER(fuse_file_info))),
('init', CFUNCTYPE(c_voidp, c_voidp)),
('destroy', CFUNCTYPE(c_voidp, c_voidp)),
('access', CFUNCTYPE(c_int, c_char_p, c_int)),
('create', CFUNCTYPE(c_int, c_char_p, c_mode_t, POINTER(fuse_file_info))),
('ftruncate', CFUNCTYPE(c_int, c_char_p, c_off_t, POINTER(fuse_file_info))),
('fgetattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat),
POINTER(fuse_file_info))),
('lock', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info), c_int, c_voidp)),
('utimens', CFUNCTYPE(c_int, c_char_p, POINTER(c_utimbuf))),
('bmap', CFUNCTYPE(c_int, c_char_p, c_size_t, POINTER(c_ulonglong)))]
def time_of_timespec(ts):
return ts.tv_sec + ts.tv_nsec / 10 ** 9
def set_st_attrs(st, attrs):
for key, val in attrs.items():
if key in ('st_atime', 'st_mtime', 'st_ctime'):
timespec = getattr(st, key + 'spec')
timespec.tv_sec = int(val)
timespec.tv_nsec = int((val - timespec.tv_sec) * 10 ** 9)
elif hasattr(st, key):
setattr(st, key, val)
_libfuse_path = find_library('fuse')
if not _libfuse_path:
raise EnvironmentError('Unable to find libfuse')
_libfuse = CDLL(_libfuse_path)
_libfuse.fuse_get_context.restype = POINTER(fuse_context)
def fuse_get_context():
"""Returns a (uid, gid, pid) tuple"""
ctxp = _libfuse.fuse_get_context()
ctx = ctxp.contents
return ctx.uid, ctx.gid, ctx.pid
class FUSE(object):
"""This class is the lower level interface and should not be subclassed
under normal use. Its methods are called by fuse.
Assumes API version 2.6 or later."""
def __init__(self, operations, mountpoint, raw_fi=False, **kwargs):
"""Setting raw_fi to True will cause FUSE to pass the fuse_file_info
class as is to Operations, instead of just the fh field.
This gives you access to direct_io, keep_cache, etc."""
self.operations = operations
self.raw_fi = raw_fi
args = ['fuse']
if kwargs.pop('foreground', False):
args.append('-f')
if kwargs.pop('debug', False):
args.append('-d')
if kwargs.pop('nothreads', False):
args.append('-s')
kwargs.setdefault('fsname', operations.__class__.__name__)
args.append('-o')
args.append(','.join(key if val == True else '%s=%s' % (key, val)
for key, val in kwargs.items()))
args.append(mountpoint)
argv = (c_char_p * len(args))(*args)
fuse_ops = fuse_operations()
for name, prototype in fuse_operations._fields_:
if prototype != c_voidp and getattr(operations, name, None):
op = partial(self._wrapper_, getattr(self, name))
setattr(fuse_ops, name, prototype(op))
_libfuse.fuse_main_real(len(args), argv, pointer(fuse_ops),
sizeof(fuse_ops), None)
del self.operations # Invoke the destructor
def _wrapper_(self, func, *args, **kwargs):
"""Decorator for the methods that follow"""
try:
return func(*args, **kwargs) or 0
except OSError as e:
return -(e.errno or EFAULT)
except:
print_exc()
return -EFAULT
def getattr(self, path, buf):
return self.fgetattr(path, buf, None)
def readlink(self, path, buf, bufsize):
ret = self.operations('readlink', path).encode('utf-8')
data = create_string_buffer(ret[:bufsize - 1])
memmove(buf, data, len(data))
return 0
def mknod(self, path, mode, dev):
return self.operations('mknod', path, mode, dev)
def mkdir(self, path, mode):
return self.operations('mkdir', path, mode)
def unlink(self, path):
return self.operations('unlink', path)
def rmdir(self, path):
return self.operations('rmdir', path)
def symlink(self, source, target):
return self.operations('symlink', target, source)
def rename(self, old, new):
return self.operations('rename', old, new)
def link(self, source, target):
return self.operations('link', target, source)
def chmod(self, path, mode):
return self.operations('chmod', path, mode)
def chown(self, path, uid, gid):
return self.operations('chown', path, uid, gid)
def truncate(self, path, length):
return self.operations('truncate', path, length)
def open(self, path, fip):
fi = fip.contents
if self.raw_fi:
return self.operations('open', path, fi)
else:
fi.fh = self.operations('open', path, fi.flags)
return 0
def read(self, path, buf, size, offset, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
ret = self.operations('read', path, size, offset, fh)
if not ret:
return 0
data = create_string_buffer(ret[:size], size)
memmove(buf, data, size)
return size
def write(self, path, buf, size, offset, fip):
data = string_at(buf, size)
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('write', path, data, offset, fh)
def statfs(self, path, buf):
stv = buf.contents
attrs = self.operations('statfs', path)
for key, val in attrs.items():
if hasattr(stv, key):
setattr(stv, key, val)
return 0
def flush(self, path, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('flush', path, fh)
def release(self, path, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('release', path, fh)
def fsync(self, path, datasync, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('fsync', path, datasync, fh)
def setxattr(self, path, name, value, size, options, *args):
data = string_at(value, size)
return self.operations('setxattr', path, name, data, options, *args)
def getxattr(self, path, name, value, size, *args):
ret = self.operations('getxattr', path, name, *args)
retsize = len(ret)
buf = create_string_buffer(ret, retsize) # Does not add trailing 0
if bool(value):
if retsize > size:
return -ERANGE
memmove(value, buf, retsize)
return retsize
def listxattr(self, path, namebuf, size):
ret = self.operations('listxattr', path)
buf = create_string_buffer('\x00'.join(ret)) if ret else ''
bufsize = len(buf)
if bool(namebuf):
if bufsize > size:
return -ERANGE
memmove(namebuf, buf, bufsize)
return bufsize
def removexattr(self, path, name):
return self.operations('removexattr', path, name)
def opendir(self, path, fip):
# Ignore raw_fi
fip.contents.fh = self.operations('opendir', path)
return 0
def readdir(self, path, buf, filler, offset, fip):
# Ignore raw_fi
for item in self.operations('readdir', path, fip.contents.fh):
if isinstance(item, str):
name, st, offset = item, None, 0
else:
name, attrs, offset = item
if attrs:
st = c_stat()
set_st_attrs(st, attrs)
else:
st = None
if filler(buf, name.encode('utf-8'), st, offset) != 0:
break
return 0
def releasedir(self, path, fip):
# Ignore raw_fi
return self.operations('releasedir', path, fip.contents.fh)
def fsyncdir(self, path, datasync, fip):
# Ignore raw_fi
return self.operations('fsyncdir', path, datasync, fip.contents.fh)
def init(self, conn):
return self.operations('init', '/')
def destroy(self, private_data):
return self.operations('destroy', '/')
def access(self, path, amode):
return self.operations('access', path, amode)
def create(self, path, mode, fip):
fi = fip.contents
if self.raw_fi:
return self.operations('create', path, mode, fi)
else:
fi.fh = self.operations('create', path, mode)
return 0
def ftruncate(self, path, length, fip):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('truncate', path, length, fh)
def fgetattr(self, path, buf, fip):
memset(buf, 0, sizeof(c_stat))
st = buf.contents
fh = fip and (fip.contents if self.raw_fi else fip.contents.fh)
attrs = self.operations('getattr', path, fh)
set_st_attrs(st, attrs)
return 0
def lock(self, path, fip, cmd, lock):
fh = fip.contents if self.raw_fi else fip.contents.fh
return self.operations('lock', path, fh, cmd, lock)
def utimens(self, path, buf):
if buf:
atime = time_of_timespec(buf.contents.actime)
mtime = time_of_timespec(buf.contents.modtime)
times = (atime, mtime)
else:
times = None
return self.operations('utimens', path, times)
def bmap(self, path, blocksize, idx):
return self.operations('bmap', path, blocksize, idx)
class Operations(object):
"""This class should be subclassed and passed as an argument to FUSE on
initialization. All operations should raise an OSError exception on
error.
When in doubt of what an operation should do, check the FUSE header
file or the corresponding system call man page."""
def __call__(self, op, *args):
if not hasattr(self, op):
raise OSError(EFAULT, '')
return getattr(self, op)(*args)
def access(self, path, amode):
return 0
bmap = None
def chmod(self, path, mode):
raise OSError(EROFS, '')
def chown(self, path, uid, gid):
raise OSError(EROFS, '')
def create(self, path, mode, fi=None):
"""When raw_fi is False (default case), fi is None and create should
return a numerical file handle.
When raw_fi is True the file handle should be set directly by create
and return 0."""
raise OSError(EROFS, '')
def destroy(self, path):
"""Called on filesystem destruction. Path is always /"""
pass
def flush(self, path, fh):
return 0
def fsync(self, path, datasync, fh):
return 0
def fsyncdir(self, path, datasync, fh):
return 0
def getattr(self, path, fh=None):
"""Returns a dictionary with keys identical to the stat C structure
of stat(2).
st_atime, st_mtime and st_ctime should be floats.
NOTE: There is an incombatibility between Linux and Mac OS X concerning
st_nlink of directories. Mac OS X counts all files inside the directory,
while Linux counts only the subdirectories."""
if path != '/':
raise OSError(ENOENT, '')
return dict(st_mode=(S_IFDIR | 0o755), st_nlink=2)
def getxattr(self, path, name, position=0):
raise OSError(ENOTSUP, '')
def init(self, path):
"""Called on filesystem initialization. Path is always /
Use it instead of __init__ if you start threads on initialization."""
pass
def link(self, target, source):
raise OSError(EROFS, '')
def listxattr(self, path):
return []
lock = None
def mkdir(self, path, mode):
raise OSError(EROFS, '')
def mknod(self, path, mode, dev):
raise OSError(EROFS, '')
def open(self, path, flags):
"""When raw_fi is False (default case), open should return a numerical
file handle.
When raw_fi is True the signature of open becomes:
open(self, path, fi)
and the file handle should be set directly."""
return 0
def opendir(self, path):
"""Returns a numerical file handle."""
return 0
def read(self, path, size, offset, fh):
"""Returns a string containing the data requested."""
raise OSError(ENOENT, '')
def readdir(self, path, fh):
"""Can return either a list of names, or a list of (name, attrs, offset)
tuples. attrs is a dict as in getattr."""
return ['.', '..']
def readlink(self, path):
raise OSError(ENOENT, '')
def release(self, path, fh):
return 0
def releasedir(self, path, fh):
return 0
def removexattr(self, path, name):
raise OSError(ENOTSUP, '')
def rename(self, old, new):
raise OSError(EROFS, '')
def rmdir(self, path):
raise OSError(EROFS, '')
def setxattr(self, path, name, value, options, position=0):
raise OSError(ENOTSUP, '')
def statfs(self, path):
"""Returns a dictionary with keys identical to the statvfs C structure
of statvfs(3).
On Mac OS X f_bsize and f_frsize must be a power of 2 (minimum 512)."""
return {}
def symlink(self, target, source):
raise OSError(EROFS, '')
def truncate(self, path, length, fh=None):
raise OSError(EROFS, '')
def unlink(self, path):
raise OSError(EROFS, '')
def utimens(self, path, times=None):
"""Times is a (atime, mtime) tuple. If None use current time."""
return 0
def write(self, path, data, offset, fh):
raise OSError(EROFS, '')
class LoggingMixIn:
def __call__(self, op, path, *args):
logging.debug('-> %s %s %s', op, path, repr(args))
ret = '[Unknown Error]'
try:
ret = getattr(self, op)(path, *args)
return ret
except OSError as e:
ret = str(e)
raise
finally:
logging.debug('<- %s %s', op, repr(ret))
fs/expose/importhook.py
......@@ -42,6 +42,8 @@ from fs.opener import fsopendir, OpenerError
from fs.errors import *
from fs.path import *
from six import b
class FSImportHook(object):
"""PEP-302-compliant module finder and loader for FS objects.
......@@ -204,9 +206,9 @@ class FSImportHook(object):
if info is None:
info = self._get_module_info(fullname)
(path,type,ispkg) = info
code = self.fs.getcontents(path)
code = self.fs.getcontents(path, 'rb')
if type == imp.PY_SOURCE:
code = code.replace("\r\n","\n")
code = code.replace(b("\r\n"),b("\n"))
return compile(code,path,"exec")
elif type == imp.PY_COMPILED:
if code[:4] != imp.get_magic():
......@@ -223,12 +225,12 @@ class FSImportHook(object):
(path,type,ispkg) = info
if type != imp.PY_SOURCE:
return None
return self.fs.getcontents(path).replace("\r\n","\n")
return self.fs.getcontents(path).replace(b("\r\n"),b("\n"))
def get_data(self,path):
"""Read the specified data file."""
try:
return self.fs.getcontents(path)
return self.fs.getcontents(path, 'rb')
except FSError, e:
raise IOError(str(e))
......
fs/expose/xmlrpc.py
......@@ -19,6 +19,9 @@ import xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer
from datetime import datetime
import six
from six import PY3, b
class RPCFSInterface(object):
"""Wrapper to expose an FS via a XML-RPC compatible interface.
......@@ -37,11 +40,15 @@ class RPCFSInterface(object):
must return something that can be represented in ASCII. The default
is base64-encoded UTF-8.
"""
if PY3:
return path
return path.encode("utf8").encode("base64")
def decode_path(self, path):
"""Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8")
if PY3:
return path
return path.decode("base64").decode("utf8")
def getmeta(self, meta_name):
meta = self.fs.getmeta(meta_name)
......@@ -54,9 +61,9 @@ class RPCFSInterface(object):
def hasmeta(self, meta_name):
return self.fs.hasmeta(meta_name)
def get_contents(self, path):
def get_contents(self, path, mode="rb"):
path = self.decode_path(path)
data = self.fs.getcontents(path)
data = self.fs.getcontents(path, mode)
return xmlrpclib.Binary(data)
def set_contents(self, path, data):
......@@ -105,9 +112,10 @@ class RPCFSInterface(object):
modified_time = datetime.strptime(modified_time.value, "%Y%m%dT%H:%M:%S")
return self.fs.settimes(path, accessed_time, modified_time)
def getinfo(self, path):
def getinfo(self, path):
path = self.decode_path(path)
return self.fs.getinfo(path)
info = self.fs.getinfo(path)
return info
def desc(self, path):
path = self.decode_path(path)
......
fs/filelike.py
......@@ -502,7 +502,7 @@ class FileLikeBase(object):
# If not found, return whole string up to <size> length
# Any leftovers are pushed onto front of buffer
if indx == -1:
data = "".join(bits)
data = b("").join(bits)
if size > 0 and sizeSoFar > size:
extra = data[size:]
data = data[:size]
......
fs/ftpfs.py
......@@ -29,10 +29,16 @@ import calendar
from socket import error as socket_error
from fs.local_functools import wraps
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import six
from six import PY3
if PY3:
from six import BytesIO as StringIO
else:
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import time
......
fs/memoryfs.py
......@@ -23,7 +23,6 @@ import threading
import six
def _check_mode(mode, mode_chars):
for c in mode_chars:
if c not in mode:
......@@ -76,7 +75,8 @@ class MemoryFile(object):
def __str__(self):
return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)
__repr__ = __str__
def __repr__(self):
return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
def __unicode__(self):
return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
......@@ -89,7 +89,9 @@ class MemoryFile(object):
pass
def __iter__(self):
return self
self.mem_file.seek(self.pos)
for line in self.mem_file:
yield line
@seek_and_lock
def next(self):
......@@ -614,10 +616,10 @@ class MemoryFS(FS):
@synchronize
def setcontents(self, path, data, chunk_size=1024*64):
if not isinstance(data, str):
if not isinstance(data, six.binary_type):
return super(MemoryFS, self).setcontents(path, data, chunk_size)
if not self.exists(path):
self.open(path, 'w').close()
self.open(path, 'wb').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
......
fs/osfs/__init__.py
......@@ -31,7 +31,7 @@ from fs.osfs.watch import OSFSWatchMixin
@convert_os_errors
def _os_stat(path):
"""Replacement for os.stat that raises FSError subclasses."""
"""Replacement for os.stat that raises FSError subclasses."""
return os.stat(path)
@convert_os_errors
......@@ -204,8 +204,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
return super(OSFS, self).getmeta(meta_name, default)
@convert_os_errors
def open(self, path, mode="r", **kwargs):
#mode = filter(lambda c: c in "rwabt+",mode)
def open(self, path, mode="r", **kwargs):
mode = ''.join(c for c in mode if c in 'rwabt+')
sys_path = self.getsyspath(path)
try:
......@@ -331,7 +330,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
@convert_os_errors
def getinfo(self, path):
stats = self._stat(path)
info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') )
info = dict((k, getattr(stats, k)) for k in dir(stats) if k.startswith('st_'))
info['size'] = info['st_size']
# TODO: this doesn't actually mean 'creation time' on unix
ct = info.get('st_ctime', None)
......
fs/remote.py
......@@ -94,7 +94,8 @@ class RemoteFileBuffer(FileWrapper):
self._eof = True
if not hasattr(rfile, "read"):
rfile = StringIO(unicode(rfile))
#rfile = StringIO(unicode(rfile))
rfile = StringIO(rfile)
self._rfile = rfile
else:
......
fs/rpcfs.py
......@@ -18,6 +18,8 @@ from fs.path import *
from fs.filelike import StringIO
import six
from six import PY3, b
def re_raise_faults(func):
"""Decorator to re-raise XML-RPC faults as proper exceptions."""
......@@ -126,6 +128,9 @@ class RPCFS(FS):
def __str__(self):
return '<RPCFS: %s>' % (self.uri,)
def __repr__(self):
return '<RPCFS: %s>' % (self.uri,)
@synchronize
def __getstate__(self):
......@@ -147,11 +152,15 @@ class RPCFS(FS):
must return something that can be represented in ASCII. The default
is base64-encoded UTF8.
"""
return path.encode("utf8").encode("base64")
if PY3:
return path
return path.encode("utf8").encode("base64")
def decode_path(self, path):
"""Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8")
if PY3:
return path
return path.decode("base64").decode("utf8")
@synchronize
def getmeta(self, meta_name, default=NoDefaultMeta):
......@@ -169,18 +178,18 @@ class RPCFS(FS):
# TODO: chunked transport of large files
path = self.encode_path(path)
if "w" in mode:
self.proxy.set_contents(path,xmlrpclib.Binary(""))
self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
if "r" in mode or "a" in mode or "+" in mode:
try:
data = self.proxy.get_contents(path).data
data = self.proxy.get_contents(path, "rb").data
except IOError:
if "w" not in mode and "a" not in mode:
raise ResourceNotFoundError(path)
if not self.isdir(dirname(path)):
raise ParentDirectoryMissingError(path)
self.proxy.set_contents(path,xmlrpclib.Binary(""))
self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
else:
data = ""
data = b("")
f = StringIO(data)
if "a" not in mode:
f.seek(0,0)
......@@ -277,7 +286,7 @@ class RPCFS(FS):
@synchronize
def getinfo(self, path):
path = self.encode_path(path)
path = self.encode_path(path)
return self.proxy.getinfo(path)
@synchronize
......
fs/tests/__init__.py
......@@ -297,8 +297,8 @@ class FSTestCases(object):
self.assertRaises(ResourceInvalidError,self.fs.listdirinfo,"foo")
def test_walk(self):
self.fs.setcontents('a.txt', 'hello')
self.fs.setcontents('b.txt', 'world')
self.fs.setcontents('a.txt', b('hello'))
self.fs.setcontents('b.txt', b('world'))
self.fs.makeopendir('foo').setcontents('c', b('123'))
sorted_walk = sorted([(d,sorted(fs)) for (d,fs) in self.fs.walk()])
self.assertEquals(sorted_walk,
......@@ -730,7 +730,8 @@ class FSTestCases(object):
checkcontents("hello",b("12345"))
def test_truncate_to_larger_size(self):
print repr(self.fs)
print self.fs.__class__
with self.fs.open("hello","wb") as f:
f.truncate(30)
self.assertEquals(self.fs.getsize("hello"), 30)
......
fs/tests/test_importhook.py
......@@ -10,6 +10,7 @@ from fs.expose.importhook import FSImportHook
from fs.tempfs import TempFS
from fs.zipfs import ZipFS
from six import b
class TestFSImportHook(unittest.TestCase):
......@@ -32,23 +33,23 @@ class TestFSImportHook(unittest.TestCase):
sys.path_importer_cache.clear()
def _init_modules(self,fs):
fs.setcontents("fsih_hello.py",dedent("""
fs.setcontents("fsih_hello.py",b(dedent("""
message = 'hello world!'
"""))
""")))
fs.makedir("fsih_pkg")
fs.setcontents("fsih_pkg/__init__.py",dedent("""
fs.setcontents("fsih_pkg/__init__.py",b(dedent("""
a = 42
"""))
fs.setcontents("fsih_pkg/sub1.py",dedent("""
""")))
fs.setcontents("fsih_pkg/sub1.py",b(dedent("""
import fsih_pkg
from fsih_hello import message
a = fsih_pkg.a
"""))
fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(dedent("""
""")))
fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(b(dedent("""
import fsih_pkg
from fsih_hello import message
a = fsih_pkg.a * 2
""")))
"""))))
def _getpyc(self,src):
"""Get the .pyc contents to match th given .py source code."""
......
fs/tests/test_watch.py
......@@ -178,7 +178,7 @@ class WatcherTestCases:
while not isinstance(event,CLOSED):
event = changes.next(timeout=1)
# That should be the last event in the list
self.assertRaises(StopIteration,changes.next,timeout=1)
self.assertRaises(StopIteration,getattr(changes, "next"),timeout=1)
changes.close()
......
fs/tests/test_xattr.py
......@@ -11,6 +11,7 @@ from fs.path import *
from fs.errors import *
from fs.tests import FSTestCases
from six import b
class XAttrTestCases:
"""Testcases for filesystems providing extended attribute support.
......@@ -26,11 +27,11 @@ class XAttrTestCases:
self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
self.fs.delxattr(p,"xattr1")
self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
self.fs.setcontents("test.txt","hello")
self.fs.setcontents("test.txt",b("hello"))
do_getsetdel("test.txt")
self.assertRaises(ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
self.fs.makedir("mystuff")
self.fs.setcontents("/mystuff/test.txt","")
self.fs.setcontents("/mystuff/test.txt",b(""))
do_getsetdel("mystuff")
do_getsetdel("mystuff/test.txt")
......@@ -49,15 +50,15 @@ class XAttrTestCases:
self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2"])
self.fs.delxattr(p,"attr2")
self.assertEquals(sorted(self.fs.listxattrs(p)),[])
self.fs.setcontents("test.txt","hello")
self.fs.setcontents("test.txt",b("hello"))
do_list("test.txt")
self.fs.makedir("mystuff")
self.fs.setcontents("/mystuff/test.txt","")
self.fs.setcontents("/mystuff/test.txt",b(""))
do_list("mystuff")
do_list("mystuff/test.txt")
def test_copy_xattrs(self):
self.fs.setcontents("a.txt","content")
self.fs.setcontents("a.txt",b("content"))
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
......@@ -75,7 +76,7 @@ class XAttrTestCases:
self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
def test_move_xattrs(self):
self.fs.setcontents("a.txt","content")
self.fs.setcontents("a.txt",b("content"))
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
......
fs/tests/test_zipfs.py
......@@ -13,9 +13,10 @@ import shutil
import fs.tests
from fs.path import *
from fs import zipfs
from six import b
from fs import zipfs
class TestReadZipFS(unittest.TestCase):
def setUp(self):
......@@ -24,11 +25,11 @@ class TestReadZipFS(unittest.TestCase):
self.zf = zipfile.ZipFile(self.temp_filename, "w")
zf = self.zf
zf.writestr("a.txt", "Hello, World!")
zf.writestr("b.txt", "b")
zf.writestr("1.txt", "1")
zf.writestr("foo/bar/baz.txt", "baz")
zf.writestr("foo/second.txt", "hai")
zf.writestr("a.txt", b("Hello, World!"))
zf.writestr("b.txt", b("b"))
zf.writestr("1.txt", b("1"))
zf.writestr("foo/bar/baz.txt", b("baz"))
zf.writestr("foo/second.txt", b("hai"))
zf.close()
self.fs = zipfs.ZipFS(self.temp_filename, "r")
......@@ -50,18 +51,18 @@ class TestReadZipFS(unittest.TestCase):
return contents
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
check_contents("a.txt", b("Hello, World!"))
check_contents("1.txt", b("1"))
check_contents("foo/bar/baz.txt", b("baz"))
def test_getcontents(self):
def read_contents(path):
return self.fs.getcontents(path)
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
check_contents("a.txt", b("Hello, World!"))
check_contents("1.txt", b("1"))
check_contents("foo/bar/baz.txt", b("baz"))
def test_is(self):
self.assert_(self.fs.isfile('a.txt'))
......@@ -98,15 +99,15 @@ class TestWriteZipFS(unittest.TestCase):
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f = zip_fs.open(filename, 'wb')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
makefile("foo/bar/baz.txt", "baz")
makefile("foo/second.txt", "hai")
makefile("a.txt", b("Hello, World!"))
makefile("b.txt", b("b"))
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
makefile("foo/bar/baz.txt", b("baz"))
makefile("foo/second.txt", b("hai"))
zip_fs.close()
......@@ -123,11 +124,11 @@ class TestWriteZipFS(unittest.TestCase):
def check_contents(filename, contents):
zcontents = zf.read(filename.encode("CP437"))
self.assertEqual(contents, zcontents)
check_contents("a.txt", "Hello, World!")
check_contents("b.txt", "b")
check_contents("foo/bar/baz.txt", "baz")
check_contents("foo/second.txt", "hai")
check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
check_contents("a.txt", b()"Hello, World!")
check_contents("b.txt", b("b"))
check_contents("foo/bar/baz.txt", b("baz"))
check_contents("foo/second.txt", b("hai"))
check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
class TestAppendZipFS(TestWriteZipFS):
......@@ -141,19 +142,19 @@ class TestAppendZipFS(TestWriteZipFS):
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f = zip_fs.open(filename, 'wb')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
makefile("a.txt", b("Hello, World!"))
makefile("b.txt", b("b"))
zip_fs.close()
zip_fs = zipfs.ZipFS(self.temp_filename, 'a')
makefile("foo/bar/baz.txt", "baz")
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
makefile("foo/second.txt", "hai")
makefile("foo/bar/baz.txt", b("baz"))
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
makefile("foo/second.txt", b("hai"))
zip_fs.close()
......@@ -168,7 +169,7 @@ class TestZipFSErrors(unittest.TestCase):
def test_bogus_zipfile(self):
badzip = os.path.join(self.workdir,"bad.zip")
f = open(badzip,"wb")
f.write("I'm not really a zipfile")
f.write(b("I'm not really a zipfile"))
f.close()
self.assertRaises(zipfs.ZipOpenError,zipfs.ZipFS,badzip)
......
fs/zipfs.py
......@@ -19,6 +19,7 @@ from memoryfs import MemoryFS
import tempfs
from six import PY3
class ZipOpenError(CreateFailedError):
"""Thrown when the zip file could not be opened"""
......@@ -144,9 +145,20 @@ class ZipFS(FS):
def __unicode__(self):
return u"<ZipFS: %s>" % self.zip_path
def _decode_path(self, path):
if PY3:
return path
return path.decode(self.encoding)
def _encode_path(self, path):
if PY3:
return path
return path.encode(self.encoding)
def _parse_resource_list(self):
for path in self.zf.namelist():
self._add_resource(path.decode(self.encoding))
#self._add_resource(path.decode(self.encoding))
self._add_resource(self._decode_path(path))
def _add_resource(self, path):
if path.endswith('/'):
......@@ -185,9 +197,9 @@ class ZipFS(FS):
msg="1 Zip file must be opened for reading ('r') or appending ('a')")
try:
if hasattr(self.zf, 'open') and self._zip_file_string:
return self.zf.open(path.encode(self.encoding))
return self.zf.open(self._encode_path(path), "r")
else:
contents = self.zf.read(path.encode(self.encoding))
contents = self.zf.read(self._encode_path(path))
except KeyError:
raise ResourceNotFoundError(path)
return StringIO(contents)
......@@ -214,7 +226,7 @@ class ZipFS(FS):
raise ResourceNotFoundError(path)
path = normpath(relpath(path))
try:
contents = self.zf.read(path.encode(self.encoding))
contents = self.zf.read(self._encode_path(path))
except KeyError:
raise ResourceNotFoundError(path)
except RuntimeError:
......@@ -224,7 +236,7 @@ class ZipFS(FS):
@synchronize
def _on_write_close(self, filename):
sys_path = self.temp_fs.getsyspath(filename)
self.zf.write(sys_path, filename.encode(self.encoding))
self.zf.write(sys_path, self._encode_path(filename))
def desc(self, path):
return "%s in zip file %s" % (path, self.zip_path)
......@@ -256,14 +268,14 @@ class ZipFS(FS):
raise ResourceNotFoundError(path)
path = normpath(path).lstrip('/')
try:
zi = self.zf.getinfo(path.encode(self.encoding))
zi = self.zf.getinfo(self._encode_path(path))
zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
for k, v in zinfo.iteritems():
if callable(v):
zinfo[k] = v()
except KeyError:
zinfo = {'file_size':0}
info = {'size' : zinfo['file_size'] }
info = {'size' : zinfo['file_size']}
if 'date_time' in zinfo:
info['created_time'] = datetime.datetime(*zinfo['date_time'])
info.update(zinfo)
......
tox.ini
......@@ -8,7 +8,7 @@ deps = distribute
boto
nose
mako
pyftpdlib
pyftpdlib
changedir=.tox
commands = nosetests fs.tests -v \
[]
......@@ -22,7 +22,7 @@ deps = distribute
nose
mako
pyftpdlib
simplejson
simplejson
[testenv:py32]
commands = nosetests fs.tests -v \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment