Skip to content

P
pyfs
Commit 75fe116f by willmcgugan
Options

Mostly doc changes, and some pedantic pep-8 tweaks

parent 4ded92bc
Showing with 353 additions and 238 deletions
fs/__init__.py
...@@ -28,13 +28,13 @@ import path ...@@ -28,13 +28,13 @@ import path
_thread_synchronize_default = True _thread_synchronize_default = True
def set_thread_synchronize_default(sync): def set_thread_synchronize_default(sync):
"""Sets the default thread synctonisation flag. """Sets the default thread synchronisation flag.
FS objects are made thread-safe through the use of a per-FS threading Lock FS objects are made thread-safe through the use of a per-FS threading Lock
object. Since this can introduce an small overhead it can be disabled with object. Since this can introduce an small overhead it can be disabled with
this function if the code is single-threaded. this function if the code is single-threaded.
:param sync: Set wether to use thread synchronization for new FS objects :param sync: Set whether to use thread synchronisation for new FS objects
""" """
global _thread_synchronization_default global _thread_synchronization_default
......
fs/base.py
...@@ -998,7 +998,6 @@ class SubFS(FS): ...@@ -998,7 +998,6 @@ class SubFS(FS):
def flags_to_mode(flags): def flags_to_mode(flags):
"""Convert an os.O_* flag bitmask into an FS mode string.""" """Convert an os.O_* flag bitmask into an FS mode string."""
print flags
if flags & os.O_EXCL: if flags & os.O_EXCL:
raise UnsupportedError("open",msg="O_EXCL is not supported") raise UnsupportedError("open",msg="O_EXCL is not supported")
if flags & os.O_WRONLY: if flags & os.O_WRONLY:
...@@ -1017,5 +1016,4 @@ def flags_to_mode(flags): ...@@ -1017,5 +1016,4 @@ def flags_to_mode(flags):
mode = "r+" mode = "r+"
else: else:
mode = "r" mode = "r"
print mode
return mode return mode
fs/expose/django_storage.py
...@@ -8,6 +8,7 @@ Use an FS object for Django File Storage ...@@ -8,6 +8,7 @@ Use an FS object for Django File Storage
from django.conf import settings from django.conf import settings
from django.core.files.storage import Storage from django.core.files.storage import Storage
from django.core.files import File
from fs.path import abspath, dirname from fs.path import abspath, dirname
from fs.errors import convert_fs_errors from fs.errors import convert_fs_errors
...@@ -15,45 +16,49 @@ from fs.errors import convert_fs_errors ...@@ -15,45 +16,49 @@ from fs.errors import convert_fs_errors
class FSStorage(Storage): class FSStorage(Storage):
"""Expose an FS object as a Django File Storage object.""" """Expose an FS object as a Django File Storage object."""
def __init__(self,fs=None,base_url=None): def __init__(self, fs=None, base_url=None):
"""
:param fs: an FS object
:param base_url: The url to prepend to the path
"""
if fs is None: if fs is None:
fs = settings.DEFAULT_FILE_STORAGE_FS fs = settings.DEFAULT_FILE_STORAGE_FS
if base_url is None: if base_url is None:
base_url = settings.MEDIA_URL base_url = settings.MEDIA_URL
while base_url.endswith("/"): base_url = base_url.rstrip('/')
base_url = base_url[:-1]
self.fs = fs self.fs = fs
self.base_url = base_url self.base_url = base_url
def exists(self,name): def exists(self, name):
return self.fs.isfile(name) return self.fs.isfile(name)
def path(self,name): def path(self, name):
path = self.fs.getsyspath(name) path = self.fs.getsyspath(name)
if path is None: if path is None:
raise NotImplementedError raise NotImplementedError
return path return path
@convert_fs_errors @convert_fs_errors
def size(self,name): def size(self, name):
return self.fs.getsize(name) return self.fs.getsize(name)
@convert_fs_errors @convert_fs_errors
def url(self,name): def url(self, name):
return self.base_url + abspath(name) return self.base_url + abspath(name)
@convert_fs_errors @convert_fs_errors
def _open(self,name,mode): def _open(self, name, mode):
return self.fs.open(name,mode) return File(self.fs.open(name, mode))
@convert_fs_errors @convert_fs_errors
def _save(self,name,content): def _save(self, name, content):
self.fs.makedir(dirname(name),allow_recreate=True,recursive=True) self.fs.makedir(dirname(name), allow_recreate=True, recursive=True)
self.fs.setcontents(name,content) self.fs.setcontents(name, content)
return name return name
@convert_fs_errors @convert_fs_errors
def delete(self,name): def delete(self, name):
try: try:
self.fs.remove(name) self.fs.remove(name)
except ResourceNotFoundError: except ResourceNotFoundError:
......
fs/expose/fuse/__init__.py
...@@ -45,6 +45,7 @@ fuse.py code from Giorgos Verigakis: ...@@ -45,6 +45,7 @@ fuse.py code from Giorgos Verigakis:
""" """
import datetime
import os import os
import sys import sys
import signal import signal
...@@ -83,6 +84,7 @@ def handle_fs_errors(func): ...@@ -83,6 +84,7 @@ def handle_fs_errors(func):
equivalent OSError. It also makes the function return zero instead equivalent OSError. It also makes the function return zero instead
of None as an indication of successful execution. of None as an indication of successful execution.
""" """
name = func.__name__
func = convert_fs_errors(func) func = convert_fs_errors(func)
@wraps(func) @wraps(func)
def wrapper(*args,**kwds): def wrapper(*args,**kwds):
...@@ -97,7 +99,7 @@ def handle_fs_errors(func): ...@@ -97,7 +99,7 @@ def handle_fs_errors(func):
class FSOperations(Operations): class FSOperations(Operations):
"""FUSE Operations interface delegating all activities to an FS object.""" """FUSE Operations interface delegating all activities to an FS object."""
def __init__(self,fs,on_init=None,on_destroy=None): def __init__(self, fs, on_init=None, on_destroy=None):
self.fs = fs self.fs = fs
self._on_init = on_init self._on_init = on_init
self._on_destroy = on_destroy self._on_destroy = on_destroy
...@@ -110,13 +112,13 @@ class FSOperations(Operations): ...@@ -110,13 +112,13 @@ class FSOperations(Operations):
# This dict is indexed by path, then file handle. # This dict is indexed by path, then file handle.
self._files_size_written = {} self._files_size_written = {}
def _get_file(self,fh): def _get_file(self, fh):
try: try:
return self._files_by_handle[fh.fh] return self._files_by_handle[fh.fh]
except KeyError: except KeyError:
raise FSError("invalid file handle") raise FSError("invalid file handle")
def _reg_file(self,f,path): def _reg_file(self, f, path):
self._files_lock.acquire() self._files_lock.acquire()
try: try:
fh = self._next_handle fh = self._next_handle
...@@ -130,7 +132,7 @@ class FSOperations(Operations): ...@@ -130,7 +132,7 @@ class FSOperations(Operations):
finally: finally:
self._files_lock.release() self._files_lock.release()
def _del_file(self,fh): def _del_file(self, fh):
self._files_lock.acquire() self._files_lock.acquire()
try: try:
(f,path,lock) = self._files_by_handle.pop(fh.fh) (f,path,lock) = self._files_by_handle.pop(fh.fh)
...@@ -140,31 +142,31 @@ class FSOperations(Operations): ...@@ -140,31 +142,31 @@ class FSOperations(Operations):
finally: finally:
self._files_lock.release() self._files_lock.release()
def init(self,conn): def init(self, conn):
if self._on_init: if self._on_init:
self._on_init() self._on_init()
def destroy(self,data): def destroy(self, data):
if self._on_destroy: if self._on_destroy:
self._on_destroy() self._on_destroy()
@handle_fs_errors @handle_fs_errors
def chmod(self,path,mode): def chmod(self, path, mode):
raise UnsupportedError("chmod") raise UnsupportedError("chmod")
@handle_fs_errors @handle_fs_errors
def chown(self,path,uid,gid): def chown(self, path, uid, gid):
raise UnsupportedError("chown") raise UnsupportedError("chown")
@handle_fs_errors @handle_fs_errors
def create(self,path,mode,fi): def create(self, path, mode, fi):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
fh = self._reg_file(self.fs.open(path,"w"),path) fh = self._reg_file(self.fs.open(path,"w"),path)
fi.fh = fh fi.fh = fh
fi.keep_cache = 0 fi.keep_cache = 0
@handle_fs_errors @handle_fs_errors
def flush(self,path,fh): def flush(self, path, fh):
(file,_,lock) = self._get_file(fh) (file,_,lock) = self._get_file(fh)
lock.acquire() lock.acquire()
try: try:
...@@ -173,11 +175,12 @@ class FSOperations(Operations): ...@@ -173,11 +175,12 @@ class FSOperations(Operations):
lock.release() lock.release()
@handle_fs_errors @handle_fs_errors
def getattr(self,path,fh=None): def getattr(self, path, fh=None):
return self._get_stat_dict(path.decode(NATIVE_ENCODING)) attrs = self._get_stat_dict(path.decode(NATIVE_ENCODING))
return attrs
@handle_fs_errors @handle_fs_errors
def getxattr(self,path,name,position=0): def getxattr(self, path, name, position=0):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
name = name.decode(NATIVE_ENCODING) name = name.decode(NATIVE_ENCODING)
try: try:
...@@ -190,11 +193,11 @@ class FSOperations(Operations): ...@@ -190,11 +193,11 @@ class FSOperations(Operations):
return value return value
@handle_fs_errors @handle_fs_errors
def link(self,target,souce): def link(self, target, souce):
raise UnsupportedError("link") raise UnsupportedError("link")
@handle_fs_errors @handle_fs_errors
def listxattr(self,path): def listxattr(self, path):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
try: try:
return self.fs.listxattrs(path) return self.fs.listxattrs(path)
...@@ -202,7 +205,7 @@ class FSOperations(Operations): ...@@ -202,7 +205,7 @@ class FSOperations(Operations):
return [] return []
@handle_fs_errors @handle_fs_errors
def mkdir(self,path,mode): def mkdir(self, path, mode):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
try: try:
self.fs.makedir(path,mode) self.fs.makedir(path,mode)
...@@ -210,11 +213,11 @@ class FSOperations(Operations): ...@@ -210,11 +213,11 @@ class FSOperations(Operations):
self.fs.makedir(path) self.fs.makedir(path)
@handle_fs_errors @handle_fs_errors
def mknod(self,path,mode,dev): def mknod(self, path, mode, dev):
raise UnsupportedError("mknod") raise UnsupportedError("mknod")
@handle_fs_errors @handle_fs_errors
def open(self,path,fi): def open(self, path, fi):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
mode = flags_to_mode(fi.flags) mode = flags_to_mode(fi.flags)
fi.fh = self._reg_file(self.fs.open(path,mode),path) fi.fh = self._reg_file(self.fs.open(path,mode),path)
...@@ -222,7 +225,7 @@ class FSOperations(Operations): ...@@ -222,7 +225,7 @@ class FSOperations(Operations):
return 0 return 0
@handle_fs_errors @handle_fs_errors
def read(self,path,size,offset,fh): def read(self, path, size, offset, fh):
(file,_,lock) = self._get_file(fh) (file,_,lock) = self._get_file(fh)
lock.acquire() lock.acquire()
try: try:
...@@ -233,7 +236,7 @@ class FSOperations(Operations): ...@@ -233,7 +236,7 @@ class FSOperations(Operations):
lock.release() lock.release()
@handle_fs_errors @handle_fs_errors
def readdir(self,path,fh=None): def readdir(self, path, fh=None):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
# If listdir() can return info dicts directly, it will save FUSE # If listdir() can return info dicts directly, it will save FUSE
# having to call getinfo() on each entry individually. # having to call getinfo() on each entry individually.
...@@ -252,11 +255,11 @@ class FSOperations(Operations): ...@@ -252,11 +255,11 @@ class FSOperations(Operations):
return entries return entries
@handle_fs_errors @handle_fs_errors
def readlink(self,path): def readlink(self, path):
raise UnsupportedError("readlink") raise UnsupportedError("readlink")
@handle_fs_errors @handle_fs_errors
def release(self,path,fh): def release(self, path, fh):
(file,_,lock) = self._get_file(fh) (file,_,lock) = self._get_file(fh)
lock.acquire() lock.acquire()
try: try:
...@@ -266,7 +269,7 @@ class FSOperations(Operations): ...@@ -266,7 +269,7 @@ class FSOperations(Operations):
lock.release() lock.release()
@handle_fs_errors @handle_fs_errors
def removexattr(self,path,name): def removexattr(self, path, name):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
name = name.decode(NATIVE_ENCODING) name = name.decode(NATIVE_ENCODING)
try: try:
...@@ -275,7 +278,7 @@ class FSOperations(Operations): ...@@ -275,7 +278,7 @@ class FSOperations(Operations):
raise UnsupportedError("removexattr") raise UnsupportedError("removexattr")
@handle_fs_errors @handle_fs_errors
def rename(self,old,new): def rename(self, old, new):
old = old.decode(NATIVE_ENCODING) old = old.decode(NATIVE_ENCODING)
new = new.decode(NATIVE_ENCODING) new = new.decode(NATIVE_ENCODING)
try: try:
...@@ -292,7 +295,7 @@ class FSOperations(Operations): ...@@ -292,7 +295,7 @@ class FSOperations(Operations):
self.fs.removedir(path) self.fs.removedir(path)
@handle_fs_errors @handle_fs_errors
def setxattr(self,path,name,value,options,position=0): def setxattr(self, path, name, value, options, position=0):
path = path.decode(NATIVE_ENCODING) path = path.decode(NATIVE_ENCODING)
name = name.decode(NATIVE_ENCODING) name = name.decode(NATIVE_ENCODING)
try: try:
...@@ -343,7 +346,12 @@ class FSOperations(Operations): ...@@ -343,7 +346,12 @@ class FSOperations(Operations):
@handle_fs_errors @handle_fs_errors
def utimens(self, path, times=None): def utimens(self, path, times=None):
raise UnsupportedError("utimens") accessed_time, modified_time = times
if accessed_time is not None:
accessed_time = datetime.datetime.fromtimestamp(accessed_time)
if modified_time is not None:
modified_time = datetime.datetime.fromtimestamp(modified_time)
self.fs.settimes(path, accessed_time, modified_time)
@handle_fs_errors @handle_fs_errors
def write(self, path, data, offset, fh): def write(self, path, data, offset, fh):
...@@ -358,13 +366,13 @@ class FSOperations(Operations): ...@@ -358,13 +366,13 @@ class FSOperations(Operations):
finally: finally:
lock.release() lock.release()
def _get_stat_dict(self,path): def _get_stat_dict(self, path):
"""Build a 'stat' dictionary for the given file.""" """Build a 'stat' dictionary for the given file."""
info = self.fs.getinfo(path) info = self.fs.getinfo(path)
self._fill_stat_dict(path,info) self._fill_stat_dict(path,info)
return info return info
def _fill_stat_dict(self,path,info): def _fill_stat_dict(self, path, info):
"""Fill default values in the stat dict.""" """Fill default values in the stat dict."""
uid, gid, pid = fuse_get_context() uid, gid, pid = fuse_get_context()
private_keys = [k for k in info if k.startswith("_")] private_keys = [k for k in info if k.startswith("_")]
...@@ -379,7 +387,12 @@ class FSOperations(Operations): ...@@ -379,7 +387,12 @@ class FSOperations(Operations):
info.setdefault("st_blksize",1024) info.setdefault("st_blksize",1024)
info.setdefault("st_blocks",1) info.setdefault("st_blocks",1)
# The interesting stuff # The interesting stuff
mode = info.get("st_mode",0700) if 'st_mode' not in info:
if self.fs.isdir(path):
info['st_mode'] = 0755
else:
info['st_mode'] = 0666
mode = info['st_mode']
if not statinfo.S_ISDIR(mode) and not statinfo.S_ISREG(mode): if not statinfo.S_ISDIR(mode) and not statinfo.S_ISREG(mode):
if self.fs.isdir(path): if self.fs.isdir(path):
info["st_mode"] = mode | statinfo.S_IFDIR info["st_mode"] = mode | statinfo.S_IFDIR
...@@ -405,7 +418,7 @@ class FSOperations(Operations): ...@@ -405,7 +418,7 @@ class FSOperations(Operations):
return info return info
def mount(fs,path,foreground=False,ready_callback=None,unmount_callback=None,**kwds): def mount(fs, path, foreground=False, ready_callback=None, unmount_callback=None, **kwds):
"""Mount the given FS at the given path, using FUSE. """Mount the given FS at the given path, using FUSE.
By default, this function spawns a new background process to manage the By default, this function spawns a new background process to manage the
...@@ -425,11 +438,12 @@ def mount(fs,path,foreground=False,ready_callback=None,unmount_callback=None,**k ...@@ -425,11 +438,12 @@ def mount(fs,path,foreground=False,ready_callback=None,unmount_callback=None,**k
* fsname Name to display in the mount info table * fsname Name to display in the mount info table
""" """
path = os.path.expanduser(path)
if foreground: if foreground:
op = FSOperations(fs,on_init=ready_callback,on_destroy=unmount_callback) op = FSOperations(fs, on_init=ready_callback, on_destroy=unmount_callback)
return FUSE(op,path,raw_fi=True,foreground=foreground,**kwds) return FUSE(op, path, raw_fi=True, foreground=foreground, **kwds)
else: else:
mp = MountProcess(fs,path,kwds) mp = MountProcess(fs, path, kwds)
if ready_callback: if ready_callback:
ready_callback() ready_callback()
if unmount_callback: if unmount_callback:
...@@ -498,7 +512,7 @@ class MountProcess(subprocess.Popen): ...@@ -498,7 +512,7 @@ class MountProcess(subprocess.Popen):
unmount_timeout = 5 unmount_timeout = 5
def __init__(self,fs,path,fuse_opts={},nowait=False,**kwds): def __init__(self, fs, path, fuse_opts={}, nowait=False, **kwds):
self.path = path self.path = path
if nowait or kwds.get("close_fds",False): if nowait or kwds.get("close_fds",False):
cmd = 'from fs.expose.fuse import MountProcess; ' cmd = 'from fs.expose.fuse import MountProcess; '
...@@ -533,12 +547,12 @@ class MountProcess(subprocess.Popen): ...@@ -533,12 +547,12 @@ class MountProcess(subprocess.Popen):
self.wait() self.wait()
tmr.cancel() tmr.cancel()
if not hasattr(subprocess.Popen,"terminate"): if not hasattr(subprocess.Popen, "terminate"):
def terminate(self): def terminate(self):
"""Gracefully terminate the subprocess.""" """Gracefully terminate the subprocess."""
os.kill(self.pid,signal.SIGTERM) os.kill(self.pid,signal.SIGTERM)
if not hasattr(subprocess.Popen,"kill"): if not hasattr(subprocess.Popen, "kill"):
def kill(self): def kill(self):
"""Forcibly terminate the subprocess.""" """Forcibly terminate the subprocess."""
os.kill(self.pid,signal.SIGKILL) os.kill(self.pid,signal.SIGKILL)
...@@ -582,10 +596,10 @@ class MountProcess(subprocess.Popen): ...@@ -582,10 +596,10 @@ class MountProcess(subprocess.Popen):
if __name__ == "__main__": if __name__ == "__main__":
import os, os.path import os, os.path
from fs.tempfs import TempFS from fs.tempfs import TempFS
mount_point = os.path.join(os.environ["HOME"],"fs.expose.fuse") mount_point = os.path.join(os.environ["HOME"], "fs.expose.fuse")
if not os.path.exists(mount_point): if not os.path.exists(mount_point):
os.makedirs(mount_point) os.makedirs(mount_point)
def ready_callback(): def ready_callback():
print "READY" print "READY"
mount(TempFS(),mount_point,foreground=True,ready_callback=ready_callback) mount(TempFS(), mount_point, foreground=True, ready_callback=ready_callback)
fs/expose/sftp.py
...@@ -70,7 +70,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): ...@@ -70,7 +70,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
This SFTPServerInterface subclass expects a single additional argument, This SFTPServerInterface subclass expects a single additional argument,
the fs object to be exposed. Use it to set up a transport subsystem the fs object to be exposed. Use it to set up a transport subsystem
handler like so: handler like so::
t.set_subsystem_handler("sftp",SFTPServer,SFTPServerInterface,fs) t.set_subsystem_handler("sftp",SFTPServer,SFTPServerInterface,fs)
...@@ -79,7 +79,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): ...@@ -79,7 +79,7 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
paramiko server infrastructure. paramiko server infrastructure.
""" """
def __init__(self,server,fs,encoding=None,*args,**kwds): def __init__(self, server, fs, encoding=None, *args, **kwds):
self.fs = fs self.fs = fs
if encoding is None: if encoding is None:
encoding = "utf8" encoding = "utf8"
...@@ -87,12 +87,12 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): ...@@ -87,12 +87,12 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
super(SFTPServerInterface,self).__init__(server,*args,**kwds) super(SFTPServerInterface,self).__init__(server,*args,**kwds)
@report_sftp_errors @report_sftp_errors
def open(self,path,flags,attr): def open(self, path, flags, attr):
return SFTPHandle(self,path,flags) return SFTPHandle(self, path, flags)
@report_sftp_errors @report_sftp_errors
def list_folder(self,path): def list_folder(self, path):
if not isinstance(path,unicode): if not isinstance(path, unicode):
path = path.decode(self.encoding) path = path.decode(self.encoding)
stats = [] stats = []
for entry in self.fs.listdir(path,absolute=True): for entry in self.fs.listdir(path,absolute=True):
...@@ -100,8 +100,8 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): ...@@ -100,8 +100,8 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
return stats return stats
@report_sftp_errors @report_sftp_errors
def stat(self,path): def stat(self, path):
if not isinstance(path,unicode): if not isinstance(path, unicode):
path = path.decode(self.encoding) path = path.decode(self.encoding)
info = self.fs.getinfo(path) info = self.fs.getinfo(path)
stat = paramiko.SFTPAttributes() stat = paramiko.SFTPAttributes()
...@@ -115,52 +115,52 @@ class SFTPServerInterface(paramiko.SFTPServerInterface): ...@@ -115,52 +115,52 @@ class SFTPServerInterface(paramiko.SFTPServerInterface):
stat.st_mode = 0777 | statinfo.S_IFREG stat.st_mode = 0777 | statinfo.S_IFREG
return stat return stat
def lstat(self,path): def lstat(self, path):
return self.stat(path) return self.stat(path)
@report_sftp_errors @report_sftp_errors
def remove(self,path): def remove(self, path):
if not isinstance(path,unicode): if not isinstance(path,unicode):
path = path.decode(self.encoding) path = path.decode(self.encoding)
self.fs.remove(path) self.fs.remove(path)
return paramiko.SFTP_OK return paramiko.SFTP_OK
@report_sftp_errors @report_sftp_errors
def rename(self,oldpath,newpath): def rename(self, oldpath, newpath):
if not isinstance(oldpath,unicode): if not isinstance(oldpath, unicode):
oldpath = oldpath.decode(self.encoding) oldpath = oldpath.decode(self.encoding)
if not isinstance(newpath,unicode): if not isinstance(newpath, unicode):
newpath = newpath.decode(self.encoding) newpath = newpath.decode(self.encoding)
if self.fs.isfile(oldpath): if self.fs.isfile(oldpath):
self.fs.move(oldpath,newpath) self.fs.move(oldpath, newpath)
else: else:
self.fs.movedir(oldpath,newpath) self.fs.movedir(oldpath, newpath)
return paramiko.SFTP_OK return paramiko.SFTP_OK
@report_sftp_errors @report_sftp_errors
def mkdir(self,path,attr): def mkdir(self, path, attr):
if not isinstance(path,unicode): if not isinstance(path,unicode):
path = path.decode(self.encoding) path = path.decode(self.encoding)
self.fs.makedir(path) self.fs.makedir(path)
return paramiko.SFTP_OK return paramiko.SFTP_OK
@report_sftp_errors @report_sftp_errors
def rmdir(self,path): def rmdir(self, path):
if not isinstance(path,unicode): if not isinstance(path,unicode):
path = path.decode(self.encoding) path = path.decode(self.encoding)
self.fs.removedir(path) self.fs.removedir(path)
return paramiko.SFTP_OK return paramiko.SFTP_OK
def canonicalize(self,path): def canonicalize(self, path):
return abspath(normpath(path)) return abspath(normpath(path))
def chattr(self,path,attr): def chattr(self, path, attr):
return paramiko.SFTP_OP_UNSUPPORTED return paramiko.SFTP_OP_UNSUPPORTED
def readlink(self,path): def readlink(self, path):
return paramiko.SFTP_OP_UNSUPPORTED return paramiko.SFTP_OP_UNSUPPORTED
def symlink(self,path): def symlink(self, path):
return paramiko.SFTP_OP_UNSUPPORTED return paramiko.SFTP_OP_UNSUPPORTED
...@@ -171,7 +171,7 @@ class SFTPHandle(paramiko.SFTPHandle): ...@@ -171,7 +171,7 @@ class SFTPHandle(paramiko.SFTPHandle):
and write requests directly through the to underlying file from the FS. and write requests directly through the to underlying file from the FS.
""" """
def __init__(self,owner,path,flags): def __init__(self, owner, path, flags):
super(SFTPHandle,self).__init__(flags) super(SFTPHandle,self).__init__(flags)
mode = flags_to_mode(flags) + "b" mode = flags_to_mode(flags) + "b"
self.owner = owner self.owner = owner
...@@ -186,12 +186,12 @@ class SFTPHandle(paramiko.SFTPHandle): ...@@ -186,12 +186,12 @@ class SFTPHandle(paramiko.SFTPHandle):
return paramiko.SFTP_OK return paramiko.SFTP_OK
@report_sftp_errors @report_sftp_errors
def read(self,offset,length): def read(self, offset, length):
self._file.seek(offset) self._file.seek(offset)
return self._file.read(length) return self._file.read(length)
@report_sftp_errors @report_sftp_errors
def write(self,offset,data): def write(self, offset, data):
self._file.seek(offset) self._file.seek(offset)
self._file.write(data) self._file.write(data)
return paramiko.SFTP_OK return paramiko.SFTP_OK
...@@ -215,7 +215,7 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler): ...@@ -215,7 +215,7 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler):
def handle(self): def handle(self):
t = paramiko.Transport(self.request) t = paramiko.Transport(self.request)
t.add_server_key(self.server.host_key) t.add_server_key(self.server.host_key)
t.set_subsystem_handler("sftp",paramiko.SFTPServer,SFTPServerInterface,self.server.fs,getattr(self.server,"encoding",None)) t.set_subsystem_handler("sftp", paramiko.SFTPServer, SFTPServerInterface, self.server.fs, getattr(self.server,"encoding",None))
# Note that this actually spawns a new thread to handle the requests. # Note that this actually spawns a new thread to handle the requests.
# (Actually, paramiko.Transport is a subclass of Thread) # (Actually, paramiko.Transport is a subclass of Thread)
t.start_server(server=self.server) t.start_server(server=self.server)
...@@ -249,7 +249,7 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface): ...@@ -249,7 +249,7 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
""" """
def __init__(self,address,fs=None,encoding=None,host_key=None,RequestHandlerClass=None): def __init__(self, address, fs=None, encoding=None, host_key=None, RequestHandlerClass=None):
self.fs = fs self.fs = fs
self.encoding = encoding self.encoding = encoding
if host_key is None: if host_key is None:
...@@ -259,25 +259,25 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface): ...@@ -259,25 +259,25 @@ class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
RequestHandlerClass = SFTPRequestHandler RequestHandlerClass = SFTPRequestHandler
sockserv.TCPServer.__init__(self,address,RequestHandlerClass) sockserv.TCPServer.__init__(self,address,RequestHandlerClass)
def close_request(self,request): def close_request(self, request):
# paramiko.Transport closes itself when finished. # paramiko.Transport closes itself when finished.
# If we close it here, we'll break the Transport thread. # If we close it here, we'll break the Transport thread.
pass pass
def check_channel_request(self,kind,chanid): def check_channel_request(self, kind, chanid):
if kind == 'session': if kind == 'session':
return paramiko.OPEN_SUCCEEDED return paramiko.OPEN_SUCCEEDED
return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
def check_auth_none(self,username): def check_auth_none(self, username):
"""Check whether the user can proceed without authentication.""" """Check whether the user can proceed without authentication."""
return paramiko.AUTH_SUCCESSFUL return paramiko.AUTH_SUCCESSFUL
def check_auth_publickey(self,username,key): def check_auth_publickey(self, username,key):
"""Check whether the given public key is valid for authentication.""" """Check whether the given public key is valid for authentication."""
return paramiko.AUTH_FAILED return paramiko.AUTH_FAILED
def check_auth_password(self,username,password): def check_auth_password(self, username, password):
"""Check whether the given password is valid for authentication.""" """Check whether the given password is valid for authentication."""
return paramiko.AUTH_FAILED return paramiko.AUTH_FAILED
......
fs/expose/xmlrpc.py
...@@ -26,10 +26,10 @@ class RPCFSInterface(object): ...@@ -26,10 +26,10 @@ class RPCFSInterface(object):
the contents of files. the contents of files.
""" """
def __init__(self,fs): def __init__(self, fs):
self.fs = fs self.fs = fs
def encode_path(self,path): def encode_path(self, path):
"""Encode a filesystem path for sending over the wire. """Encode a filesystem path for sending over the wire.
Unfortunately XMLRPC only supports ASCII strings, so this method Unfortunately XMLRPC only supports ASCII strings, so this method
...@@ -38,99 +38,99 @@ class RPCFSInterface(object): ...@@ -38,99 +38,99 @@ class RPCFSInterface(object):
""" """
return path.encode("utf8").encode("base64") return path.encode("utf8").encode("base64")
def decode_path(self,path): def decode_path(self, path):
"""Decode paths arriving over the wire.""" """Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8") return path.decode("base64").decode("utf8")
def get_contents(self,path): def get_contents(self, path):
path = self.decode_path(path) path = self.decode_path(path)
data = self.fs.getcontents(path) data = self.fs.getcontents(path)
return xmlrpclib.Binary(data) return xmlrpclib.Binary(data)
def set_contents(self,path,data): def set_contents(self, path, data):
path = self.decode_path(path) path = self.decode_path(path)
self.fs.createfile(path,data.data) self.fs.createfile(path,data.data)
def exists(self,path): def exists(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.exists(path) return self.fs.exists(path)
def isdir(self,path): def isdir(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.isdir(path) return self.fs.isdir(path)
def isfile(self,path): def isfile(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.isfile(path) return self.fs.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = self.decode_path(path) path = self.decode_path(path)
entries = self.fs.listdir(path,wildcard,full,absolute,dirs_only,files_only) entries = self.fs.listdir(path,wildcard,full,absolute,dirs_only,files_only)
return [self.encode_path(e) for e in entries] return [self.encode_path(e) for e in entries]
def makedir(self,path,recursive=False,allow_recreate=False): def makedir(self, path, recursive=False, allow_recreate=False):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.makedir(path,recursive,allow_recreate) return self.fs.makedir(path, recursive, allow_recreate)
def remove(self,path): def remove(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.remove(path) return self.fs.remove(path)
def removedir(self,path,recursive=False,force=False): def removedir(self, path, recursive=False, force=False):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.removedir(path,recursive,force) return self.fs.removedir(path, recursive, force)
def rename(self,src,dst): def rename(self, src, dst):
src = self.decode_path(src) src = self.decode_path(src)
dst = self.decode_path(dst) dst = self.decode_path(dst)
return self.fs.rename(src,dst) return self.fs.rename(src, dst)
def getinfo(self,path): def getinfo(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.getinfo(path) return self.fs.getinfo(path)
def desc(self,path): def desc(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return self.fs.desc(path) return self.fs.desc(path)
def getxattr(self,path,attr,default=None): def getxattr(self, path, attr, default=None):
path = self.decode_path(path) path = self.decode_path(path)
attr = self.decode_path(attr) attr = self.decode_path(attr)
return self.fs.getxattr(path,attr,default) return self.fs.getxattr(path, attr, default)
def setxattr(self,path,attr,value): def setxattr(self, path, attr, value):
path = self.decode_path(path) path = self.decode_path(path)
attr = self.decode_path(attr) attr = self.decode_path(attr)
return self.fs.setxattr(path,attr,value) return self.fs.setxattr(path, attr, value)
def delxattr(self,path,attr): def delxattr(self, path, attr):
path = self.decode_path(path) path = self.decode_path(path)
attr = self.decode_path(attr) attr = self.decode_path(attr)
return self.fs.delxattr(path,attr) return self.fs.delxattr(path, attr)
def listxattrs(self,path): def listxattrs(self, path):
path = self.decode_path(path) path = self.decode_path(path)
return [self.encode_path(a) for a in self.fs.listxattrs(path)] return [self.encode_path(a) for a in self.fs.listxattrs(path)]
def copy(self,src,dst,overwrite=False,chunk_size=16384): def copy(self, src, dst, overwrite=False, chunk_size=16384):
src = self.decode_path(src) src = self.decode_path(src)
dst = self.decode_path(dst) dst = self.decode_path(dst)
return self.fs.copy(src,dst,overwrite,chunk_size) return self.fs.copy(src, dst, overwrite, chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384): def move(self,src,dst,overwrite=False,chunk_size=16384):
src = self.decode_path(src) src = self.decode_path(src)
dst = self.decode_path(dst) dst = self.decode_path(dst)
return self.fs.move(src,dst,overwrite,chunk_size) return self.fs.move(src, dst, overwrite, chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.decode_path(src) src = self.decode_path(src)
dst = self.decode_path(dst) dst = self.decode_path(dst)
return self.fs.movedir(src,dst,overwrite,ignore_errors,chunk_size) return self.fs.movedir(src, dst, overwrite, ignore_errors, chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.decode_path(src) src = self.decode_path(src)
dst = self.decode_path(dst) dst = self.decode_path(dst)
return self.fs.copydir(src,dst,overwrite,ignore_errors,chunk_size) return self.fs.copydir(src, dst, overwrite, ignore_errors, chunk_size)
class RPCFSServer(SimpleXMLRPCServer): class RPCFSServer(SimpleXMLRPCServer):
...@@ -148,7 +148,7 @@ class RPCFSServer(SimpleXMLRPCServer): ...@@ -148,7 +148,7 @@ class RPCFSServer(SimpleXMLRPCServer):
attribute "serve_more_requests" to False. attribute "serve_more_requests" to False.
""" """
def __init__(self,fs,addr,requestHandler=None,logRequests=None): def __init__(self, fs, addr, requestHandler=None, logRequests=None):
kwds = dict(allow_none=True) kwds = dict(allow_none=True)
if requestHandler is not None: if requestHandler is not None:
kwds['requestHandler'] = requestHandler kwds['requestHandler'] = requestHandler
......
fs/memoryfs.py
...@@ -306,7 +306,6 @@ class MemoryFS(FS): ...@@ -306,7 +306,6 @@ class MemoryFS(FS):
if dir_item is None: if dir_item is None:
parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname) parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
return self
def _orphan_files(self, file_dir_entry): def _orphan_files(self, file_dir_entry):
for f in file_dir_entry.open_files: for f in file_dir_entry.open_files:
...@@ -505,11 +504,11 @@ class MemoryFS(FS): ...@@ -505,11 +504,11 @@ class MemoryFS(FS):
info['modified_time'] = dir_entry.modified_time info['modified_time'] = dir_entry.modified_time
info['accessed_time'] = dir_entry.accessed_time info['accessed_time'] = dir_entry.accessed_time
if dir_entry.isfile(): if dir_entry.isdir():
info['st_mode'] = 0755
else:
info['size'] = len(dir_entry.data or '') info['size'] = len(dir_entry.data or '')
info['st_mode'] = 0666 info['st_mode'] = 0666
else:
info['st_mode'] = 0700
return info return info
......
fs/mountfs.py
...@@ -2,7 +2,43 @@ ...@@ -2,7 +2,43 @@
fs.mountfs fs.mountfs
========== ==========
Contains MountFS class which is a virtual Filesystem which can have other Filesystems linked as branched directories, much like a symlink in Linux Contains MountFS class which is a virtual filesystem which can have other filesystems linked as branched directories.
For example, lets say we have two filesystems containing config files and resource respectively::
[config_fs]
|-- config.cfg
`-- defaults.cfg
[resources_fs]
|-- images
| |-- logo.jpg
| `-- photo.jpg
`-- data.dat
We can combine these filesystems in to a single filesystem with the following code::
from fs.mountfs import MountFS
combined_fs = MountFS
combined_fs.mountdir('config', config_fs)
combined_fs.mountdir('resources', resources_fs)
This will create a single filesystem where paths under `config` map to `config_fs`, and paths under `resources` map to `resources_fs`::
[combined_fs]
|-- config
| |-- config.cfg
| `-- defaults.cfg
`-- resources
|-- images
| |-- logo.jpg
| `-- photo.jpg
`-- data.dat
Now both filesystems can be accessed with the same path structure::
print combined_fs.getcontents('/config/defaults.cfg')
read_jpg(combined_fs.open('/resources/images/logo.jpg')
""" """
...@@ -278,7 +314,13 @@ class MountFS(FS): ...@@ -278,7 +314,13 @@ class MountFS(FS):
@synchronize @synchronize
def mountfile(self, path, open_callable=None, info_callable=None): def mountfile(self, path, open_callable=None, info_callable=None):
"""Mounts a single file path. """ """Mounts a single file path.
:param path: A path within the MountFS
:param open_Callable: A callable that returns a file-like object
:param info_callable: A callable that returns a dictionary with information regarding the file-like object
"""
path = normpath(path) path = normpath(path)
self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable) self.mount_tree[path] = MountFS.FileMount(path, callable, info_callable)
......
fs/multifs.py
...@@ -9,7 +9,26 @@ FS in order, until it either finds a path that exists or raises a ResourceNotFou ...@@ -9,7 +9,26 @@ FS in order, until it either finds a path that exists or raises a ResourceNotFou
One use for such a filesystem would be to selectively override a set of files, One use for such a filesystem would be to selectively override a set of files,
to customize behaviour. For example, to create a filesystem that could be used to customize behaviour. For example, to create a filesystem that could be used
to *theme* a web application:: to *theme* a web application. We start with the following directories::
`-- templates
|-- snippets
| `-- panel.html
|-- index.html
|-- profile.html
`-- base.html
`-- theme
|-- snippets
| |-- widget.html
| `-- extra.html
|-- index.html
`-- theme.html
And we want to create a single filesystem that looks for files in `templates` if
they don't exist in `theme`. We can do this with the following code::
from fs.osfs import OSFS from fs.osfs import OSFS
from fs.multifs import MultiFS from fs.multifs import MultiFS
...@@ -17,10 +36,19 @@ to *theme* a web application:: ...@@ -17,10 +36,19 @@ to *theme* a web application::
themed_template_fs.addfs('templates', OSFS('templates')) themed_template_fs.addfs('templates', OSFS('templates'))
themed_template_fs.addfs('theme', OSFS('themes')) themed_template_fs.addfs('theme', OSFS('themes'))
index_template = themed_template_fs.getcontent('index.html')
This will read the contents of *themes/index.html*, if it exists, otherwise Now we have a `themed_template_fs` FS object presents a single view of both
it will look for it in *templates/index.html*. directories::
|-- snippets
| |-- panel.html
| |-- widget.html
| `-- extra.html
|-- index.html
|-- profile.html
|-- base.html
`-- theme.html
""" """
......
fs/osfs/__init__.py
...@@ -31,7 +31,7 @@ def _os_stat(path): ...@@ -31,7 +31,7 @@ def _os_stat(path):
return os.stat(path) return os.stat(path)
class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS): class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
"""Expose the underlying operating-system filesystem as an FS object. """Expose the underlying operating-system filesystem as an FS object.
This is the most basic of filesystems, which simply shadows the underlaying This is the most basic of filesystems, which simply shadows the underlaying
...@@ -39,15 +39,15 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS): ...@@ -39,15 +39,15 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
methods in the os and os.path modules. methods in the os and os.path modules.
""" """
def __init__(self, root_path, dir_mode=0700, thread_synchronize=_thread_synchronize_default, encoding=None, create=False): def __init__(self, root_path, thread_synchronize=_thread_synchronize_default, encoding=None, create=False, dir_mode=0700):
""" """
Creates an FS object that represents the OS Filesystem under a given root path Creates an FS object that represents the OS Filesystem under a given root path
:param root_path: The root OS path :param root_path: The root OS path
:param dir_mode: srt
:param thread_synchronize: If True, this object will be thread-safe by use of a threading.Lock object :param thread_synchronize: If True, this object will be thread-safe by use of a threading.Lock object
:param encoding: The encoding method for path strings :param encoding: The encoding method for path strings
:param create: Of True, then root_path will be created (if necessary) :param create: If True, then root_path will be created if it doesn't already exist
:param dir_mode: The mode to use when creating the directory
""" """
...@@ -90,11 +90,16 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS): ...@@ -90,11 +90,16 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
path = self._decode_path(path) path = self._decode_path(path)
return path return path
def unsyspath(self,path): def unsyspath(self, path):
"""Convert a system-level path into an FS-level path. """Convert a system-level path into an FS-level path.
This basically the reverse of getsyspath(). If the path does not This basically the reverse of getsyspath(). If the path does not
refer to a location within this filesystem, ValueError is raised. refer to a location within this filesystem, ValueError is raised.
:param path: a system path
:returns: a path within this FS object
:rtype: string
""" """
path = os.path.normpath(os.path.abspath(path)) path = os.path.normpath(os.path.abspath(path))
if not path.startswith(self.root_path + os.path.sep): if not path.startswith(self.root_path + os.path.sep):
...@@ -208,7 +213,6 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS): ...@@ -208,7 +213,6 @@ class OSFS(OSFSXAttrMixin,OSFSWatchMixin,FS):
raise ParentDirectoryMissingError(dst) raise ParentDirectoryMissingError(dst)
raise raise
def _stat(self,path): def _stat(self,path):
"""Stat the given path, normalising error codes.""" """Stat the given path, normalising error codes."""
sys_path = self.getsyspath(path) sys_path = self.getsyspath(path)
......
fs/path.py
...@@ -4,8 +4,8 @@ fs.path ...@@ -4,8 +4,8 @@ fs.path
Useful functions for FS path manipulation. Useful functions for FS path manipulation.
This is broadly similar to the standard 'os.path' module but works with This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (backslash-separated, paths in the canonical format expected by all FS objects (forwardslash-separated,
optional leading slash). optional leading slash).
""" """
...@@ -19,7 +19,8 @@ def normpath(path): ...@@ -19,7 +19,8 @@ def normpath(path):
tries very hard to return a new path string the canonical FS format. tries very hard to return a new path string the canonical FS format.
If the path is invalid, ValueError will be raised. If the path is invalid, ValueError will be raised.
:param path: Path to normalize :param path: path to normalize
:returns: a valid FS path
>>> normpath(r"foo\\bar\\baz") >>> normpath(r"foo\\bar\\baz")
'foo/bar/baz' 'foo/bar/baz'
...@@ -50,7 +51,7 @@ def normpath(path): ...@@ -50,7 +51,7 @@ def normpath(path):
if path[0] in "\\/": if path[0] in "\\/":
if not components: if not components:
components = [""] components = [""]
components.insert(0,"") components.insert(0, "")
if isinstance(path, unicode): if isinstance(path, unicode):
return u"/".join(components) return u"/".join(components)
else: else:
...@@ -73,7 +74,14 @@ def iteratepath(path, numsplits=None): ...@@ -73,7 +74,14 @@ def iteratepath(path, numsplits=None):
return map(None, path.split('/', numsplits)) return map(None, path.split('/', numsplits))
def recursepath(path, reverse=False): def recursepath(path, reverse=False):
"""Iterate from root to path, returning intermediate paths""" """Returns intermediate paths from the root to the given path
:param reverse: reverses the order of the paths
>>> recursepath('a/b/c')
['/', u'/a', u'/a/b', u'/a/b/c']
"""
if reverse: if reverse:
paths = [] paths = []
path = abspath(path).rstrip("/") path = abspath(path).rstrip("/")
...@@ -107,6 +115,9 @@ def relpath(path): ...@@ -107,6 +115,9 @@ def relpath(path):
:param path: Path to adjust :param path: Path to adjust
>>> relpath('/a/b')
'a/b'
""" """
while path and path[0] == "/": while path and path[0] == "/":
path = path[1:] path = path[1:]
...@@ -147,9 +158,9 @@ join = pathjoin ...@@ -147,9 +158,9 @@ join = pathjoin
def pathsplit(path): def pathsplit(path):
"""Splits a path into (head,tail) pair. """Splits a path into (head, tail) pair.
This function splits a path into a pair (head,tail) where 'tail' is the This function splits a path into a pair (head, tail) where 'tail' is the
last pathname component and 'head' is all preceeding components. last pathname component and 'head' is all preceeding components.
:param path: Path to split :param path: Path to split
...@@ -269,7 +280,7 @@ class PathMap(object): ...@@ -269,7 +280,7 @@ class PathMap(object):
A PathMap is like a dictionary where the keys are all FS paths. It allows A PathMap is like a dictionary where the keys are all FS paths. It allows
various dictionary operations (e.g. listing values, clearing values) to various dictionary operations (e.g. listing values, clearing values) to
be performed on a subset of the keys sharing some common prefix, e.g.: be performed on a subset of the keys sharing some common prefix, e.g.::
# list all values in the map # list all values in the map
pm.values() pm.values()
......
fs/rpcfs.py
""" """
fs.rpcfs
fs.rpcfs: client to access an FS via XML-RPC ========
This module provides the class 'RPCFS' to access a remote FS object over This module provides the class 'RPCFS' to access a remote FS object over
XML-RPC. You probably want to use this in conjunction with the 'RPCFSServer' XML-RPC. You probably want to use this in conjunction with the 'RPCFSServer'
class from the fs.expose.xmlrpc module. class from the :mod:`fs.expose.xmlrpc` module.
""" """
...@@ -90,7 +90,7 @@ class RPCFS(FS): ...@@ -90,7 +90,7 @@ class RPCFS(FS):
This class provides the client-side logic for accessing a remote FS This class provides the client-side logic for accessing a remote FS
object, and is dual to the RPCFSServer class defined in fs.expose.xmlrpc. object, and is dual to the RPCFSServer class defined in fs.expose.xmlrpc.
Example: Example::
fs = RPCFS("http://my.server.com/filesystem/location/") fs = RPCFS("http://my.server.com/filesystem/location/")
...@@ -102,6 +102,9 @@ class RPCFS(FS): ...@@ -102,6 +102,9 @@ class RPCFS(FS):
The only required argument is the uri of the server to connect The only required argument is the uri of the server to connect
to. This will be passed to the underlying XML-RPC server proxy to. This will be passed to the underlying XML-RPC server proxy
object, along with the 'transport' argument if it is provided. object, along with the 'transport' argument if it is provided.
:param uri: address of the server
""" """
self.uri = uri self.uri = uri
self._transport = transport self._transport = transport
...@@ -127,12 +130,12 @@ class RPCFS(FS): ...@@ -127,12 +130,12 @@ class RPCFS(FS):
pass pass
return state return state
def __setstate__(self,state): def __setstate__(self, state):
for (k,v) in state.iteritems(): for (k,v) in state.iteritems():
self.__dict__[k] = v self.__dict__[k] = v
self.proxy = self._make_proxy() self.proxy = self._make_proxy()
def encode_path(self,path): def encode_path(self, path):
"""Encode a filesystem path for sending over the wire. """Encode a filesystem path for sending over the wire.
Unfortunately XMLRPC only supports ASCII strings, so this method Unfortunately XMLRPC only supports ASCII strings, so this method
...@@ -141,11 +144,11 @@ class RPCFS(FS): ...@@ -141,11 +144,11 @@ class RPCFS(FS):
""" """
return path.encode("utf8").encode("base64") return path.encode("utf8").encode("base64")
def decode_path(self,path): def decode_path(self, path):
"""Decode paths arriving over the wire.""" """Decode paths arriving over the wire."""
return path.decode("base64").decode("utf8") return path.decode("base64").decode("utf8")
def open(self,path,mode="r"): def open(self, path, mode="r"):
# TODO: chunked transport of large files # TODO: chunked transport of large files
path = self.encode_path(path) path = self.encode_path(path)
if "w" in mode: if "w" in mode:
...@@ -178,83 +181,83 @@ class RPCFS(FS): ...@@ -178,83 +181,83 @@ class RPCFS(FS):
f.close = newclose f.close = newclose
return f return f
def exists(self,path): def exists(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.exists(path) return self.proxy.exists(path)
def isdir(self,path): def isdir(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.isdir(path) return self.proxy.isdir(path)
def isfile(self,path): def isfile(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.isfile(path) return self.proxy.isfile(path)
def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
path = self.encode_path(path) path = self.encode_path(path)
entries = self.proxy.listdir(path,wildcard,full,absolute,dirs_only,files_only) entries = self.proxy.listdir(path,wildcard,full,absolute,dirs_only,files_only)
return [self.decode_path(e) for e in entries] return [self.decode_path(e) for e in entries]
def makedir(self,path,recursive=False,allow_recreate=False): def makedir(self, path, recursive=False, allow_recreate=False):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.makedir(path,recursive,allow_recreate) return self.proxy.makedir(path,recursive,allow_recreate)
def remove(self,path): def remove(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.remove(path) return self.proxy.remove(path)
def removedir(self,path,recursive=False,force=False): def removedir(self, path, recursive=False, force=False):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.removedir(path,recursive,force) return self.proxy.removedir(path,recursive,force)
def rename(self,src,dst): def rename(self, src, dst):
src = self.encode_path(src) src = self.encode_path(src)
dst = self.encode_path(dst) dst = self.encode_path(dst)
return self.proxy.rename(src,dst) return self.proxy.rename(src,dst)
def getinfo(self,path): def getinfo(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.getinfo(path) return self.proxy.getinfo(path)
def desc(self,path): def desc(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return self.proxy.desc(path) return self.proxy.desc(path)
def getxattr(self,path,attr,default=None): def getxattr(self, path, attr, default=None):
path = self.encode_path(path) path = self.encode_path(path)
attr = self.encode_path(attr) attr = self.encode_path(attr)
return self.fs.getxattr(path,attr,default) return self.fs.getxattr(path,attr,default)
def setxattr(self,path,attr,value): def setxattr(self, path, attr, value):
path = self.encode_path(path) path = self.encode_path(path)
attr = self.encode_path(attr) attr = self.encode_path(attr)
return self.fs.setxattr(path,attr,value) return self.fs.setxattr(path,attr,value)
def delxattr(self,path,attr): def delxattr(self, path, attr):
path = self.encode_path(path) path = self.encode_path(path)
attr = self.encode_path(attr) attr = self.encode_path(attr)
return self.fs.delxattr(path,attr) return self.fs.delxattr(path,attr)
def listxattrs(self,path): def listxattrs(self, path):
path = self.encode_path(path) path = self.encode_path(path)
return [self.decode_path(a) for a in self.fs.listxattrs(path)] return [self.decode_path(a) for a in self.fs.listxattrs(path)]
def copy(self,src,dst,overwrite=False,chunk_size=16384): def copy(self, src, dst, overwrite=False, chunk_size=16384):
src = self.encode_path(src) src = self.encode_path(src)
dst = self.encode_path(dst) dst = self.encode_path(dst)
return self.proxy.copy(src,dst,overwrite,chunk_size) return self.proxy.copy(src,dst,overwrite,chunk_size)
def move(self,src,dst,overwrite=False,chunk_size=16384): def move(self, src, dst, overwrite=False, chunk_size=16384):
src = self.encode_path(src) src = self.encode_path(src)
dst = self.encode_path(dst) dst = self.encode_path(dst)
return self.proxy.move(src,dst,overwrite,chunk_size) return self.proxy.move(src,dst,overwrite,chunk_size)
def movedir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.encode_path(src) src = self.encode_path(src)
dst = self.encode_path(dst) dst = self.encode_path(dst)
return self.proxy.movedir(src,dst,overwrite,ignore_errors,chunk_size) return self.proxy.movedir(src, dst, overwrite, ignore_errors, chunk_size)
def copydir(self,src,dst,overwrite=False,ignore_errors=False,chunk_size=16384): def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
src = self.encode_path(src) src = self.encode_path(src)
dst = self.encode_path(dst) dst = self.encode_path(dst)
return self.proxy.copydir(src,dst,overwrite,ignore_errors,chunk_size) return self.proxy.copydir(src,dst,overwrite,ignore_errors,chunk_size)
......
fs/sftpfs.py
...@@ -42,7 +42,7 @@ class SFTPFS(FS): ...@@ -42,7 +42,7 @@ class SFTPFS(FS):
class in the paramiko module. class in the paramiko module.
""" """
def __init__(self,connection,root_path="/",encoding=None,**credentials): def __init__(self, connection, root_path="/", encoding=None, **credentials):
"""SFTPFS constructor. """SFTPFS constructor.
The only required argument is 'connection', which must be something The only required argument is 'connection', which must be something
...@@ -58,6 +58,10 @@ class SFTPFS(FS): ...@@ -58,6 +58,10 @@ class SFTPFS(FS):
machine - access to files outsite this root wil be prevented. Any machine - access to files outsite this root wil be prevented. Any
other keyword arguments are assumed to be credentials to be used when other keyword arguments are assumed to be credentials to be used when
connecting the transport. connecting the transport.
:param connection: a connection string
:param root_path: The root path to open
""" """
if encoding is None: if encoding is None:
encoding = "utf8" encoding = "utf8"
......
fs/utils.py
...@@ -181,7 +181,7 @@ def find_duplicates(fs, ...@@ -181,7 +181,7 @@ def find_duplicates(fs,
other attributes not take in to account). other attributes not take in to account).
:param fs: A filesystem object :param fs: A filesystem object
:param compare_paths: An iterable of paths within the FS object, or all files if omited :param compare_paths: An iterable of paths within the FS object, or all files if omitted
:param quick: If set to True, the quick method of finding duplicates will be used, which can potentially return false positives if the files have the same size and start with the same data. Do not use when deleting files! :param quick: If set to True, the quick method of finding duplicates will be used, which can potentially return false positives if the files have the same size and start with the same data. Do not use when deleting files!
:param signature_chunk_size: The number of bytes to read before generating a signature checksum value :param signature_chunk_size: The number of bytes to read before generating a signature checksum value
:param signature_size: The total number of bytes read to generate a signature :param signature_size: The total number of bytes read to generate a signature
...@@ -281,7 +281,7 @@ def find_duplicates(fs, ...@@ -281,7 +281,7 @@ def find_duplicates(fs,
paths = list(set(paths).difference(dups)) paths = list(set(paths).difference(dups))
def print_fs(fs, path='/', max_levels=5, file_out=sys.stdout, terminal_colors=None): def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
"""Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid. """Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid.
Be careful about printing a OSFS, or any other large filesystem. Be careful about printing a OSFS, or any other large filesystem.
Without max_levels set, this function will traverse the entire directory tree. Without max_levels set, this function will traverse the entire directory tree.
...@@ -303,6 +303,9 @@ def print_fs(fs, path='/', max_levels=5, file_out=sys.stdout, terminal_colors=No ...@@ -303,6 +303,9 @@ def print_fs(fs, path='/', max_levels=5, file_out=sys.stdout, terminal_colors=No
""" """
if file_out is None:
file_out = sys.stdout
if terminal_colors is None: if terminal_colors is None:
if sys.platform == 'win32': if sys.platform == 'win32':
terminal_colors = False terminal_colors = False
......
fs/wrapfs/__init__.py
...@@ -47,20 +47,20 @@ class WrapFS(FS): ...@@ -47,20 +47,20 @@ class WrapFS(FS):
The following methods can be overridden to control how files are The following methods can be overridden to control how files are
accessed in the underlying FS object: accessed in the underlying FS object:
_file_wrap(file,mode): called for each file that is opened from * _file_wrap(file, mode): called for each file that is opened from
the underlying FS; may return a modified the underlying FS; may return a modified
file-like object. file-like object.
_encode(path): encode a path for access in the underlying FS * _encode(path): encode a path for access in the underlying FS
_decode(path): decode a path from the underlying FS * _decode(path): decode a path from the underlying FS
If the required path translation proceeds one component at a time, If the required path translation proceeds one component at a time,
it may be simpler to override the _encode_name() and _decode_name() it may be simpler to override the _encode_name() and _decode_name()
methods. methods.
""" """
def __init__(self,fs): def __init__(self, fs):
super(WrapFS,self).__init__() super(WrapFS,self).__init__()
try: try:
self._lock = fs._lock self._lock = fs._lock
...@@ -68,19 +68,19 @@ class WrapFS(FS): ...@@ -68,19 +68,19 @@ class WrapFS(FS):
self._lock = None self._lock = None
self.wrapped_fs = fs self.wrapped_fs = fs
def _file_wrap(self,f,mode): def _file_wrap(self, f, mode):
"""Apply wrapping to an opened file.""" """Apply wrapping to an opened file."""
return f return f
def _encode_name(self,name): def _encode_name(self, name):
"""Encode path component for the underlying FS.""" """Encode path component for the underlying FS."""
return name return name
def _decode_name(self,name): def _decode_name(self, name):
"""Decode path component from the underlying FS.""" """Decode path component from the underlying FS."""
return name return name
def _encode(self,path): def _encode(self, path):
"""Encode path for the underlying FS.""" """Encode path for the underlying FS."""
names = path.split("/") names = path.split("/")
e_names = [] e_names = []
...@@ -91,7 +91,7 @@ class WrapFS(FS): ...@@ -91,7 +91,7 @@ class WrapFS(FS):
e_names.append(self._encode_name(name)) e_names.append(self._encode_name(name))
return "/".join(e_names) return "/".join(e_names)
def _decode(self,path): def _decode(self, path):
"""Decode path from the underlying FS.""" """Decode path from the underlying FS."""
names = path.split("/") names = path.split("/")
d_names = [] d_names = []
...@@ -102,7 +102,7 @@ class WrapFS(FS): ...@@ -102,7 +102,7 @@ class WrapFS(FS):
d_names.append(self._decode_name(name)) d_names.append(self._decode_name(name))
return "/".join(d_names) return "/".join(d_names)
def _adjust_mode(self,mode): def _adjust_mode(self, mode):
"""Adjust the mode used to open a file in the underlying FS. """Adjust the mode used to open a file in the underlying FS.
This method takes the mode given when opening a file, and should This method takes the mode given when opening a file, and should
...@@ -116,11 +116,11 @@ class WrapFS(FS): ...@@ -116,11 +116,11 @@ class WrapFS(FS):
return (mode,mode) return (mode,mode)
@rewrite_errors @rewrite_errors
def getsyspath(self,path,allow_none=False): def getsyspath(self, path, allow_none=False):
return self.wrapped_fs.getsyspath(self._encode(path),allow_none) return self.wrapped_fs.getsyspath(self._encode(path),allow_none)
@rewrite_errors @rewrite_errors
def hassyspath(self,path): def hassyspath(self, path):
return self.wrapped_fs.hassyspath(self._encode(path)) return self.wrapped_fs.hassyspath(self._encode(path))
@rewrite_errors @rewrite_errors
...@@ -130,19 +130,19 @@ class WrapFS(FS): ...@@ -130,19 +130,19 @@ class WrapFS(FS):
return self._file_wrap(f, mode) return self._file_wrap(f, mode)
@rewrite_errors @rewrite_errors
def exists(self,path): def exists(self, path):
return self.wrapped_fs.exists(self._encode(path)) return self.wrapped_fs.exists(self._encode(path))
@rewrite_errors @rewrite_errors
def isdir(self,path): def isdir(self, path):
return self.wrapped_fs.isdir(self._encode(path)) return self.wrapped_fs.isdir(self._encode(path))
@rewrite_errors @rewrite_errors
def isfile(self,path): def isfile(self, path):
return self.wrapped_fs.isfile(self._encode(path)) return self.wrapped_fs.isfile(self._encode(path))
@rewrite_errors @rewrite_errors
def listdir(self,path="",**kwds): def listdir(self, path="", **kwds):
wildcard = kwds.pop("wildcard","*") wildcard = kwds.pop("wildcard","*")
info = kwds.get("info",False) info = kwds.get("info",False)
entries = [] entries = []
...@@ -160,74 +160,74 @@ class WrapFS(FS): ...@@ -160,74 +160,74 @@ class WrapFS(FS):
return entries return entries
@rewrite_errors @rewrite_errors
def makedir(self,path,*args,**kwds): def makedir(self, path, *args, **kwds):
return self.wrapped_fs.makedir(self._encode(path),*args,**kwds) return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)
@rewrite_errors @rewrite_errors
def remove(self,path): def remove(self, path):
return self.wrapped_fs.remove(self._encode(path)) return self.wrapped_fs.remove(self._encode(path))
@rewrite_errors @rewrite_errors
def removedir(self,path,*args,**kwds): def removedir(self, path, *args, **kwds):
return self.wrapped_fs.removedir(self._encode(path),*args,**kwds) return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)
@rewrite_errors @rewrite_errors
def rename(self,src,dst): def rename(self, src, dst):
return self.wrapped_fs.rename(self._encode(src),self._encode(dst)) return self.wrapped_fs.rename(self._encode(src),self._encode(dst))
@rewrite_errors @rewrite_errors
def getinfo(self,path): def getinfo(self, path):
return self.wrapped_fs.getinfo(self._encode(path)) return self.wrapped_fs.getinfo(self._encode(path))
@rewrite_errors @rewrite_errors
def desc(self,path): def desc(self, path):
return self.wrapped_fs.desc(self._encode(path)) return self.wrapped_fs.desc(self._encode(path))
@rewrite_errors @rewrite_errors
def copy(self,src,dst,**kwds): def copy(self, src, dst, **kwds):
return self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds) return self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors @rewrite_errors
def move(self,src,dst,**kwds): def move(self, src, dst, **kwds):
return self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds) return self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors @rewrite_errors
def movedir(self,src,dst,**kwds): def movedir(self, src, dst, **kwds):
return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),**kwds) return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors @rewrite_errors
def copydir(self,src,dst,**kwds): def copydir(self, src, dst, **kwds):
return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),**kwds) return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),**kwds)
@rewrite_errors @rewrite_errors
def getxattr(self,path,name,default=None): def getxattr(self, path, name, default=None):
try: try:
return self.wrapped_fs.getxattr(self._encode(path),name,default) return self.wrapped_fs.getxattr(self._encode(path),name,default)
except AttributeError: except AttributeError:
raise UnsupportedError("getxattr") raise UnsupportedError("getxattr")
@rewrite_errors @rewrite_errors
def setxattr(self,path,name,value): def setxattr(self, path, name, value):
try: try:
return self.wrapped_fs.setxattr(self._encode(path),name,value) return self.wrapped_fs.setxattr(self._encode(path),name,value)
except AttributeError: except AttributeError:
raise UnsupportedError("setxattr") raise UnsupportedError("setxattr")
@rewrite_errors @rewrite_errors
def delxattr(self,path,name): def delxattr(self, path, name):
try: try:
return self.wrapped_fs.delxattr(self._encode(path),name) return self.wrapped_fs.delxattr(self._encode(path),name)
except AttributeError: except AttributeError:
raise UnsupportedError("delxattr") raise UnsupportedError("delxattr")
@rewrite_errors @rewrite_errors
def listxattrs(self,path): def listxattrs(self, path):
try: try:
return self.wrapped_fs.listxattrs(self._encode(path)) return self.wrapped_fs.listxattrs(self._encode(path))
except AttributeError: except AttributeError:
raise UnsupportedError("listxattrs") raise UnsupportedError("listxattrs")
def __getattr__(self,attr): def __getattr__(self, attr):
# These attributes can be used by the destructor, but may not be # These attributes can be used by the destructor, but may not be
# defined if there are errors in the constructor. # defined if there are errors in the constructor.
if attr == "closed": if attr == "closed":
...@@ -243,18 +243,18 @@ class WrapFS(FS): ...@@ -243,18 +243,18 @@ class WrapFS(FS):
super(WrapFS,self).close() super(WrapFS,self).close()
def wrap_fs_methods(decorator,cls=None,exclude=[]): def wrap_fs_methods(decorator, cls=None, exclude=[]):
"""Apply the given decorator to all FS methods on the given class. """Apply the given decorator to all FS methods on the given class.
This function can be used in two ways. When called with two arguments it This function can be used in two ways. When called with two arguments it
applies the given function 'decorator' to each FS method of the given applies the given function 'decorator' to each FS method of the given
class. When called with just a single argument, it creates and returns class. When called with just a single argument, it creates and returns
a class decorator which will do the same thing when applied. So you can a class decorator which will do the same thing when applied. So you can
use it like this: use it like this::
wrap_fs_methods(mydecorator,MyFSClass) wrap_fs_methods(mydecorator,MyFSClass)
Or on more recent Python versions, like this: Or on more recent Python versions, like this::
@wrap_fs_methods(mydecorator) @wrap_fs_methods(mydecorator)
class MyFSClass(FS): class MyFSClass(FS):
......
fs/wrapfs/hidedotfilesfs.py
""" """
fs.wrapfs.hidedotfiles fs.wrapfs.hidedotfilesfs
====================== ========================
An FS wrapper class for hiding dot-files in directory listings. An FS wrapper class for hiding dot-files in directory listings.
...@@ -17,17 +17,17 @@ class HideDotFilesFS(WrapFS): ...@@ -17,17 +17,17 @@ class HideDotFilesFS(WrapFS):
It is False by default. It is False by default.
""" """
def is_hidden(self,path): def is_hidden(self, path):
"""Check whether the given path should be hidden.""" """Check whether the given path should be hidden."""
return path and basename(path)[0] == "." return path and basename(path)[0] == "."
def _encode(self,path): def _encode(self, path):
return path return path
def _decode(self,path): def _decode(self, path):
return path return path
def listdir(self,path="",**kwds): def listdir(self, path="", **kwds):
hidden = kwds.pop("hidden",True) hidden = kwds.pop("hidden",True)
entries = self.wrapped_fs.listdir(path,**kwds) entries = self.wrapped_fs.listdir(path,**kwds)
if not hidden: if not hidden:
......
fs/wrapfs/lazyfs.py
...@@ -26,7 +26,7 @@ class LazyFS(WrapFS): ...@@ -26,7 +26,7 @@ class LazyFS(WrapFS):
the first time it is accessed. the first time it is accessed.
""" """
def __init__(self,fs): def __init__(self, fs):
super(LazyFS,self).__init__(fs) super(LazyFS,self).__init__(fs)
self._lazy_creation_lock = Lock() self._lazy_creation_lock = Lock()
...@@ -35,7 +35,7 @@ class LazyFS(WrapFS): ...@@ -35,7 +35,7 @@ class LazyFS(WrapFS):
del state["_lazy_creation_lock"] del state["_lazy_creation_lock"]
return state return state
def __setstate__(self,state): def __setstate__(self, state):
self.__dict__.update(state) self.__dict__.update(state)
self._lazy_creation_lock = Lock() self._lazy_creation_lock = Lock()
...@@ -55,7 +55,7 @@ class LazyFS(WrapFS): ...@@ -55,7 +55,7 @@ class LazyFS(WrapFS):
finally: finally:
self._lazy_creation_lock.release() self._lazy_creation_lock.release()
def _set_wrapped_fs(self,fs): def _set_wrapped_fs(self, fs):
if isinstance(fs,FS): if isinstance(fs,FS):
self.__dict__["wrapped_fs"] = fs self.__dict__["wrapped_fs"] = fs
elif isinstance(fs,type): elif isinstance(fs,type):
...@@ -75,7 +75,7 @@ class LazyFS(WrapFS): ...@@ -75,7 +75,7 @@ class LazyFS(WrapFS):
wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs) wrapped_fs = property(_get_wrapped_fs,_set_wrapped_fs)
def setcontents(self,path,data): def setcontents(self, path, data):
return self.wrapped_fs.setcontents(path,data) return self.wrapped_fs.setcontents(path,data)
def close(self): def close(self):
......
fs/wrapfs/limitsizefs.py
...@@ -18,14 +18,14 @@ from fs.wrapfs import WrapFS ...@@ -18,14 +18,14 @@ from fs.wrapfs import WrapFS
class LimitSizeFS(WrapFS): class LimitSizeFS(WrapFS):
"""FS wrapper class to limit total size of files stored.""" """FS wrapper class to limit total size of files stored."""
def __init__(self,fs,max_size): def __init__(self, fs, max_size):
super(LimitSizeFS,self).__init__(fs) super(LimitSizeFS,self).__init__(fs)
self.max_size = max_size self.max_size = max_size
self.cur_size = sum(self.getsize(f) for f in self.walkfiles()) self.cur_size = sum(self.getsize(f) for f in self.walkfiles())
self._size_lock = threading.Lock() self._size_lock = threading.Lock()
self._file_sizes = {} self._file_sizes = {}
def _decr_size(self,decr): def _decr_size(self, decr):
with self._size_lock: with self._size_lock:
self.cur_size -= decr self.cur_size -= decr
...@@ -35,16 +35,16 @@ class LimitSizeFS(WrapFS): ...@@ -35,16 +35,16 @@ class LimitSizeFS(WrapFS):
del state["_file_sizes"] del state["_file_sizes"]
return state return state
def __setstate__(self,state): def __setstate__(self, state):
super(LimitSizeFS,self).__setstate__(state) super(LimitSizeFS,self).__setstate__(state)
self._size_lock = threading.Lock() self._size_lock = threading.Lock()
def getsyspath(self,path,allow_none=False): def getsyspath(self, path, allow_none=False):
if not allow_none: if not allow_none:
raise NoSysPathError(path) raise NoSysPathError(path)
return None return None
def open(self,path,mode="r"): def open(self, path, mode="r"):
path = relpath(normpath(path)) path = relpath(normpath(path))
with self._size_lock: with self._size_lock:
try: try:
...@@ -60,7 +60,7 @@ class LimitSizeFS(WrapFS): ...@@ -60,7 +60,7 @@ class LimitSizeFS(WrapFS):
self._file_sizes[path] = 0 self._file_sizes[path] = 0
return LimitSizeFile(self,path,f,mode,size) return LimitSizeFile(self,path,f,mode,size)
def _ensure_file_size(self,path,size): def _ensure_file_size(self, path, size):
path = relpath(normpath(path)) path = relpath(normpath(path))
with self._size_lock: with self._size_lock:
if path not in self._file_sizes: if path not in self._file_sizes:
...@@ -73,22 +73,22 @@ class LimitSizeFS(WrapFS): ...@@ -73,22 +73,22 @@ class LimitSizeFS(WrapFS):
self.cur_size += diff self.cur_size += diff
self._file_sizes[path] = size self._file_sizes[path] = size
def copy(self,src,dst,**kwds): def copy(self, src, dst, **kwds):
FS.copy(self,src,dst,**kwds) FS.copy(self,src,dst,**kwds)
def copydir(self,src,dst,**kwds): def copydir(self, src, dst, **kwds):
FS.copydir(self,src,dst,**kwds) FS.copydir(self,src,dst,**kwds)
def move(self,src,dst,**kwds): def move(self, src, dst, **kwds):
FS.move(self,src,dst,**kwds) FS.move(self,src,dst,**kwds)
path = relpath(normpath(src)) path = relpath(normpath(src))
with self._size_lock: with self._size_lock:
self._file_sizes.pop(path,None) self._file_sizes.pop(path,None)
def movedir(self,src,dst,**kwds): def movedir(self, src, dst, **kwds):
FS.movedir(self,src,dst,**kwds) FS.movedir(self,src,dst,**kwds)
def remove(self,path): def remove(self, path):
size = self.getsize(path) size = self.getsize(path)
super(LimitSizeFS,self).remove(path) super(LimitSizeFS,self).remove(path)
self._decr_size(size) self._decr_size(size)
...@@ -96,12 +96,12 @@ class LimitSizeFS(WrapFS): ...@@ -96,12 +96,12 @@ class LimitSizeFS(WrapFS):
with self._size_lock: with self._size_lock:
self._file_sizes.pop(path,None) self._file_sizes.pop(path,None)
def removedir(self,path,recursive=False,force=False): def removedir(self, path, recursive=False, force=False):
size = sum(self.getsize(f) for f in self.walkfiles(path)) size = sum(self.getsize(f) for f in self.walkfiles(path))
super(LimitSizeFS,self).removedir(path,recursive=recursive,force=force) super(LimitSizeFS,self).removedir(path,recursive=recursive,force=force)
self._decr_size(size) self._decr_size(size)
def rename(self,src,dst): def rename(self, src, dst):
try: try:
size = self.getsize(dst) size = self.getsize(dst)
except ResourceNotFoundError: except ResourceNotFoundError:
...@@ -116,7 +116,7 @@ class LimitSizeFS(WrapFS): ...@@ -116,7 +116,7 @@ class LimitSizeFS(WrapFS):
class LimitSizeFile(object): class LimitSizeFile(object):
"""Filelike wrapper class for use by LimitSizeFS.""" """Filelike wrapper class for use by LimitSizeFS."""
def __init__(self,fs,path,file,mode,size): def __init__(self, fs, path, file, mode, size):
self._lock = fs._lock self._lock = fs._lock
self.fs = fs self.fs = fs
self.path = path self.path = path
...@@ -126,17 +126,17 @@ class LimitSizeFile(object): ...@@ -126,17 +126,17 @@ class LimitSizeFile(object):
self.closed = False self.closed = False
@synchronize @synchronize
def write(self,data): def write(self, data):
pos = self.file.tell() pos = self.file.tell()
self.size = self.fs._ensure_file_size(self.path,pos+len(data)) self.size = self.fs._ensure_file_size(self.path,pos+len(data))
self.file.write(data) self.file.write(data)
def writelines(self,lines): def writelines(self, lines):
for line in lines: for line in lines:
self.write(line) self.write(line)
@synchronize @synchronize
def truncate(self,size=None): def truncate(self, size=None):
pos = self.file.tell() pos = self.file.tell()
if size is None: if size is None:
size = pos size = pos
...@@ -145,7 +145,7 @@ class LimitSizeFile(object): ...@@ -145,7 +145,7 @@ class LimitSizeFile(object):
self.size = size self.size = size
# This is lifted straight from the stdlib's tempfile.py # This is lifted straight from the stdlib's tempfile.py
def __getattr__(self,name): def __getattr__(self, name):
file = self.__dict__['file'] file = self.__dict__['file']
a = getattr(file, name) a = getattr(file, name)
if not issubclass(type(a), type(0)): if not issubclass(type(a), type(0)):
...@@ -156,7 +156,7 @@ class LimitSizeFile(object): ...@@ -156,7 +156,7 @@ class LimitSizeFile(object):
self.file.__enter__() self.file.__enter__()
return self return self
def __exit__(self,exc,value,tb): def __exit__(self, exc, value, tb):
self.close() self.close()
return False return False
......
fs/wrapfs/readonlyfs.py
...@@ -15,7 +15,7 @@ class ReadOnlyFS(WrapFS): ...@@ -15,7 +15,7 @@ class ReadOnlyFS(WrapFS):
Note that this isn't a secure sandbox, untrusted code could work around the Note that this isn't a secure sandbox, untrusted code could work around the
read-only restrictions by getting the base class. Its main purpose is to read-only restrictions by getting the base class. Its main purpose is to
provide a degree of safety if you want to protect an FS object from provide a degree of safety if you want to protect an FS object from
modification. accidental modification.
""" """
......
fs/xattrs.py
...@@ -128,6 +128,7 @@ class SimulateXAttr(WrapFS): ...@@ -128,6 +128,7 @@ class SimulateXAttr(WrapFS):
pass pass
self._set_attr_dict(path, attrs) self._set_attr_dict(path, attrs)
@synchronize
def listxattrs(self,path): def listxattrs(self,path):
"""List all the extended attribute keys set on the given path.""" """List all the extended attribute keys set on the given path."""
if not self.exists(path): if not self.exists(path):
......
fs/zipfs.py
...@@ -61,7 +61,7 @@ class ZipFS(FS): ...@@ -61,7 +61,7 @@ class ZipFS(FS):
:param zip_file: A (system) path, or a file-like object :param zip_file: A (system) path, or a file-like object
:param mode: Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending :param mode: Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
:param compression: Can be 'deflated' (default) to compress data or 'stored' to just store date :param compression: Can be 'deflated' (default) to compress data or 'stored' to just store date
:param allow_zip_64: -- Set to True to use zip files greater than 2 MB, default is False :param allow_zip_64: -- Set to True to use zip files greater than 2 GB, default is False
:param encoding: -- The encoding to use for unicode filenames :param encoding: -- The encoding to use for unicode filenames
:param thread_synchronize: -- Set to True (default) to enable thread-safety :param thread_synchronize: -- Set to True (default) to enable thread-safety
...@@ -203,6 +203,9 @@ class ZipFS(FS): ...@@ -203,6 +203,9 @@ class ZipFS(FS):
try: try:
zi = self.zf.getinfo(path.encode(self.encoding)) zi = self.zf.getinfo(path.encode(self.encoding))
zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_')) zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
for k, v in zinfo.iteritems():
if callable(v):
zinfo[k] = v()
except KeyError: except KeyError:
zinfo = {'file_size':0} zinfo = {'file_size':0}
info = {'size' : zinfo['file_size'] } info = {'size' : zinfo['file_size'] }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment