Work in progress with an FS->Fuse adapter. Added a default implementation of…

Work in progress with an FS->Fuse adapter. Added a default implementation of xattr to base.py, fixed some issues in memoryfs

fs/base.py

@@ -12,6 +12,11 @@ except ImportError:
     import dummy_threading as threading
 import dummy_threading
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 error_msgs = {
 
     "UNKNOWN_ERROR" :   "No information on error: %(path)s",
@@ -430,6 +435,7 @@ class FS(object):
         finally:
             if f is not None:
                 f.close()
+    setcontents = createfile
 
     def opendir(self, path):
         """Opens a directory and returns a FS object representing its contents.
@@ -613,6 +619,42 @@ class FS(object):
             self.remove(src)
 
 
+    def _get_attr_path(self, path):
+        if self.isdir(path):
+            return pathjoin(path, '.dirattrs')
+        else:
+            dir_path, file_path = pathsplit(path)
+            return pathjoin(path, '.attrs.'+file_path)
+
+    def _get_attr_dict(self, path):
+        attr_path = self._get_attr_path(path)
+        if self.exists(attr_path):
+            return pickle.loads(self.getcontents(attr_path))
+        else:
+            return {}
+
+
+    def _set_attr_dict(self, path, attrs):
+        attr_path = self._get_attr_path(path)
+        self.setcontents(path, self.pickle.dumps(attrs))
+
+    def setattr(self, path, key, value):
+        attrs = self._get_attr_dict(path)
+        attrs[key] = value
+        self._set_attr_dict(path, attrs)
+
+    def getattr(self, path, key, default):
+        return self._get_attr_dict(path).get(key, default)
+
+    def removeattr(self, path, key):
+        attrs = self._get_attrs()
+        if path in self._get_attrs():
+            del attrs[key]
+
+    def listattrs(self, path):
+        return self._get_attr_dict(path).keys()
+
+
     def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
         """Moves a directory from one location to another.
 

fs/fuseserver.py

+#!/usr/bin/env python
+
+
+import base
+
+import fuse
+fuse.fuse_python_api = (0, 2)
+
+
+from datetime import datetime
+import time
+from os import errno
+
+import sys
+from stat import *
+
+def showtb(f):
+
+    def run(*args, **kwargs):
+        print
+        print "-"*80
+        print f, args, kwargs
+        try:
+            ret = f(*args, **kwargs)
+            print "\tReturned:", repr(ret)
+            return ret
+        except Exception, e:
+            print e
+            raise
+        print "-"*80
+        print
+    return run
+
+"""
+::*'''<code>open(path, flags)</code>'''
+
+::*'''<code>create(path, flags, mode)</code>'''
+
+::*'''<code>read(path, length, offset, fh=None)</code>'''
+
+::*'''<code>write(path, buf, offset, fh=None)</code>'''
+
+::*'''<code>fgetattr(path, fh=None)</code>'''
+
+::*'''<code>ftruncate(path, len, fh=None)</code>'''
+
+::*'''<code>flush(path, fh=None)</code>'''
+
+::*'''<code>release(path, fh=None)</code>'''
+
+::*'''<code>fsync(path, fdatasync, fh=None)</code>'''
+
+"""
+
+
+class FuseFile(object):
+
+    def __init__(self, f):
+        self.f = f
+
+
+
+
+
+_run_t = time.time()
+class FSFUSE(fuse.Fuse):
+
+    def __init__(self, fs, *args, **kwargs):
+        fuse.Fuse.__init__(self, *args, **kwargs)
+        self._fs = fs
+
+    @showtb
+    def fsinit(self):
+        return 0
+
+    def __getattr__(self, name):
+        print name
+        raise AttributeError
+
+    #@showtb
+    def getattr(self, path):
+
+        if not self._fs.exists(path):
+            return -errno.ENOENT
+
+        class Stat(fuse.Stat):
+            def __init__(self, context, fs, path):
+                fuse.Stat.__init__(self)
+                info = fs.getinfo(path)
+                isdir = fs.isdir(path)
+
+                fsize = fs.getsize(path) or 1024
+                self.st_ino = 0
+                self.st_dev = 0
+                self.st_nlink = 2 if isdir else 1
+                self.st_blksize = fsize
+                self.st_mode = info.get('st_mode', S_IFDIR | 0755 if isdir else S_IFREG | 0666)
+                print self.st_mode
+                self.st_uid = context['uid']
+                self.st_gid = context['gid']
+                self.st_rdev = 0
+                self.st_size = fsize
+                self.st_blocks = 1
+
+                for key, value in info.iteritems():
+                    if not key.startswith('_'):
+                        setattr(self, key, value)
+
+                def do_time(attr, key):
+                    if not hasattr(self, attr):
+                        if key in info:
+                            info_t = info[key]
+                            setattr(self, attr, time.mktime(info_t.timetuple()))
+                        else:
+                            setattr(self, attr, _run_t)
+
+                do_time('st_atime', 'accessed_time')
+                do_time('st_mtime', 'modified_time')
+                do_time('st_ctime', 'created_time')
+
+                #for v in dir(self):
+                #    if not v.startswith('_'):
+                #        print v, getattr(self, v)
+
+        return Stat(self.GetContext(), self._fs, path)
+
+    @showtb
+    def chmod(self, path, mode):
+        return 0
+
+    @showtb
+    def chown(self, path, user, group):
+        return 0
+
+    @showtb
+    def utime(self, path, times):
+        return 0
+
+    @showtb
+    def utimens(self, path, times):
+        return 0
+
+    @showtb
+    def fsyncdir(self):
+        pass
+
+    @showtb
+    def bmap(self):
+        return 0
+
+    @showtb
+    def ftruncate(self, path, flags, fh):
+        if fh is not None:
+            fh.truncate()
+            fh.flush()
+        return 0
+
+    def fsdestroy(self):
+        return 0
+
+    @showtb
+    def statfs(self):
+        return (0, 0, 0, 0, 0, 0, 0)
+
+
+
+    #def setattr
+    #
+    #
+    #@showtb
+    #def getdir(self, path, offset):
+    #    paths = ['.', '..']
+    #    paths += self._fs.listdir(path)
+    #    print repr(paths)
+    #
+    #    for p in paths:
+    #        yield fuse.Direntry(p)
+
+    @showtb
+    def opendir(self, path):
+        return 0
+
+    @showtb
+    def getxattr(self, path, name, default):
+        return self._fs.getattr(path, name, default)
+
+    @showtb
+    def setxattr(self, path, name, value):
+        self._fs.setattr(path, name)
+        return 0
+
+    @showtb
+    def removeattr(self, path, name):
+        self._fs.removeattr(path, name)
+        return 0
+
+    @showtb
+    def listxattr(self, path, something):
+        return self._fs.listattrs(path)
+
+    @showtb
+    def open(self, path, flags):
+        return self._fs.open(path, flags=flags)
+
+    @showtb
+    def create(self, path, flags, mode):
+        return self._fs.open(path, "w")
+
+    @showtb
+    def read(self, path, length, offset, fh=None):
+        if fh:
+            fh.seek(offset)
+            return fh.read(length)
+
+    @showtb
+    def write(self, path, buf, offset, fh=None):
+        if fh:
+            fh.seek(offset)
+            # FUSE seems to expect a return value of the number of bytes written,
+            # but Python file objects don't return that information,
+            # so we will assume all bytes are written...
+            bytes_written = fh.write(buf) or len(buf)
+            return bytes_written
+
+    @showtb
+    def release(self, path, flags, fh=None):
+        if fh:
+            fh.close()
+            return 0
+
+    @showtb
+    def flush(self, path, fh=None):
+        if fh:
+            try:
+                fh.flush()
+            except base.FSError:
+                return 0
+            return 0
+
+    @showtb
+    def access(self, path, *args, **kwargs):
+        return 0
+
+
+    #@showtb
+    def readdir(self, path, offset):
+        paths = ['.', '..']
+        paths += self._fs.listdir(path)
+        return [fuse.Direntry(p) for p in paths]
+
+    #@showtb
+    #def fgetattr(self, path, fh=None):
+    #    fh.flush()
+    #    return self.getattr(path)
+
+    @showtb
+    def readlink(self, path):
+        return path
+
+    @showtb
+    def symlink(self, path, path1):
+        return 0
+
+
+    @showtb
+    def mknod(self, path, mode, rdev):
+        f = None
+        try:
+            f = self._fs.open(path, mode)
+        finally:
+            f.close()
+        return 0
+
+    @showtb
+    def mkdir(self, path, mode):
+        self._fs.mkdir(path, mode)
+        return 0
+
+    @showtb
+    def rmdir(self, path):
+        self._fs.removedir(path, True)
+        return 0
+
+    @showtb
+    def unlink(self, path):
+        try:
+            self._fs.remove(path)
+        except base.FSError:
+            return 0
+        return 0
+
+    #symlink(target, name)
+
+    @showtb
+    def rename(self, old, new):
+        self._fs.rename(old, new)
+        return 0
+
+
+
+    #@showtb
+    #def read(self, path, size, offset):
+    #    pass
+
+
+
+def main(fs):
+    usage="""
+        FSFS: Exposes an FS
+    """ + fuse.Fuse.fusage
+
+    server = FSFUSE(fs, version="%prog 0.1",
+                    usage=usage, dash_s_do='setsingle')
+
+    #server.readdir('.', 0)
+
+    server.parse(errex=1)
+    server.main()
+
+
+if __name__ == "__main__":
+
+    import memoryfs
+    import osfs
+    mem_fs = memoryfs.MemoryFS()
+    mem_fs.makedir("test")
+    mem_fs.createfile("a.txt", "This is a test")
+    mem_fs.createfile("test/b.txt", "This is in a sub-dir")
+
+
+    #fs = osfs.OSFS('/home/will/fusetest/')
+    #main(fs)
+
+    main(mem_fs)

fs/memoryfs.py

@@ -23,6 +23,7 @@ def _check_mode(mode, mode_chars):
 class MemoryFile(object):
 
     def __init__(self, path, memory_fs, value, mode):
+        self.closed = False
         self.path = path
         self.memory_fs = memory_fs
         self.mode = mode
@@ -54,7 +55,6 @@ class MemoryFile(object):
                 self.mem_file = StringIO()
 
         assert self.mem_file is not None, "self.mem_file should have a value"
-        self.closed = False
 
     def __str__(self):
         return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)
@@ -69,7 +69,8 @@ class MemoryFile(object):
             self.close()
 
     def flush(self):
-        pass
+        value = self.mem_file.getvalue()
+        self.memory_fs._on_flush_memory_file(self.path, value)
 
     def __iter__(self):
         return iter(self.mem_file)
@@ -81,9 +82,9 @@ class MemoryFile(object):
         return self.mem_file.readline(*args, **kwargs)
 
     def close(self):
-        if not self.closed:
+        if not self.closed and self.mem_file is not None:
             value = self.mem_file.getvalue()
-            self.memory_fs._on_close_memory_file(self.path, value)
+            self.memory_fs._on_close_memory_file(self, self.path, value)
             self.mem_file.close()
             self.closed = True
 
@@ -123,6 +124,7 @@ class MemoryFS(FS):
             if contents is None and type == "dir":
                 contents = {}
 
+            self.open_files = []
             self.contents = contents
             self.data = None
             self.locks = 0
@@ -285,6 +287,10 @@ class MemoryFS(FS):
         finally:
             self._lock.release()
 
+    def _orphan_files(self, file_dir_entry):
+        for f in file_dir_entry.open_files:
+            f.close()
+
     def _lock_dir_entry(self, path):
         self._lock.acquire()
         try:
@@ -329,6 +335,7 @@ class MemoryFS(FS):
 
                 self._lock_dir_entry(path)
                 mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
+                file_dir_entry.open_files.append(mem_file)
                 return mem_file
 
             elif 'w' in mode:
@@ -344,6 +351,7 @@ class MemoryFS(FS):
                 self._lock_dir_entry(path)
 
                 mem_file = self.file_factory(path, self, None, mode)
+                file_dir_entry.open_files.append(mem_file)
                 return mem_file
 
             if parent_dir_entry is None:
@@ -360,7 +368,8 @@ class MemoryFS(FS):
                 raise ResourceNotFoundError("NO_FILE", path)
 
             if dir_entry.islocked():
-                raise ResourceLockedError("FILE_LOCKED", path)
+                self._orphan_files(dir_entry)
+                #raise ResourceLockedError("FILE_LOCKED", path)
 
             pathname, dirname = pathsplit(path)
 
@@ -401,14 +410,22 @@ class MemoryFS(FS):
 
     def rename(self, src, dst):
         if not issamedir(src, dst):
-            raise ValueError("Destination path must the same directory (user the move method for moving to a different directory)")
+            raise ValueError("Destination path must the same directory (use the move method for moving to a different directory)")
         self._lock.acquire()
         try:
+
+            dst = pathsplit(dst)[-1]
+
             dir_entry = self._get_dir_entry(src)
             if dir_entry is None:
                 raise ResourceNotFoundError("NO_DIR", src)
-            if dir_entry.islocked():
-                raise ResourceLockedError("FILE_LOCKED", src)
+            #if dir_entry.islocked():
+            #    raise ResourceLockedError("FILE_LOCKED", src)
+
+            open_files = dir_entry.open_files[:]
+            for f in open_files:
+                f.flush()
+                f.path = dst
 
             dst_dir_entry = self._get_dir_entry(dst)
             if dst_dir_entry is not None:
@@ -417,21 +434,36 @@ class MemoryFS(FS):
             pathname, dirname = pathsplit(src)
             parent_dir = self._get_dir_entry(pathname)
             parent_dir.contents[dst] = parent_dir.contents[dirname]
+            parent_dir.name = dst
             del parent_dir.contents[dirname]
+
         finally:
             self._lock.release()
 
 
-    def _on_close_memory_file(self, path, value):
+    def _on_close_memory_file(self, open_file, path, value):
+        self._lock.acquire()
+        try:
+            filepath, filename = pathsplit(path)
+            dir_entry = self._get_dir_entry(path)
+            if dir_entry is not None and value is not None:
+                dir_entry.data = value
+                dir_entry.open_files.remove(open_file)
+                self._unlock_dir_entry(path)
+        finally:
+            self._lock.release()
+
+    def _on_flush_memory_file(self, path, value):
         self._lock.acquire()
         try:
             filepath, filename = pathsplit(path)
             dir_entry = self._get_dir_entry(path)
             dir_entry.data = value
-            self._unlock_dir_entry(path)
         finally:
             self._lock.release()
 
+
+
     def listdir(self, path="/", wildcard=None, full=False, absolute=False, hidden=True, dirs_only=False, files_only=False):
         self._lock.acquire()
         try:
@@ -455,7 +487,7 @@ class MemoryFS(FS):
             info['created_time'] = dir_entry.created_time
 
             if dir_entry.isfile():
-                info['size'] = len(dir_entry.data)
+                info['size'] = len(dir_entry.data or '')
 
             return info
         finally: