adding latest code for TahoeFS

ChangeLog

@@ -34,8 +34,9 @@
 0.4:
 
     * New FS implementations (under fs.contrib):
-        * BigFS:   read contents of a BIG file (C&C game file format)
-        * DAVFS:   access remote files stored on a WebDAV server
+        * BigFS:    read contents of a BIG file (C&C game file format)
+        * DAVFS:    access remote files stored on a WebDAV server
+        * TahoeFS:  access files stored in a Tahoe-LAFS grid
     * New fs.expose implementations:
         * dokan:   mount an FS object as a drive using Dokan (win32-only)
     * Modified listdir and walk methods to accept callables as well as strings
@@ -52,7 +53,7 @@
         * watch_win32:  don't create immortal reference cycles.
     * MountFS: added support for mounting at the root directory, and for
       mounting over an existing mount.
-    * Added 'getpathurl' and 'haspathurl' methods
+    * Added 'getpathurl' and 'haspathurl' methods.
     * Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these
       can often determine whether a path is a file or directory by inspecting
       the info dict and avoid an additional query to the filesystem.

fs/contrib/tahoefs/connection.py

+import platform
+import logging
+
+import fs.errors as errors
+from fs import SEEK_END
+
+python3 = int(platform.python_version_tuple()[0]) > 2
+
+if python3:
+    from urllib.parse import urlencode, pathname2url, quote
+    from urllib.request import Request, urlopen
+else:
+    from urllib import urlencode, pathname2url
+    from urllib2 import Request, urlopen, quote
+
+class PutRequest(Request):
+    def __init__(self, *args, **kwargs):
+        self.get_method = lambda: u'PUT'
+        Request.__init__(self, *args, **kwargs)
+         
+class DeleteRequest(Request):
+    def __init__(self, *args, **kwargs):
+        self.get_method = lambda: u'DELETE'
+        Request.__init__(self, *args, **kwargs)
+              
+class Connection:
+    def __init__(self, webapi):
+        self.webapi = webapi
+        self.headers = {'Accept': 'text/plain'}
+
+    def _get_headers(self, f, size=None):
+        '''
+            Retrieve length of string or file object and prepare HTTP headers.
+        '''
+        if isinstance(f, basestring):
+            # Just set up content length
+            size = len(f)
+        elif getattr(f, 'read', None):
+            if size == None:
+                # When size is already known, skip this           
+                f.seek(0, SEEK_END)
+                size = f.tell()
+                f.seek(0)
+        else:
+            raise errors.UnsupportedError("Cannot handle type %s" % type(f))
+        
+        headers = {'Content-Length': size}
+        headers.update(self.headers)
+        return headers
+
+    def _urlencode(self, data):
+        _data = {}
+        for k, v in data.items():
+            _data[k.encode('utf-8')] = v.encode('utf-8')
+        return urlencode(_data)
+
+    def _quotepath(self, path, params={}):
+        q = quote(path.encode('utf-8'), safe='/')
+        if params:
+            return u"%s?%s" % (q, self._urlencode(params))
+        return q
+        
+    def _urlopen(self, req):
+        try:
+            #print req.get_full_url()
+            return urlopen(req)
+        except Exception, e:
+            if not getattr(e, 'getcode', None):
+                raise errors.RemoteConnectionError(str(e))
+            code = e.getcode()
+            if code == 500:
+                # Probably out of space or unhappiness error
+                raise errors.StorageSpaceError(e.fp.read())
+            elif code in (400, 404, 410):
+                # Standard not found
+                 raise errors.ResourceNotFoundError(e.fp.read())
+            raise errors.ResourceInvalidError(e.fp.read())
+        
+    def post(self, path, data={}, params={}):
+        data = self._urlencode(data)
+        path = self._quotepath(path, params)
+        req = Request(''.join([self.webapi, path]), data, headers=self.headers)
+        return self._urlopen(req)
+    
+    def get(self, path, data={}, offset=None, length=None):
+        data = self._urlencode(data)
+        path = self._quotepath(path)
+        if data: 
+            path = u'?'.join([path, data])
+
+        headers = {}
+        headers.update(self.headers)
+        if offset:
+            if length:
+                headers['Range'] = 'bytes=%d-%d' % \
+                                    (int(offset), int(offset+length))
+            else:
+                headers['Range'] = 'bytes=%d-' % int(offset)
+            
+        req = Request(''.join([self.webapi, path]), headers=headers)
+        return self._urlopen(req)
+
+    def put(self, path, data, size=None, params={}):
+        path = self._quotepath(path, params)
+        headers = self._get_headers(data, size=size)
+        req = PutRequest(''.join([self.webapi, path]), data, headers=headers)    
+        return self._urlopen(req)
+    
+    def delete(self, path, data={}):   
+        path = self._quotepath(path)
+        req = DeleteRequest(''.join([self.webapi, path]), data, headers=self.headers)
+        return self._urlopen(req)

fs/contrib/tahoefs/test_tahoefs.py

+#!/usr/bin/python
+"""
+    Test the TahoeFS
+    
+    @author: Marek Palatinus <marek@palatinus.cz>
+"""
+
+import sys
+import logging
+import unittest
+
+from fs.base import FS
+import fs.errors as errors
+from fs.tests import FSTestCases, ThreadingTestCases
+from fs.contrib.tahoefs import TahoeFS, Connection
+
+logging.getLogger().setLevel(logging.DEBUG)
+logging.getLogger('fs.tahoefs').addHandler(logging.StreamHandler(sys.stdout))
+
+WEBAPI = 'http://pubgrid.tahoe-lafs.org'
+
+class TestTahoeFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
+    def setUp(self):
+        self.dircap = TahoeFS.createdircap(WEBAPI)
+        self.fs = TahoeFS(self.dircap, timeout=0, webapi=WEBAPI)
+             
+    def tearDown(self):
+        self.fs.close()
+         
+    def test_dircap(self):
+        # Is dircap in correct format?
+        self.assert_(self.dircap.startswith('URI:DIR2:') and len(self.dircap) > 50)
+     
+    def test_concurrent_copydir(self):
+        #  makedir() on TahoeFS is currently not atomic
+        pass
+
+    def test_makedir_winner(self):
+        #  makedir() on TahoeFS is currently not atomic
+        pass
+    
+    def test_big_file(self):
+        pass
+
+if __name__ == '__main__':
+    unittest.main()

fs/contrib/tahoefs/util.py

+'''
+Created on 25.9.2010
+
+@author: marekp
+'''
+
+import sys
+import platform
+import stat as statinfo
+
+import fs.errors as errors
+from fs.path import pathsplit
+try:
+    # For non-CPython or older CPython versions.
+    # Simplejson also comes with C speedup module which
+    # is not in standard CPython >=2.6 library.
+    import simplejson as json
+except ImportError:
+    import json
+    
+from .connection import Connection
+
+python3 = int(platform.python_version_tuple()[0]) > 2
+
+if python3:
+    from urllib.error import HTTPError
+else:
+    from urllib2 import HTTPError 
+    
+class TahoeUtil:
+    def __init__(self, webapi):
+        self.connection = Connection(webapi)
+        
+    def createdircap(self):
+        return self.connection.post(u'/uri', params={u't': u'mkdir'}).read()
+            
+    def unlink(self, dircap, path=None):
+        path = self.fixwinpath(path, False)
+        self.connection.delete(u'/uri/%s%s' % (dircap, path))
+    
+    def info(self, dircap, path):
+        path = self.fixwinpath(path, False)
+        meta = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))
+        return self._info(path, meta)
+            
+    def fixwinpath(self, path, direction=True):
+        '''
+            No, Tahoe really does not support file streams...
+            This is ugly hack, because it is not Tahoe-specific.
+            Should be move to middleware if will be any.
+        '''
+        if platform.system() != 'Windows':
+            return path
+        
+        if direction and ':' in path:
+                path = path.replace(':', '__colon__')
+        elif not direction and '__colon__' in path:
+                path = path.replace('__colon__', ':')
+        return path
+    
+    def _info(self, path, data):
+        if isinstance(data, list):
+            type = data[0]
+            data = data[1]
+        elif isinstance(data, dict):
+            type = data['type']
+        else:
+            raise errors.ResourceInvalidError('Metadata in unknown format!')
+        
+        if type == 'unknown':
+            raise errors.ResourceNotFoundError(path)
+        
+        info = {'name': unicode(self.fixwinpath(path, True)),
+                'type': type,
+                'size': data.get('size', 0),
+                'ctime': None,
+                'uri': data.get('rw_uri', data.get('ro_uri'))}
+        if 'metadata' in data:
+            info['ctime'] = data['metadata'].get('ctime')
+    
+        if info['type'] == 'dirnode':
+            info['st_mode'] = 0777 |  statinfo.S_IFDIR 
+        else:
+            info['st_mode'] = 0644
+
+        return info
+        
+    def list(self, dircap, path=None):
+        path = self.fixwinpath(path, False)
+        
+        data = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))     
+
+        if len(data) < 2 or data[0] != 'dirnode':
+            raise errors.ResourceInvalidError('Metadata in unknown format!')
+        
+        data = data[1]['children']
+        for i in data.keys():
+            x = self._info(i, data[i])
+            yield x
+
+    def mkdir(self, dircap, path):
+        path = self.fixwinpath(path, False)    
+        path = pathsplit(path)
+        
+        self.connection.post(u"/uri/%s%s" % (dircap, path[0]), data={u't': u'mkdir', u'name': path[1]})
+    
+    def move(self, dircap, src, dst):
+        if src == '/' or dst == '/':
+            raise errors.UnsupportedError("Too dangerous operation, aborting")
+
+        src = self.fixwinpath(src, False)
+        dst = self.fixwinpath(dst, False)
+        
+        src_tuple = pathsplit(src)
+        dst_tuple = pathsplit(dst)
+        
+        if src_tuple[0] == dst_tuple[0]:
+            # Move inside one directory
+            self.connection.post(u"/uri/%s%s" % (dircap, src_tuple[0]), data={u't': u'rename',
+                                        u'from_name': src_tuple[1], u'to_name': dst_tuple[1]})
+            return
+
+        # Move to different directory. Firstly create link on dst, then remove from src
+        try:
+            self.info(dircap, dst)
+        except errors.ResourceNotFoundError:
+            pass
+        else:
+            self.unlink(dircap, dst)
+         
+        uri = self.info(dircap, src)['uri']
+        self.connection.put(u"/uri/%s%s" % (dircap, dst), data=uri, params={u't': u'uri'})        
+        if uri != self.info(dircap, dst)['uri']:
+            raise errors.OperationFailedError('Move failed')
+        
+        self.unlink(dircap, src)