Fixes and documentation

docs/concepts.rst

 Concepts
 ========
 
-It is generally quite easy to get in to the mind-set of using PyFilesystem interface over lower level interfaces (since the code tends to be simpler) but there are a few concepts which you will need to keep in mind.
+Working with PyFilesystem is generally easier than working with lower level interfaces, as long as you are aware these simple concepts. 
 
 Sandboxing
 ----------
@@ -25,7 +25,9 @@ We can open the `foo` directory with the following code::
 	from fs.osfs import OSFS
 	foo_fs = OSFS('foo')
 
-The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable, especially if we are passing `foo_fs` to an untrusted function or one that could potentially delete files. Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method::
+The `foo_fs` object can work with any of the contents of `bar` and `baz`, which may not be desirable,
+especially if we are passing `foo_fs` to an untrusted function or to a function that has the potential to delete files.
+Fortunately we can isolate a single sub-directory with then :meth:`~fs.base.FS.opendir` method::
 
 	bar_fs = foo_fs.opendir('bar')
 
@@ -54,7 +56,7 @@ When working with paths in FS objects, keep in mind the following:
  * A double dot means 'previous directory'
  
 Note that paths used by the FS interface will use this format, but the constructor or additional methods may not.
-Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which can be platform-dependent.
+Notably the :mod:`~fs.osfs.OSFS` constructor which requires an OS path -- the format of which is platform-dependent.
 
 There are many helpful functions for working with paths in the :mod:`fs.path` module.
 

docs/getting_started.rst

@@ -14,13 +14,21 @@ Add the -U switch if you want to upgrade a previous installation::
 
 	easy_install -U fs
 	
-This will install the latest stable release. If you would prefer to install the cutting edge release then you can get the latest copy of the source via SVN::
+If you prefer to use Pip (http://pypi.python.org/pypi/pip) to install Python packages, the procedure is much the same::
+
+	pip install fs
+	
+Or to upgrade::
+
+	pip install fs --upgrade
+
+You can also install the cutting edge release by checking out the source via SVN::
 
     svn checkout http://pyfilesystem.googlecode.com/svn/trunk/ pyfilesystem-read-only
     cd pyfilesystem-read-only
     python setup.py install
 
-You should now have the `fs` module on your path:
+Whichever method you use, you should now have the `fs` module on your path (version number may vary)::
 
     >>> import fs
     >>> fs.__version__

fs/expose/serve/server.py

@@ -19,10 +19,28 @@ class _SocketFile(object):
     def write(self, data):
         self.socket.sendall(data)
 
-class ConnectionThread(threading.Thread):
+
+def remote_call(method_name=None):
+    method = method_name    
+    def deco(f):
+        if not hasattr(f, '_remote_call_names'):
+            f._remote_call_names = []        
+        f._remote_call_names.append(method or f.__name__)
+        return f      
+    return deco
+
+
+class RemoteResponse(Exception):
+    def __init__(self, header, payload):
+        self.header = header
+        self.payload = payload
+
+class ConnectionHandlerBase(threading.Thread):
+    
+    _methods = {}
     
     def __init__(self, server, connection_id, socket, address):
-        super(ConnectionThread, self).__init__()
+        super(ConnectionHandlerBase, self).__init__()
         self.server = server
         self.connection_id = connection_id        
         self.socket = socket
@@ -34,6 +52,16 @@ class ConnectionThread(threading.Thread):
         self._lock = threading.RLock()
         self.socket_error = None
           
+        if not self._methods:  
+            for method_name in dir(self):
+                method = getattr(self, method_name)
+                if callable(method) and hasattr(method, '_remote_call_names'):
+                    for name in method._remote_call_names:
+            
+                        self._methods[name] = method
+                        
+        print self._methods
+            
         self.fs = None        
     
     def run(self):
@@ -71,16 +99,36 @@ class ConnectionThread(threading.Thread):
         print '-' * 30
         print repr(header)
         print repr(payload)
-        
-        if header['method'] == 'ping':
-            self.encoder.write({'client_ref':header['client_ref']}, payload)
-         
+        if header['type'] == 'rpc':
+            method = header['method']
+            args = header['args']
+            kwargs = header['kwargs']
+            method_callable = self._methods[method]
+            remote = dict(type='rpcresult',
+                          client_ref = header['client_ref'])
+            try:
+                response = method_callable(*args, **kwargs)
+                remote['response'] = response
+                self.encoder.write(remote, '')
+            except RemoteResponse, response:
+                self.encoder.write(response.header, response.payload)                                
+
+class RemoteFSConnection(ConnectionHandlerBase):
+
+    @remote_call()
+    def auth(self, username, password, resource):
+        self.username = username
+        self.password = password
+        self.resource = resource
+        from fs.memoryfs import MemoryFS
+        self.fs = MemoryFS()
         
 class Server(object):
     
-    def __init__(self, addr='', port=3000):
+    def __init__(self, addr='', port=3000, connection_factory=RemoteFSConnection):
         self.addr = addr
         self.port = port
+        self.connection_factory = connection_factory
         self.socket = None
         self.connection_id = 0
         self.threads = {}
@@ -124,7 +172,7 @@ class Server(object):
         print "Connection from", address
         with self._lock:
             self.connection_id += 1
-            thread = ConnectionThread(self,
+            thread = self.connection_factory(self,
                                              self.connection_id,
                                              clientsocket,
                                              address)            

fs/expose/sftp.py

@@ -245,7 +245,10 @@ class SFTPRequestHandler(sockserv.StreamRequestHandler):
         t.start_server(server=self.server)
 
 
-class BaseSFTPServer(sockserv.TCPServer,paramiko.ServerInterface):
+class ThreadedTCPServer(sockserv.TCPServer, sockserv.ThreadingMixIn):
+    pass
+
+class BaseSFTPServer(ThreadedTCPServer, paramiko.ServerInterface):
     """SocketServer.TCPServer subclass exposing an FS via SFTP.
 
     BaseSFTPServer combines a simple SocketServer.TCPServer subclass with an
@@ -318,6 +321,7 @@ if __name__ == "__main__":
     from fs.tempfs import TempFS
     server = BaseSFTPServer(("localhost",8022),TempFS())
     try:
+        #import rpdb2; rpdb2.start_embedded_debugger('password')
         server.serve_forever()
     except (SystemExit,KeyboardInterrupt):
         server.server_close()

fs/ftpfs.py

@@ -1209,7 +1209,7 @@ class FTPFS(FS):
     @ftperrors
     def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):                
         path = normpath(path)
-        self.clear_dircache(path)
+        #self.clear_dircache(path)
         if not self.exists(path):
             raise ResourceNotFoundError(path)
         if not self.isdir(path):

fs/opener.py

@@ -174,7 +174,7 @@ class OpenerRegistry(object):
         for name in opener.names:
             self.registry[name] = index
     
-    def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
+    def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False, cache_hint=True):
         """Parses a FS url and returns an fs object a path within that FS object
         (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
         
@@ -217,6 +217,7 @@ class OpenerRegistry(object):
             raise OpenerError("Unable to parse '%s'" % orig_url)        
 
         fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)        
+        fs.cache_hint(cache_hint)
         
         if fs_path and iswildcard(fs_path):
             pathname, resourcename = pathsplit(fs_path or '')

fs/path.py

@@ -237,6 +237,7 @@ def splitext(path):
     path = pathjoin(parent_path, pathname)
     return path, '.' + ext
 
+
 def isdotfile(path):
     """Detects if a path references a dot file, i.e. a resource who's name
     starts with a '.'
@@ -255,6 +256,7 @@ def isdotfile(path):
     """
     return basename(path).startswith('.')
 
+
 def dirname(path):
     """Returns the parent directory of a path.
 
@@ -266,10 +268,14 @@ def dirname(path):
     >>> dirname('foo/bar/baz')
     'foo/bar'
     
+    >>> dirname('/foo/bar')
+    '/foo'
+    
+    >>> dirname('/foo')
+    '/'
+
     """
-    if '/' not in path:
-        return ''
-    return path.rsplit('/', 1)[0]
+    return pathsplit(path)[0]
 
 
 def basename(path):
@@ -290,9 +296,7 @@ def basename(path):
     ''
 
     """
-    if '/' not in path:
-        return path
-    return path.rsplit('/', 1)[-1]
+    return pathsplit(path)[1]
 
 
 def issamedir(path1, path2):
@@ -309,11 +313,13 @@ def issamedir(path1, path2):
     """
     return dirname(normpath(path1)) == dirname(normpath(path2))
 
+
 def isbase(path1, path2):
     p1 = forcedir(abspath(path1))
     p2 = forcedir(abspath(path2))
     return p1 == p2 or p1.startswith(p2) 
 
+
 def isprefix(path1, path2):
     """Return true is path1 is a prefix of path2.
     
@@ -341,6 +347,7 @@ def isprefix(path1, path2):
             return False
     return True
 
+
 def forcedir(path):
     """Ensure the path ends with a trailing /
     

fs/remotefs.py

 # Work in Progress - Do not use
-
+from __future__ import with_statement
 from fs.base import FS
 from fs.expose.serve import packetstream
 
@@ -109,17 +109,38 @@ class _SocketFile(object):
         self.socket.close()
      
 
+class _RemoteFile(object):
+    
+    def __init__(self, path, connection):
+        self.path = path
+        self.connection = connection  
+
 class RemoteFS(FS):
     
-    def __init__(self, addr='', port=3000, transport=None):
+    _meta = { 'thead_safe' : True,
+              'network' : True,
+              'virtual' : False,
+              'read_only' : False,
+              'unicode_paths' : True,
+              }
+    
+    def __init__(self, addr='', port=3000, username=None, password=None, resource=None, transport=None):
         self.addr = addr
         self.port = port
+        self.username = None
+        self.password = None
+        self.resource = None
         self.transport = transport
         if self.transport is None:
             self.transport = self._open_connection()
         self.packet_handler = PacketHandler(self.transport)  
         self.packet_handler.start()  
         
+        self._remote_call('auth',
+                          username=username,
+                          password=password,
+                          resource=resource)
+        
     def _open_connection(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.connect((self.addr, self.port))
@@ -127,6 +148,19 @@ class RemoteFS(FS):
         socket_file.write('pyfs/0.1\n')
         return socket_file
     
+    def _make_call(self, method_name, *args, **kwargs):
+        call = dict(type='rpc',
+                    method=method_name,
+                    args=args,
+                    kwargs=kwargs)
+        return call        
+        
+    def _remote_call(self, method_name, *args, **kwargs):
+        call = self._make_call(method_name, *args, **kwargs)
+        call_id = self.packet_handler.send_packet(call)
+        header, payload = self.packet_handler.get_packet(call_id)
+        return header, payload
+    
     def ping(self, msg):
         call_id = self.packet_handler.send_packet({'type':'rpc', 'method':'ping'}, msg)
         header, payload = self.packet_handler.get_packet(call_id)
@@ -138,10 +172,17 @@ class RemoteFS(FS):
         self.transport.close()
         self.packet_handler.join()
     
+    def open(self, path, mode="r", **kwargs):
+        pass
+    
+    def exists(self, path):
+        remote = self._remote_call('exists', path)
+        return remote.get('response')
+        
+        
 
 if __name__ == "__main__":
     
     rfs = RemoteFS()    
-    rfs.ping("Hello, World!")
     rfs.close()
     
\ No newline at end of file

fs/sftpfs.py

@@ -222,6 +222,8 @@ class SFTPFS(FS):
     @property
     @synchronize
     def client(self):
+        if self.closed:
+            return None
         client = getattr(self._tlocal, 'client', None)
         if client is None:
             if self._transport is None:
@@ -242,9 +244,10 @@ class SFTPFS(FS):
     def close(self):
         """Close the connection to the remote server."""
         if not self.closed:
-            if self.client:
-                self.client.close()
-            if self._owns_transport and self._transport:
+            self._tlocal = None
+            #if self.client:
+            #    self.client.close()
+            if self._owns_transport and self._transport and self._transport.is_active:
                 self._transport.close()
             self.closed = True
 

fs/tests/test_expose.py

@@ -29,6 +29,22 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
         port = 3000
         self.temp_fs = TempFS()
         self.server = None
+        
+        self.serve_more_requests = True
+        self.server_thread = threading.Thread(target=self.runServer)
+        self.server_thread.setDaemon(True) 
+        
+        self.start_event = threading.Event()
+        self.end_event = threading.Event()
+                   
+        self.server_thread.start()
+        
+        self.start_event.wait()
+
+    def runServer(self):
+        """Run the server, swallowing shutdown-related execptions."""
+        
+        port = 3000
         while not self.server:
             try:
                 self.server = self.makeServer(self.temp_fs,("127.0.0.1",port))
@@ -37,37 +53,48 @@ class TestRPCFS(unittest.TestCase, FSTestCases, ThreadingTestCases):
                     port += 1
                 else:
                     raise
-        self.server_addr = ("127.0.0.1",port)
-        self.serve_more_requests = True
-        self.server_thread = threading.Thread(target=self.runServer)
-        self.server_thread.daemon = True            
-        self.server_thread.start()
+        self.server_addr = ("127.0.0.1", port)
+        
+        self.server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        
+#        if sys.platform != "win32":
+#            try:
+#                self.server.socket.settimeout(1)
+#            except socket.error:
+#                pass
+#        
+        self.start_event.set()
         
-    def runServer(self):
-        """Run the server, swallowing shutdown-related execptions."""
-        if sys.platform != "win32":
-            try:
-                self.server.socket.settimeout(0.1)
-            except socket.error:
-                pass
         try:
+            #self.server.serve_forever()
             while self.serve_more_requests:
                 self.server.handle_request()
         except Exception, e:
             pass
         
+        self.end_event.set()
+
     def setUp(self):
         self.startServer()
         self.fs = rpcfs.RPCFS("http://%s:%d" % self.server_addr)
 
     def tearDown(self):
         self.serve_more_requests = False
+        #self.server.socket.close()
+#            self.server.socket.shutdown(socket.SHUT_RDWR)
+#            self.server.socket.close()
+#            self.temp_fs.close()
+        #self.server_thread.join()
+        
+        #self.end_event.wait()
+        #return
+        
         try:
             self.bump()
             self.server.server_close()
         except Exception:
             pass
-        self.server_thread.join()
+        #self.server_thread.join()
         self.temp_fs.close()
 
     def bump(self):

fs/tests/test_path.py

@@ -105,7 +105,6 @@ class TestPathFunctions(unittest.TestCase):
         self.assertEquals(recursepath("hello",reverse=True),["/hello","/"])
         self.assertEquals(recursepath("",reverse=True),["/"])
         
-        
     def test_isdotfile(self):
         for path in ['.foo',
                      '.svn',
@@ -125,7 +124,9 @@ class TestPathFunctions(unittest.TestCase):
         tests = [('foo', ''),
                  ('foo/bar', 'foo'),
                  ('foo/bar/baz', 'foo/bar'),
-                 ('/', '')]
+                 ('/foo/bar', '/foo'),
+                 ('/foo', '/'),
+                 ('/', '/')]
         for path, test_dirname in tests:
             self.assertEqual(dirname(path), test_dirname)