add fs.expose.importhook

This module allows you to import python modules from the files in an FS.
So you can now import modules directly over HTTP, SFTP, or even from a
zipfile ;-)

ChangeLog

@@ -39,6 +39,7 @@
         * TahoeFS:  access files stored in a Tahoe-LAFS grid
     * New fs.expose implementations:
         * dokan:   mount an FS object as a drive using Dokan (win32-only)
+        * importhook:  import modules from files in an FS object
     * Modified listdir and walk methods to accept callables as well as strings
       for wildcards.
     * Added listdirinfo method, which yields both the entry names and the

docs/expose.rst

@@ -20,6 +20,10 @@ XMLRPC
 ------
 Makes an FS object available over XMLRPC. See :mod:`fs.expose.xmlrpc`
 
+Import Hook
+-----------
+Allows importing python modules from the files in an FS object. See :mod:`fs.expose.importhook`
+
 Django Storage
 --------------
 Connects FS objects to Django. See :mod:`fs.expose.django_storage`

docs/expose/importhook.rst

+.. automodule:: fs.expose.importhook
+    :members:

docs/expose/index.rst

@@ -10,4 +10,5 @@ The ``fs.expose`` module contains a number of options for making an FS implement
    dokan.rst
    sftp.rst
    xmlrpc.rst
+   importhook.rst
    django_storage.rst

fs/contrib/davfs/__init__.py

@@ -254,7 +254,7 @@ class DAVFS(FS):
                 resp = con.getresponse()
                 self._cookiejar.extract_cookies(FakeResp(resp),FakeReq(con,url.scheme,url.path))
             except Exception, e:
-                #logger.debug("DAVFS <ERR %s %s/%s",resp.status,method,url.hostname,url.path)
+                #logger.debug("DAVFS <ERR %s %s/%s",e,method,url.hostname,url.path)
                 self._del_connection(con)
                 raise
             else:

fs/expose/importhook.py

+"""
+fs.expose.importhook
+====================
+
+Expose an FS object to the python import machinery, via a PEP-302 loader.
+
+This module allows you to import python modules from an arbitrary FS object,
+by placing FS urls on sys.path and/or inserting objects into sys.meta_path.
+
+The main class in this module is FSImportHook, which is a PEP-302-compliant
+module finder and loader.  If you place an instance of FSImportHook on
+sys.meta_path, you will be able to import modules from the exposed filesystem::
+
+    >>> from fs.memoryfs import MemoryFS
+    >>> m = MemoryFS()
+    >>> m.setcontents("helloworld.py","print 'hello world!")
+    >>>
+    >>> import sys
+    >>> from fs.expose.importhook import FSImportHook
+    >>> sys.meta_path.append(FSImportHook(m))
+    >>> import helloworld
+    hello world!
+
+It is also possible to install FSImportHook as an import path handler. This
+allows you to place filesystem URLs on sys.path and have them automagically
+opened for importing.  This example would allow modules to be imported from
+an SFTP server::
+
+    >>> from fs.expose.importhook import FSImportHook
+    >>> FSImportHook.install()
+    >>> sys.path.append("sftp://some.remote.machine/mypath/")
+
+"""
+
+import sys
+import imp
+import marshal
+
+from fs.base import FS
+from fs.opener import fsopendir, OpenerError
+from fs.errors import *
+from fs.path import *
+
+class FSImportHook(object):
+    """PEP-302-compliant module finder and loader for FS objects.
+
+    FSImportHook is a module finder and loader that takes its data from an
+    arbitrary FS object.  The FS must have .py or .pyc files stored in the
+    standard module structure.
+
+    For easy use with sys.path, FSImportHook will also accept a filesystem
+    URL, which is automatically opened using fs.opener.
+    """
+
+    _VALID_MODULE_TYPES = set((imp.PY_SOURCE,imp.PY_COMPILED))
+
+    def __init__(self,fs_or_url):
+        #  If given a string, try to open it as an FS url.
+        #  Don't open things on the local filesystem though.
+        if isinstance(fs_or_url,basestring):
+            if ":" not in fs_or_url:
+                raise ImportError
+            try:
+                self.fs = fsopendir(fs_or_url)
+            except OpenerError:
+                raise ImportError
+            self.path = fs_or_url
+        #  Otherwise, it must be an FS object of some sort.
+        else:
+            if not isinstance(fs_or_url,FS):
+                raise ImportError
+            self.fs = fs_or_url
+            self.path = None
+
+    @classmethod
+    def install(cls):
+        """Install this class into the import machinery.
+
+        This classmethod installs the custom FSImportHook class into the
+        import machinery of the running process, if it is not already
+        installed.
+        """
+        for i,imp in enumerate(sys.path_hooks):
+            try:
+                if issubclass(cls,imp):
+                    break
+            except TypeError:
+                pass
+        else:
+            sys.path_hooks.append(cls)
+            sys.path_importer_cache.clear()
+
+    @classmethod
+    def uninstall(cls):
+        """Uninstall this class from the import machinery.
+
+        This classmethod uninstalls the custom FSImportHook class from the
+        import machinery of the running process.
+        """
+        to_rem = []
+        for i,imp in enumerate(sys.path_hooks):
+            try:
+                if issubclass(cls,imp):
+                    to_rem.append(imp)
+                    break
+            except TypeError:
+                pass
+        for imp in to_rem:
+            sys.path_hooks.remove(imp)
+            sys.path_importer_cache.clear()
+
+    def find_module(self,fullname,path=None):
+        """Find the FS loader for the given module.
+
+        This object is always its own loader, so this really just checks
+        whether it's a valid module on the exposed filesystem.
+        """
+        try:
+            self._get_module_info(fullname)
+        except ImportError:
+            return None
+        else:
+            return self
+
+    def _get_module_info(self,fullname):
+        """Get basic information about the given module.
+
+        If the specified module exists, this method returns a tuple giving
+        its filepath, file type and whether it's a package.  Otherwise,
+        it raise ImportError.
+        """
+        prefix = fullname.replace(".","/")
+        #  Is it a regular module?
+        (path,type) = self._find_module_file(prefix)
+        if path is not None:
+            return (path,type,False)
+        #  Is it a package?
+        prefix = pathjoin(prefix,"__init__")
+        (path,type) = self._find_module_file(prefix)
+        if path is not None:
+            return (path,type,True)
+        #  No, it's nothing
+        raise ImportError(fullname)
+
+    def _find_module_file(self,prefix):
+        """Find a module file from the given path prefix.
+
+        This method iterates over the possible module suffixes, checking each
+        in turn and returning the first match found.  It returns a two-tuple
+        (path,type) or (None,None) if there's no module.
+        """
+        for (suffix,mode,type) in imp.get_suffixes():
+            if type in self._VALID_MODULE_TYPES:
+                path = prefix + suffix
+                if self.fs.isfile(path):
+                    return (path,type)
+        return (None,None)
+
+    def load_module(self,fullname):
+        """Load the specified module.
+
+        This method locates the file for the specified module, loads and
+        executes it and returns the created module object.
+        """
+        #  Reuse an existing module if present.
+        try:
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        #  Try to create from source or bytecode.
+        info = self._get_module_info(fullname)
+        code = self.get_code(fullname,info)
+        if code is None:
+            raise ImportError(fullname)
+        mod = imp.new_module(fullname)
+        mod.__file__ = "<loading>"
+        mod.__loader__ = self
+        sys.modules[fullname] = mod
+        try:
+            exec code in mod.__dict__
+            mod.__file__ = self.get_filename(fullname,info)
+            if self.is_package(fullname,info):
+                if self.path is None:
+                    mod.__path__ = []
+                else:
+                    mod.__path__ = [self.path]
+            return mod
+        except Exception:
+            sys.modules.pop(fullname,None)
+            raise
+
+    def is_package(self,fullname,info=None):
+        """Check whether the specified module is a package."""
+        if info is None:
+            info = self._get_module_info(fullname)
+        (path,type,ispkg) = info
+        return ispkg
+
+    def get_code(self,path,info=None):
+        """Get the bytecode for the specified module."""
+        if info is None:
+            info = self._get_module_info(fullname)
+        (path,type,ispkg) = info
+        code = self.fs.getcontents(path)
+        if type == imp.PY_SOURCE:
+            code = code.replace("\r\n","\n")
+            return compile(code,path,"exec")
+        elif type == imp.PY_COMPILED:
+            if code[:4] != imp.get_magic():
+                return None
+            return marshal.loads(code[8:])
+        else:
+            return None
+        return code
+
+    def get_source(self,fullname,info=None):
+        """Get the sourcecode for the specified module, if present."""
+        if info is None:
+            info = self._get_module_info(fullname)
+        (path,type,ispkg) = info
+        if type != imp.PY_SOURCE:
+            return None
+        return self.fs.getcontents(path).replace("\r\n","\n")
+
+    def get_data(self,path):
+        """Read the specified data file."""
+        try:
+            return self.fs.getcontents(path)
+        except FSError, e:
+            raise IOError(str(e))
+
+    def get_filename(self,fullname,info=None):
+        """Get the __file__ attribute for the specified module."""
+        if info is None:
+            info = self._get_module_info(fullname)
+        (path,type,ispkg) = info
+        return path
+ 

fs/opener.py

@@ -538,7 +538,8 @@ example:
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create_dir):
         from fs.tempfs import TempFS        
-        fs = TempFS(identifier=fs_name_params)                
+        from fs.wrapfs.lazyfs import LazyFS
+        fs = LazyFS((TempFS,(),{"identifier":fs_name_params}))                
         return fs, fs_path
 
 class S3Opener(Opener):

fs/tests/test_importhook.py

+
+import sys
+import unittest
+import marshal
+import imp
+import struct
+from textwrap import dedent
+
+from fs.expose.importhook import FSImportHook
+from fs.tempfs import TempFS
+from fs.zipfs import ZipFS
+
+
+class TestFSImportHook(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        for mph in list(sys.meta_path):
+            if isinstance(mph,FSImportHook):
+                sys.meta_path.remove(mph)
+        for ph in list(sys.path_hooks):
+            if issubclass(ph,FSImportHook):
+                sys.path_hooks.remove(mph)
+        for (k,v) in sys.modules.items():
+            if k.startswith("fsih_"):
+                del sys.modules[k]
+            elif hasattr(v,"__loader__"):
+                if isinstance(v.__loader__,FSImportHook):
+                    del sys.modules[k]
+        sys.path_importer_cache.clear()
+
+    def _init_modules(self,fs):
+        fs.setcontents("fsih_hello.py",dedent("""
+            message = 'hello world!'
+        """))
+        fs.makedir("fsih_pkg")
+        fs.setcontents("fsih_pkg/__init__.py",dedent("""
+            a = 42
+        """))
+        fs.setcontents("fsih_pkg/sub1.py",dedent("""
+            import fsih_pkg
+            from fsih_hello import message
+            a = fsih_pkg.a
+        """))
+        fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(dedent("""
+            import fsih_pkg
+            from fsih_hello import message
+            a = fsih_pkg.a * 2
+        """)))
+
+    def _getpyc(self,src):
+        """Get the .pyc contents to match th given .py source code."""
+        code = imp.get_magic() + struct.pack("<i",0)
+        code += marshal.dumps(compile(src,__file__,"exec"))
+        return code
+
+    def test_loader_methods(self):
+        t = TempFS()
+        self._init_modules(t)
+        ih = FSImportHook(t)
+        sys.meta_path.append(ih)
+        try:
+            self.assertEquals(ih.find_module("fsih_hello"),ih)
+            self.assertEquals(ih.find_module("fsih_helo"),None)
+            self.assertEquals(ih.find_module("fsih_pkg"),ih)
+            self.assertEquals(ih.find_module("fsih_pkg.sub1"),ih)
+            self.assertEquals(ih.find_module("fsih_pkg.sub2"),ih)
+            self.assertEquals(ih.find_module("fsih_pkg.sub3"),None)
+            m = ih.load_module("fsih_hello")
+            self.assertEquals(m.message,"hello world!")
+            self.assertRaises(ImportError,ih.load_module,"fsih_helo")
+            m = ih.load_module("fsih_pkg.sub1")
+            self.assertEquals(m.message,"hello world!")
+            self.assertEquals(m.a,42)
+            m = ih.load_module("fsih_pkg.sub2")
+            self.assertEquals(m.message,"hello world!")
+            self.assertEquals(m.a,42 * 2)
+            self.assertRaises(ImportError,ih.load_module,"fsih_pkg.sub3")
+        finally:
+            sys.meta_path.remove(ih)
+            t.close()
+
+    def _check_imports_are_working(self):
+        try:
+            import fsih_hello
+            self.assertEquals(fsih_hello.message,"hello world!")
+            try:
+                import fsih_helo
+            except ImportError:
+                pass
+            else:
+                assert False, "ImportError not raised"
+            import fsih_pkg
+            import fsih_pkg.sub1
+            self.assertEquals(fsih_pkg.sub1.message,"hello world!")
+            self.assertEquals(fsih_pkg.sub1.a,42)
+            import fsih_pkg.sub2
+            self.assertEquals(fsih_pkg.sub2.message,"hello world!")
+            self.assertEquals(fsih_pkg.sub2.a,42 * 2)
+            try:
+                import fsih_pkg.sub3
+            except ImportError:
+                pass
+            else:
+                assert False, "ImportError not raised"
+        finally:
+            for k in sys.modules.keys():
+                if k.startswith("fsih_"):
+                    del sys.modules[k]
+
+    def test_importer_on_meta_path(self):
+        t = TempFS()
+        self._init_modules(t)
+        ih = FSImportHook(t)
+        sys.meta_path.append(ih)
+        try:
+            self._check_imports_are_working()
+        finally:
+            sys.meta_path.remove(ih)
+            t.close()
+
+    def test_url_on_sys_path(self):
+        t = TempFS()
+        zpath = t.getsyspath("modules.zip")
+        z = ZipFS(zpath,"w")
+        self._init_modules(z)
+        z.close()
+        z = ZipFS(zpath,"r")
+        assert z.isfile("fsih_hello.py")
+        z.close()
+        sys.path.append("zip://" + zpath)
+        FSImportHook.install()
+        try:
+            self._check_imports_are_working()
+        finally:
+            sys.path_hooks.remove(FSImportHook)
+            sys.path.pop()
+            t.close()
+