Python3 compatibility

fs/expose/fuse/__init__.py

@@ -66,8 +66,13 @@ from fs.errors import *
 from fs.path import *
 from fs.local_functools import wraps
 
+from six import PY3
+
 try:
-    import fuse_ctypes as fuse
+    if PY3:
+        import fuse3 as fuse
+    else:
+        import fuse
 except NotImplementedError:
     raise ImportError("FUSE found but not usable")
 try:
@@ -178,7 +183,7 @@ class FSOperations(Operations):
         # FUSE doesn't seem to pass correct mode information here - at least,
         # I haven't figured out how to distinguish between "w" and "w+".
         # Go with the most permissive option.
-        fh = self._reg_file(self.fs.open(path,"w+"),path)
+        fh = self._reg_file(self.fs.open(path,"wb+"),path)
         fi.fh = fh
         fi.keep_cache = 0
 
@@ -318,10 +323,10 @@ class FSOperations(Operations):
     def truncate(self, path, length, fh=None):
         path = path.decode(NATIVE_ENCODING)
         if fh is None and length == 0:
-            self.fs.open(path,"w").close()
+            self.fs.open(path,"wb").close()
         else:
             if fh is None:
-                f = self.fs.open(path,"r+")
+                f = self.fs.open(path,"rb+")
                 if not hasattr(f,"truncate"):
                     raise UnsupportedError("truncate")
                 f.truncate(length)
@@ -544,7 +549,7 @@ class MountProcess(subprocess.Popen):
             os.close(w)
             if os.read(r,1) != "S":
                 self.terminate()
-                raise RuntimeError("FUSE error: " + os.read(r,20)).decode(NATIVE_ENCODING)
+                raise RuntimeError("FUSE error: " + os.read(r,20).decode(NATIVE_ENCODING))
 
     def unmount(self):
         """Cleanly unmount the FUSE filesystem, terminating this subprocess."""

fs/expose/importhook.py

@@ -42,6 +42,8 @@ from fs.opener import fsopendir, OpenerError
 from fs.errors import *
 from fs.path import *
 
+from six import b
+
 class FSImportHook(object):
     """PEP-302-compliant module finder and loader for FS objects.
 
@@ -204,9 +206,9 @@ class FSImportHook(object):
         if info is None:
             info = self._get_module_info(fullname)
         (path,type,ispkg) = info
-        code = self.fs.getcontents(path)
+        code = self.fs.getcontents(path, 'rb')
         if type == imp.PY_SOURCE:
-            code = code.replace("\r\n","\n")
+            code = code.replace(b("\r\n"),b("\n"))
             return compile(code,path,"exec")
         elif type == imp.PY_COMPILED:
             if code[:4] != imp.get_magic():
@@ -223,12 +225,12 @@ class FSImportHook(object):
         (path,type,ispkg) = info
         if type != imp.PY_SOURCE:
             return None
-        return self.fs.getcontents(path).replace("\r\n","\n")
+        return self.fs.getcontents(path).replace(b("\r\n"),b("\n"))
 
     def get_data(self,path):
         """Read the specified data file."""
         try:
-            return self.fs.getcontents(path)
+            return self.fs.getcontents(path, 'rb')
         except FSError, e:
             raise IOError(str(e))
 

fs/expose/xmlrpc.py

@@ -19,6 +19,9 @@ import xmlrpclib
 from SimpleXMLRPCServer import SimpleXMLRPCServer
 from datetime import datetime
 
+import six
+from six import PY3, b
+
 class RPCFSInterface(object):
     """Wrapper to expose an FS via a XML-RPC compatible interface.
 
@@ -37,10 +40,14 @@ class RPCFSInterface(object):
         must return something that can be represented in ASCII.  The default
         is base64-encoded UTF-8.
         """
+        if PY3:
+            return path        
         return path.encode("utf8").encode("base64")
 
     def decode_path(self, path):
         """Decode paths arriving over the wire."""
+        if PY3:
+            return path
         return path.decode("base64").decode("utf8")        
 
     def getmeta(self, meta_name):
@@ -54,9 +61,9 @@ class RPCFSInterface(object):
     def hasmeta(self, meta_name):
         return self.fs.hasmeta(meta_name)
 
-    def get_contents(self, path):
+    def get_contents(self, path, mode="rb"):
         path = self.decode_path(path)
-        data = self.fs.getcontents(path)
+        data = self.fs.getcontents(path, mode)
         return xmlrpclib.Binary(data)
 
     def set_contents(self, path, data):
@@ -107,7 +114,8 @@ class RPCFSInterface(object):
 
     def getinfo(self, path):        
         path = self.decode_path(path)
-        return self.fs.getinfo(path)
+        info = self.fs.getinfo(path)        
+        return info
 
     def desc(self, path):
         path = self.decode_path(path)

fs/filelike.py

@@ -502,7 +502,7 @@ class FileLikeBase(object):
         # If not found, return whole string up to <size> length
         # Any leftovers are pushed onto front of buffer
         if indx == -1:
-            data = "".join(bits)
+            data = b("").join(bits)
             if size > 0 and sizeSoFar > size:
                 extra = data[size:]
                 data = data[:size]

fs/ftpfs.py

@@ -29,9 +29,15 @@ import calendar
 from socket import error as socket_error
 from fs.local_functools import wraps
 
-try:
+import six
+from six import PY3
+
+if PY3:
+    from six import BytesIO as StringIO
+else:
+    try:
         from cStringIO import StringIO
-except ImportError:
+    except ImportError:
         from StringIO import StringIO
 
 import time

fs/memoryfs.py

@@ -23,7 +23,6 @@ import threading
 import six
 
 
-
 def _check_mode(mode, mode_chars):
     for c in mode_chars:
         if c not in mode:
@@ -76,7 +75,8 @@ class MemoryFile(object):
     def __str__(self):
         return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)
 
-    __repr__ = __str__
+    def __repr__(self):
+        return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path) 
 
     def __unicode__(self):
         return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
@@ -89,7 +89,9 @@ class MemoryFile(object):
         pass
 
     def __iter__(self):
-        return self
+        self.mem_file.seek(self.pos)
+        for line in self.mem_file:
+            yield line        
 
     @seek_and_lock
     def next(self):        
@@ -614,10 +616,10 @@ class MemoryFS(FS):
     
     @synchronize
     def setcontents(self, path, data, chunk_size=1024*64):
-        if not isinstance(data, str):
+        if not isinstance(data, six.binary_type):        
             return super(MemoryFS, self).setcontents(path, data, chunk_size)        
         if not self.exists(path):      
-            self.open(path, 'w').close()                    
+            self.open(path, 'wb').close()                    
             
         dir_entry = self._get_dir_entry(path)
         if not dir_entry.isfile():

fs/osfs/__init__.py

@@ -205,7 +205,6 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
 
     @convert_os_errors
     def open(self, path, mode="r", **kwargs):        
-        #mode = filter(lambda c: c in "rwabt+",mode)
         mode = ''.join(c for c in mode if c in 'rwabt+')
         sys_path = self.getsyspath(path)
         try:
@@ -331,7 +330,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
     @convert_os_errors
     def getinfo(self, path):
         stats = self._stat(path)
-        info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') )
+        info = dict((k, getattr(stats, k)) for k in dir(stats) if k.startswith('st_'))
         info['size'] = info['st_size']
         #  TODO: this doesn't actually mean 'creation time' on unix
         ct = info.get('st_ctime', None)

fs/remote.py

@@ -94,7 +94,8 @@ class RemoteFileBuffer(FileWrapper):
                 self._eof = True
                 
             if not hasattr(rfile, "read"):
-                rfile = StringIO(unicode(rfile))
+                #rfile = StringIO(unicode(rfile))
+                rfile = StringIO(rfile)
                 
             self._rfile = rfile
         else:

fs/rpcfs.py

@@ -18,6 +18,8 @@ from fs.path import *
 
 from fs.filelike import StringIO
 
+import six
+from six import PY3, b
 
 def re_raise_faults(func):
     """Decorator to re-raise XML-RPC faults as proper exceptions."""
@@ -127,6 +129,9 @@ class RPCFS(FS):
     def __str__(self):
         return '<RPCFS: %s>' % (self.uri,)
     
+    def __repr__(self):
+        return '<RPCFS: %s>' % (self.uri,)
+
     @synchronize
     def __getstate__(self):
         state = super(RPCFS,self).__getstate__()
@@ -147,10 +152,14 @@ class RPCFS(FS):
         must return something that can be represented in ASCII.  The default
         is base64-encoded UTF8.
         """
+        if PY3:
+            return path
         return path.encode("utf8").encode("base64")      
 
     def decode_path(self, path):
         """Decode paths arriving over the wire."""
+        if PY3:
+            return path        
         return path.decode("base64").decode("utf8")        
     
     @synchronize
@@ -169,18 +178,18 @@ class RPCFS(FS):
         # TODO: chunked transport of large files
         path = self.encode_path(path)
         if "w" in mode:
-            self.proxy.set_contents(path,xmlrpclib.Binary(""))
+            self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
         if "r" in mode or "a" in mode or "+" in mode:
             try:
-                data = self.proxy.get_contents(path).data
+                data = self.proxy.get_contents(path, "rb").data
             except IOError:
                 if "w" not in mode and "a" not in mode:
                     raise ResourceNotFoundError(path)
                 if not self.isdir(dirname(path)):
                     raise ParentDirectoryMissingError(path)
-                self.proxy.set_contents(path,xmlrpclib.Binary(""))
+                self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
         else:
-            data = ""
+            data = b("")
         f = StringIO(data)
         if "a" not in mode:
             f.seek(0,0)

fs/tests/__init__.py

@@ -297,8 +297,8 @@ class FSTestCases(object):
         self.assertRaises(ResourceInvalidError,self.fs.listdirinfo,"foo")
 
     def test_walk(self):
-        self.fs.setcontents('a.txt', 'hello')
-        self.fs.setcontents('b.txt', 'world')
+        self.fs.setcontents('a.txt', b('hello'))
+        self.fs.setcontents('b.txt', b('world'))
         self.fs.makeopendir('foo').setcontents('c', b('123'))
         sorted_walk = sorted([(d,sorted(fs)) for (d,fs) in self.fs.walk()])
         self.assertEquals(sorted_walk,
@@ -730,7 +730,8 @@ class FSTestCases(object):
         checkcontents("hello",b("12345"))
 
     def test_truncate_to_larger_size(self):
-        
+        print repr(self.fs)
+        print self.fs.__class__
         with self.fs.open("hello","wb") as f:
             f.truncate(30)
         self.assertEquals(self.fs.getsize("hello"), 30)

fs/tests/test_importhook.py

@@ -10,6 +10,7 @@ from fs.expose.importhook import FSImportHook
 from fs.tempfs import TempFS
 from fs.zipfs import ZipFS
 
+from six import b
 
 class TestFSImportHook(unittest.TestCase):
 
@@ -32,23 +33,23 @@ class TestFSImportHook(unittest.TestCase):
         sys.path_importer_cache.clear()
 
     def _init_modules(self,fs):
-        fs.setcontents("fsih_hello.py",dedent("""
+        fs.setcontents("fsih_hello.py",b(dedent("""
             message = 'hello world!'
-        """))
+        """)))
         fs.makedir("fsih_pkg")
-        fs.setcontents("fsih_pkg/__init__.py",dedent("""
+        fs.setcontents("fsih_pkg/__init__.py",b(dedent("""
             a = 42
-        """))
-        fs.setcontents("fsih_pkg/sub1.py",dedent("""
+        """)))
+        fs.setcontents("fsih_pkg/sub1.py",b(dedent("""
             import fsih_pkg
             from fsih_hello import message
             a = fsih_pkg.a
-        """))
-        fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(dedent("""
+        """)))
+        fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(b(dedent("""
             import fsih_pkg
             from fsih_hello import message
             a = fsih_pkg.a * 2
-        """)))
+        """))))
 
     def _getpyc(self,src):
         """Get the .pyc contents to match th given .py source code."""

fs/tests/test_watch.py

@@ -178,7 +178,7 @@ class WatcherTestCases:
         while not isinstance(event,CLOSED):
             event = changes.next(timeout=1)
         #  That should be the last event in the list
-        self.assertRaises(StopIteration,changes.next,timeout=1)
+        self.assertRaises(StopIteration,getattr(changes, "next"),timeout=1)
         changes.close()
 
 

fs/tests/test_xattr.py

@@ -11,6 +11,7 @@ from fs.path import *
 from fs.errors import *
 from fs.tests import FSTestCases
 
+from six import b 
 
 class XAttrTestCases:
     """Testcases for filesystems providing extended attribute support.
@@ -26,11 +27,11 @@ class XAttrTestCases:
             self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
             self.fs.delxattr(p,"xattr1")
             self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
-        self.fs.setcontents("test.txt","hello")
+        self.fs.setcontents("test.txt",b("hello"))
         do_getsetdel("test.txt")
         self.assertRaises(ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
         self.fs.makedir("mystuff")
-        self.fs.setcontents("/mystuff/test.txt","")
+        self.fs.setcontents("/mystuff/test.txt",b(""))
         do_getsetdel("mystuff")
         do_getsetdel("mystuff/test.txt")
 
@@ -49,15 +50,15 @@ class XAttrTestCases:
             self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2"])
             self.fs.delxattr(p,"attr2")
             self.assertEquals(sorted(self.fs.listxattrs(p)),[])
-        self.fs.setcontents("test.txt","hello")
+        self.fs.setcontents("test.txt",b("hello"))
         do_list("test.txt")
         self.fs.makedir("mystuff")
-        self.fs.setcontents("/mystuff/test.txt","")
+        self.fs.setcontents("/mystuff/test.txt",b(""))
         do_list("mystuff")
         do_list("mystuff/test.txt")
 
     def test_copy_xattrs(self):
-        self.fs.setcontents("a.txt","content")
+        self.fs.setcontents("a.txt",b("content"))
         self.fs.setxattr("a.txt","myattr","myvalue")
         self.fs.setxattr("a.txt","testattr","testvalue")
         self.fs.makedir("stuff")
@@ -75,7 +76,7 @@ class XAttrTestCases:
         self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
 
     def test_move_xattrs(self):
-        self.fs.setcontents("a.txt","content")
+        self.fs.setcontents("a.txt",b("content"))
         self.fs.setxattr("a.txt","myattr","myvalue")
         self.fs.setxattr("a.txt","testattr","testvalue")
         self.fs.makedir("stuff")

fs/tests/test_zipfs.py

@@ -13,9 +13,10 @@ import shutil
 
 import fs.tests
 from fs.path import *
+from fs import zipfs
 
+from six import b
 
-from fs import zipfs
 class TestReadZipFS(unittest.TestCase):
 
     def setUp(self):
@@ -24,11 +25,11 @@ class TestReadZipFS(unittest.TestCase):
 
         self.zf = zipfile.ZipFile(self.temp_filename, "w")
         zf = self.zf
-        zf.writestr("a.txt", "Hello, World!")
-        zf.writestr("b.txt", "b")
-        zf.writestr("1.txt", "1")
-        zf.writestr("foo/bar/baz.txt", "baz")
-        zf.writestr("foo/second.txt", "hai")
+        zf.writestr("a.txt", b("Hello, World!"))
+        zf.writestr("b.txt", b("b"))
+        zf.writestr("1.txt", b("1"))
+        zf.writestr("foo/bar/baz.txt", b("baz"))
+        zf.writestr("foo/second.txt", b("hai"))
         zf.close()
         self.fs = zipfs.ZipFS(self.temp_filename, "r")
 
@@ -50,18 +51,18 @@ class TestReadZipFS(unittest.TestCase):
             return contents
         def check_contents(path, expected):
             self.assert_(read_contents(path)==expected)
-        check_contents("a.txt", "Hello, World!")
-        check_contents("1.txt", "1")
-        check_contents("foo/bar/baz.txt", "baz")
+        check_contents("a.txt", b("Hello, World!"))
+        check_contents("1.txt", b("1"))
+        check_contents("foo/bar/baz.txt", b("baz"))
 
     def test_getcontents(self):
         def read_contents(path):
             return self.fs.getcontents(path)
         def check_contents(path, expected):
             self.assert_(read_contents(path)==expected)
-        check_contents("a.txt", "Hello, World!")
-        check_contents("1.txt", "1")
-        check_contents("foo/bar/baz.txt", "baz")
+        check_contents("a.txt", b("Hello, World!"))
+        check_contents("1.txt", b("1"))
+        check_contents("foo/bar/baz.txt", b("baz"))
 
     def test_is(self):
         self.assert_(self.fs.isfile('a.txt'))
@@ -98,15 +99,15 @@ class TestWriteZipFS(unittest.TestCase):
         def makefile(filename, contents):
             if dirname(filename):
                 zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
-            f = zip_fs.open(filename, 'w')
+            f = zip_fs.open(filename, 'wb')
             f.write(contents)
             f.close()
 
-        makefile("a.txt", "Hello, World!")
-        makefile("b.txt", "b")
-        makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
-        makefile("foo/bar/baz.txt", "baz")
-        makefile("foo/second.txt", "hai")
+        makefile("a.txt", b("Hello, World!"))
+        makefile("b.txt", b("b"))
+        makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
+        makefile("foo/bar/baz.txt", b("baz"))
+        makefile("foo/second.txt", b("hai"))
 
         zip_fs.close()
 
@@ -123,11 +124,11 @@ class TestWriteZipFS(unittest.TestCase):
         def check_contents(filename, contents):
             zcontents = zf.read(filename.encode("CP437"))
             self.assertEqual(contents, zcontents)
-        check_contents("a.txt", "Hello, World!")
-        check_contents("b.txt", "b")
-        check_contents("foo/bar/baz.txt", "baz")
-        check_contents("foo/second.txt", "hai")
-        check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
+        check_contents("a.txt", b()"Hello, World!")
+        check_contents("b.txt", b("b"))
+        check_contents("foo/bar/baz.txt", b("baz"))
+        check_contents("foo/second.txt", b("hai"))
+        check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
 
 
 class TestAppendZipFS(TestWriteZipFS):
@@ -141,19 +142,19 @@ class TestAppendZipFS(TestWriteZipFS):
         def makefile(filename, contents):
             if dirname(filename):
                 zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
-            f = zip_fs.open(filename, 'w')
+            f = zip_fs.open(filename, 'wb')
             f.write(contents)
             f.close()
 
-        makefile("a.txt", "Hello, World!")
-        makefile("b.txt", "b")
+        makefile("a.txt", b("Hello, World!"))
+        makefile("b.txt", b("b"))
 
         zip_fs.close()
         zip_fs = zipfs.ZipFS(self.temp_filename, 'a')
 
-        makefile("foo/bar/baz.txt", "baz")
-        makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
-        makefile("foo/second.txt", "hai")
+        makefile("foo/bar/baz.txt", b("baz"))
+        makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
+        makefile("foo/second.txt", b("hai"))
 
         zip_fs.close()
 
@@ -168,7 +169,7 @@ class TestZipFSErrors(unittest.TestCase):
     def test_bogus_zipfile(self):
         badzip = os.path.join(self.workdir,"bad.zip")
         f = open(badzip,"wb")
-        f.write("I'm not really a zipfile")
+        f.write(b("I'm not really a zipfile"))
         f.close()
         self.assertRaises(zipfs.ZipOpenError,zipfs.ZipFS,badzip)
 

fs/zipfs.py

@@ -19,6 +19,7 @@ from memoryfs import MemoryFS
 
 import tempfs
 
+from six import PY3
 
 class ZipOpenError(CreateFailedError):
     """Thrown when the zip file could not be opened"""
@@ -144,9 +145,20 @@ class ZipFS(FS):
     def __unicode__(self):
         return u"<ZipFS: %s>" % self.zip_path
 
+    def _decode_path(self, path):
+        if PY3:
+            return path
+        return path.decode(self.encoding)
+    
+    def _encode_path(self, path):
+        if PY3:
+            return path
+        return path.encode(self.encoding)
+
     def _parse_resource_list(self):
         for path in self.zf.namelist():
-            self._add_resource(path.decode(self.encoding))
+            #self._add_resource(path.decode(self.encoding))
+            self._add_resource(self._decode_path(path))
 
     def _add_resource(self, path):
         if path.endswith('/'):
@@ -185,9 +197,9 @@ class ZipFS(FS):
                                            msg="1 Zip file must be opened for reading ('r') or appending ('a')")
             try:
                 if hasattr(self.zf, 'open') and self._zip_file_string:
-                    return self.zf.open(path.encode(self.encoding))
+                    return self.zf.open(self._encode_path(path), "r")
                 else:
-                    contents = self.zf.read(path.encode(self.encoding))
+                    contents = self.zf.read(self._encode_path(path))
             except KeyError:
                 raise ResourceNotFoundError(path)
             return StringIO(contents)
@@ -214,7 +226,7 @@ class ZipFS(FS):
             raise ResourceNotFoundError(path)
         path = normpath(relpath(path))
         try:
-            contents = self.zf.read(path.encode(self.encoding))
+            contents = self.zf.read(self._encode_path(path))
         except KeyError:
             raise ResourceNotFoundError(path)
         except RuntimeError:
@@ -224,7 +236,7 @@ class ZipFS(FS):
     @synchronize
     def _on_write_close(self, filename):
         sys_path = self.temp_fs.getsyspath(filename)
-        self.zf.write(sys_path, filename.encode(self.encoding))
+        self.zf.write(sys_path, self._encode_path(filename))
 
     def desc(self, path):        
         return "%s in zip file %s" % (path, self.zip_path)        
@@ -256,14 +268,14 @@ class ZipFS(FS):
             raise ResourceNotFoundError(path)
         path = normpath(path).lstrip('/')
         try:
-            zi = self.zf.getinfo(path.encode(self.encoding))
+            zi = self.zf.getinfo(self._encode_path(path))
             zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
             for k, v in zinfo.iteritems():
                 if callable(v):
                     zinfo[k] = v()
         except KeyError:
             zinfo = {'file_size':0}
-        info = {'size' : zinfo['file_size'] }
+        info = {'size' : zinfo['file_size']}
         if 'date_time' in zinfo:
             info['created_time'] = datetime.datetime(*zinfo['date_time'])
         info.update(zinfo)