Skip to content

P
pyfs
Commit 3b900b1c by willmcgugan
Options

Python3 compatibility

parent 754fcc98
Showing with 130 additions and 82 deletions
fs/expose/fuse/__init__.py
......@@ -66,8 +66,13 @@ from fs.errors import *
from fs.path import *
from fs.local_functools import wraps
from six import PY3
try:
import fuse_ctypes as fuse
if PY3:
import fuse3 as fuse
else:
import fuse
except NotImplementedError:
raise ImportError("FUSE found but not usable")
try:
......@@ -178,7 +183,7 @@ class FSOperations(Operations):
# FUSE doesn't seem to pass correct mode information here - at least,
# I haven't figured out how to distinguish between "w" and "w+".
# Go with the most permissive option.
fh = self._reg_file(self.fs.open(path,"w+"),path)
fh = self._reg_file(self.fs.open(path,"wb+"),path)
fi.fh = fh
fi.keep_cache = 0
......@@ -318,10 +323,10 @@ class FSOperations(Operations):
def truncate(self, path, length, fh=None):
path = path.decode(NATIVE_ENCODING)
if fh is None and length == 0:
self.fs.open(path,"w").close()
self.fs.open(path,"wb").close()
else:
if fh is None:
f = self.fs.open(path,"r+")
f = self.fs.open(path,"rb+")
if not hasattr(f,"truncate"):
raise UnsupportedError("truncate")
f.truncate(length)
......@@ -544,7 +549,7 @@ class MountProcess(subprocess.Popen):
os.close(w)
if os.read(r,1) != "S":
self.terminate()
raise RuntimeError("FUSE error: " + os.read(r,20)).decode(NATIVE_ENCODING)
raise RuntimeError("FUSE error: " + os.read(r,20).decode(NATIVE_ENCODING))
def unmount(self):
"""Cleanly unmount the FUSE filesystem, terminating this subprocess."""
......
fs/expose/importhook.py
......@@ -42,6 +42,8 @@ from fs.opener import fsopendir, OpenerError
from fs.errors import *
from fs.path import *
from six import b
class FSImportHook(object):
"""PEP-302-compliant module finder and loader for FS objects.
......@@ -204,9 +206,9 @@ class FSImportHook(object):
if info is None:
info = self._get_module_info(fullname)
(path,type,ispkg) = info
code = self.fs.getcontents(path)
code = self.fs.getcontents(path, 'rb')
if type == imp.PY_SOURCE:
code = code.replace("\r\n","\n")
code = code.replace(b("\r\n"),b("\n"))
return compile(code,path,"exec")
elif type == imp.PY_COMPILED:
if code[:4] != imp.get_magic():
......@@ -223,12 +225,12 @@ class FSImportHook(object):
(path,type,ispkg) = info
if type != imp.PY_SOURCE:
return None
return self.fs.getcontents(path).replace("\r\n","\n")
return self.fs.getcontents(path).replace(b("\r\n"),b("\n"))
def get_data(self,path):
"""Read the specified data file."""
try:
return self.fs.getcontents(path)
return self.fs.getcontents(path, 'rb')
except FSError, e:
raise IOError(str(e))
......
fs/expose/xmlrpc.py
......@@ -19,6 +19,9 @@ import xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer
from datetime import datetime
import six
from six import PY3, b
class RPCFSInterface(object):
"""Wrapper to expose an FS via a XML-RPC compatible interface.
......@@ -37,10 +40,14 @@ class RPCFSInterface(object):
must return something that can be represented in ASCII. The default
is base64-encoded UTF-8.
"""
if PY3:
return path
return path.encode("utf8").encode("base64")
def decode_path(self, path):
"""Decode paths arriving over the wire."""
if PY3:
return path
return path.decode("base64").decode("utf8")
def getmeta(self, meta_name):
......@@ -54,9 +61,9 @@ class RPCFSInterface(object):
def hasmeta(self, meta_name):
return self.fs.hasmeta(meta_name)
def get_contents(self, path):
def get_contents(self, path, mode="rb"):
path = self.decode_path(path)
data = self.fs.getcontents(path)
data = self.fs.getcontents(path, mode)
return xmlrpclib.Binary(data)
def set_contents(self, path, data):
......@@ -107,7 +114,8 @@ class RPCFSInterface(object):
def getinfo(self, path):
path = self.decode_path(path)
return self.fs.getinfo(path)
info = self.fs.getinfo(path)
return info
def desc(self, path):
path = self.decode_path(path)
......
fs/filelike.py
......@@ -502,7 +502,7 @@ class FileLikeBase(object):
# If not found, return whole string up to <size> length
# Any leftovers are pushed onto front of buffer
if indx == -1:
data = "".join(bits)
data = b("").join(bits)
if size > 0 and sizeSoFar > size:
extra = data[size:]
data = data[:size]
......
fs/ftpfs.py
......@@ -29,9 +29,15 @@ import calendar
from socket import error as socket_error
from fs.local_functools import wraps
try:
import six
from six import PY3
if PY3:
from six import BytesIO as StringIO
else:
try:
from cStringIO import StringIO
except ImportError:
except ImportError:
from StringIO import StringIO
import time
......
fs/memoryfs.py
......@@ -23,7 +23,6 @@ import threading
import six
def _check_mode(mode, mode_chars):
for c in mode_chars:
if c not in mode:
......@@ -76,7 +75,8 @@ class MemoryFile(object):
def __str__(self):
return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)
__repr__ = __str__
def __repr__(self):
return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
def __unicode__(self):
return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
......@@ -89,7 +89,9 @@ class MemoryFile(object):
pass
def __iter__(self):
return self
self.mem_file.seek(self.pos)
for line in self.mem_file:
yield line
@seek_and_lock
def next(self):
......@@ -614,10 +616,10 @@ class MemoryFS(FS):
@synchronize
def setcontents(self, path, data, chunk_size=1024*64):
if not isinstance(data, str):
if not isinstance(data, six.binary_type):
return super(MemoryFS, self).setcontents(path, data, chunk_size)
if not self.exists(path):
self.open(path, 'w').close()
self.open(path, 'wb').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
......
fs/osfs/__init__.py
......@@ -205,7 +205,6 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
@convert_os_errors
def open(self, path, mode="r", **kwargs):
#mode = filter(lambda c: c in "rwabt+",mode)
mode = ''.join(c for c in mode if c in 'rwabt+')
sys_path = self.getsyspath(path)
try:
......@@ -331,7 +330,7 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
@convert_os_errors
def getinfo(self, path):
stats = self._stat(path)
info = dict((k, getattr(stats, k)) for k in dir(stats) if not k.startswith('__') )
info = dict((k, getattr(stats, k)) for k in dir(stats) if k.startswith('st_'))
info['size'] = info['st_size']
# TODO: this doesn't actually mean 'creation time' on unix
ct = info.get('st_ctime', None)
......
fs/remote.py
......@@ -94,7 +94,8 @@ class RemoteFileBuffer(FileWrapper):
self._eof = True
if not hasattr(rfile, "read"):
rfile = StringIO(unicode(rfile))
#rfile = StringIO(unicode(rfile))
rfile = StringIO(rfile)
self._rfile = rfile
else:
......
fs/rpcfs.py
......@@ -18,6 +18,8 @@ from fs.path import *
from fs.filelike import StringIO
import six
from six import PY3, b
def re_raise_faults(func):
"""Decorator to re-raise XML-RPC faults as proper exceptions."""
......@@ -127,6 +129,9 @@ class RPCFS(FS):
def __str__(self):
return '<RPCFS: %s>' % (self.uri,)
def __repr__(self):
return '<RPCFS: %s>' % (self.uri,)
@synchronize
def __getstate__(self):
state = super(RPCFS,self).__getstate__()
......@@ -147,10 +152,14 @@ class RPCFS(FS):
must return something that can be represented in ASCII. The default
is base64-encoded UTF8.
"""
if PY3:
return path
return path.encode("utf8").encode("base64")
def decode_path(self, path):
"""Decode paths arriving over the wire."""
if PY3:
return path
return path.decode("base64").decode("utf8")
@synchronize
......@@ -169,18 +178,18 @@ class RPCFS(FS):
# TODO: chunked transport of large files
path = self.encode_path(path)
if "w" in mode:
self.proxy.set_contents(path,xmlrpclib.Binary(""))
self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
if "r" in mode or "a" in mode or "+" in mode:
try:
data = self.proxy.get_contents(path).data
data = self.proxy.get_contents(path, "rb").data
except IOError:
if "w" not in mode and "a" not in mode:
raise ResourceNotFoundError(path)
if not self.isdir(dirname(path)):
raise ParentDirectoryMissingError(path)
self.proxy.set_contents(path,xmlrpclib.Binary(""))
self.proxy.set_contents(path,xmlrpclib.Binary(b("")))
else:
data = ""
data = b("")
f = StringIO(data)
if "a" not in mode:
f.seek(0,0)
......
fs/tests/__init__.py
......@@ -297,8 +297,8 @@ class FSTestCases(object):
self.assertRaises(ResourceInvalidError,self.fs.listdirinfo,"foo")
def test_walk(self):
self.fs.setcontents('a.txt', 'hello')
self.fs.setcontents('b.txt', 'world')
self.fs.setcontents('a.txt', b('hello'))
self.fs.setcontents('b.txt', b('world'))
self.fs.makeopendir('foo').setcontents('c', b('123'))
sorted_walk = sorted([(d,sorted(fs)) for (d,fs) in self.fs.walk()])
self.assertEquals(sorted_walk,
......@@ -730,7 +730,8 @@ class FSTestCases(object):
checkcontents("hello",b("12345"))
def test_truncate_to_larger_size(self):
print repr(self.fs)
print self.fs.__class__
with self.fs.open("hello","wb") as f:
f.truncate(30)
self.assertEquals(self.fs.getsize("hello"), 30)
......
fs/tests/test_importhook.py
......@@ -10,6 +10,7 @@ from fs.expose.importhook import FSImportHook
from fs.tempfs import TempFS
from fs.zipfs import ZipFS
from six import b
class TestFSImportHook(unittest.TestCase):
......@@ -32,23 +33,23 @@ class TestFSImportHook(unittest.TestCase):
sys.path_importer_cache.clear()
def _init_modules(self,fs):
fs.setcontents("fsih_hello.py",dedent("""
fs.setcontents("fsih_hello.py",b(dedent("""
message = 'hello world!'
"""))
""")))
fs.makedir("fsih_pkg")
fs.setcontents("fsih_pkg/__init__.py",dedent("""
fs.setcontents("fsih_pkg/__init__.py",b(dedent("""
a = 42
"""))
fs.setcontents("fsih_pkg/sub1.py",dedent("""
""")))
fs.setcontents("fsih_pkg/sub1.py",b(dedent("""
import fsih_pkg
from fsih_hello import message
a = fsih_pkg.a
"""))
fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(dedent("""
""")))
fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(b(dedent("""
import fsih_pkg
from fsih_hello import message
a = fsih_pkg.a * 2
""")))
"""))))
def _getpyc(self,src):
"""Get the .pyc contents to match th given .py source code."""
......
fs/tests/test_watch.py
......@@ -178,7 +178,7 @@ class WatcherTestCases:
while not isinstance(event,CLOSED):
event = changes.next(timeout=1)
# That should be the last event in the list
self.assertRaises(StopIteration,changes.next,timeout=1)
self.assertRaises(StopIteration,getattr(changes, "next"),timeout=1)
changes.close()
......
fs/tests/test_xattr.py
......@@ -11,6 +11,7 @@ from fs.path import *
from fs.errors import *
from fs.tests import FSTestCases
from six import b
class XAttrTestCases:
"""Testcases for filesystems providing extended attribute support.
......@@ -26,11 +27,11 @@ class XAttrTestCases:
self.assertEqual(self.fs.getxattr(p,"xattr1"),"value1")
self.fs.delxattr(p,"xattr1")
self.assertEqual(self.fs.getxattr(p,"xattr1"),None)
self.fs.setcontents("test.txt","hello")
self.fs.setcontents("test.txt",b("hello"))
do_getsetdel("test.txt")
self.assertRaises(ResourceNotFoundError,self.fs.getxattr,"test2.txt","xattr1")
self.fs.makedir("mystuff")
self.fs.setcontents("/mystuff/test.txt","")
self.fs.setcontents("/mystuff/test.txt",b(""))
do_getsetdel("mystuff")
do_getsetdel("mystuff/test.txt")
......@@ -49,15 +50,15 @@ class XAttrTestCases:
self.assertEquals(sorted(self.fs.listxattrs(p)),["attr2"])
self.fs.delxattr(p,"attr2")
self.assertEquals(sorted(self.fs.listxattrs(p)),[])
self.fs.setcontents("test.txt","hello")
self.fs.setcontents("test.txt",b("hello"))
do_list("test.txt")
self.fs.makedir("mystuff")
self.fs.setcontents("/mystuff/test.txt","")
self.fs.setcontents("/mystuff/test.txt",b(""))
do_list("mystuff")
do_list("mystuff/test.txt")
def test_copy_xattrs(self):
self.fs.setcontents("a.txt","content")
self.fs.setcontents("a.txt",b("content"))
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
......@@ -75,7 +76,7 @@ class XAttrTestCases:
self.assertEquals(self.fs.getxattr("stuff","dirattr"),"a directory")
def test_move_xattrs(self):
self.fs.setcontents("a.txt","content")
self.fs.setcontents("a.txt",b("content"))
self.fs.setxattr("a.txt","myattr","myvalue")
self.fs.setxattr("a.txt","testattr","testvalue")
self.fs.makedir("stuff")
......
fs/tests/test_zipfs.py
......@@ -13,9 +13,10 @@ import shutil
import fs.tests
from fs.path import *
from fs import zipfs
from six import b
from fs import zipfs
class TestReadZipFS(unittest.TestCase):
def setUp(self):
......@@ -24,11 +25,11 @@ class TestReadZipFS(unittest.TestCase):
self.zf = zipfile.ZipFile(self.temp_filename, "w")
zf = self.zf
zf.writestr("a.txt", "Hello, World!")
zf.writestr("b.txt", "b")
zf.writestr("1.txt", "1")
zf.writestr("foo/bar/baz.txt", "baz")
zf.writestr("foo/second.txt", "hai")
zf.writestr("a.txt", b("Hello, World!"))
zf.writestr("b.txt", b("b"))
zf.writestr("1.txt", b("1"))
zf.writestr("foo/bar/baz.txt", b("baz"))
zf.writestr("foo/second.txt", b("hai"))
zf.close()
self.fs = zipfs.ZipFS(self.temp_filename, "r")
......@@ -50,18 +51,18 @@ class TestReadZipFS(unittest.TestCase):
return contents
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
check_contents("a.txt", b("Hello, World!"))
check_contents("1.txt", b("1"))
check_contents("foo/bar/baz.txt", b("baz"))
def test_getcontents(self):
def read_contents(path):
return self.fs.getcontents(path)
def check_contents(path, expected):
self.assert_(read_contents(path)==expected)
check_contents("a.txt", "Hello, World!")
check_contents("1.txt", "1")
check_contents("foo/bar/baz.txt", "baz")
check_contents("a.txt", b("Hello, World!"))
check_contents("1.txt", b("1"))
check_contents("foo/bar/baz.txt", b("baz"))
def test_is(self):
self.assert_(self.fs.isfile('a.txt'))
......@@ -98,15 +99,15 @@ class TestWriteZipFS(unittest.TestCase):
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f = zip_fs.open(filename, 'wb')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
makefile("foo/bar/baz.txt", "baz")
makefile("foo/second.txt", "hai")
makefile("a.txt", b("Hello, World!"))
makefile("b.txt", b("b"))
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
makefile("foo/bar/baz.txt", b("baz"))
makefile("foo/second.txt", b("hai"))
zip_fs.close()
......@@ -123,11 +124,11 @@ class TestWriteZipFS(unittest.TestCase):
def check_contents(filename, contents):
zcontents = zf.read(filename.encode("CP437"))
self.assertEqual(contents, zcontents)
check_contents("a.txt", "Hello, World!")
check_contents("b.txt", "b")
check_contents("foo/bar/baz.txt", "baz")
check_contents("foo/second.txt", "hai")
check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
check_contents("a.txt", b()"Hello, World!")
check_contents("b.txt", b("b"))
check_contents("foo/bar/baz.txt", b("baz"))
check_contents("foo/second.txt", b("hai"))
check_contents(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
class TestAppendZipFS(TestWriteZipFS):
......@@ -141,19 +142,19 @@ class TestAppendZipFS(TestWriteZipFS):
def makefile(filename, contents):
if dirname(filename):
zip_fs.makedir(dirname(filename), recursive=True, allow_recreate=True)
f = zip_fs.open(filename, 'w')
f = zip_fs.open(filename, 'wb')
f.write(contents)
f.close()
makefile("a.txt", "Hello, World!")
makefile("b.txt", "b")
makefile("a.txt", b("Hello, World!"))
makefile("b.txt", b("b"))
zip_fs.close()
zip_fs = zipfs.ZipFS(self.temp_filename, 'a')
makefile("foo/bar/baz.txt", "baz")
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", "this is the alpha and the omega")
makefile("foo/second.txt", "hai")
makefile("foo/bar/baz.txt", b("baz"))
makefile(u"\N{GREEK SMALL LETTER ALPHA}/\N{GREEK CAPITAL LETTER OMEGA}.txt", b("this is the alpha and the omega"))
makefile("foo/second.txt", b("hai"))
zip_fs.close()
......@@ -168,7 +169,7 @@ class TestZipFSErrors(unittest.TestCase):
def test_bogus_zipfile(self):
badzip = os.path.join(self.workdir,"bad.zip")
f = open(badzip,"wb")
f.write("I'm not really a zipfile")
f.write(b("I'm not really a zipfile"))
f.close()
self.assertRaises(zipfs.ZipOpenError,zipfs.ZipFS,badzip)
......
fs/zipfs.py
......@@ -19,6 +19,7 @@ from memoryfs import MemoryFS
import tempfs
from six import PY3
class ZipOpenError(CreateFailedError):
"""Thrown when the zip file could not be opened"""
......@@ -144,9 +145,20 @@ class ZipFS(FS):
def __unicode__(self):
return u"<ZipFS: %s>" % self.zip_path
def _decode_path(self, path):
if PY3:
return path
return path.decode(self.encoding)
def _encode_path(self, path):
if PY3:
return path
return path.encode(self.encoding)
def _parse_resource_list(self):
for path in self.zf.namelist():
self._add_resource(path.decode(self.encoding))
#self._add_resource(path.decode(self.encoding))
self._add_resource(self._decode_path(path))
def _add_resource(self, path):
if path.endswith('/'):
......@@ -185,9 +197,9 @@ class ZipFS(FS):
msg="1 Zip file must be opened for reading ('r') or appending ('a')")
try:
if hasattr(self.zf, 'open') and self._zip_file_string:
return self.zf.open(path.encode(self.encoding))
return self.zf.open(self._encode_path(path), "r")
else:
contents = self.zf.read(path.encode(self.encoding))
contents = self.zf.read(self._encode_path(path))
except KeyError:
raise ResourceNotFoundError(path)
return StringIO(contents)
......@@ -214,7 +226,7 @@ class ZipFS(FS):
raise ResourceNotFoundError(path)
path = normpath(relpath(path))
try:
contents = self.zf.read(path.encode(self.encoding))
contents = self.zf.read(self._encode_path(path))
except KeyError:
raise ResourceNotFoundError(path)
except RuntimeError:
......@@ -224,7 +236,7 @@ class ZipFS(FS):
@synchronize
def _on_write_close(self, filename):
sys_path = self.temp_fs.getsyspath(filename)
self.zf.write(sys_path, filename.encode(self.encoding))
self.zf.write(sys_path, self._encode_path(filename))
def desc(self, path):
return "%s in zip file %s" % (path, self.zip_path)
......@@ -256,14 +268,14 @@ class ZipFS(FS):
raise ResourceNotFoundError(path)
path = normpath(path).lstrip('/')
try:
zi = self.zf.getinfo(path.encode(self.encoding))
zi = self.zf.getinfo(self._encode_path(path))
zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
for k, v in zinfo.iteritems():
if callable(v):
zinfo[k] = v()
except KeyError:
zinfo = {'file_size':0}
info = {'size' : zinfo['file_size'] }
info = {'size' : zinfo['file_size']}
if 'date_time' in zinfo:
info['created_time'] = datetime.datetime(*zinfo['date_time'])
info.update(zinfo)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment