Skip to content

P
pyfs
Commit 4dff3320 by willmcgugan
Options

Changed syntax for commands to be more url like, optimized sftps to use fewer queries for listdir

parent f1f224c1
Showing with 381 additions and 121 deletions
fs/commands/fscp.py
from fs.opener import opener
from fs.utils import copyfile, copystructure
from fs.path import pathjoin
from fs.path import pathjoin, iswildcard
from fs.errors import FSError
from fs.commands.runner import Command
import sys
......@@ -92,7 +92,7 @@ Copy SOURCE to DESTINATION"""
if src_path is None:
src_path = '/'
if self.is_wildcard(src_path):
if iswildcard(src_path):
for file_path in src_fs.listdir(wildcard=src_path, full=True):
copy_fs_paths.append((self.FILE, src_fs, file_path, file_path))
......
fs/commands/fsls.py
#!/usr/bin/env python
from fs.opener import opener
from fs.path import pathsplit, abspath, isdotfile
from fs.path import pathsplit, abspath, isdotfile, iswildcard
from fs.commands.runner import Command
from collections import defaultdict
import sys
......@@ -43,10 +43,10 @@ List contents of [PATH]"""
path = path or '.'
wildcard = None
if self.is_wildcard(path):
if iswildcard(path):
path, wildcard = pathsplit(path)
if fs.isfile(path):
if path != '.' and fs.isfile(path):
if not options.dirsonly:
file_paths.append(path)
else:
......
fs/commands/fstree.py
......@@ -14,7 +14,9 @@ Recursively display the contents of PATH in an ascii tree"""
def get_optparse(self):
optparse = super(FSTree, self).get_optparse()
optparse.add_option('-d', '--depth', dest='depth', type="int", default=5,
help="Maximum depth to display", metavar="DEPTH")
help="Maximum depth to display", metavar="DEPTH")
optparse.add_option('-a', '--all', dest='all', action='store_true', default=False,
help="do not hide dot files")
return optparse
def do_run(self, options, args):
......@@ -31,7 +33,8 @@ Recursively display the contents of PATH in an ascii tree"""
print_fs(fs, path or '',
file_out=self.output_file,
max_levels=options.depth,
terminal_colors=self.is_terminal())
terminal_colors=self.is_terminal(),
hide_dotfiles=not options.all)
def run():
return FSTree().run()
......
fs/commands/runner.py
......@@ -2,7 +2,7 @@ import sys
from optparse import OptionParser
from fs.opener import opener, OpenerError
from fs.errors import FSError
from fs.path import splitext, pathsplit, isdotfile
from fs.path import splitext, pathsplit, isdotfile, iswildcard
import platform
from collections import defaultdict
......@@ -55,11 +55,6 @@ class Command(object):
self.terminal_width = w
self.name = self.__class__.__name__.lower()
def is_wildcard(self, path):
if path is None:
return False
return '*' in path or '?' in path
def is_terminal(self):
try:
return self.output_file.isatty()
......@@ -111,7 +106,7 @@ class Command(object):
if path is None:
return [], []
pathname, resourcename = pathsplit(path)
if self.is_wildcard(resourcename):
if iswildcard(resourcename):
dir_paths = fs.listdir(pathname,
wildcard=resourcename,
absolute=True,
......@@ -137,7 +132,7 @@ class Command(object):
resources = []
for fs, path in fs_paths:
if self.is_wildcard(path):
if path and iswildcard(path):
if not files_only:
dir_paths = fs.listdir(wildcard=path, dirs_only=True)
for path in dir_paths:
......@@ -227,8 +222,8 @@ class Command(object):
if self.is_terminal():
self.output("\n")
return 0
except ValueError:
pass
#except ValueError:
# pass
except SystemExit:
return 0
except IOError:
......
fs/ftpfs.py
......@@ -751,8 +751,7 @@ class FTPFS(FS):
def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT,
port=21,
dircache=True,
max_buffer_size=128*1024*1024):
dircache=True):
""" Connect to a FTP server.
:param host: Host to connect to
......@@ -765,8 +764,7 @@ class FTPFS(FS):
which will speed up operations such as getinfo, isdi, isfile, but
changes to the ftp file structure will not be visible until
`~fs.ftpfs.FTPFS.clear_dircache` is called
:param dircache: If True directory information will be cached for fast access
:param max_buffer_size: Number of bytes to hold before blocking write operations
:param dircache: If True directory information will be cached for fast access
"""
......@@ -780,9 +778,7 @@ class FTPFS(FS):
self.timeout = timeout
self.use_dircache = dircache
self.get_dircache()
self.max_buffer_size = max_buffer_size
self.get_dircache()
self._cache_hint = False
self._locals._ftp = None
......
fs/memoryfs.py
......@@ -156,7 +156,7 @@ class DirEntry(object):
self.locks += 1
def unlock(self):
self.locks -=1
self.locks -= 1
assert self.locks >=0, "Lock / Unlock mismatch!"
def desc_contents(self):
......@@ -494,7 +494,7 @@ class MemoryFS(FS):
if dir_entry is None:
raise ResourceNotFoundError(path)
if dir_entry.isfile():
raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
raise ResourceInvalidError(path, msg="not a directory: %(path)s")
paths = dir_entry.contents.keys()
for (i,p) in enumerate(paths):
if not isinstance(p,unicode):
......@@ -522,7 +522,7 @@ class MemoryFS(FS):
return info
@synchronize
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
......@@ -533,7 +533,7 @@ class MemoryFS(FS):
dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
......@@ -544,7 +544,7 @@ class MemoryFS(FS):
dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
def copy(self, src, dst, overwrite=False, chunk_size=16384):
def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
......@@ -555,7 +555,7 @@ class MemoryFS(FS):
dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
def move(self, src, dst, overwrite=False, chunk_size=16384):
def move(self, src, dst, overwrite=False, chunk_size=1024*64):
src_dir_entry = self._get_dir_entry(src)
if src_dir_entry is None:
raise ResourceNotFoundError(src)
......@@ -566,6 +566,27 @@ class MemoryFS(FS):
dst_dir_entry.xattrs.update(src_xattrs)
@synchronize
def getcontents(self, path):
dir_entry = self._get_dir_entry(path)
if dir_entry is None:
raise ResourceNotFoundError(path)
if not dir_entry.isfile():
raise ResourceInvalidError(path, msg="not a directory: %(path)s")
return dir_entry.data or ''
@synchronize
def setcontents(self, path, data, chunk_size=1024*64):
if not isinstance(data, str):
return super(MemoryFS, self).setcontents(path, data, chunk_size)
if not self.exists(path):
self.open(path, 'w').close()
dir_entry = self._get_dir_entry(path)
if not dir_entry.isfile():
raise ResourceInvalidError('Not a directory %(path)s', path)
dir_entry.data = data
@synchronize
def setxattr(self, path, key, value):
dir_entry = self._dir_entry(path)
key = unicode(key)
......
fs/opener.py
import sys
from fs.osfs import OSFS
from fs.path import pathsplit
from fs.path import pathsplit, basename, join, iswildcard
import os
import os.path
import re
from urlparse import urlparse
class OpenerError(Exception):
pass
......@@ -31,11 +33,15 @@ def _expand_syspath(path):
return path
class OpenerRegistry(object):
re_fs_url = re.compile(r'''
^
(?:\[(.*?)\])*
(.*?)
:\/\/
(?:
\((.*?)\)
......@@ -46,6 +52,8 @@ class OpenerRegistry(object):
\+(.*?)$
)*$
''', re.VERBOSE)
def __init__(self, openers=[]):
self.registry = {}
......@@ -56,13 +64,12 @@ class OpenerRegistry(object):
@classmethod
def split_segments(self, fs_url):
match = self.re_fs_url.match(fs_url)
assert match is not None, "broken re?"
return match.groups()
match = self.re_fs_url.match(fs_url)
return match
def get_opener(self, name):
if name not in self.registry:
raise NoOpenerError("No opener for [%s]" % name)
raise NoOpenerError("No opener for %s" % name)
index = self.registry[name]
return self.openers[index]
......@@ -73,34 +80,56 @@ class OpenerRegistry(object):
self.registry[name] = index
def parse(self, fs_url, default_fs_name=None, writeable=False, create=False):
fs_name, paren_url, fs_url, path = self.split_segments(fs_url)
orig_url = fs_url
match = self.split_segments(fs_url)
fs_url = fs_url or paren_url
if fs_name is None and path is None:
fs_url = os.path.expanduser(os.path.expandvars(fs_url))
fs_url = os.path.normpath(os.path.abspath(fs_url))
fs_url, path = pathsplit(fs_url)
if not fs_url:
fs_url = '/'
if match:
fs_name, paren_url, fs_url, path = match.groups()
fs_url = fs_url or paren_url or ''
if ':' in fs_name:
fs_name, sub_protocol = fs_name.split(':', 1)
fs_url = '%s://%s' % (sub_protocol, fs_url)
fs_name = fs_name or self.default_opener
else:
fs_name = default_fs_name or self.default_opener
fs_url = _expand_syspath(fs_url)
path = ''
fs_name, fs_name_params = self.parse_name(fs_name)
opener = self.get_opener(fs_name)
fs_name = fs_name or self.default_opener
if fs_url is None:
raise OpenerError("Unable to parse '%s'" % orig_url)
if fs_name is None:
fs_name = fs_default_name
wildcard = None
if iswildcard(fs_url):
fs_url, wildcard = pathsplit(fs_url)
fs_name, fs_name_params = self.parse_name(fs_name)
opener = self.get_opener(fs_name)
fs = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
if wildcard:
fs_path = join(fs_path or '', wildcard)
else:
path = join(fs_path or '', path)
if path:
pathname, resourcename = pathsplit(path)
if pathname:
fs = fs.opendir(pathname)
path = resourcename
if not iswildcard(path):
if fs.isdir(path):
fs = fs.opendir(path)
fs_path = ''
else:
fs_path = path
return fs, path
return fs, fs_path
def parse_credentials(self, url):
......@@ -145,11 +174,21 @@ class OSFSOpener(Opener):
names = ['osfs', 'file']
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
username, password, fs_path = registry.parse_credentials(fs_path)
from fs.osfs import OSFS
osfs = OSFS(fs_path, create=create)
return osfs
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
from fs.osfs import OSFS
username, password, fs_path = registry.parse_credentials(fs_path)
path = _expand_syspath(fs_path)
if create:
sys.makedirs(fs_path)
if os.path.isdir(path):
osfs = OSFS(path)
filepath = None
else:
path, filepath = pathsplit(path)
osfs = OSFS(path, create=create)
return osfs, filepath
class ZipOpener(Opener):
......@@ -157,22 +196,21 @@ class ZipOpener(Opener):
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
create_zip = fs_name_params == 'new'
append_zip = fs_name_params == 'append'
zip_file = None
if fs_path.startswith('['):
container_fs, container_path = registry.parse(fs_path)
if not container_path:
raise OpenerError("Not a file")
container_mode = 'r+b'
if create_zip:
container_mode = 'w+b'
elif writeable:
container_mode = 'w+b'
append_zip = fs_name_params == 'add'
zip_file = container_fs.open(container_path, mode=container_mode)
zip_fs, zip_path = registry.parse(fs_path)
if zip_path is None:
raise OpenerError('File required for zip opener')
if create:
open_mode = 'wb'
if append_zip:
open_mode = 'r+b'
else:
open_mode = 'rb'
zip_file = zip_fs.open(zip_path, mode=open_mode)
username, password, fs_path = registry.parse_credentials(fs_path)
......@@ -182,21 +220,18 @@ class ZipOpener(Opener):
if append_zip:
mode = 'a'
elif create_zip or create:
elif create:
mode = 'w'
else:
if writeable:
mode = 'w'
else:
mode = 'a'
if fs_name == 'zip64':
allow_zip_64 = True
else:
allow_zip_64 = False
allow_zip_64 = fs_name == 'zip64'
zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64)
return zipfs
return zipfs, None
class RPCOpener(Opener):
names = ['rpc']
......@@ -206,9 +241,16 @@ class RPCOpener(Opener):
from fs.rpcfs import RPCFS
username, password, fs_path = registry.parse_credentials(fs_path)
if not fs_path.startswith('http://'):
fs_path = 'http://' + fs_path
rpcfs = RPCFS(fs_path)
return rpcfs
fs_path = 'http://' + fs_path
scheme, netloc, path, params, query, fragment = urlparse(fs_path)
rpcfs = RPCFS('%s://%s' % (scheme, netloc))
if create and path:
rpcfs.makedir(path, recursive=True, allow_recreate=True)
return rpcfs, path or None
class FTPOpener(Opener):
names = ['ftp']
......@@ -217,22 +259,28 @@ class FTPOpener(Opener):
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
from fs.ftpfs import FTPFS
username, password, fs_path = registry.parse_credentials(fs_path)
if '/' in fs_path:
url, root_path = fs_path.split('/', 1)
else:
url = fs_path
root_path = ''
scheme, netloc, path, params, query, fragment = urlparse(fs_path)
if not scheme:
fs_path = 'ftp://' + fs_path
scheme, netloc, path, params, query, fragment = urlparse(fs_path)
dirpath, resourcepath = pathsplit(path)
url = netloc
ftpfs = FTPFS(url, user=username or '', passwd=password or '')
ftpfs.cache_hint(True)
if root_path not in ('', '/'):
if not ftpfs.isdir(root_path):
raise OpenerError("'%s' is not a directory on the server" % root_path)
return ftpfs.opendir(root_path)
if create and path:
ftpfs.makedir(path, recursive=True, allow_recreate=True)
return ftpfs
if dirpath:
ftpfs = ftpfs.opendir(dirpath)
if not resourcepath:
return ftpfs, None
else:
return ftpfs, resourcepath
class SFTPOpener(Opener):
......@@ -256,6 +304,8 @@ class SFTPOpener(Opener):
addr = fs_path
fs_path = '/'
fs_path, resourcename = pathsplit(fs_path)
host = addr
port = None
if ':' in host:
......@@ -265,10 +315,25 @@ class SFTPOpener(Opener):
except ValueError:
pass
else:
host = (addr, port)
host = (addr, port)
#if not username or not password:
# raise OpenerError('SFTP requires authentication')
if create:
sftpfs = SFTPFS(host, root_path='/', **credentials)
if not sftpfs._transport.is_authenticated():
sftpfs.close()
raise OpenerError('SFTP requires authentication')
sftpfs = sfspfs.makeopendir(fs_path)
return sftpfs, None
sftpfs = SFTPFS(host, root_path=fs_path, **credentials)
return sftpfs
if not sftpfs._transport.is_authenticated():
sftpfs.close()
raise OpenerError('SFTP requires authentication')
return sftpfs, resourcename
class MemOpener(Opener):
......@@ -277,7 +342,10 @@ class MemOpener(Opener):
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
from fs.memoryfs import MemoryFS
return MemoryFS()
memfs = MemoryFS()
if create:
memfs = memfs.makeopendir(fs_path)
return memfs, None
class DebugOpener(Opener):
names = ['debug']
......@@ -287,13 +355,13 @@ class DebugOpener(Opener):
from fs.wrapfs.debugfs import DebugFS
if fs_path:
fs, path = registry.parse(fs_path, writeable=writeable, create=create)
return DebugFS(fs, verbose=False)
return DebugFS(fs, verbose=False), None
if fs_name_params == 'ram':
from fs.memoryfs import MemoryFS
return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False)
return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False), None
else:
from fs.tempfs import TempFS
return DebugFS(TempFS(), identifier=fs_name_params, verbose=False)
return DebugFS(TempFS(), identifier=fs_name_params, verbose=False), None
class TempOpener(Opener):
names = ['temp']
......@@ -301,7 +369,7 @@ class TempOpener(Opener):
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
from fs.tempfs import TempFS
return TempFS(identifier=fs_name_params, temp_dir=fs_path)
return TempFS(identifier=fs_name_params, temp_dir=fs_path), None
opener = OpenerRegistry([OSFSOpener,
......@@ -317,7 +385,7 @@ opener = OpenerRegistry([OSFSOpener,
def main():
fs, path = opener.parse('galleries.zip')
fs, path = opener.parse('sftp://willmcgugan.com')
print fs, path
if __name__ == "__main__":
......
fs/path.py
......@@ -515,3 +515,16 @@ class PathMap(object):
def names(self,root="/"):
return list(self.iternames(root))
_wild_chars = frozenset('*?[]!{}')
def iswildcard(path):
"""Check if a path ends with a wildcard
>>> is_wildcard('foo/bar/baz.*')
True
>>> is_wildcard('foo/bar')
False
"""
assert path is not None
base_chars = frozenset(basename(path))
return not base_chars.isdisjoint(_wild_chars)
fs/sftpfs.py
......@@ -9,12 +9,15 @@ Filesystem accessing an SFTP server (via paramiko)
import datetime
import stat as statinfo
import threading
import os
import paramiko
from getpass import getuser
from binascii import hexlify
from fs.base import *
from fs.path import *
from fs.errors import *
from fs.utils import isdir, isfile
# SFTPClient appears to not be thread-safe, so we use an instance per thread
if hasattr(threading, "local"):
......@@ -58,6 +61,7 @@ class SFTPFS(FS):
'atomic.setcontents' : False
}
def __init__(self, connection, root_path="/", encoding=None, **credentials):
"""SFTPFS constructor.
......@@ -88,18 +92,81 @@ class SFTPFS(FS):
self._tlocal = thread_local()
self._transport = None
self._client = None
hostname = None
if isinstance(connection, basestring):
hostname = connection
else:
try:
hostname, port = connection
except ValueError:
pass
hostkeytype = None
hostkey = None
if hostname is not None:
try:
host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
except IOError:
try:
# try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts'))
except IOError:
host_keys = {}
if host_keys.has_key(hostname):
hostkeytype = host_keys[hostname].keys()[0]
hostkey = host_keys[hostname][hostkeytype]
credentials['hostkey'] = hostkey
if not credentials.get('username'):
credentials['username'] = getuser()
super(SFTPFS, self).__init__()
if isinstance(connection,paramiko.Channel):
self._transport = None
self._client = paramiko.SFTPClient(connection)
else:
if not isinstance(connection,paramiko.Transport):
else:
if not isinstance(connection, paramiko.Transport):
connection = paramiko.Transport(connection)
self._owns_transport = True
if not connection.is_authenticated():
connection.connect(**credentials)
try:
if not connection.is_authenticated():
connection.connect(**credentials)
if not connection.is_authenticated():
self._agent_auth(connection, credentials.get('username'))
if not connection.is_authenticated():
connection.close()
raise RemoteConnectionError('No auth')
except paramiko.AuthenticationException:
raise RemoteConnectionError('Auth rejected')
self._transport = connection
self.root_path = abspath(normpath(root_path))
super(SFTPFS, self).__init__()
@classmethod
def _agent_auth(cls, transport, username):
"""
Attempt to authenticate to the given transport using any of the private
keys available from an SSH agent.
"""
agent = paramiko.Agent()
agent_keys = agent.get_keys()
if len(agent_keys) == 0:
return False
for key in agent_keys:
try:
transport.auth_publickey(username, key)
return key
except paramiko.SSHException:
pass
return None
def __del__(self):
self.close()
......@@ -184,6 +251,8 @@ class SFTPFS(FS):
@convert_os_errors
def isdir(self,path):
if path == '/':
return True
npath = self._normpath(path)
try:
stat = self.client.stat(npath)
......@@ -209,6 +278,10 @@ class SFTPFS(FS):
npath = self._normpath(path)
try:
paths = self.client.listdir(npath)
if dirs_only or files_only:
path_attrs = self.client.listdir_attr(npath)
else:
path_attrs = None
except IOError, e:
if getattr(e,"errno",None) == 2:
if self.isfile(path):
......@@ -217,10 +290,72 @@ class SFTPFS(FS):
elif self.isfile(path):
raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
raise
if path_attrs is not None:
if dirs_only:
filter_paths = []
for path, attr in zip(paths, path_attrs):
if isdir(self, path, attr.__dict__):
filter_paths.append(path)
paths = filter_paths
elif files_only:
filter_paths = []
for path, attr in zip(paths, path_attrs):
if isfile(self, path, attr.__dict__):
filter_paths.append(path)
paths = filter_paths
for (i,p) in enumerate(paths):
if not isinstance(p,unicode):
paths[i] = p.decode(self.encoding)
return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
return self._listdir_helper(path, paths, wildcard, full, absolute, False, False)
@convert_os_errors
def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
npath = self._normpath(path)
try:
paths = self.client.listdir(npath)
attrs = self.client.listdir_attr(npath)
attrs_map = dict(zip(paths, attrs))
except IOError, e:
if getattr(e,"errno",None) == 2:
if self.isfile(path):
raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
raise ResourceNotFoundError(path)
elif self.isfile(path):
raise ResourceInvalidError(path,msg="Can't list directory contents of a file: %(path)s")
raise
if dirs_only:
filter_paths = []
for path, attr in zip(paths, attrs):
if isdir(self, path, attr.__dict__):
filter_paths.append(path)
paths = filter_paths
elif files_only:
filter_paths = []
for path, attr in zip(paths, attrs):
if isfile(self, path, attr.__dict__):
filter_paths.append(path)
paths = filter_paths
for (i, p) in enumerate(paths):
if not isinstance(p, unicode):
paths[i] = p.decode(self.encoding)
def getinfo(p):
resourcename = basename(p)
info = attrs_map.get(resourcename)
if info is None:
return self.getinfo(pathjoin(path, p))
return self._extract_info(info.__dict__)
return [(p, getinfo(p)) for p in
self._listdir_helper(path, paths, wildcard, full, absolute, False, False)]
@convert_os_errors
def makedir(self,path,recursive=False,allow_recreate=False):
......@@ -335,6 +470,23 @@ class SFTPFS(FS):
raise ParentDirectoryMissingError(dst,msg="Destination directory does not exist: %(path)s")
raise
_info_vars = frozenset('st_size st_uid st_gid st_mode st_atime st_mtime'.split())
@classmethod
def _extract_info(cls, stats):
fromtimestamp = datetime.datetime.fromtimestamp
info = dict((k, v) for k, v in stats.iteritems() if k in cls._info_vars)
info['size'] = info['st_size']
ct = info.get('st_ctime')
if ct is not None:
info['created_time'] = fromtimestamp(ct)
at = info.get('st_atime')
if at is not None:
info['accessed_time'] = fromtimestamp(at)
mt = info.get('st_mtime')
if mt is not None:
info['modified_time'] = fromtimestamp(mt)
return info
@convert_os_errors
def getinfo(self, path):
npath = self._normpath(path)
......
fs/utils.py
......@@ -56,10 +56,13 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
src = None
try:
# Chunk copy
src = src_fs.open(src_path, 'rb')
if src_fs.getsize(src_path) < chunk_size:
src = src_fs.getcontents(src_path)
else:
src = src_fs.open(src_path, 'rb')
dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
finally:
if src is not None:
if src is not None and hasattr(src, 'close'):
src.close()
......@@ -89,14 +92,18 @@ def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
FS._shutil_movefile(src_syspath, dst_syspath)
return
src = None
try:
src = src_fs.open(src_path, 'rb')
dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
src_fs.remove(src_path)
src = None
try:
# Chunk copy
if src_fs.getsize(src_path) < chunk_size:
src = src_fs.getcontents(src_path)
else:
src = src_fs.open(src_path, 'rb')
dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
src_fs.remove(src_path)
finally:
if src is not None:
if src is not None and hasattr(src, 'close'):
src.close()
def movedir(fs1, fs2, overwrite=False, ignore_errors=False, chunk_size=64*1024):
......@@ -324,7 +331,7 @@ def find_duplicates(fs,
paths = list(set(paths).difference(dups))
def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None, hide_dotfiles=False):
"""Prints a filesystem listing to stdout (including sub dirs). Useful as a debugging aid.
Be careful about printing a OSFS, or any other large filesystem.
Without max_levels set, this function will traverse the entire directory tree.
......@@ -343,13 +350,14 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
:param file_out: File object to write output to (defaults to sys.stdout)
:param terminal_colors: If True, terminal color codes will be written, set to False for non-console output.
The default (None) will select an appropriate setting for the platform.
:param hide_dotfiles: if True, files or directories begining with '.' will be removed
"""
if file_out is None:
file_out = sys.stdout
file_encoding = getattr(file_out, 'encoding', 'utf-8')
file_encoding = getattr(file_out, 'encoding', 'utf-8') or 'utf-8'
if terminal_colors is None:
if sys.platform.startswith('win'):
......@@ -388,12 +396,16 @@ def print_fs(fs, path='/', max_levels=5, file_out=None, terminal_colors=None):
def print_dir(fs, path, levels=[]):
try:
dir_listing = [(fs.isdir(pathjoin(path,p)), p) for p in fs.listdir(path)]
dir_listing = ( [(True, p) for p in fs.listdir(path, dirs_only=True)] +
[(False, p) for p in fs.listdir(path, files_only=True)] )
except Exception, e:
prefix = ''.join([('| ', ' ')[last] for last in levels]) + ' '
write(wrap_prefix(prefix[:-1] + ' ') + wrap_error("unabled to retrieve directory list (%s) ..." % str(e)))
return 0
if hide_dotfiles:
dir_listing = [(isdir, p) for isdir, p in dir_listing if not p.startswith('.')]
dir_listing.sort(key = lambda (isdir, p):(not isdir, p.lower()))
for i, (is_dir, item) in enumerate(dir_listing):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment