Skip to content

P
pyfs
Commit a16c90b9 by willmcgugan
Options

Command fixes

parent d9986180
Showing with 64 additions and 23 deletions
fs/base.py
......@@ -579,7 +579,7 @@ class FS(object):
sys_path = self.getsyspath(path)
except NoSysPathError:
return "No description available"
return "OS resource, maps to %s" % sys_path
return sys_path
def getcontents(self, path):
......
fs/commands/fscp.py
......@@ -53,6 +53,9 @@ Copy SOURCE to DESTINATION"""
def get_action(self):
return copyfile
def get_verb(self):
return 'copying...'
def get_optparse(self):
optparse = super(FScp, self).get_optparse()
optparse.add_option('-p', '--progress', dest='progress', action="store_true", default=False,
......@@ -85,8 +88,12 @@ Copy SOURCE to DESTINATION"""
progress = options.progress
if progress:
sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...'))
sys.stdout.flush()
self.root_dirs = []
for fs_url in srcs:
for i, fs_url in enumerate(srcs):
src_fs, src_path = self.open_fs(fs_url)
if src_path is None:
......@@ -113,6 +120,14 @@ Copy SOURCE to DESTINATION"""
self.error('%s is not a file or directory\n' % src_path)
return 1
if progress:
sys.stdout.write(self.progress_bar(len(srcs), i + 1, 'scanning...'))
sys.stdout.flush()
if progress:
sys.stdout.write(self.progress_bar(len(copy_fs_paths), 0, self.get_verb()))
sys.stdout.flush()
if self.options.threads > 1:
copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]
......@@ -174,7 +189,7 @@ Copy SOURCE to DESTINATION"""
dst_fs.close()
if complete and options.progress:
sys.stdout.write(self.progress_bar(self.total_files, self.done_files))
sys.stdout.write(self.progress_bar(self.total_files, self.done_files, ''))
sys.stdout.write('\n')
sys.stdout.flush()
......@@ -191,7 +206,7 @@ Copy SOURCE to DESTINATION"""
print "%s -> %s" % (src_fs.desc(src_path), dst_fs.desc(dst_path))
elif self.options.progress:
self.done_files += 1
sys.stdout.write(self.progress_bar(self.total_files, self.done_files))
sys.stdout.write(self.progress_bar(self.total_files, self.done_files, self.get_verb()))
sys.stdout.flush()
finally:
self.lock.release()
......@@ -219,12 +234,15 @@ Copy SOURCE to DESTINATION"""
done_steps = int(done * bar_width)
bar_steps = ('#' * done_steps).ljust(bar_width)
msg = '%i%% ' % int(done * 100.0)
msg = '%s %i%%' % (msg, int(done * 100.0))
msg = msg.ljust(20)
if total == remaining:
throb = ''
bar = '\r%s[%s] %s' % (throb, bar_steps, msg)
bar = '\r%s[%s] %s\r' % (throb, bar_steps, msg.lstrip())
return bar
def run():
......
fs/commands/fsmv.py
......@@ -7,6 +7,9 @@ class FSMove(fscp.FScp):
usage = """fsmv [OPTION]... [SOURCE] [DESTINATION]
Move files from SOURCE to DESTINATION"""
def get_verb(self):
return 'moving...'
def get_action(self):
return movefile
......
fs/ftpfs.py
......@@ -522,7 +522,19 @@ def _skip(s, i, c):
return i
def fileftperrors(f):
@wraps(f)
def deco(self, *args, **kwargs):
self._lock.acquire()
try:
try:
ret = f(self, *args, **kwargs)
except Exception, e:
self.ftpfs._translate_exception(args[0] if args else '', e)
finally:
self._lock.release()
return ret
return deco
......@@ -530,6 +542,8 @@ class _FTPFile(object):
""" A file-like that provides access to a file being streamed over ftp."""
blocksize = 1024 * 64
def __init__(self, ftpfs, ftp, path, mode):
if not hasattr(self, '_lock'):
self._lock = threading.RLock()
......@@ -549,6 +563,7 @@ class _FTPFile(object):
#self._lock = ftpfs._lock
self._start_file(mode, path)
@fileftperrors
def _start_file(self, mode, path):
self.read_pos = 0
self.write_pos = 0
......@@ -564,7 +579,7 @@ class _FTPFile(object):
else:
self.conn = self.ftp.transfercmd('STOR '+path)
@synchronize
@fileftperrors
def read(self, size=None):
if self.conn is None:
return ''
......@@ -572,7 +587,7 @@ class _FTPFile(object):
chunks = []
if size is None:
while 1:
data = self.conn.recv(4096)
data = self.conn.recv(self.blocksize)
if not data:
self.conn.close()
self.conn = None
......@@ -584,7 +599,7 @@ class _FTPFile(object):
remaining_bytes = size
while remaining_bytes:
read_size = min(remaining_bytes, 4096)
read_size = min(remaining_bytes, self.blocksize)
data = self.conn.recv(read_size)
if not data:
self.conn.close()
......@@ -597,14 +612,14 @@ class _FTPFile(object):
return ''.join(chunks)
@synchronize
@fileftperrors
def write(self, data):
data_pos = 0
remaining_data = len(data)
while remaining_data:
chunk_size = min(remaining_data, 4096)
chunk_size = min(remaining_data, self.blocksize)
self.conn.sendall(data[data_pos:data_pos+chunk_size])
data_pos += chunk_size
remaining_data -= chunk_size
......@@ -617,11 +632,11 @@ class _FTPFile(object):
def __exit__(self,exc_type,exc_value,traceback):
self.close()
#@synchronize
@fileftperrors
def flush(self):
self.ftpfs._on_file_written(self.path)
@synchronize
@fileftperrors
def seek(self, pos, where=fs.SEEK_SET):
# Ftp doesn't support a real seek, so we close the transfer and resume
# it at the new position with the REST command
......@@ -662,14 +677,14 @@ class _FTPFile(object):
#raise UnsupportedError('ftp seek')
@synchronize
@fileftperrors
def tell(self):
if 'r' in self.mode:
return self.read_pos
else:
return self.write_pos
@synchronize
@fileftperrors
def truncate(self, size=None):
self.ftpfs._on_file_written(self.path)
# Inefficient, but I don't know how else to implement this
......@@ -697,16 +712,22 @@ class _FTPFile(object):
self.write('\0' * (size - len(data)))
@synchronize
@fileftperrors
def close(self):
if 'w' in self.mode or 'a' in self.mode or '+' in self.mode:
self.ftpfs._on_file_written(self.path)
if self.conn is not None:
try:
self.conn.close()
self.conn = None
self.ftp.voidresp()
except error_temp, error_perm:
pass
if self.ftp is not None:
try:
self.ftp.close()
except error_temp, error_perm:
pass
self.closed = True
def __iter__(self):
......@@ -998,7 +1019,7 @@ class FTPFS(FS):
code, message = str(exception).split(' ', 1)
code = int(code)
if code == 550:
raise ResourceNotFoundError(path)
pass
if code == 552:
raise StorageSpaceError
raise PermissionDeniedError(str(exception), path=path, msg="FTP error: %s" % str(exception), details=exception)
......@@ -1019,7 +1040,7 @@ class FTPFS(FS):
mode = mode.lower()
if self.isdir(path):
raise ResourceInvalidError(path)
if 'r' in mode:
if 'r' in mode or 'a' in mode:
if not self.isfile(path):
raise ResourceNotFoundError(path)
if 'w' in mode or 'a' in mode or '+' in mode:
......@@ -1029,7 +1050,7 @@ class FTPFS(FS):
return f
@ftperrors
def setcontents(self, path, data, chunk_size=8192):
def setcontents(self, path, data, chunk_size=1024*64):
path = normpath(path)
if isinstance(data, basestring):
data = StringIO(data)
......@@ -1039,7 +1060,7 @@ class FTPFS(FS):
@ftperrors
def getcontents(self, path):
contents = StringIO()
self.ftp.retrbinary('RETR %s' % _encode(normpath(path)), contents.write, blocksize=1024*16)
self.ftp.retrbinary('RETR %s' % _encode(normpath(path)), contents.write, blocksize=1024*64)
return contents.getvalue()
@ftperrors
......
fs/utils.py
......@@ -56,9 +56,8 @@ def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1
src = None
dst = None
try:
# Chunk copy
src = src_fs.open(src_path, 'rb')
dst = src_fs.open(dst_path, 'wb')
dst = dst_fs.open(dst_path, 'wb')
write = dst.write
read = src.read
chunk = read(chunk_size)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment