Skip to content

P
pyfs
Commit db293aab by willmcgugan
Options

Some optimizations

parent 3500b1ce
Showing with 56 additions and 35 deletions
fs/base.py
......@@ -512,6 +512,7 @@ class FS(object):
directory entries is returned.
"""
path = normpath(path)
if dirs_only and files_only:
raise ValueError("dirs_only and files_only can not both be True")
......@@ -522,14 +523,17 @@ class FS(object):
entries = [p for p in entries if wildcard(p)]
if dirs_only:
entries = [p for p in entries if self.isdir(pathjoin(path, p))]
isdir = self.isdir
entries = [p for p in entries if isdir(pathcombine(path, p))]
elif files_only:
entries = [p for p in entries if self.isfile(pathjoin(path, p))]
isfile = self.isfile
entries = [p for p in entries if isfile(pathcombine(path, p))]
if full:
entries = [pathjoin(path, p) for p in entries]
entries = [pathcombine(path, p) for p in entries]
elif absolute:
entries = [abspath(pathjoin(path, p)) for p in entries]
path = abspath(path)
entries = [(pathcombine(path, p)) for p in entries]
return entries
......@@ -912,6 +916,7 @@ class FS(object):
"""
path = normpath(path)
def listdir(path, *args, **kwargs):
if ignore_errors:
try:
......@@ -936,18 +941,22 @@ class FS(object):
if search == "breadth":
dirs = [path]
dirs_append = dirs.append
dirs_pop = dirs.pop
isdir = self.isdir
while dirs:
current_path = dirs.pop()
current_path = dirs_pop()
paths = []
paths_append = paths.append
try:
for filename in listdir(current_path):
path = pathjoin(current_path, filename)
if self.isdir(path):
path = pathcombine(current_path, filename)
if isdir(path):
if dir_wildcard(path):
dirs.append(path)
dirs_append(path)
else:
if wildcard(filename):
paths.append(filename)
paths_append(filename)
except ResourceNotFoundError:
# Could happen if another thread / process deletes something whilst we are walking
pass
......@@ -997,9 +1006,9 @@ class FS(object):
:rtype: iterator of file paths
"""
for path, files in self.walk(path, wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors):
for path, files in self.walk(normpath(path), wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors):
for f in files:
yield pathjoin(path, f)
yield pathcombine(path, f)
def walkdirs(self,
path="/",
......
fs/osfs/__init__.py
......@@ -15,6 +15,7 @@ For example, to print all the files and directories in the OS root::
import os
import os.path
from os.path import exists as _exists, isdir as _isdir, isfile as _isfile
import sys
import errno
import datetime
......@@ -222,22 +223,20 @@ class OSFS(OSFSXAttrMixin, OSFSWatchMixin, FS):
@convert_os_errors
def exists(self, path):
path = self.getsyspath(path)
return os.path.exists(path)
return _exists(self.getsyspath(path))
@convert_os_errors
def isdir(self, path):
path = self.getsyspath(path)
return os.path.isdir(path)
return _isdir(self.getsyspath(path))
@convert_os_errors
def isfile(self, path):
path = self.getsyspath(path)
return os.path.isfile(path)
return _isfile(self.getsyspath(path))
@convert_os_errors
def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
paths = [self._decode_path(p) for p in os.listdir(self.getsyspath(path))]
_decode_path = self._decode_path
paths = [_decode_path(p) for p in os.listdir(self.getsyspath(path))]
return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
@convert_os_errors
......
fs/path.py
......@@ -169,6 +169,17 @@ def pathjoin(*paths):
path = abspath(path)
return path
def pathcombine(path1, path2):
"""Joins two paths together.
This is faster than `pathjoin`, but only works when the second path is relative,
and there are no backreferences in either path.
>>> pathcombine("foo/bar", "baz")
'foo/bar/baz'
"""
return "%s/%s" % (path1.rstrip('/'), path2.lstrip('/'))
def join(*paths):
"""Joins any number of paths together, returning a new path string.
......@@ -510,9 +521,9 @@ class PathMap(object):
return
for (nm,subm) in m.iteritems():
if not nm:
yield abspath(normpath(root))
yield abspath(root)
else:
k = pathjoin(root,nm)
k = pathcombine(root,nm)
for subk in self.iterkeys(k,subm):
yield subk
......@@ -524,6 +535,7 @@ class PathMap(object):
def itervalues(self,root="/",m=None):
"""Iterate over all values whose keys begin with the given root path."""
root = normpath(root)
if m is None:
m = self._map
for name in iteratepath(root):
......@@ -535,7 +547,7 @@ class PathMap(object):
if not nm:
yield subm
else:
k = pathjoin(root,nm)
k = pathcombine(root,nm)
for subv in self.itervalues(k,subm):
yield subv
......@@ -544,6 +556,7 @@ class PathMap(object):
def iteritems(self,root="/",m=None):
"""Iterate over all (key,value) pairs beginning with the given root."""
root = normpath(root)
if m is None:
m = self._map
for name in iteratepath(root):
......@@ -555,7 +568,7 @@ class PathMap(object):
if not nm:
yield (abspath(normpath(root)),subm)
else:
k = pathjoin(root,nm)
k = pathcombine(root,nm)
for (subk,subv) in self.iteritems(k,subm):
yield (subk,subv)
......
fs/wrapfs/__init__.py
......@@ -195,13 +195,13 @@ class WrapFS(FS):
entries = []
enc_path = self._encode(path)
for e in self.wrapped_fs.listdir(enc_path,**kwds):
e = basename(self._decode(pathjoin(enc_path,e)))
e = basename(self._decode(pathcombine(enc_path,e)))
if not wildcard(e):
continue
if full:
e = pathjoin(path,e)
e = pathcombine(path,e)
elif absolute:
e = abspath(pathjoin(path,e))
e = abspath(pathcombine(path,e))
entries.append(e)
return entries
......@@ -222,13 +222,13 @@ class WrapFS(FS):
wildcard = lambda fn:bool (wildcard_re.match(fn))
enc_path = self._encode(path)
for e in self.wrapped_fs.ilistdir(enc_path,**kwds):
e = basename(self._decode(pathjoin(enc_path,e)))
e = basename(self._decode(pathcombine(enc_path,e)))
if not wildcard(e):
continue
if full:
e = pathjoin(path,e)
e = pathcombine(path,e)
elif absolute:
e = abspath(pathjoin(path,e))
e = abspath(pathcombine(path,e))
yield e
@rewrite_errors
......@@ -249,13 +249,13 @@ class WrapFS(FS):
entries = []
enc_path = self._encode(path)
for (nm,info) in self.wrapped_fs.listdirinfo(enc_path,**kwds):
nm = basename(self._decode(pathjoin(enc_path,nm)))
nm = basename(self._decode(pathcombine(enc_path,nm)))
if not wildcard(nm):
continue
if full:
nm = pathjoin(path,nm)
nm = pathcombine(path,nm)
elif absolute:
nm = abspath(pathjoin(path,nm))
nm = abspath(pathcombine(path,nm))
entries.append((nm,info))
return entries
......@@ -276,13 +276,13 @@ class WrapFS(FS):
wildcard = lambda fn:bool (wildcard_re.match(fn))
enc_path = self._encode(path)
for (nm,info) in self.wrapped_fs.ilistdirinfo(enc_path,**kwds):
nm = basename(self._decode(pathjoin(enc_path,nm)))
nm = basename(self._decode(pathcombine(enc_path,nm)))
if not wildcard(nm):
continue
if full:
nm = pathjoin(path,nm)
nm = pathcombine(path,nm)
elif absolute:
nm = abspath(pathjoin(path,nm))
nm = abspath(pathcombine(path,nm))
yield (nm,info)
@rewrite_errors
......@@ -300,7 +300,7 @@ class WrapFS(FS):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
for (dirpath,filepaths) in self.wrapped_fs.walk(self._encode(path),search=search,ignore_errors=ignore_errors):
filepaths = [basename(self._decode(pathjoin(dirpath,p)))
filepaths = [basename(self._decode(pathcombine(dirpath,p)))
for p in filepaths]
dirpath = abspath(self._decode(dirpath))
if wildcard is not None:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment