Skip to content

P
pyfs
Commit 6641f314 by willmcgugan
Options

Modified listdir and walk to take callables in addition to wildcards

parent 30c928cc
Showing with 81 additions and 28 deletions
ChangeLog
......@@ -31,3 +31,6 @@
* New implementation of print_fs, accessible through tree method on base class
0.4:
* Modified listdir and walk methods to accept callables as well as strings
for wildcards
fs/base.py
......@@ -23,6 +23,7 @@ import shutil
import fnmatch
import datetime
import time
import re
try:
import threading
except ImportError:
......@@ -51,7 +52,6 @@ class DummyLock(object):
pass
def silence_fserrors(f, *args, **kwargs):
"""Perform a function call and return None if FSError is thrown
......@@ -282,7 +282,7 @@ class FS(object):
:param path: root of the path to list
:type path: string
:param wildcard: Only returns paths that match this wildcard
:type wildcard: string
:type wildcard: string containing a wildcard, or a callable that accepts a path and returns a boolean
:param full: returns full paths (relative to the root)
:type full: bool
:param absolute: returns absolute paths (paths begining with /)
......@@ -350,8 +350,10 @@ class FS(object):
raise ValueError("dirs_only and files_only can not both be True")
if wildcard is not None:
match = fnmatch.fnmatch
entries = [p for p in entries if match(p, wildcard)]
if not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
entries = [p for p in entries if wildcard(p)]
if dirs_only:
entries = [p for p in entries if self.isdir(pathjoin(path, p))]
......@@ -529,7 +531,9 @@ class FS(object):
:param path: root path to start walking
:param wildcard: if given, only return files that match this wildcard
:type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard
:type dir_wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
:param search: -- a string dentifying the method used to walk the directories. There are two such methods:
* 'breadth' Yields paths in the top directories first
* 'depth' Yields the deepest paths first
......@@ -537,6 +541,19 @@ class FS(object):
"""
if wildcard is None:
wildcard = lambda f:True
elif not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
if dir_wildcard is None:
dir_wildcard = lambda f:True
elif not callable(dir_wildcard):
dir_wildcard_re = re.compile(fnmatch.translate(dir_wildcard))
dir_wildcard = lambda fn:bool (dir_wildcard_re.match(fn))
def listdir(path, *args, **kwargs):
if ignore_errors:
try:
......@@ -547,26 +564,20 @@ class FS(object):
return self.listdir(path, *args, **kwargs)
if search == "breadth":
dirs = [path]
while dirs:
current_path = dirs.pop()
paths = []
for filename in listdir(current_path):
path = pathjoin(current_path, filename)
if self.isdir(path):
if dir_wildcard is not None:
if fnmatch.fnmatch(path, dir_wilcard):
if dir_wildcard(path):
dirs.append(path)
else:
dirs.append(path)
else:
if wildcard is not None:
if fnmatch.fnmatch(path, wildcard):
paths.append(filename)
else:
if wildcard(filename):
paths.append(filename)
yield (current_path, paths)
elif search == "depth":
......@@ -579,6 +590,7 @@ class FS(object):
for p in recurse(path):
yield p
else:
raise ValueError("Search should be 'breadth' or 'depth'")
......@@ -588,11 +600,13 @@ class FS(object):
dir_wildcard=None,
search="breadth",
ignore_errors=False ):
"""Like the 'walk' method, but just yields files.
"""Like the 'walk' method, but just yields file paths.
:param path: root path to start walking
:param wildcard: if given, only return files that match this wildcard
:type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the file path and returns a boolean
:param dir_wildcard: if given, only walk directories that match the wildcard
:type dir_wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
:param search: same as walk method
:param ignore_errors: ignore any errors reading the directory
"""
......@@ -609,6 +623,7 @@ class FS(object):
:param path: root path to start walking
:param wildcard: if given, only return dictories that match this wildcard
:type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
:param search: same as the walk method
:param ignore_errors: ignore any errors reading the directory
"""
......
fs/memoryfs.py
......@@ -306,8 +306,9 @@ class MemoryFS(FS):
parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
@synchronize
def _orphan_files(self, file_dir_entry):
for f in file_dir_entry.open_files:
for f in file_dir_entry.open_files[:]:
f.close()
@synchronize
......@@ -414,7 +415,6 @@ class MemoryFS(FS):
parent_dir = self._get_dir_entry(pathname)
del parent_dir.contents[dirname]
@synchronize
def rename(self, src, dst):
src_dir,src_name = pathsplit(src)
......@@ -441,7 +441,7 @@ class MemoryFS(FS):
dst_dir_entry.xattrs.update(src_xattrs)
del src_dir_entry.contents[src_name]
@synchronize
def settimes(self, path, accessed_time=None, modified_time=None):
now = datetime.datetime.now()
if accessed_time is None:
......@@ -456,7 +456,6 @@ class MemoryFS(FS):
return True
return False
@synchronize
def _on_close_memory_file(self, open_file, path, value):
dir_entry = self._get_dir_entry(path)
......@@ -475,7 +474,6 @@ class MemoryFS(FS):
dir_entry = self._get_dir_entry(path)
dir_entry.modified_time = datetime.datetime.now()
@synchronize
def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
dir_entry = self._get_dir_entry(path)
......
fs/tests/test_walk.py 0 → 100644
"""
Test the walk function and related code
"""
import unittest
from fs.memoryfs import MemoryFS
class TestWalk(unittest.TestCase):
def setUp(self):
self.fs = MemoryFS()
self.fs.createfile('a.txt', 'hello')
self.fs.createfile('b.txt', 'world')
self.fs.makeopendir('foo').createfile('c', '123')
self.fs.makeopendir('.svn').createfile('ignored')
def test_wildcard(self):
for dir_path, paths in self.fs.walk(wildcard='*.txt'):
for path in paths:
self.assert_(path.endswith('.txt'))
for dir_path, paths in self.fs.walk(wildcard=lambda fn:fn.endswith('.txt')):
for path in paths:
self.assert_(path.endswith('.txt'))
def test_dir_wildcard(self):
for dir_path, paths in self.fs.walk(dir_wildcard=lambda fn:not fn.endswith('.svn')):
for path in paths:
self.assert_('.svn' not in path)
\ No newline at end of file
fs/tests/test_zipfs.py
......@@ -174,5 +174,5 @@ class TestZipFSErrors(unittest.TestCase):
def test_missing_zipfile(self):
missingzip = os.path.join(self.workdir,"missing.zip")
self.assertRaises(zipfs.ZipMissingError,zipfs.ZipFS,missingzip)
self.assertRaises(zipfs.ZipNotFoundError,zipfs.ZipFS,missingzip)
fs/wrapfs/__init__.py
......@@ -15,8 +15,9 @@ standard unix shell functionality of hiding dot-files in directory listings.
"""
import re
import sys
from fnmatch import fnmatch
import fnmatch
from fs.base import FS, threading, synchronize
from fs.errors import *
......@@ -146,17 +147,22 @@ class WrapFS(FS):
@rewrite_errors
def listdir(self, path="", **kwds):
wildcard = kwds.pop("wildcard","*")
if wildcard is None:
wildcard = lambda fn:True
elif not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn:bool (wildcard_re.match(fn))
info = kwds.get("info",False)
entries = []
for e in self.wrapped_fs.listdir(self._encode(path),**kwds):
if info:
e = e.copy()
e["name"] = self._decode(e["name"])
if wildcard is not None and not fnmatch(e["name"],wildcard):
if not wildcard(e["name"]):
continue
else:
e = self._decode(e)
if wildcard is not None and not fnmatch(e,wildcard):
if not wildcard(e):
continue
entries.append(e)
return entries
......
fs/wrapfs/lazyfs.py
......@@ -16,7 +16,6 @@ except ImportError:
from fs.base import FS
from fs.wrapfs import WrapFS
from fs.path import *
class LazyFS(WrapFS):
......
fs/zipfs.py
......@@ -26,7 +26,7 @@ class ZipOpenError(CreateFailedError):
pass
class ZipMissingError(CreateFailedError):
class ZipNotFoundError(CreateFailedError):
"""Thrown when the requested zip file does not exist"""
pass
......@@ -80,7 +80,7 @@ class ZipFS(FS):
:param encoding: -- The encoding to use for unicode filenames
:param thread_synchronize: -- Set to True (default) to enable thread-safety
:raises ZipOpenError: Thrown when the zip file could not be opened
:raises ZipMissingError: Thrown when the zip file does not exist (derived from ZipOpenError)
:raises ZipNotFoundError: Thrown when the zip file does not exist (derived from ZipOpenError)
"""
super(ZipFS, self).__init__(thread_synchronize=thread_synchronize)
......@@ -105,7 +105,7 @@ class ZipFS(FS):
if str(ioe).startswith('[Errno 22] Invalid argument'):
raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
details=bzf)
raise ZipMissingError("Zip file not found (%s)" % str(zip_file),
raise ZipNotFoundError("Zip file not found (%s)" % str(zip_file),
details=ioe)
self.zip_path = str(zip_file)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment