Modified listdir and walk to take callables in addition to wildcards

ChangeLog

@@ -31,3 +31,6 @@
     * New implementation of print_fs, accessible through tree method on base class
 
 0.4:
+
+	* Modified listdir and walk methods to accept callables as well as strings
+		for wildcards

fs/base.py

@@ -23,6 +23,7 @@ import shutil
 import fnmatch
 import datetime
 import time
+import re
 try:
     import threading
 except ImportError:
@@ -51,7 +52,6 @@ class DummyLock(object):
         pass
 
 
-
 def silence_fserrors(f, *args, **kwargs):
     """Perform a function call and return None if FSError is thrown
 
@@ -282,7 +282,7 @@ class FS(object):
         :param path: root of the path to list
         :type path: string
         :param wildcard: Only returns paths that match this wildcard
-        :type wildcard: string
+        :type wildcard: string containing a wildcard, or a callable that accepts a path and returns a boolean
         :param full: returns full paths (relative to the root)
         :type full: bool
         :param absolute: returns absolute paths (paths begining with /)
@@ -350,8 +350,10 @@ class FS(object):
             raise ValueError("dirs_only and files_only can not both be True")
 
         if wildcard is not None:
-            match = fnmatch.fnmatch
-            entries = [p for p in entries if match(p, wildcard)]
+            if not callable(wildcard):
+                wildcard_re = re.compile(fnmatch.translate(wildcard))
+                wildcard = lambda fn:bool (wildcard_re.match(fn))          
+            entries = [p for p in entries if wildcard(p)]
 
         if dirs_only:
             entries = [p for p in entries if self.isdir(pathjoin(path, p))]
@@ -529,7 +531,9 @@ class FS(object):
 
         :param path: root path to start walking
         :param wildcard: if given, only return files that match this wildcard
+        :type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the file path and returns a boolean
         :param dir_wildcard: if given, only walk directories that match the wildcard
+        :type dir_wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
         :param search: -- a string dentifying the method used to walk the directories. There are two such methods:
             * 'breadth' Yields paths in the top directories first
             * 'depth' Yields the deepest paths first
@@ -537,6 +541,19 @@ class FS(object):
 
         """
         
+        
+        if wildcard is None:
+            wildcard = lambda f:True
+        elif not callable(wildcard):
+            wildcard_re = re.compile(fnmatch.translate(wildcard))
+            wildcard = lambda fn:bool (wildcard_re.match(fn))
+            
+        if dir_wildcard is None:
+            dir_wildcard = lambda f:True
+        elif not callable(dir_wildcard):
+            dir_wildcard_re = re.compile(fnmatch.translate(dir_wildcard))
+            dir_wildcard = lambda fn:bool (dir_wildcard_re.match(fn))                    
+        
         def listdir(path, *args, **kwargs):
             if ignore_errors:
                 try:
@@ -547,26 +564,20 @@ class FS(object):
                 return self.listdir(path, *args, **kwargs)
         
         if search == "breadth":
+            
             dirs = [path]
             while dirs:
                 current_path = dirs.pop()
-
                 paths = []
                 for filename in listdir(current_path):
-
                     path = pathjoin(current_path, filename)
                     if self.isdir(path):                        
-                        if dir_wildcard is not None:
-                            if fnmatch.fnmatch(path, dir_wilcard):
+                        if dir_wildcard(path):
                             dirs.append(path)                        
                     else:                        
-                            dirs.append(path)
-                    else:
-                        if wildcard is not None:
-                            if fnmatch.fnmatch(path, wildcard):
-                                paths.append(filename)
-                        else:
+                        if wildcard(filename):
                             paths.append(filename)
+                        
                 yield (current_path, paths)
 
         elif search == "depth":
@@ -579,6 +590,7 @@ class FS(object):
 
             for p in recurse(path):
                 yield p
+                
         else:
             raise ValueError("Search should be 'breadth' or 'depth'")
 
@@ -588,11 +600,13 @@ class FS(object):
                   dir_wildcard=None,
                   search="breadth",
                   ignore_errors=False ):
-        """Like the 'walk' method, but just yields files.
+        """Like the 'walk' method, but just yields file paths.
 
         :param path: root path to start walking
         :param wildcard: if given, only return files that match this wildcard
+        :type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the file path and returns a boolean
         :param dir_wildcard: if given, only walk directories that match the wildcard
+        :type dir_wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
         :param search: same as walk method
         :param ignore_errors: ignore any errors reading the directory
         """
@@ -609,6 +623,7 @@ class FS(object):
 
         :param path: root path to start walking
         :param wildcard: if given, only return dictories that match this wildcard
+        :type wildcard: A string containing a wildcard (e.g. *.txt) or a callable that takes the directory name and returns a boolean
         :param search: same as the walk method
         :param ignore_errors: ignore any errors reading the directory
         """

fs/memoryfs.py

@@ -306,8 +306,9 @@ class MemoryFS(FS):
             parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
         
 
+    @synchronize
     def _orphan_files(self, file_dir_entry):
-        for f in file_dir_entry.open_files:
+        for f in file_dir_entry.open_files[:]:
             f.close()
 
     @synchronize
@@ -414,7 +415,6 @@ class MemoryFS(FS):
             parent_dir = self._get_dir_entry(pathname)
             del parent_dir.contents[dirname]
 
-
     @synchronize
     def rename(self, src, dst):
         src_dir,src_name = pathsplit(src)
@@ -441,7 +441,7 @@ class MemoryFS(FS):
         dst_dir_entry.xattrs.update(src_xattrs)
         del src_dir_entry.contents[src_name]
 
-
+    @synchronize
     def settimes(self, path, accessed_time=None, modified_time=None):
         now = datetime.datetime.now()
         if accessed_time is None:
@@ -456,7 +456,6 @@ class MemoryFS(FS):
             return True
         return False
         
-
     @synchronize
     def _on_close_memory_file(self, open_file, path, value):        
         dir_entry = self._get_dir_entry(path)
@@ -475,7 +474,6 @@ class MemoryFS(FS):
         dir_entry = self._get_dir_entry(path)
         dir_entry.modified_time = datetime.datetime.now()        
 
-
     @synchronize
     def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
         dir_entry = self._get_dir_entry(path)

fs/tests/test_walk.py

+"""
+
+    Test the walk function and related code
+
+"""
+import unittest
+from fs.memoryfs import MemoryFS
+
+class TestWalk(unittest.TestCase):
+    
+    def setUp(self):
+        self.fs = MemoryFS()
+        self.fs.createfile('a.txt', 'hello')
+        self.fs.createfile('b.txt', 'world')
+        self.fs.makeopendir('foo').createfile('c', '123')
+        self.fs.makeopendir('.svn').createfile('ignored')
+    
+    def test_wildcard(self):
+        for dir_path, paths in self.fs.walk(wildcard='*.txt'):
+            for path in paths:
+                self.assert_(path.endswith('.txt'))
+        for dir_path, paths in self.fs.walk(wildcard=lambda fn:fn.endswith('.txt')):
+            for path in paths:
+                self.assert_(path.endswith('.txt'))
+    
+    def test_dir_wildcard(self):
+        
+        for dir_path, paths in self.fs.walk(dir_wildcard=lambda fn:not fn.endswith('.svn')):            
+            for path in paths:                
+                self.assert_('.svn' not in path)
+        
\ No newline at end of file

fs/tests/test_zipfs.py

@@ -174,5 +174,5 @@ class TestZipFSErrors(unittest.TestCase):
 
     def test_missing_zipfile(self):
         missingzip = os.path.join(self.workdir,"missing.zip")
-        self.assertRaises(zipfs.ZipMissingError,zipfs.ZipFS,missingzip)
+        self.assertRaises(zipfs.ZipNotFoundError,zipfs.ZipFS,missingzip)
 

fs/wrapfs/__init__.py

@@ -15,8 +15,9 @@ standard unix shell functionality of hiding dot-files in directory listings.
 
 """
 
+import re
 import sys
-from fnmatch import fnmatch
+import fnmatch
 
 from fs.base import FS, threading, synchronize
 from fs.errors import *
@@ -146,17 +147,22 @@ class WrapFS(FS):
     @rewrite_errors
     def listdir(self, path="", **kwds):
         wildcard = kwds.pop("wildcard","*")
+        if wildcard is None:
+            wildcard = lambda fn:True
+        elif not callable(wildcard):
+            wildcard_re = re.compile(fnmatch.translate(wildcard))
+            wildcard = lambda fn:bool (wildcard_re.match(fn))         
         info = kwds.get("info",False)
         entries = []
         for e in self.wrapped_fs.listdir(self._encode(path),**kwds):
             if info:
                 e = e.copy()
                 e["name"] = self._decode(e["name"])
-                if wildcard is not None and not fnmatch(e["name"],wildcard):
+                if not wildcard(e["name"]):
                     continue
             else:
                 e = self._decode(e)
-                if wildcard is not None and not fnmatch(e,wildcard):
+                if not wildcard(e):
                     continue
             entries.append(e) 
         return entries

fs/wrapfs/lazyfs.py

@@ -16,7 +16,6 @@ except ImportError:
 
 from fs.base import FS
 from fs.wrapfs import WrapFS
-from fs.path import *
 
 
 class LazyFS(WrapFS):

fs/zipfs.py

@@ -26,7 +26,7 @@ class ZipOpenError(CreateFailedError):
     pass
 
 
-class ZipMissingError(CreateFailedError):
+class ZipNotFoundError(CreateFailedError):
     """Thrown when the requested zip file does not exist"""
     pass
 
@@ -80,7 +80,7 @@ class ZipFS(FS):
         :param encoding: --  The encoding to use for unicode filenames
         :param thread_synchronize: -- Set to True (default) to enable thread-safety
         :raises ZipOpenError: Thrown when the zip file could not be opened
-        :raises ZipMissingError: Thrown when the zip file does not exist (derived from ZipOpenError)
+        :raises ZipNotFoundError: Thrown when the zip file does not exist (derived from ZipOpenError)
 
         """
         super(ZipFS, self).__init__(thread_synchronize=thread_synchronize)
@@ -105,7 +105,7 @@ class ZipFS(FS):
             if str(ioe).startswith('[Errno 22] Invalid argument'):
                 raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                                    details=bzf)
-            raise ZipMissingError("Zip file not found (%s)" % str(zip_file),
+            raise ZipNotFoundError("Zip file not found (%s)" % str(zip_file),
                                   details=ioe)
                 
         self.zip_path = str(zip_file)