added ilistdir() and ilistdirinfo() methods

ChangeLog

@@ -48,9 +48,9 @@
       fs.wrapfs.subfs.
     * OSFSWatchMixin improvements:
         * watch_inotify:  allow more than one watcher on a single path.
+        * watch_win32:  don't create immortal reference cycles.
         * watch_win32:  report errors if the filesystem does't support
                         ReadDirectoryChangesW.
-        * watch_win32:  don't create immortal reference cycles.
     * MountFS: added support for mounting at the root directory, and for
       mounting over an existing mount.
     * Added 'getpathurl' and 'haspathurl' methods.
@@ -68,4 +68,6 @@
     * Optimized listdir and listdirinfo in SFTPFS
     * Made memoryfs work with threads
     * Added copyfile_non_atomic and movefile_non_atomic for improved performance of multi-threaded copies
+    * Added ilistdir() and ilistdirinfo() methods, which are generator-based
+      variants of listdir() and listdirinfo().
 

fs/base.py

@@ -203,17 +203,17 @@ class FS(object):
                 self._lock = DummyLock()
 
     def getmeta(self, meta_name, default=NoDefaultMeta):
-        """Retrieve a meta value associated with an FS object. Meta values are
-        a way for an FS implementation to report potentially useful information
-        associated with the file system.
+        """Retrieve a meta value associated with an FS object.
+
+        Meta values are a way for an FS implementation to report potentially
+        useful information associated with the file system.
         
         A meta key is a lower case string with no spaces. Meta keys may also
         be grouped in namespaces in a dotted notation, e.g. 'atomic.namespaces'.                
-        
         FS implementations aren't obliged to return any meta values, but the
         following are common:
         
-         * *read_only* True if the file system can not be modified
+         * *read_only* True if the file system cannot be modified
          * *network* True if the file system requires network access
          * *unicode_paths* True if the file system supports unicode paths
          * *case_insensitive_paths* True if the file system ignores the case of paths        
@@ -407,9 +407,14 @@ class FS(object):
                           absolute=False,
                           dirs_only=False,
                           files_only=False):
+        """Retrieves a list of paths and path info under a given path.
+
+        This method behaves like listdir() but instead of just returning
+        the name of each item in the directory, it returns a tuple of the
+        name and the info dict as returned by getinfo.
 
-        """Retrieves an iterable of paths and path info (as returned by getinfo) under
-        a given path.
+        Depending on the filesystem, this may be more efficient than calling
+        getinfo() on each individual item returned by listdir().
 
         :param path: root of the path to list
         :param wildcard: filter paths that match this wildcard
@@ -474,6 +479,35 @@ class FS(object):
 
         return entries
 
+    def ilistdir(self, path="./",
+                       wildcard=None,
+                       full=False,
+                       absolute=False,
+                       dirs_only=False,
+                       files_only=False):
+        """Generator yielding the files and directories under a given path.
+
+        This method behaves identically to listdir() but returns a generator
+        instead of a list.  Depending on the filesystem this may be more
+        efficient than calling listdir() and iterating over the resulting list.
+        """
+        return iter(self.listdir(path,wildcard,full,absolute,dirs_only,files_only))
+
+    def ilistdirinfo(self, path="./",
+                           wildcard=None,
+                           full=False,
+                           absolute=False,
+                           dirs_only=False,
+                           files_only=False):
+        """Generator yielding paths and path info under a given path.
+
+        This method behaves identically to listdirinfo() but returns a generator
+        instead of a list.  Depending on the filesystem this may be more
+        efficient than calling listdirinfo() and iterating over the resulting
+        list.
+        """
+        return iter(self.listdirinfo(path,wildcard,full,absolute,dirs_only,files_only))
+
     def makedir(self, path, recursive=False, allow_recreate=False):
         """Make a directory on the filesystem.
 

fs/contrib/davfs/__init__.py

@@ -374,6 +374,9 @@ class DAVFS(FS):
             response.close()
          
     def listdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        return list(self.ilistdir(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only))
+
+    def ilistdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
         pf = propfind(prop="<prop xmlns='DAV:'><resourcetype /></prop>")
         response = self._request(path,"PROPFIND",pf.render(),{"Depth":"1"})
         try:
@@ -381,7 +384,6 @@ class DAVFS(FS):
                 raise ResourceNotFoundError(path)
             if response.status != 207:
                 raise_generic_error(response,"listdir",path)
-            entries = []
             msres = multistatus.parse(response.read())
             dir_ok = False
             for res in msres.responses:
@@ -393,32 +395,44 @@ class DAVFS(FS):
                           break
                 else:
                     nm = basename(self._url2path(res.href))
+                    entry_ok = False
                     if dirs_only:
                         for ps in res.propstats:
                             if ps.props.getElementsByTagNameNS("DAV:","collection"):
-                                entries.append(nm)
+                                entry_ok = True
                                 break
                     elif files_only:
                         for ps in res.propstats:
                             if ps.props.getElementsByTagNameNS("DAV:","collection"):
                                 break
                         else:
-                            entries.append(nm)
+                            entry_ok = True
                     else:
-                        entries.append(nm)
-            if not dir_ok:
-                raise ResourceInvalidError(path)
+                        entry_ok = True
+                    if not entry_ok:
+                        continue
                     if wildcard is not None:
-                entries = [e for e in entries if fnmatch.fnmatch(e,wildcard)]
+                        if isinstance(wildcard,basestring):
+                            if not fnmatch.fnmatch(nm,wildcard):
+                                continue
+                        else:
+                            if not wildcard(nm):
+                                continue
                     if full:
-                entries = [relpath(pathjoin(path,e)) for e in entries]
+                        yield relpath(pathjoin(path,nm))
                     elif absolute:
-                entries = [abspath(pathjoin(path,e)) for e in entries]
-            return entries
+                        yield abspath(pathjoin(path,nm))
+                    else:
+                        yield nm
+            if not dir_ok:
+                raise ResourceInvalidError(path)
         finally:
             response.close()
 
     def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
+        return list(self.ilistdirinfo(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only))
+
+    def ilistdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
         pf = propfind(prop="<prop xmlns='DAV:'><resourcetype /><getcontentlength /><getlastmodified /><getetag /></prop>")
         response = self._request(path,"PROPFIND",pf.render(),{"Depth":"1"})
         try:
@@ -426,7 +440,6 @@ class DAVFS(FS):
                 raise ResourceNotFoundError(path)
             if response.status != 207:
                 raise_generic_error(response,"listdir",path)
-            entries = []
             msres = multistatus.parse(response.read())
             dir_ok = False
             for res in msres.responses:
@@ -441,28 +454,37 @@ class DAVFS(FS):
                     # appropriate type and add to entries list as required.
                     info = self._info_from_propfind(res)
                     nm = basename(self._url2path(res.href))
+                    entry_ok = False
                     if dirs_only:
                         for ps in res.propstats:
                             if ps.props.getElementsByTagNameNS("DAV:","collection"):
-                                entries.append((nm,info))
+                                entry_ok = True
                                 break
                     elif files_only:
                         for ps in res.propstats:
                             if ps.props.getElementsByTagNameNS("DAV:","collection"):
                                 break
                         else:
-                            entries.append((nm,info))
+                            entry_ok = True
                     else:
-                        entries.append((nm,info))
-            if not dir_ok:
-                raise ResourceInvalidError(path)
+                        entry_ok = True
+                    if not entry_ok:
+                        continue
                     if wildcard is not None:
-                entries = [(e,info) for (e,info) in entries if fnmatch.fnmatch(e,wildcard)]
+                        if isinstance(wildcard,basestring):
+                            if not fnmatch.fnmatch(nm,wildcard):
+                                continue
+                        else:
+                            if not wildcard(nm):
+                                continue
                     if full:
-                entries = [(relpath(pathjoin(path,e)),info) for (e,info) in entries]
+                        yield (relpath(pathjoin(path,nm)),info)
                     elif absolute:
-                entries = [(abspath(pathjoin(path,e)),info) for (e,info) in entries]
-            return entries
+                        yield (abspath(pathjoin(path,nm)),info)
+                    else:
+                        yield (nm,info)
+            if not dir_ok:
+                raise ResourceInvalidError(path)
         finally:
             response.close()