Skip to content

P
pyfs
Commit 95460ba6 by rfkelly0
Options

DAVFS: incremental parsing of PROPFIND results, for lower-latency streaming 

This will really only benefit if the server sends the results incrementally
(e.g. via chunked-encoding) but it can't hurt in the general case anyway.
parent 18201481
Showing with 109 additions and 98 deletions
fs/contrib/davfs/__init__.py
...@@ -29,6 +29,7 @@ import re ...@@ -29,6 +29,7 @@ import re
import datetime import datetime
import cookielib import cookielib
import fnmatch import fnmatch
import xml.dom.pulldom
from fs.base import * from fs.base import *
from fs.path import * from fs.path import *
...@@ -36,6 +37,7 @@ from fs.errors import * ...@@ -36,6 +37,7 @@ from fs.errors import *
from fs.remote import RemoteFileBuffer from fs.remote import RemoteFileBuffer
from fs.contrib.davfs.util import * from fs.contrib.davfs.util import *
from fs.contrib.davfs import xmlobj
from fs.contrib.davfs.xmlobj import * from fs.contrib.davfs.xmlobj import *
import logging import logging
...@@ -378,116 +380,99 @@ class DAVFS(FS): ...@@ -378,116 +380,99 @@ class DAVFS(FS):
return list(self.ilistdir(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only)) return list(self.ilistdir(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only))
def ilistdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): def ilistdir(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
pf = propfind(prop="<prop xmlns='DAV:'><resourcetype /></prop>") props = "<D:resourcetype />"
response = self._request(path,"PROPFIND",pf.render(),{"Depth":"1"}) dir_ok = False
try: for res in self._do_propfind(path,props):
if response.status == 404: if self._isurl(path,res.href):
raise ResourceNotFoundError(path) # The directory itself, check it's actually a directory
if response.status != 207: for ps in res.propstats:
raise_generic_error(response,"listdir",path) if ps.props.getElementsByTagNameNS("DAV:","collection"):
msres = multistatus.parse(response.read()) dir_ok = True
dir_ok = False break
for res in msres.responses: else:
if self._isurl(path,res.href): nm = basename(self._url2path(res.href))
# The directory itself, check it's actually a directory entry_ok = False
for ps in res.propstats: if dirs_only:
if ps.props.getElementsByTagNameNS("DAV:","collection"): for ps in res.propstats:
dir_ok = True if ps.props.getElementsByTagNameNS("DAV:","collection"):
break
else:
nm = basename(self._url2path(res.href))
entry_ok = False
if dirs_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
entry_ok = True
break
elif files_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
break
else:
entry_ok = True entry_ok = True
break
elif files_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
break
else: else:
entry_ok = True entry_ok = True
if not entry_ok: else:
continue entry_ok = True
if wildcard is not None: if not entry_ok:
if isinstance(wildcard,basestring): continue
if not fnmatch.fnmatch(nm,wildcard): if wildcard is not None:
continue if isinstance(wildcard,basestring):
else: if not fnmatch.fnmatch(nm,wildcard):
if not wildcard(nm): continue
continue
if full:
yield relpath(pathjoin(path,nm))
elif absolute:
yield abspath(pathjoin(path,nm))
else: else:
yield nm if not wildcard(nm):
if not dir_ok: continue
raise ResourceInvalidError(path) if full:
finally: yield relpath(pathjoin(path,nm))
response.close() elif absolute:
yield abspath(pathjoin(path,nm))
else:
yield nm
if not dir_ok:
raise ResourceInvalidError(path)
def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): def listdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
return list(self.ilistdirinfo(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only)) return list(self.ilistdirinfo(path=path,wildcard=wildcard,full=full,absolute=absolute,dirs_only=dirs_only,files_only=files_only))
def ilistdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False): def ilistdirinfo(self,path="./",wildcard=None,full=False,absolute=False,dirs_only=False,files_only=False):
pf = propfind(prop="<prop xmlns='DAV:'><resourcetype /><getcontentlength /><getlastmodified /><getetag /></prop>") props = "<D:resourcetype /><D:getcontentlength />" \
response = self._request(path,"PROPFIND",pf.render(),{"Depth":"1"}) "<D:getlastmodified /><D:getetag />"
try: dir_ok = False
if response.status == 404: for res in self._do_propfind(path,props):
raise ResourceNotFoundError(path) if self._isurl(path,res.href):
if response.status != 207: # The directory itself, check it's actually a directory
raise_generic_error(response,"listdir",path) for ps in res.propstats:
msres = multistatus.parse(response.read()) if ps.props.getElementsByTagNameNS("DAV:","collection"):
dir_ok = False dir_ok = True
for res in msres.responses: break
if self._isurl(path,res.href): else:
# The directory itself, check it's actually a directory # An entry in the directory, check if it's of the
for ps in res.propstats: # appropriate type and add to entries list as required.
if ps.props.getElementsByTagNameNS("DAV:","collection"): info = self._info_from_propfind(res)
dir_ok = True nm = basename(self._url2path(res.href))
break entry_ok = False
else: if dirs_only:
# An entry in the directory, check if it's of the for ps in res.propstats:
# appropriate type and add to entries list as required. if ps.props.getElementsByTagNameNS("DAV:","collection"):
info = self._info_from_propfind(res)
nm = basename(self._url2path(res.href))
entry_ok = False
if dirs_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
entry_ok = True
break
elif files_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
break
else:
entry_ok = True entry_ok = True
break
elif files_only:
for ps in res.propstats:
if ps.props.getElementsByTagNameNS("DAV:","collection"):
break
else: else:
entry_ok = True entry_ok = True
if not entry_ok: else:
continue entry_ok = True
if wildcard is not None: if not entry_ok:
if isinstance(wildcard,basestring): continue
if not fnmatch.fnmatch(nm,wildcard): if wildcard is not None:
continue if isinstance(wildcard,basestring):
else: if not fnmatch.fnmatch(nm,wildcard):
if not wildcard(nm): continue
continue
if full:
yield (relpath(pathjoin(path,nm)),info)
elif absolute:
yield (abspath(pathjoin(path,nm)),info)
else: else:
yield (nm,info) if not wildcard(nm):
if not dir_ok: continue
raise ResourceInvalidError(path) if full:
finally: yield (relpath(pathjoin(path,nm)),info)
response.close() elif absolute:
yield (abspath(pathjoin(path,nm)),info)
else:
yield (nm,info)
if not dir_ok:
raise ResourceInvalidError(path)
def makedir(self,path,recursive=False,allow_recreate=False): def makedir(self,path,recursive=False,allow_recreate=False):
response = self._request(path,"MKCOL") response = self._request(path,"MKCOL")
...@@ -560,6 +545,31 @@ class DAVFS(FS): ...@@ -560,6 +545,31 @@ class DAVFS(FS):
finally: finally:
response.close() response.close()
def _do_propfind(self,path,props,incremental=True):
"""Incremental PROPFIND parsing, for use with ilistdir/ilistdirinfo.
This generator method incrementally parses the results returned by
a PROPFIND, yielding each <response> object as it becomes available.
If the server is able to send responses in chunked encoding, then
this can substantially speed up iterating over the results.
"""
pf = propfind(prop="<prop xmlns:D='DAV:'>" + props + "</prop>")
response = self._request(path,"PROPFIND",pf.render(),{"Depth":"1"})
try:
if response.status == 404:
raise ResourceNotFoundError(path)
if response.status != 207:
raise_generic_error(response,"listdir",path)
xmlevents = xml.dom.pulldom.parse(response,bufsize=1024)
for (evt,node) in xmlevents:
if evt == xml.dom.pulldom.START_ELEMENT:
if node.namespaceURI == "DAV:":
if node.localName == "response":
xmlevents.expandNode(node)
yield xmlobj.response.parse(node)
finally:
response.close()
def _info_from_propfind(self,res): def _info_from_propfind(self,res):
info = {} info = {}
for ps in res.propstats: for ps in res.propstats:
...@@ -745,7 +755,8 @@ class DAVFS(FS): ...@@ -745,7 +755,8 @@ class DAVFS(FS):
props.append(propname) props.append(propname)
return props return props
# TODO: getxattrs() and setxattrs() # TODO: bulk getxattrs() and setxattrs() methods
def raise_generic_error(response,opname,path): def raise_generic_error(response,opname,path):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment