Tweaking daisychain internals to allow get_url to modify the path destination…

Tweaking daisychain internals to allow get_url to modify the path destination when downloading to a directory.
Minor module refactoring.

examples/playbooks/get_url.yml

@@ -12,5 +12,5 @@
     - jquery.min.js
     - mobile/latest/jquery.mobile.min.js
     - ui/jquery-ui-git.css
-  - name: Pass urlencoded name to CGI
-    action: get_url url=http://example.com/name.cgi?name='${person}' dest=/tmp/test
+  #- name: Pass urlencoded name to CGI
+  #  action: get_url url=http://example.com/name.cgi?name='${person}' dest=/tmp/test

lib/ansible/module_common.py

@@ -128,22 +128,27 @@ class AnsibleModule(object):
         log_args = re.sub(r'password=.+ (.*)', r"password=NOT_LOGGING_PASSWORD \1", self.args)
         syslog.syslog(syslog.LOG_NOTICE, 'Invoked with %s' % log_args)
 
+    def jsonify(self, data):
+        return json.dumps(data)
+
     def exit_json(self, **kwargs):
         ''' return from the module, without error '''
-        print json.dumps(kwargs)
+        print self.jsonify(kwargs)
         sys.exit(0)
 
     def fail_json(self, **kwargs):
         ''' return from the module, with an error message '''
         assert 'msg' in kwargs, "implementation error -- msg to explain the error is required"
         kwargs['failed'] = True
-        print json.dumps(kwargs)
+        print self.jsonify(kwargs)
         sys.exit(1)
 
     def md5(self, filename):
         ''' Return MD5 hex digest of local file, or None if file is not present. '''
         if not os.path.exists(filename):
             return None
+        if os.path.isdir(filename):
+            self.fail_json(msg="attempted to take md5sum of directory: %s" % filename)
         digest = _md5()
         blocksize = 64 * 1024
         infile = open(filename, 'rb')

lib/ansible/runner/__init__.py

@@ -537,6 +537,14 @@ class Runner(object):
             group_hosts[g.name] = [ h.name for h in g.hosts ]
         inject['groups'] = group_hosts
 
+        # allow module args to work as a dictionary
+        # though it is usually a string
+        new_args = ""
+        if type(self.module_args) == dict:
+            for (k,v) in self.module_args.iteritems():
+                new_args = new_args + "%s='%s' " % (k,v)
+            self.module_args = new_args
+
         conditional = utils.template(self.conditional, inject)
         if not eval(conditional):
             result = utils.jsonify(dict(skipped=True))
@@ -568,6 +576,8 @@ class Runner(object):
         if result.is_successful() and 'daisychain' in result.result:
             chained = True
             self.module_name = result.result['daisychain']
+            if 'daisychain_args' in result.result:
+                self.module_args = result.result['daisychain_args']
             result2 = self._executor_internal_inner(host, inject, port)
             changed = result.result.get('changed',False) or result2.result.get('changed',False)
             result.result.update(result2.result)

library/get_url

@@ -17,50 +17,21 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
-# Synopsis:
-#        ansible -m get_url -a "url=http://some.place/some.file dest=/tmp/file"
-#
-# Arguments:
-#    url=                (mandatory, no default)
-#    dest=                (mandatory, no default)
-#        if dest= is a file, url is copied to that file
-#        if dest= is a directory, determine name from url= and store it in dest/
-#    mode, owner, group, ... from the "file" module are also supported
-#
-# Playbook:
-#   The dest=<directory> feature lets you do this in a Playbook:
-#
-#          - name: Grab a bunch of jQuery stuff
-#            action: get_url url=http://code.jquery.com/$item  dest=${jquery_directory} mode=0444
-#            with_items:
-#            - jquery.min.js
-#            - mobile/latest/jquery.mobile.min.js
-#            - ui/jquery-ui-git.css
-#
-# TODO:
-#    timeout=
-#    Support gzip compression? 
-#        http://www.diveintopython.net/http_web_services/gzip_compression.html
+# see examples/playbooks/get_url.yml
 
-import sys
 import os
-import shlex
 import shutil
 import syslog
 import datetime
 import tempfile
 
-try:
-    from hashlib import md5 as _md5
-except ImportError: 
-    from md5 import md5 as _md5
-
 HAS_URLLIB2=True
 try:
     import urllib2
 except ImportError: 
     HAS_URLLIB2=False
 HAS_URLPARSE=True
+
 try:
     import urlparse
     import socket
@@ -68,118 +39,96 @@ except ImportError:
     HAS_URLPARSE=False
 
 # ==============================================================
-# support
-
-def md5(filename):
-    ''' Return MD5 hex digest of local file, or None if file is not present. '''
-    if not os.path.exists(filename):
-        return None
-    digest = _md5()
-    blocksize = 64 * 1024
-    infile = open(filename, 'rb')
-    block = infile.read(blocksize)
-    while block:
-        digest.update(block)
-        block = infile.read(blocksize)
-    infile.close()
-    return digest.hexdigest()
-
-# ==============================================================
 # url handling
 
 def url_filename(url):
-    return os.path.basename(urlparse.urlsplit(url)[2])
+    fn = os.path.basename(urlparse.urlsplit(url)[2])
+    if fn == '':
+        return 'index.html'
+    return fn
 
-def url_do_get(url, dest):
-    """Get url and return request and info
-       Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp
+def url_do_get(module, url, dest):
+    """
+    Get url and return request and info
+    Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp
     """
+
     USERAGENT = 'ansible-httpget'
-    info = {}
-    info['url'] = url
+    info = dict(url=url)
     r = None
+    actualdest = None
 
-    if dest:
-        if os.path.isdir(dest):
-            destpath = "%s/%s" % (dest, url_filename(url))
-        else:
-            destpath = dest
+    if os.path.isdir(dest):
+        urlfilename = url_filename(url)
+        actualdest = "%s/%s" % (dest, url_filename(url))
+        module.params['path'] = actualdest
     else:
-        destpath = url_filename(url)
-
-    info['destpath'] = destpath
+         actualdest = dest
+    info['daisychain_args'] = module.params
+    info['actualdest'] = actualdest
 
     request = urllib2.Request(url)
     request.add_header('User-agent', USERAGENT)
 
-    if os.path.exists(destpath):
-        t = datetime.datetime.utcfromtimestamp(os.path.getmtime(destpath))
+    if os.path.exists(actualdest):
+        t = datetime.datetime.utcfromtimestamp(os.path.getmtime(actualdest))
         tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000')
         request.add_header('If-Modified-Since', tstamp)
 
     try:
         r = urllib2.urlopen(request)
-
-        dinfo = dict(r.info())
-        for x in dinfo:
-             info[x] = dinfo[x]
-
-        info['msg'] = "OK %s octets" % r.headers.get('Content-Length', 'unknown')
-        info['status'] = 200
+        info.update(r.info())
+        info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
     except urllib2.HTTPError as e:
         # Must not fail_json() here so caller can handle HTTP 304 unmodified
-        info['msg'] = "%s" % e
-        info['status'] = e.code
+        info.update(dict(msg=str(e), status=e.code))
         return r, info
     except urllib2.URLError as e:
-        if 'code' in e:
-            co = e.code
-        else:
-            co = -1
-        resp = "%s" % e
-        module.fail_json(msg="Request failed", status_code=co, response=resp)
+        code = getattr(e, 'code', -1)
+        module.fail_json(msg="Request failed: %s" % str(e), status_code=code)
 
     return r, info
 
-def url_get(url, dest):
-    """Get url and store at dest. If dest is a directory, determine filename
-       from url, otherwise dest is a file
-       Return info about the request.
+def url_get(module, url, dest):
+    """
+    Download url and store at dest. 
+    If dest is a directory, determine filename from url.
+    Return (tempfile, info about the request)
     """
 
-    req, info = url_do_get(url, dest)
-
-    # TODO: should really handle 304, but how? src file could exist (and be
-    #       newer) but be empty ...
+    req, info = url_do_get(module, url, dest)
 
+    # TODO: should really handle 304, but how? src file could exist (and be newer) but empty
     if info['status'] == 304:
-        module.exit_json(url=url, dest=info.get('destpath', dest), changed=False, msg=info.get('msg', ''))
-
-    # We have the data. Create a temporary file and copy content into that
-    # to do the MD5-thing
+        module.exit_json(url=url, dest=info.get('actualdest', dest), changed=False, msg=info.get('msg', ''))
 
-    if info['status'] == 200:
-        destpath = info['destpath']
-
-        fd, tempname = tempfile.mkstemp()
-        f = os.fdopen(fd, 'wb')
-        try:
-            shutil.copyfileobj(req, f)
-        except Exception, err:
-            os.remove(tempname)
-            module.fail_json(msg="failed to create temporary content file: %s" % str(err))
-        f.close()
-        req.close()
-
-        return tempname, info
-    else:
+    # create a temporary file and copy content to do md5-based replacement
+    if info['status'] != 200:
         module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=url)
+    actualdest = info['actualdest']
+
+    fd, tempname = tempfile.mkstemp()
+    f = os.fdopen(fd, 'wb')
+    try:
+        shutil.copyfileobj(req, f)
+    except Exception, err:
+        os.remove(tempname)
+        module.fail_json(msg="failed to create temporary content file: %s" % str(err))
+    f.close()
+    req.close()
+    return tempname, info
 
 # ==============================================================
 # main
 
 def main():
-    global module
+    
+    # does this really happen on non-ancient python? 
+    if not HAS_URLLIB2:
+        module.fail_json(msg="urllib2 is not installed")
+    if not HAS_URLPARSE:
+        module.fail_json(msg="urlparse is not installed")
+
     module = AnsibleModule(
         argument_spec = dict(
             url = dict(required=True),
@@ -187,31 +136,14 @@ def main():
         )
     )
     
-    url = module.params.get('url', '')
-    dest = module.params.get('dest', '')
-    
-    if url == "":
-        module.fail_json(msg="url= URL missing")
-    if dest == "":
-        module.fail_json(msg="dest= missing")
-    
-    dest = os.path.expanduser(dest)
-    
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-    if not HAS_URLPARSE:
-        module.fail_json(msg="urlparse is not installed")
-    
-    
-    # Here we go... if this succeeds, tmpsrc is the name of a temporary file
-    # containing slurped content. If it fails, we've already raised an error
-    # to Ansible
-    
-    tmpsrc, info = url_get(url, dest)
-    
-    md5sum_src = None
+    url  = module.params['url']
+    dest = os.path.expanduser(module.params['dest'])
     
-    dest = info.get('destpath', None)
+    # download to tmpsrc
+    tmpsrc, info = url_get(module, url, dest)
+    md5sum_src   = None
+    md5sum_dest  = None
+    dest         = info['actualdest']
     
     # raise an error if there is no tmpsrc file
     if not os.path.exists(tmpsrc):
@@ -220,10 +152,8 @@ def main():
     if not os.access(tmpsrc, os.R_OK):
         os.remove(tmpsrc)
         module.fail_json( msg="Source %s not readable" % (tmpsrc))
-    md5sum_src = md5(tmpsrc)
-    
-    
-    md5sum_dest = None
+    md5sum_src = module.md5(tmpsrc)
+     
     # check if there is no dest file
     if os.path.exists(dest):
         # raise an error if copy has no permission on dest
@@ -233,14 +163,13 @@ def main():
         if not os.access(dest, os.R_OK):
             os.remove(tmpsrc)
             module.fail_json( msg="Destination %s not readable" % (dest))
-        md5sum_dest = md5(dest)
+        md5sum_dest = module.md5(dest)
     else:
         if not os.access(os.path.dirname(dest), os.W_OK):
             os.remove(tmpsrc)
             module.fail_json( msg="Destination %s not writable" % (os.path.dirname(dest)))
     
     if md5sum_src != md5sum_dest:
-        # was os.system("cp %s %s" % (src, dest))
         try:
             shutil.copyfile(tmpsrc, dest)
         except Exception, err:
@@ -250,12 +179,12 @@ def main():
     else:
         changed = False
     
-    # Mission complete
-    
     os.remove(tmpsrc)
-    module.exit_json(url=url, dest=dest, src=tmpsrc,
-        md5sum=md5sum_src, changed=changed, msg=info.get('msg', ''),
-        daisychain="file")
+
+    # Mission complete
+    module.exit_json(url=url, dest=dest, src=tmpsrc, md5sum=md5sum_src, 
+        changed=changed, msg=info.get('msg',''), 
+        daisychain="file", daisychain_args=info.get('daisychain_args',''))
 
 # this is magic, see lib/ansible/module_common.py
 #<<INCLUDE_ANSIBLE_MODULE_COMMON>>