implement lookup plugins for arbitrary enumeration over arbitrary things. See…

implement lookup plugins for arbitrary enumeration over arbitrary things.   See the mailing list for some cool examples.

CHANGELOG.md

@@ -8,6 +8,7 @@ Highlighted Core Changes:
 * fireball mode -- ansible can bootstrap a ephemeral 0mq (zeromq) daemon that runs as a given user and expires after X period of time.  It is very fast.
 * playbooks with errors now return 2 on failure.  1 indicates a more fatal syntax error.  Similar for /usr/bin/ansible
 * server side action code (template, etc) are now fully pluggable
+* ability to write lookup plugins, like the code powering "with_fileglob" (see below)
 
 Other Core Changes:
 
@@ -44,6 +45,7 @@ Highlighted playbook changes:
 * when using a $list variable with $var or ${var} syntax it will automatically join with commas
 * setup is not run more than once when we know it is has already been run in a play that included another play, etc
 * can set/override sudo and sudo_user on individual tasks in a play, defaults to what is set in the play if not present
+* ability to use with_fileglob to iterate over local file patterns
 
 Other playbook changes:
 

lib/ansible/playbook/__init__.py

@@ -26,6 +26,8 @@ from play import Play
 
 SETUP_CACHE = collections.defaultdict(dict)
 
+plugins_dir = os.path.join(os.path.dirname(__file__), '..', 'runner')
+
 class PlayBook(object):
     '''
     runs an ansible playbook, given as a datastructure or YAML filename.
@@ -105,9 +107,12 @@ class PlayBook(object):
         self.private_key_file = private_key_file
         self.only_tags        = only_tags
 
-        self.inventory        = ansible.inventory.Inventory(host_list)
+        self.inventory           = ansible.inventory.Inventory(host_list)
         self.inventory.subset(subset)
-        self.modules_list = utils.get_available_modules(self.module_path)
+
+        self.modules_list        = utils.get_available_modules(self.module_path)
+        lookup_plugins_dir = os.path.join(plugins_dir, 'lookup_plugins')
+        self.lookup_plugins_list = utils.import_plugins(lookup_plugins_dir)
 
         if not self.inventory._is_script:
             self.global_vars.update(self.inventory.get_group_variables('all'))

lib/ansible/playbook/task.py

@@ -26,7 +26,8 @@ class Task(object):
         'notify', 'module_name', 'module_args', 'module_vars',
         'play', 'notified_by', 'tags', 'register', 'with_items', 
         'delegate_to', 'first_available_file', 'ignore_errors',
-        'local_action', 'transport', 'sudo', 'sudo_user', 'sudo_pass'
+        'local_action', 'transport', 'sudo', 'sudo_user', 'sudo_pass',
+        'items_lookup_plugin', 'items_lookup_terms'
     ]
 
     # to prevent typos and such
@@ -40,11 +41,23 @@ class Task(object):
     def __init__(self, play, ds, module_vars=None):
         ''' constructor loads from a task or handler datastructure '''
 
-        # code to allow for saying "modulename: args" versus "action: modulename args"
         for x in ds.keys():
+
+            # code to allow for saying "modulename: args" versus "action: modulename args"
             if x in play.playbook.modules_list:
-                ds['action'] = x + " " + ds.get(x, None)
+                ds['action'] = x + " " + ds[x]
                 ds.pop(x)
+
+            # code to allow "with_glob" and to reference a lookup plugin named glob
+            elif x.startswith("with_") and x != 'with_items':
+                plugin_name = x.replace("with_","")
+                if plugin_name in play.playbook.lookup_plugins_list:
+                    ds['items_lookup_plugin'] = plugin_name
+                    ds['items_lookup_terms'] = ds[x]
+                    ds.pop(x)
+                else:
+                    raise errors.AnsibleError("cannot find lookup plugin named %s for usage in with_%s" % (plugin_name, plugin_name))
+
             elif not x in Task.VALID_KEYS:
                 raise errors.AnsibleError("%s is not a legal parameter in an Ansible task or handler" % x)
 
@@ -101,6 +114,10 @@ class Task(object):
         self.first_available_file = ds.get('first_available_file', None)
         self.with_items = ds.get('with_items', None)
 
+        self.items_lookup_plugin = ds.get('items_lookup_plugin', None)
+        self.items_lookup_terms  = ds.get('items_lookup_terms', None)
+     
+
         self.ignore_errors = ds.get('ignore_errors', False)
 
         # notify can be a string or a list, store as a list
@@ -125,8 +142,9 @@ class Task(object):
         self.action = utils.template(None, self.action, self.module_vars)
 
         # handle mutually incompatible options
-        if self.with_items is not None and self.first_available_file is not None:
-            raise errors.AnsibleError("with_items and first_available_file are mutually incompatible in a single task")
+        incompatibles = [ x for x in [ self.with_items, self.first_available_file, self.items_lookup_plugin ] if x is not None ]
+        if len(incompatibles) > 1:
+            raise errors.AnsibleError("with_items, with_(plugin), and first_available_file are mutually incompatible in a single task")
 
         # make first_available_file accessable to Runner code
         if self.first_available_file:
@@ -137,6 +155,10 @@ class Task(object):
             self.with_items = [ ]
         self.module_vars['items'] = self.with_items
 
+        if self.items_lookup_plugin is not None:
+            self.module_vars['items_lookup_plugin'] = self.items_lookup_plugin
+            self.module_vars['items_lookup_terms'] = self.items_lookup_terms
+
         # allow runner to see delegate_to option
         self.module_vars['delegate_to'] = self.delegate_to
 

lib/ansible/runner/__init__.py

@@ -47,7 +47,7 @@ except ImportError:
 
 dirname = os.path.dirname(__file__)
 action_plugin_list = utils.import_plugins(os.path.join(dirname, 'action_plugins'))
-    
+lookup_plugin_list = utils.import_plugins(os.path.join(dirname, 'lookup_plugins')) 
         
 ################################################
 
@@ -71,8 +71,8 @@ def _executor_hook(job_queue, result_queue):
             traceback.print_exc()
 
 class HostVars(dict):
-    ''' A special view of setup_cache that adds values from the inventory
-        when needed. '''
+    ''' A special view of setup_cache that adds values from the inventory when needed. '''
+
     def __init__(self, setup_cache, inventory):
         self.setup_cache = setup_cache
         self.inventory = inventory
@@ -82,9 +82,10 @@ class HostVars(dict):
 
     def __getitem__(self, host):
         if not host in self.lookup:
-            self.lookup[host] = self.inventory.get_variables(host)
-        self.setup_cache[host].update(self.lookup[host])
-        return self.setup_cache[host]
+            result = self.inventory.get_variables(host)
+            result.update(self.setup_cache.get(host, {}))
+            self.lookup[host] = result
+        return self.lookup[host]
 
 class Runner(object):
     ''' core API interface to ansible '''
@@ -160,8 +161,11 @@ class Runner(object):
 
         # instantiate plugin classes
         self.action_plugins = {}
+        self.lookup_plugins = {}
         for (k,v) in action_plugin_list.iteritems():
             self.action_plugins[k] = v.ActionModule(self)
+        for (k,v) in lookup_plugin_list.iteritems():
+            self.lookup_plugins[k] = v.LookupModule(self)
 
     # *****************************************************
 
@@ -189,7 +193,10 @@ class Runner(object):
 
         afd, afile = tempfile.mkstemp()
         afo = os.fdopen(afd, 'w')
-        afo.write(data.encode('utf8'))
+        try:
+            afo.write(data.encode('utf8'))
+        except:
+            raise errors.AnsibleError("failure encoding into utf-8")
         afo.flush()
         afo.close()
 
@@ -283,10 +290,18 @@ class Runner(object):
 
         # allow with_items to work in playbooks...
         # apt and yum are converted into a single call, others run in a loop
-
         items = self.module_vars.get('items', [])
         if isinstance(items, basestring) and items.startswith("$"):
             items = utils.varReplaceWithItems(self.basedir, items, inject)
+
+        # if we instead said 'with_foo' and there is a lookup module named foo...
+        items_plugin = self.module_vars.get('items_lookup_plugin', None)
+        if items_plugin is not None:
+            items_terms = self.module_vars.get('items_lookup_terms', '')
+            if items_plugin in self.lookup_plugins:
+                items_terms = utils.template(self.basedir, items_terms, inject)
+                items = self.lookup_plugins[items_plugin].run(items_terms)
+
         if type(items) != list:
             raise errors.AnsibleError("with_items only takes a list: %s" % items)
 
@@ -313,6 +328,7 @@ class Runner(object):
                 results.append(result.result)
                 if result.comm_ok == False:
                     all_comm_ok = False
+                    all_failed = True
                     break
                 for x in results:
                     if x.get('changed') == True:
@@ -320,7 +336,7 @@ class Runner(object):
                     if (x.get('failed') == True) or (('rc' in x) and (x['rc'] != 0)):
                         all_failed = True
                         break
-            msg = 'All items succeeded'
+            msg = 'All items completed'
             if all_failed:
                 msg = "One or more items failed."
             rd_result = dict(failed=all_failed, changed=all_changed, results=results, msg=msg)

lib/ansible/runner/action_plugins/copy.py

@@ -39,6 +39,11 @@ class ActionModule(object):
         options = utils.parse_kv(module_args)
         source  = options.get('src', None)
         dest    = options.get('dest', None)
+
+        if dest.endswith("/"):
+            base = os.path.basename(source)
+            dest = os.path.join(dest, base)
+
         if (source is None and not 'first_available_file' in inject) or dest is None:
             result=dict(failed=True, msg="src and dest are required")
             return ReturnData(conn=conn, result=result)
@@ -78,10 +83,16 @@ class ActionModule(object):
 
             # run the copy module
             module_args = "%s src=%s" % (module_args, tmp_src)
+            print "CALLING FILE WITH: %s" % module_args
             return self.runner._execute_module(conn, tmp, 'copy', module_args, inject=inject).daisychain('file', module_args)
 
         else:
-            # no need to transfer the file, already correct md5
+            # no need to transfer the file, already correct md5, but still need to set src so the file module
+            # does not freak out.  It's just the basename of the file.
+
+            tmp_src = tmp + os.path.basename(source)
+            module_args = "%s src=%s" % (module_args, tmp_src)
             result = dict(changed=False, md5sum=remote_md5, transferred=False)
+            print "CALLING FILE WITH: %s" % module_args
             return ReturnData(conn=conn, result=result).daisychain('file', module_args)
 

lib/ansible/runner/action_plugins/template.py

@@ -42,6 +42,11 @@ class ActionModule(object):
         options  = utils.parse_kv(module_args)
         source   = options.get('src', None)
         dest     = options.get('dest', None)
+
+        if dest.endswith("/"):
+            base = os.path.basename(source)
+            dest = os.path.join(dest, base)
+
         if (source is None and 'first_available_file' not in inject) or dest is None:
             result = dict(failed=True, msg="src and dest are required")
             return ReturnData(conn=conn, comm_ok=False, result=result)

lib/ansible/runner/lookup_plugins/fileglob.py

+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import glob
+
+class LookupModule(object):
+
+    def __init__(self, runner):
+        self.runner = runner
+
+    def run(self, terms):
+        return [ f for f in glob.glob(terms) if os.path.isfile(f) ]
+
+
+

lib/ansible/utils.py

@@ -398,7 +398,10 @@ def template_from_file(basedir, path, vars):
     environment.filters['from_json'] = json.loads
     environment.filters['to_yaml'] = yaml.dump
     environment.filters['from_yaml'] = yaml.load
-    data = codecs.open(realpath, encoding="utf8").read()
+    try:
+        data = codecs.open(realpath, encoding="utf8").read()
+    except:
+        raise errors.AnsibleError("unable to process as utf-8: %s" % realpath)
     t = environment.from_string(data)
     vars = vars.copy()
     try:
@@ -668,7 +671,8 @@ def filter_leading_non_json_lines(buf):
 
 def import_plugins(directory):
     modules = {}
-    for path in glob.glob(os.path.join(directory, '*.py')):
+    python_files = os.path.join(directory, '*.py')
+    for path in glob.glob(python_files):
         if path.startswith("_"):
             continue
         name, ext = os.path.splitext(os.path.basename(path))

library/copy

@@ -89,9 +89,9 @@ def main():
             module.fail_json(msg="Destination %s not writable" % (dest))
         if not os.access(dest, os.R_OK):
             module.fail_json(msg="Destination %s not readable" % (dest))
-        # Allow dest to be directory without compromising md5 check
         if (os.path.isdir(dest)):
-            module.fail_json(msg="Destination %s cannot be a directory" % (dest))
+            basename = os.path.basename(src)
+            dest = os.path.join(dest, basename)
         md5sum_dest = module.md5(dest)
     else:
         if not os.path.exists(os.path.dirname(dest)):

library/file

@@ -339,11 +339,18 @@ def main():
     params = module.params
     state  = params['state']
     path   = os.path.expanduser(params['path'])
+
+    # source is both the source of a symlink or an informational passing of the src for a template module
+    # or copy module, even if this module never uses it, it is needed to key off some things
+
     src    = params.get('src', None)
     if src:
         src = os.path.expanduser(src)
     force = module.boolean(params['force'])
 
+    if src is not None and os.path.isdir(path):
+        params['path'] = path = os.path.join(path, os.path.basename(src))
+
     mode   = params.get('mode', None)
     owner  = params.get('owner', None)
     group  = params.get('group', None)
@@ -370,6 +377,7 @@ def main():
     changed = False
 
     prev_state = 'absent'
+
     if os.path.lexists(path):
         if os.path.islink(path):
             prev_state = 'link'
@@ -392,7 +400,7 @@ def main():
         module_exit_json(path=path, changed=True)
 
     if prev_state != 'absent' and prev_state != state and not force:
-        module_fail_json(path=path, msg='refusing to convert between %s and %s' % (prev_state, state))
+        module_fail_json(path=path, msg='refusing to convert between %s and %s for %s' % (prev_state, state, src))
 
     if prev_state == 'absent' and state == 'absent':
         module_exit_json(path=path, changed=False)
@@ -400,7 +408,7 @@ def main():
     if state == 'file':
 
         if prev_state != 'file':
-            module_fail_json(path=path, msg='file does not exist, use copy or template module to create')
+            module_fail_json(path=path, msg='file (%s) does not exist, use copy or template module to create' % path) 
 
         # set modes owners and context as needed
         changed = set_context_if_different(path, secontext, changed)

library/fireball

@@ -199,7 +199,6 @@ def serve(module, password, port, minutes):
     # password as a variable in ansible is never logged though, so it serves well
 
     key = AesKey.Read(password)
-    log("DEBUG KEY=%s" % key) # REALLY NEED TO REMOVE THIS, DEBUG/DEV ONLY!
  
     while True:
 

test/TestRunner.py

@@ -193,8 +193,11 @@ class TestRunner(unittest.TestCase):
         assert self._run('file', ['dest=' + filedemo, 'state=file'])['failed']
         assert os.path.isdir(filedemo)
 
-        assert self._run('file', ['dest=' + filedemo, 'src=/dev/null', 'state=link'])['failed']
-        assert os.path.isdir(filedemo)
+        # this used to fail but will now make a 'null' symlink in the directory pointing to dev/null.
+        # I feel this is ok but don't want to enforce it with a test.
+        #result = self._run('file', ['dest=' + filedemo, 'src=/dev/null', 'state=link'])
+        #assert result['failed']
+        #assert os.path.isdir(filedemo)
 
         assert self._run('file', ['dest=' + filedemo, 'mode=701', 'state=directory'])['changed']
         assert os.path.isdir(filedemo) and os.stat(filedemo).st_mode == 040701
@@ -245,7 +248,9 @@ class TestRunner(unittest.TestCase):
         assert self._run('file', ['dest=' + filedemo, 'state=file', 'force=yes'])['failed']
         assert os.path.isdir(filedemo)
 
-        assert self._run('file', ['dest=' + filedemo, 'src=/dev/null', 'state=link', 'force=yes'])['changed']
+        result = self._run('file', ['dest=' + filedemo, 'src=/dev/null', 'state=link', 'force=yes'])
+        assert result['changed']
+        print result
         assert os.path.islink(filedemo)
         os.unlink(filedemo)