Merge pull request #1522 from dhozac/LOOKUP-templating

Add $LOOKUP(<lookup plugin>,<data>) as a templating option

lib/ansible/playbook/task.py

@@ -149,9 +149,6 @@ class Task(object):
             # allow the user to list comma delimited tags
             import_tags = import_tags.split(",")
 
-        self.name = utils.template(None, self.name, self.module_vars)
-        self.action = utils.template(None, self.action, self.module_vars)
-
         # handle mutually incompatible options
         incompatibles = [ x for x in [ self.first_available_file, self.items_lookup_plugin ] if x is not None ]
         if len(incompatibles) > 1:

lib/ansible/runner/lookup_plugins/file.py

+# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible import utils, errors
+import os
+
+class LookupModule(object):
+
+    def __init__(self, basedir=None, **kwargs):
+        self.basedir = basedir
+
+    def run(self, terms, **kwargs):
+        path = utils.path_dwim(self.basedir, terms)
+        if not os.path.exists(path):
+            raise errors.AnsibleError("%s does not exist" % path)
+        return [open(path).read().rstrip()]

lib/ansible/runner/lookup_plugins/lines.py

+# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+from ansible import utils, errors
+
+class LookupModule(object):
+
+    def __init__(self, basedir=None, **kwargs):
+        self.basedir = basedir
+
+    def run(self, terms, **kwargs):
+        p = subprocess.Popen(terms, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        (stdout, stderr) = p.communicate()
+        if p.returncode == 0:
+            return stdout.splitlines()
+        else:
+            raise errors.AnsibleError("lookup_plugin.lines(%s) returned %d" % (terms, p.returncode))

lib/ansible/runner/lookup_plugins/pipe.py

+# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+from ansible import utils, errors
+
+class LookupModule(object):
+
+    def __init__(self, basedir=None, **kwargs):
+        self.basedir = basedir
+
+    def run(self, terms, **kwargs):
+        p = subprocess.Popen(terms, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        (stdout, stderr) = p.communicate()
+        if p.returncode == 0:
+            return [stdout.rstrip()]
+        else:
+            raise errors.AnsibleError("lookup_plugin.pipe(%s) returned %d" % (terms, p.returncode))

lib/ansible/utils/template.py

@@ -141,8 +141,9 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
 
     return ''.join(done)
 
-_FILEPIPECRE = re.compile(r"\$(?P<special>FILE|PIPE)\(([^\)]+)\)")
-def _varReplaceFilesAndPipes(basedir, raw):
+_FILEPIPECRE = re.compile(r"\$(?P<special>FILE|PIPE|LOOKUP)\(([^\)]+)\)")
+def _varReplaceFilesAndPipes(basedir, raw, vars):
+    from ansible import utils
     done = [] # Completed chunks to return
 
     while raw:
@@ -156,21 +157,18 @@ def _varReplaceFilesAndPipes(basedir, raw):
 
         replacement = m.group()
         if m.group(1) == "FILE":
-            from ansible import utils
-            path = utils.path_dwim(basedir, m.group(2))
-            try:
-                f = open(path, "r")
-                replacement = f.read()
-                f.close()
-            except IOError:
-                raise errors.AnsibleError("$FILE(%s) failed" % path)
+            module_name = "file"
+            args = m.group(2)
         elif m.group(1) == "PIPE":
-            p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE)
-            (stdout, stderr) = p.communicate()
-            if p.returncode == 0:
-                replacement = stdout
-            else:
-                raise errors.AnsibleError("$PIPE(%s) returned %d" % (m.group(2), p.returncode))
+            module_name = "pipe"
+            args = m.group(2)
+        elif m.group(1) == "LOOKUP":
+            module_name, args = m.group(2).split(",", 1)
+            args = args.strip()
+        instance = utils.plugins.lookup_loader.get(module_name, basedir=basedir)
+        replacement = instance.run(args, inject=vars)
+        if not isinstance(replacement, basestring):
+            replacement = ",".join(replacement)
 
         start, end = m.span()
         done.append(raw[:start])    # Keep stuff leading up to token
@@ -211,7 +209,7 @@ def template(basedir, text, vars, expand_lists=False):
     except UnicodeEncodeError:
         pass # already unicode
     text = varReplace(unicode(text), vars, expand_lists=expand_lists)
-    text = _varReplaceFilesAndPipes(basedir, text)
+    text = _varReplaceFilesAndPipes(basedir, text, vars)
     return text
 
 def template_from_file(basedir, path, vars):

test/TestPlayBook.py

@@ -113,6 +113,10 @@ class TestPlaybook(unittest.TestCase):
 
    def _run(self, test_playbook, host_list='test/ansible_hosts'):
        ''' run a module and get the localhost results '''
+       # This ensures tests are independent of eachother
+       ansible.playbook.SETUP_CACHE.clear()
+       EVENTS = []
+
        self.test_callbacks = TestCallbacks()
        self.playbook = ansible.playbook.PlayBook(
            playbook     = test_playbook,
@@ -177,6 +181,29 @@ class TestPlaybook(unittest.TestCase):
 
        assert utils.jsonify(expected, format=True) == utils.jsonify(actual, format=True)
 
+   def test_lookups(self):
+       pb = os.path.join(self.test_dir, 'lookup_plugins.yml')
+       actual = self._run(pb)
+
+       # if different, this will output to screen
+       print "**ACTUAL**"
+       print utils.jsonify(actual, format=True)
+       expected =  {
+           "localhost": {
+               "changed": 7,
+               "failures": 0,
+               "ok": 9,
+               "skipped": 1,
+               "unreachable": 0
+           }   
+       }   
+       print "**EXPECTED**"
+       print utils.jsonify(expected, format=True)
+
+       assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True)
+
+       assert len(EVENTS) == 44
+
    def test_playbook_vars(self): 
        test_callbacks = TestCallbacks()
        playbook = ansible.playbook.PlayBook(

test/TestUtils.py

@@ -272,18 +272,18 @@ class TestUtils(unittest.TestCase):
         assert res == u'hello oh great one'
 
     def test_varReplace_include(self):
-        template = 'hello $FILE(world)'
+        template = 'hello $FILE(world) $LOOKUP(file, world)'
 
         res = ansible.utils.template("test", template, {})
 
-        assert res == u'hello world'
+        assert res == u'hello world world'
 
     def test_varReplace_include_script(self):
-        template = 'hello $PIPE(echo world)'
+        template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)'
 
         res = ansible.utils.template("test", template, {})
 
-        assert res == u'hello world'
+        assert res == u'hello world world'
 
     #####################################
     ### varReplaceWithItems function tests

test/lookup_plugins.yml

+# simple test of lookup plugins in with_*
+---
+- hosts: all
+  vars:
+    empty_list: []
+  tasks:
+  - name: test with_items
+    action: command true
+    with_items:
+    - 1
+    - 2
+    - 3
+  - name: test with_items with empty list
+    action: command true
+    with_items: $empty_list
+
+  - name: test with_file and FILE
+    action: command test "$item" = "$FILE(sample.j2)"
+    with_file: sample.j2
+
+  - name: test with_pipe
+    action: command test "$item" = "$PIPE(cat sample.j2)"
+    with_pipe: cat sample.j2
+
+  - name: test LOOKUP and PIPE
+    action: command test "$LOOKUP(pipe, cat sample.j2)" = "$PIPE(cat sample.j2)"
+
+  - name: ensure test file doesnt exist
+    # command because file will return differently
+    action: command rm -f /tmp/ansible-test-with_lines-data
+  - name: test with_lines
+    action: shell echo "$item" >> /tmp/ansible-test-with_lines-data
+    with_lines: cat sample.j2
+  - name: verify with_lines
+    action: copy src=sample.j2 dest=/tmp/ansible-test-with_lines-data
+  - name: cleanup test file
+    action: file path=/tmp/ansible-test-with_lines-data state=absent