Merge pull request #1607 from dhozac/j2-templated-vars

Templating fixes and features

lib/ansible/utils/template.py

@@ -33,15 +33,23 @@ import pwd
 _LISTRE = re.compile(r"(\w+)\[(\d+)\]")
 
 
-def _varFindLimitSpace(vars, space, part, depth):
+def _varFindLimitSpace(basedir, vars, space, part, depth):
+    ''' limits the search space of space to part
     
-    # TODO: comments
+    basically does space.get(part, None), but with
+    templating for part and a few more things
+    '''
 
+    # Previous part couldn't be found, nothing to limit to
     if space is None:
         return space
+    # A part with escaped .s in it is compounded by { and }, remove them
     if part[0] == '{' and part[-1] == '}':
         part = part[1:-1]
-    part = varReplace(part, vars, depth=depth + 1)
+    # Template part to resolve variables within (${var$var2})
+    part = varReplace(basedir, part, vars, depth=depth + 1)
+
+    # Now find it
     if part in space:
         space = space[part]
     elif "[" in part:
@@ -55,11 +63,30 @@ def _varFindLimitSpace(vars, space, part, depth):
                 return None
     else:
         return None
+
     return space
 
-def _varFind(text, vars, depth=0):
+def _varFind(basedir, text, vars, depth=0):
+    ''' Searches for a variable in text and finds its replacement in vars
+
+    The variables can have two formats;
+    - simple, $ followed by alphanumerics and/or underscores
+    - complex, ${ followed by alphanumerics, underscores, periods, braces and brackets, ended by a }
 
-    # TODO: comments
+    Examples:
+    - $variable: simple variable that will have vars['variable'] as its replacement
+    - ${variable.complex}: complex variable that will have vars['variable']['complex'] as its replacement
+    - $variable.complex: simple variable, identical to the first, .complex ignored
+
+    Complex variables are broken into parts by separating on periods, except if enclosed in {}.
+    ${variable.{fully.qualified.domain}} would be parsed as two parts, variable and fully.qualified.domain,
+    whereas ${variable.fully.qualified.domain} would be parsed as four parts.
+
+    Returns a dict(replacement=<value in vars>, start=<index into text where the variable stated>,
+        end=<index into text where the variable ends>)
+    or None if no variable could be found in text. If replacement is None, it should be replaced with the
+    original data in the caller.
+    '''
 
     start = text.find("$")
     if start == -1:
@@ -78,13 +105,28 @@ def _varFind(text, vars, depth=0):
         var_start += 1
     else:
         is_complex = False
-        brace_level = 0
+        brace_level = 1
+    # is_lookup is true for $FILE(...) and friends
+    is_lookup = False
+    lookup_plugin_name = None
     end = var_start
-    part_start = (var_start, brace_level)
+    # part_start is an index of where the current part started
+    part_start = var_start
     space = vars
-    while end < len(text) and ((is_complex and brace_level > 0) or not is_complex):
+    while end < len(text) and (((is_lookup or is_complex) and brace_level > 0) or (not is_complex and not is_lookup)):
         if text[end].isalnum() or text[end] == '_':
             pass
+        elif not is_complex and not is_lookup and text[end] == '(' and text[part_start:end].isupper():
+            is_lookup = True
+            lookup_plugin_name = text[part_start:end]
+            part_start = end + 1
+        elif is_lookup and text[end] == '(':
+            brace_level += 1
+        elif is_lookup and text[end] == ')':
+            brace_level -= 1
+        elif is_lookup:
+            # lookups are allowed arbitrary contents
+            pass
         elif is_complex and text[end] == '{':
             brace_level += 1
         elif is_complex and text[end] == '}':
@@ -92,23 +134,44 @@ def _varFind(text, vars, depth=0):
         elif is_complex and text[end] in ('$', '[', ']'):
             pass
         elif is_complex and text[end] == '.':
-            if brace_level == part_start[1]:
-                space = _varFindLimitSpace(vars, space, text[part_start[0]:end], depth)
-                part_start = (end + 1, brace_level)
+            if brace_level == 1:
+                space = _varFindLimitSpace(basedir, vars, space, text[part_start:end], depth)
+                part_start = end + 1
         else:
+            # This breaks out of the loop on non-variable name characters
             break
         end += 1
     var_end = end
+    # Handle "This has $ in it"
+    if var_end == part_start:
+        return {'replacement': None, 'start': start, 'end': end}
+
+    # Handle lookup plugins
+    if is_lookup:
+        # When basedir is None, handle lookup plugins later
+        if basedir is None:
+            return {'replacement': None, 'start': start, 'end': end}
+        var_end -= 1
+    	from ansible import utils
+        args = text[part_start:var_end]
+        if lookup_plugin_name == 'LOOKUP':
+            lookup_plugin_name, args = args.split(",", 1)
+            args = args.strip()
+        instance = utils.plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir)
+        if instance is not None:
+            replacement = instance.run(args, inject=vars)
+        else:
+            replacement = None
+        return {'replacement': replacement, 'start': start, 'end': end}
+
     if is_complex:
         var_end -= 1
         if text[var_end] != '}' or brace_level != 0:
             return None
-    if var_end == part_start[0]:
-        return None
-    space = _varFindLimitSpace(vars, space, text[part_start[0]:var_end], depth)
+    space = _varFindLimitSpace(basedir, vars, space, text[part_start:var_end], depth)
     return {'replacement': space, 'start': start, 'end': end}
 
-def varReplace(raw, vars, depth=0, expand_lists=False):
+def varReplace(basedir, raw, vars, depth=0, expand_lists=False):
     ''' Perform variable replacement of $variables in string raw using vars dictionary '''
     # this code originally from yum
 
@@ -118,7 +181,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
     done = [] # Completed chunks to return
 
     while raw:
-        m = _varFind(raw, vars, depth)
+        m = _varFind(basedir, raw, vars, depth)
         if not m:
             done.append(raw)
             break
@@ -130,7 +193,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
         if expand_lists and isinstance(replacement, (list, tuple)):
             replacement = ",".join(replacement)
         if isinstance(replacement, (str, unicode)):
-            replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists)
+            replacement = varReplace(basedir, replacement, vars, depth=depth+1, expand_lists=expand_lists)
         if replacement is None:
             replacement = raw[m['start']:m['end']]
 
@@ -141,53 +204,11 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
 
     return ''.join(done)
 
-_FILEPIPECRE = re.compile(r"\$(?P<special>[A-Z]+)\(([^\)]*)\)")
-def _varReplaceLookups(basedir, raw, vars):
-    from ansible import utils
-    done = [] # Completed chunks to return
-
-    while raw:
-        m = _FILEPIPECRE.search(raw)
-        if not m:
-            done.append(raw)
-            break
-
-        # Determine replacement value (if unknown lookup plugin then preserve
-        # original)
-
-        replacement = m.group()
-        if m.group(1) == "FILE":
-            module_name = "file"
-            args = m.group(2)
-        elif m.group(1) == "PIPE":
-            module_name = "pipe"
-            args = m.group(2)
-        elif m.group(1) == "LOOKUP":
-            module_name, args = m.group(2).split(",", 1)
-            args = args.strip()
-        else:
-            module_name = m.group(1).lower()
-            args = m.group(2)
-        instance = utils.plugins.lookup_loader.get(module_name, basedir=basedir)
-        if instance is not None:
-            replacement = instance.run(args, inject=vars)
-            if not isinstance(replacement, basestring):
-                replacement = ",".join(replacement)
-        else:
-            replacement = m.group(0)
-
-        start, end = m.span()
-        done.append(raw[:start])    # Keep stuff leading up to token
-        done.append(replacement.rstrip())    # Append replacement value
-        raw = raw[end:]             # Continue with remainder of string
-
-    return ''.join(done)
-
 def template_ds(basedir, varname, vars):
     ''' templates a data structure by traversing it and substituting for other data structures '''
 
     if isinstance(varname, basestring):
-        m = _varFind(varname, vars)
+        m = _varFind(basedir, varname, vars)
         if not m:
             return varname
         if m['start'] == 0 and m['end'] == len(varname):
@@ -214,11 +235,21 @@ def template(basedir, text, vars, expand_lists=False):
         text = text.decode('utf-8')
     except UnicodeEncodeError:
         pass # already unicode
-    text = varReplace(unicode(text), vars, expand_lists=expand_lists)
-    if basedir is not None:
-        text = _varReplaceLookups(basedir, text, vars)
+    text = varReplace(basedir, unicode(text), vars, expand_lists=expand_lists)
     return text
 
+class _jinja2_vars(object):
+    ''' helper class to template all variable content before jinja2 sees it '''
+    def __init__(self, basedir, vars):
+        self.basedir = basedir
+        self.vars = vars
+    def __contains__(self, k):
+        return k in self.vars
+    def __getitem__(self, varname):
+        if varname not in self.vars:
+            raise KeyError("undefined variable: %s" % varname)
+        return template_ds(self.basedir, self.vars[varname], self.vars)
+
 def template_from_file(basedir, path, vars):
     ''' run a file through the templating engine '''
 
@@ -257,7 +288,11 @@ def template_from_file(basedir, path, vars):
     vars['ansible_managed'] = time.strftime(managed_str,
                                 time.localtime(os.path.getmtime(realpath)))
 
-    res = t.render(vars)
+    # This line performs deep Jinja2 magic that uses the _jinja2_vars object for vars
+    # Ideally, this could use some API where setting shared=True and the object won't get
+    # passed through dict(o), but I have not found that yet.
+    res = jinja2.utils.concat(t.root_render_func(t.new_context(_jinja2_vars(basedir, vars), shared=True)))
+
     if data.endswith('\n') and not res.endswith('\n'):
         res = res + '\n'
     return template(basedir, res, vars)

test/TestPlayBook.py

@@ -202,7 +202,7 @@ class TestPlaybook(unittest.TestCase):
 
        assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True)
 
-       assert len(EVENTS) == 44
+       assert len(EVENTS) == 60
 
    def test_includes(self):
        pb = os.path.join(self.test_dir, 'playbook-includer.yml')

test/TestUtils.py

@@ -15,14 +15,14 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
     def test_varReplace_trailing_dollar(self):
         template = '$what $who $'
         vars = dict(what='hello', who='world')
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'hello world $'
 
     def test_varReplace_multiple(self):
@@ -32,7 +32,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
@@ -42,7 +42,7 @@ class TestUtils(unittest.TestCase):
             'whoVar': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         print res
         assert res == 'hello world'
 
@@ -52,7 +52,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world!'
 
@@ -62,7 +62,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
@@ -72,7 +72,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world}'
 
@@ -82,7 +82,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == template
 
@@ -92,7 +92,7 @@ class TestUtils(unittest.TestCase):
             'who': 'world',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world }'
 
@@ -104,7 +104,7 @@ class TestUtils(unittest.TestCase):
             },
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         print res
         assert res == template
@@ -117,7 +117,7 @@ class TestUtils(unittest.TestCase):
             },
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
@@ -130,7 +130,7 @@ class TestUtils(unittest.TestCase):
             'what': 'hello',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello 2'
 
@@ -140,7 +140,7 @@ class TestUtils(unittest.TestCase):
             'who': u'wórld',
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == u'hello wórld'
 
@@ -150,7 +150,7 @@ class TestUtils(unittest.TestCase):
             'data': [ 'no-one', 'world' ]
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
@@ -160,7 +160,7 @@ class TestUtils(unittest.TestCase):
             'data': [ 'no-one', 'world' ]
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == template
 
@@ -170,7 +170,7 @@ class TestUtils(unittest.TestCase):
             'data': [ 'no-one', 'world' ]
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == template
 
@@ -180,7 +180,7 @@ class TestUtils(unittest.TestCase):
             'data': { 'no-one': 0, 'world': 1 }
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == template
 
@@ -190,7 +190,7 @@ class TestUtils(unittest.TestCase):
             'data': [ 'no-one', {'msg': [ 'world'] } ]
         }
 
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
 
         assert res == 'hello world'
 
@@ -201,7 +201,7 @@ class TestUtils(unittest.TestCase):
         }
 
         template = '${foo}${bar}'
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'foobar'
 
     def test_varReplace_escape_dot(self):
@@ -214,7 +214,7 @@ class TestUtils(unittest.TestCase):
         }
 
         template = '${hostvars.{test.example.com}.foo}'
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'bar'
 
     def test_varReplace_list_join(self):
@@ -227,7 +227,7 @@ class TestUtils(unittest.TestCase):
         }
 
         template = 'yum pkg=${list} state=installed'
-        res = ansible.utils.varReplace(template, vars, expand_lists=True)
+        res = ansible.utils.varReplace(None, template, vars, expand_lists=True)
         assert res == 'yum pkg=foo,bar,baz state=installed'
 
     def test_varReplace_escaped_var(self):
@@ -235,7 +235,7 @@ class TestUtils(unittest.TestCase):
             'foo': 'bar',
         }
         template = 'action \$foo'
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'action $foo'
 
     def test_varReplace_var_part(self):
@@ -246,7 +246,7 @@ class TestUtils(unittest.TestCase):
             'key': 'bar',
         }
         template = 'test ${foo.$key}'
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'test result'
 
     def test_varReplace_var_partial_part(self):
@@ -257,7 +257,7 @@ class TestUtils(unittest.TestCase):
             'key': 'bar',
         }
         template = 'test ${foo.${key}baz}'
-        res = ansible.utils.varReplace(template, vars)
+        res = ansible.utils.varReplace(None, template, vars)
         assert res == 'test result'
 
     def test_template_varReplace_iterated(self):
@@ -274,14 +274,14 @@ class TestUtils(unittest.TestCase):
     def test_varReplace_include(self):
         template = 'hello $FILE(world) $LOOKUP(file, world)'
 
-        res = ansible.utils.template("test", template, {})
+        res = ansible.utils.template("test", template, {}, expand_lists=True)
 
         assert res == u'hello world world'
 
     def test_varReplace_include_script(self):
         template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)'
 
-        res = ansible.utils.template("test", template, {})
+        res = ansible.utils.template("test", template, {}, expand_lists=True)
 
         assert res == u'hello world world'