Revert templating enhancements from 73dbab70 e6c28658 d409352c 9858b1f2 4587528b…

Revert templating enhancements from 73dbab70 e6c28658 d409352c 9858b1f2 4587528b 9b1fe455 214b0b05 8d3db803 7f9504d1 5031104c 35cb9dc2 2bd8cb57 1e85c754

hacking/test-module

@@ -77,32 +77,39 @@ def write_argsfile(argstring, json=False):
 def boilerplate_module(modfile, args):
     """ simulate what ansible does with new style modules """
 
-    #module_fh = open(modfile)
-    #module_data = module_fh.read()
-    #module_fh.close()
-
-    replacer = module_common.ModuleReplacer()
-
-    #included_boilerplate = module_data.find(module_common.REPLACER) != -1 or module_data.find("import ansible.module_utils") != -1
-
-    complex_args = {}
-    inject = {}
-    (module_data, module_style, shebang) = replacer.modify_module(
-        modfile, 
-        complex_args,
-        args,
-        inject 
-    )
+    module_fh = open(modfile)
+    module_data = module_fh.read()
+    included_boilerplate = module_data.find(module_common.REPLACER) != -1
+    module_fh.close()
+
+    if included_boilerplate:
+
+        module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
+        encoded_args = repr(str(args))
+        module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
+        encoded_lang = repr(C.DEFAULT_MODULE_LANG)
+        empty_complex = repr("{}")
+        module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang)
+        module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % C.DEFAULT_SYSLOG_FACILITY)
+        module_data = module_data.replace(module_common.REPLACER_COMPLEX, empty_complex)
+
+        modfile2_path = os.path.expanduser("~/.ansible_module_generated")
+        print "* including generated source, if any, saving to: %s" % modfile2_path
+        print "* this will offset any line numbers in tracebacks/debuggers!"
+        modfile2 = open(modfile2_path, 'w')
+        modfile2.write(module_data)
+        modfile2.close()
+        modfile = modfile2_path
+
+        return (modfile2_path, included_boilerplate, False)
+    else:
 
-    modfile2_path = os.path.expanduser("~/.ansible_module_generated")
-    print "* including generated source, if any, saving to: %s" % modfile2_path
-    print "* this may offset any line numbers in tracebacks/debuggers!"
-    modfile2 = open(modfile2_path, 'w')
-    modfile2.write(module_data)
-    modfile2.close()
-    modfile = modfile2_path
+        old_style_but_json = False
+        if 'WANT_JSON' in module_data:
+           old_style_but_json = True
 
-    return (modfile2_path, module_style)
+        print "* module boilerplate substitution not requested in module, line numbers will be unaltered"
+        return (modfile, included_boilerplate, old_style_but_json)
 
 def runtest( modfile, argspath):
     """Test run a module, piping it's output for reporting."""
@@ -144,16 +151,14 @@ def rundebug(debugger, modfile, argspath):
 def main(): 
 
     options, args = parse()
-    (modfile, module_style) = boilerplate_module(options.module_path, options.module_args)
+    (modfile, is_new_style, old_style_but_json) = boilerplate_module(options.module_path, options.module_args)
 
     argspath=None
-    if module_style != 'new':
-        if module_style == 'non_native_want_json':
+    if not is_new_style:
+        if old_style_but_json:
             argspath = write_argsfile(options.module_args, json=True)
-        elif module_style == 'old':
-            argspath = write_argsfile(options.module_args, json=False)
         else:
-            raise Exception("internal error, unexpected module style: %s" % module_style)
+            argspath = write_argsfile(options.module_args, json=False)
     if options.debugger: 
         rundebug(options.debugger, modfile, argspath)
     else:

lib/ansible/playbook/__init__.py

@@ -353,7 +353,7 @@ class PlayBook(object):
         else:
             name = task.name
 
-        self.callbacks.on_task_start(template(play.basedir, name, task.module_vars, lookup_fatal=False, filter_fatal=False, lookups=False), is_handler)
+        self.callbacks.on_task_start(template(play.basedir, name, task.module_vars, lookup_fatal=False, filter_fatal=False), is_handler)
         if hasattr(self.callbacks, 'skip_task') and self.callbacks.skip_task:
             ansible.callbacks.set_task(self.callbacks, None)
             ansible.callbacks.set_task(self.runner_callbacks, None)

lib/ansible/runner/__init__.py

@@ -40,7 +40,6 @@ from ansible.utils import template
 from ansible.utils import check_conditional
 from ansible import errors
 from ansible import module_common
-from ansible.module_common import ModuleReplacer
 import poller
 import connection
 from return_data import ReturnData
@@ -52,7 +51,6 @@ try:
 except ImportError:
     HAS_ATFORK=False
 
-module_replacer = ModuleReplacer(strip_comments=False)
 multiprocessing_runner = None
         
 OUTPUT_LOCKFILE  = tempfile.TemporaryFile()
@@ -73,6 +71,11 @@ def _executor_hook(job_queue, result_queue, new_stdin):
             host = job_queue.get(block=False)
             return_data = multiprocessing_runner._executor(host, new_stdin)
             result_queue.put(return_data)
+
+            if 'LEGACY_TEMPLATE_WARNING' in return_data.flags:
+                # pass data back up across the multiprocessing fork boundary
+                template.Flags.LEGACY_TEMPLATE_WARNING = True
+
         except Queue.Empty:
             pass
         except:
@@ -368,6 +371,15 @@ class Runner(object):
     def _executor(self, host, new_stdin):
         ''' handler for multiprocessing library '''
 
+        def get_flags():
+            # flags are a way of passing arbitrary event information
+            # back up the chain, since multiprocessing forks and doesn't
+            # allow state exchange
+            flags = []
+            if template.Flags.LEGACY_TEMPLATE_WARNING:
+                flags.append('LEGACY_TEMPLATE_WARNING')
+            return flags
+
         try:
             if not new_stdin:
                 self._new_stdin = os.fdopen(os.dup(sys.stdin.fileno()))
@@ -377,6 +389,7 @@ class Runner(object):
             exec_rc = self._executor_internal(host, new_stdin)
             if type(exec_rc) != ReturnData:
                 raise Exception("unexpected return type: %s" % type(exec_rc))
+            exec_rc.flags = get_flags()
             # redundant, right?
             if not exec_rc.comm_ok:
                 self.callbacks.on_unreachable(host, exec_rc.result)
@@ -384,11 +397,11 @@ class Runner(object):
         except errors.AnsibleError, ae:
             msg = str(ae)
             self.callbacks.on_unreachable(host, msg)
-            return ReturnData(host=host, comm_ok=False, result=dict(failed=True, msg=msg))
+            return ReturnData(host=host, comm_ok=False, result=dict(failed=True, msg=msg), flags=get_flags())
         except Exception:
             msg = traceback.format_exc()
             self.callbacks.on_unreachable(host, msg)
-            return ReturnData(host=host, comm_ok=False, result=dict(failed=True, msg=msg))
+            return ReturnData(host=host, comm_ok=False, result=dict(failed=True, msg=msg), flags=get_flags())
 
     # *****************************************************
 
@@ -825,19 +838,60 @@ class Runner(object):
     def _copy_module(self, conn, tmp, module_name, module_args, inject, complex_args=None):
         ''' transfer a module over SFTP, does not run it '''
 
+        # FIXME if complex args is none, set to {}
+
+        if module_name.startswith("/"):
+            raise errors.AnsibleFileNotFound("%s is not a module" % module_name)
+
         # Search module path(s) for named module.
         in_path = utils.plugins.module_finder.find_plugin(module_name)
         if in_path is None:
             raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths()))
+
         out_path = os.path.join(tmp, module_name)
 
-        # insert shared code and arguments into the module
-        (module_data, module_style, shebang) = module_replacer.modify_module(
-            in_path, complex_args, module_args, inject
-        )
+        module_data = ""
+        module_style = 'old'
+
+        with open(in_path) as f:
+            module_data = f.read()
+            if module_common.REPLACER in module_data:
+                module_style = 'new'
+            if 'WANT_JSON' in module_data:
+                module_style = 'non_native_want_json'
+
+            complex_args_json = utils.jsonify(complex_args)
+            # We force conversion of module_args to str because module_common calls shlex.split,
+            # a standard library function that incorrectly handles Unicode input before Python 2.7.3.
+            encoded_args = repr(module_args.encode('utf-8'))
+            encoded_lang = repr(C.DEFAULT_MODULE_LANG)
+            encoded_complex = repr(complex_args_json)
+
+            module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
+            module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
+            module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang)
+            module_data = module_data.replace(module_common.REPLACER_COMPLEX, encoded_complex)
+
+            if module_style == 'new':
+                facility = C.DEFAULT_SYSLOG_FACILITY
+                if 'ansible_syslog_facility' in inject:
+                    facility = inject['ansible_syslog_facility']
+                module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility)
+
+        lines = module_data.split("\n")
+        shebang = None
+        if lines[0].startswith("#!"):
+            shebang = lines[0].strip()
+            args = shlex.split(str(shebang[2:]))
+            interpreter = args[0]
+            interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
+
+            if interpreter_config in inject:
+                lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:]))
+                module_data = "\n".join(lines)
 
-        # ship the module
         self._transfer_str(conn, tmp, module_name, module_data)
+
         return (out_path, module_style, shebang)
 
     # *****************************************************

lib/ansible/utils/__init__.py

@@ -23,6 +23,12 @@ import yaml
 import copy
 import optparse
 import operator
+from ansible import errors
+from ansible import __version__
+from ansible.utils.plugins import *
+from ansible.utils import template
+from ansible.callbacks import display
+import ansible.constants as C
 import time
 import StringIO
 import stat
@@ -37,15 +43,6 @@ import getpass
 import sys
 import textwrap
 
-# ansible
-from ansible import errors
-from ansible import __version__
-from ansible.utils.plugins import *
-from ansible.utils import template
-from ansible.callbacks import display
-import ansible.constants as C
-import filesystem
-
 VERBOSITY=0
 
 # list of all deprecation messages to prevent duplicate display
@@ -227,7 +224,7 @@ def unfrackpath(path):
     example:
     '$HOME/../../var/mail' becomes '/var/spool/mail'
     '''
-    return filesystem.unfrackpath(path)
+    return os.path.normpath(os.path.realpath(os.path.expandvars(os.path.expanduser(path))))
 
 def prepare_writeable_dir(tree,mode=0777):
     ''' make sure a directory exists and is writeable '''
@@ -253,11 +250,33 @@ def path_dwim(basedir, given):
     '''
     make relative paths work like folks expect.
     '''
-    return filesystem.path_dwim(basedir, given)
+
+    if given.startswith("/"):
+        return os.path.abspath(given)
+    elif given.startswith("~"):
+        return os.path.abspath(os.path.expanduser(given))
+    else:
+        return os.path.abspath(os.path.join(basedir, given))
 
 def path_dwim_relative(original, dirname, source, playbook_base, check=True):
     ''' find one file in a directory one level up in a dir named dirname relative to current '''
-    return filesystem.path_dwim_relative(original, dirname, source, playbook_base, check=True)
+    # (used by roles code)
+
+    basedir = os.path.dirname(original)
+    if os.path.islink(basedir):
+        basedir = unfrackpath(basedir)
+        template2 = os.path.join(basedir, dirname, source)
+    else:
+        template2 = os.path.join(basedir, '..', dirname, source)
+    source2 = path_dwim(basedir, template2)
+    if os.path.exists(source2):
+        return source2
+    obvious_local_path = path_dwim(playbook_base, source)
+    if os.path.exists(obvious_local_path):
+        return obvious_local_path
+    if check:
+        raise errors.AnsibleError("input file not found at %s or %s" % (source2, obvious_local_path))
+    return source2 # which does not exist
 
 def json_loads(data):
     ''' parse a JSON string and return a data structure '''

lib/ansible/utils/filesystem.py

-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-from ansible import errors
-from ansible import __version__
-import ansible.constants as C
-
-def path_dwim(basedir, given):
-    '''
-    make relative paths work like folks expect.
-    '''
-
-    if given.startswith("/"):
-        return os.path.abspath(given)
-    elif given.startswith("~"):
-        return os.path.abspath(os.path.expanduser(given))
-    else:
-        return os.path.abspath(os.path.join(basedir, given))
-
-def path_dwim_relative(original, dirname, source, playbook_base, check=True):
-    ''' find one file in a directory one level up in a dir named dirname relative to current '''
-    # (used by roles code)
-
-    basedir = os.path.dirname(original)
-    if os.path.islink(basedir):
-        basedir = unfrackpath(basedir)
-        template2 = os.path.join(basedir, dirname, source)
-    else:
-        template2 = os.path.join(basedir, '..', dirname, source)
-    source2 = path_dwim(basedir, template2)
-    if os.path.exists(source2):
-        return source2
-    obvious_local_path = path_dwim(playbook_base, source)
-    if os.path.exists(obvious_local_path):
-        return obvious_local_path
-    if check:
-        raise errors.AnsibleError("input file not found at %s or %s" % (source2, obvious_local_path))
-    return source2 # which does not exist
-
-def unfrackpath(path):
-    '''
-    returns a path that is free of symlinks, environment
-    variables, relative path traversals and symbols (~)
-    example:
-    '$HOME/../../var/mail' becomes '/var/spool/mail'
-    '''
-    return os.path.normpath(os.path.realpath(os.path.expandvars(os.path.expanduser(path))))
-

library/system/ping

@@ -48,9 +48,7 @@ def main():
         result['ping'] = module.params['data']
     module.exit_json(**result)
 
-### boilerplate: import common module snippets here
-from ansible.module_utils.basic import *
-
-### invoke the module
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
 main()
 

library/system/service

@@ -1205,11 +1205,7 @@ def main():
 
     module.exit_json(**result)
 
-### boilerplate: import common module snippets here
-from ansible.module_utils.basic import *
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
 
-### invoke the module
 main()
-
-
-

test/TestUtils.py

@@ -9,7 +9,6 @@ from nose.plugins.skip import SkipTest
 
 import ansible.utils
 import ansible.utils.template as template2
-import ansible.constants as C
 
 class TestUtils(unittest.TestCase):
 
@@ -17,10 +16,6 @@ class TestUtils(unittest.TestCase):
     ### varReplace function tests
 
     def test_varReplace_var_complex_var(self):
-
-        old_setting = C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES
-        C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES = True
-
         vars = {
             'x': '$y',
             'y': {
@@ -31,8 +26,6 @@ class TestUtils(unittest.TestCase):
         res = template2.template(None, template, vars)
         assert res == 'result'
 
-        C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES = old_setting
-
     #####################################
     ### template_ds function tests
 
@@ -60,9 +53,6 @@ class TestUtils(unittest.TestCase):
             ],
         }
 
-        old_setting = C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES
-        C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES = True
-
         template = '${data.var}'
         res = template2.template(None, template, vars)
         assert sorted(res) == sorted(vars['data']['var'])
@@ -79,8 +69,6 @@ class TestUtils(unittest.TestCase):
         res = template2.template(None, template, vars)
         assert res == template
 
-        C.DEFAULT_LEGACY_PLAYBOOK_VARIABLES = old_setting
-
     #####################################
     ### Template function tests
 
@@ -93,6 +81,15 @@ class TestUtils(unittest.TestCase):
 
         assert res == 'hello world'
 
+    def test_template_whitespace(self):
+        vars = {
+            'who': 'world',
+        }
+
+        res = template2.template_from_file("test", "template-whitespace", vars)
+
+        assert res == 'hello world\n'
+
     def test_template_unicode(self):
         vars = {
             'who': u'wórld',