Enable imports to work on a snippet based system, allowing for instance a…

Enable imports to work on a snippet based system, allowing for instance a library of common EC2 functions
to be reused between modules.  See library/system/service and library/system/ping for initial examples.  Can
work the old way to just import 'basic', or can import the new way to import multiple pieces of code from
module_utils/.

hacking/test-module

@@ -77,39 +77,32 @@ def write_argsfile(argstring, json=False):
 def boilerplate_module(modfile, args):
     """ simulate what ansible does with new style modules """
 
-    module_fh = open(modfile)
-    module_data = module_fh.read()
-    included_boilerplate = module_data.find(module_common.REPLACER) != -1
-    module_fh.close()
-
-    if included_boilerplate:
-
-        module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
-        encoded_args = repr(str(args))
-        module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
-        encoded_lang = repr(C.DEFAULT_MODULE_LANG)
-        empty_complex = repr("{}")
-        module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang)
-        module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % C.DEFAULT_SYSLOG_FACILITY)
-        module_data = module_data.replace(module_common.REPLACER_COMPLEX, empty_complex)
+    #module_fh = open(modfile)
+    #module_data = module_fh.read()
+    #module_fh.close()
+
+    replacer = module_common.ModuleReplacer()
+
+    #included_boilerplate = module_data.find(module_common.REPLACER) != -1 or module_data.find("import ansible.module_utils") != -1
+
+    complex_args = {}
+    inject = {}
+    (module_data, module_style, shebang) = replacer.modify_module(
+        modfile, 
+        complex_args,
+        args,
+        inject 
+    )
 
     modfile2_path = os.path.expanduser("~/.ansible_module_generated")
     print "* including generated source, if any, saving to: %s" % modfile2_path
-        print "* this will offset any line numbers in tracebacks/debuggers!"
+    print "* this may offset any line numbers in tracebacks/debuggers!"
     modfile2 = open(modfile2_path, 'w')
     modfile2.write(module_data)
     modfile2.close()
     modfile = modfile2_path
 
-        return (modfile2_path, included_boilerplate, False)
-    else:
-
-        old_style_but_json = False
-        if 'WANT_JSON' in module_data:
-           old_style_but_json = True
-
-        print "* module boilerplate substitution not requested in module, line numbers will be unaltered"
-        return (modfile, included_boilerplate, old_style_but_json)
+    return (modfile2_path, module_style)
 
 def runtest( modfile, argspath):
     """Test run a module, piping it's output for reporting."""
@@ -151,14 +144,16 @@ def rundebug(debugger, modfile, argspath):
 def main(): 
 
     options, args = parse()
-    (modfile, is_new_style, old_style_but_json) = boilerplate_module(options.module_path, options.module_args)
+    (modfile, module_style) = boilerplate_module(options.module_path, options.module_args)
 
     argspath=None
-    if not is_new_style:
-        if old_style_but_json:
+    if module_style != 'new':
+        if module_style == 'non_native_want_json':
             argspath = write_argsfile(options.module_args, json=True)
-        else:
+        elif module_style == 'old':
             argspath = write_argsfile(options.module_args, json=False)
+        else:
+            raise Exception("internal error, unexpected module style: %s" % module_style)
     if options.debugger: 
         rundebug(options.debugger, modfile, argspath)
     else:

lib/ansible/module_common.py → lib/ansible/module_utils/basic.py

@@ -27,18 +27,11 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-REPLACER = "#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
-REPLACER_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
-REPLACER_LANG = "<<INCLUDE_ANSIBLE_MODULE_LANG>>"
-REPLACER_COMPLEX = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
-
-MODULE_COMMON = """
-
 # == BEGIN DYNAMICALLY INSERTED CODE ==
 
-MODULE_ARGS = <<INCLUDE_ANSIBLE_MODULE_ARGS>>
-MODULE_LANG = <<INCLUDE_ANSIBLE_MODULE_LANG>>
-MODULE_COMPLEX_ARGS = <<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>
+MODULE_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
+MODULE_LANG = "<<INCLUDE_ANSIBLE_MODULE_LANG>>"
+MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
 
 BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1]
 BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0]
@@ -963,7 +956,3 @@ class AnsibleModule(object):
             if size >= limit:
                 break
         return '%.2f %s' % (float(size)/ limit, suffix)
-
-# == END DYNAMICALLY INSERTED CODE ===
-
-"""

lib/ansible/runner/__init__.py

@@ -40,6 +40,7 @@ from ansible.utils import template
 from ansible.utils import check_conditional
 from ansible import errors
 from ansible import module_common
+from ansible.module_common import ModuleReplacer
 import poller
 import connection
 from return_data import ReturnData
@@ -51,6 +52,7 @@ try:
 except ImportError:
     HAS_ATFORK=False
 
+module_replacer = ModuleReplacer(strip_comments=False)
 multiprocessing_runner = None
         
 OUTPUT_LOCKFILE  = tempfile.TemporaryFile()
@@ -825,60 +827,19 @@ class Runner(object):
     def _copy_module(self, conn, tmp, module_name, module_args, inject, complex_args=None):
         ''' transfer a module over SFTP, does not run it '''
 
-        # FIXME if complex args is none, set to {}
-
-        if module_name.startswith("/"):
-            raise errors.AnsibleFileNotFound("%s is not a module" % module_name)
-
         # Search module path(s) for named module.
         in_path = utils.plugins.module_finder.find_plugin(module_name)
         if in_path is None:
             raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths()))
-
         out_path = os.path.join(tmp, module_name)
 
-        module_data = ""
-        module_style = 'old'
-
-        with open(in_path) as f:
-            module_data = f.read()
-            if module_common.REPLACER in module_data:
-                module_style = 'new'
-            if 'WANT_JSON' in module_data:
-                module_style = 'non_native_want_json'
-
-            complex_args_json = utils.jsonify(complex_args)
-            # We force conversion of module_args to str because module_common calls shlex.split,
-            # a standard library function that incorrectly handles Unicode input before Python 2.7.3.
-            encoded_args = repr(module_args.encode('utf-8'))
-            encoded_lang = repr(C.DEFAULT_MODULE_LANG)
-            encoded_complex = repr(complex_args_json)
-
-            module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
-            module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
-            module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang)
-            module_data = module_data.replace(module_common.REPLACER_COMPLEX, encoded_complex)
-
-            if module_style == 'new':
-                facility = C.DEFAULT_SYSLOG_FACILITY
-                if 'ansible_syslog_facility' in inject:
-                    facility = inject['ansible_syslog_facility']
-                module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility)
-
-        lines = module_data.split("\n")
-        shebang = None
-        if lines[0].startswith("#!"):
-            shebang = lines[0].strip()
-            args = shlex.split(str(shebang[2:]))
-            interpreter = args[0]
-            interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
-
-            if interpreter_config in inject:
-                lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:]))
-                module_data = "\n".join(lines)
+        # insert shared code and arguments into the module
+        (module_data, module_style, shebang) = module_replacer.modify_module(
+            in_path, complex_args, module_args, inject
+        )
 
+        # ship the module
         self._transfer_str(conn, tmp, module_name, module_data)
-
         return (out_path, module_style, shebang)
 
     # *****************************************************

lib/ansible/utils/template.py

@@ -462,7 +462,7 @@ def template_from_file(basedir, path, vars):
         res = res + '\n'
     return template(basedir, res, vars) 
 
-def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=True):
+def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=True, filters=True):
     ''' run a string through the (Jinja2) templating engine '''
     
     def my_lookup(*args, **kwargs):
@@ -472,6 +472,8 @@ def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=T
     if type(data) == str:
         data = unicode(data, 'utf-8')
     environment = jinja2.Environment(trim_blocks=True, undefined=StrictUndefined, extensions=_get_extensions())
+
+    if filters:
         environment.filters.update(_get_filters())
 
     if '_original_file' in vars:

library/system/ping

@@ -48,7 +48,9 @@ def main():
         result['ping'] = module.params['data']
     module.exit_json(**result)
 
-# this is magic, see lib/ansible/module_common.py
-#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+### boilerplate: import common module snippets here
+from ansible.module_utils.basic import *
+
+### invoke the module
 main()
 

library/system/service

@@ -1205,7 +1205,11 @@ def main():
 
     module.exit_json(**result)
 
-# this is magic, see lib/ansible/module_common.py
-#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+### boilerplate: import common module snippets here
+from ansible.module_utils.basic import *
 
+### invoke the module
 main()
+
+
+