Allow groups to be passed to runner API to make parse_hosts optional, misc fixes…

Allow groups to be passed to runner API to make parse_hosts optional, misc fixes as a result of refactoring in Runner.
Cleanup in Playbooks module is next.

examples/playbooks/tasks/base.yml

@@ -18,5 +18,5 @@
   action: service name=iptables state=stopped
 
 - name: made up task just to show variables work here
-  action: command /bin/echo release is {{ release }}
+  action: command /bin/echo release is $release
 

lib/ansible/playbook.py

@@ -85,6 +85,9 @@ class PlayBook(object):
         self.basedir = os.path.dirname(playbook)
         self.playbook = self._parse_playbook(playbook)
 
+        self.host_list, self.groups = ansible.runner.Runner.parse_hosts(host_list)
+
+
     def _get_vars(self, play, dirname):
         vars = play.get('vars', {})
         if type(vars) != dict:
@@ -165,7 +168,6 @@ class PlayBook(object):
 
         # FIXME: TODO: use callback to reinstate per-host summary
         # and add corresponding code in /bin/ansible-playbook
-        # print results
         return results
 
     def _prune_failed_hosts(self, host_list):
@@ -285,6 +287,7 @@ class PlayBook(object):
         ''' run a particular module step in a playbook '''
         runner = ansible.runner.Runner(
             pattern=pattern,
+            groups=self.groups,
             module_name=module,
             module_args=args,
             host_list=hosts,
@@ -317,13 +320,6 @@ class PlayBook(object):
         recursively run any subtasks.
         '''
 
-        if host_list is None:
-            # pruned host lists occur when running triggered
-            # actions where not all hosts have changed
-            # though top-level tasks will pass in "None" here
-            host_list = self.host_list
-            (host_list, groups) = ansible.runner.Runner.parse_hosts(host_list)
-
         # do not continue to run tasks on hosts that have had failures
         host_list = self._prune_failed_hosts(host_list)
 
@@ -446,6 +442,8 @@ class PlayBook(object):
         #        in it, which could happen on uncontacted hosts.
 
         if vars_files is not None:
+            if type(vars_files) != list:
+                raise errors.AnsibleError("vars_files must be a list")
             self.callbacks.on_setup_secondary()
             for host in host_list:
                 cache_vars = SETUP_CACHE.get(host,{})
@@ -492,6 +490,7 @@ class PlayBook(object):
         # push any variables down to the system
         setup_results = ansible.runner.Runner(
             pattern=pattern,
+            groups=self.groups,
             module_name='setup',
             module_args=push_var_str,
             host_list=self.host_list,
@@ -540,22 +539,20 @@ class PlayBook(object):
         handlers   = pg.get('handlers', [])
         user       = pg.get('user', C.DEFAULT_REMOTE_USER)
 
-        self.host_list, groups = ansible.runner.Runner.parse_hosts(self.host_list)
-
         self.callbacks.on_play_start(pattern)
 
         # push any variables down to the system # and get facts/ohai/other data back up
-        host_list = self._do_setup_step(pattern, vars, user, self.host_list, None)
+        self.host_list = self._do_setup_step(pattern, vars, user, self.host_list, None)
          
         # now with that data, handle contentional variable file imports!
         if len(vars_files) > 0:
-            host_list = self._do_setup_step(pattern, vars, user, host_list, vars_files)
+            self.host_list = self._do_setup_step(pattern, vars, user, self.host_list, vars_files)
 
         # run all the top level tasks, these get run on every node
         for task in tasks:
             self._run_task(
                 pattern=pattern,
-                host_list=host_list,
+                host_list=self.host_list,
                 task=task, 
                 handlers=handlers,
                 remote_user=user

lib/ansible/runner.py

@@ -64,15 +64,21 @@ class Runner(object):
         forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, pattern=C.DEFAULT_PATTERN,
         remote_user=C.DEFAULT_REMOTE_USER, remote_pass=C.DEFAULT_REMOTE_PASS,
         background=0, basedir=None, setup_cache=None, transport='paramiko',
-        conditional='True', verbose=False):
+        conditional='True', groups={}, verbose=False):
     
         if setup_cache is None:
             setup_cache = {}
         if basedir is None: 
             basedir = os.getcwd()
+
         self.generated_jid = str(random.randint(0, 999999999999))
         self.connector = ansible.connection.Connection(self, transport)
-        self.host_list, self.groups = self.parse_hosts(host_list)
+
+        if type(host_list) == str:
+            self.host_list, self.groups = self.parse_hosts(host_list)
+        else:
+            self.host_list = host_list
+            self.groups    = groups
 
         self.setup_cache = setup_cache
         self.conditional = conditional
@@ -95,31 +101,37 @@ class Runner(object):
     # *****************************************************
 
     @classmethod
-    def parse_hosts_from_regular_file(cls, host_list, results, groups):
+    def parse_hosts_from_regular_file(cls, host_list):
         ''' parse a textual host file '''
 
+        results = []
+        groups = dict(ungrouped=[])
         lines = file(host_list).read().split("\n")
         group_name = 'ungrouped'
         for item in lines:
             item = item.lstrip().rstrip()
             if item.startswith("#"):
                 # ignore commented out lines
-                continue
-            if item.startswith("["):
+                pass
+            elif item.startswith("["):
                 # looks like a group
                 group_name = item.replace("[","").replace("]","").lstrip().rstrip()
                 groups[group_name] = []
             elif item != "":
                 # looks like a regular host
                 groups[group_name].append(item)
-                results.append(item)
+                if not item in results:
+                    results.append(item)
+        return (results, groups)
 
     # *****************************************************
 
     @classmethod
-    def parse_hosts_from_script(cls, host_list, results, groups):
+    def parse_hosts_from_script(cls, host_list):
         ''' evaluate a script that returns list of hosts by groups '''
 
+        results = []
+        groups = dict(ungrouped=[])
         host_list = os.path.abspath(host_list)
         cls._external_variable_script = host_list
         cmd = subprocess.Popen([host_list], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
@@ -132,6 +144,7 @@ class Runner(object):
             for host in hostlist:
                 if host not in results:
                     results.append(host)
+        return (results, groups)
 
     # *****************************************************
 
@@ -140,23 +153,21 @@ class Runner(object):
         ''' parse the host inventory file, returns (hosts, groups) '''
 
         if type(host_list) == list:
-            return (host_list, {})
+            raise Exception("function can only be called on inventory files")
 
         host_list = os.path.expanduser(host_list)
         if not os.path.exists(host_list):
             raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list)
 
-        results    = []
-        groups     = dict(ungrouped=[])
+        rc = None
         if not os.access(host_list, os.X_OK):
-            Runner.parse_hosts_from_regular_file(host_list, results, groups)
+            return Runner.parse_hosts_from_regular_file(host_list)
         else:
-            Runner.parse_hosts_from_script(host_list, results, groups)
-        return (results, groups)
+            return Runner.parse_hosts_from_script(host_list)
 
     # *****************************************************
 
-    def _matches(self, host_name, pattern=None):
+    def _matches(self, host_name, pattern):
         ''' returns if a hostname is matched by the pattern '''
 
         # a pattern is in fnmatch format but more than one pattern
@@ -168,7 +179,9 @@ class Runner(object):
         pattern = pattern.replace(";",":")
         subpatterns = pattern.split(":")
         for subpattern in subpatterns:
-            if subpattern == 'all' or fnmatch.fnmatch(host_name, subpattern):
+            if subpattern == 'all':
+                return True
+            if fnmatch.fnmatch(host_name, subpattern):
                 return True
             elif subpattern in self.groups:
                 if host_name in self.groups[subpattern]:
@@ -539,7 +552,8 @@ class Runner(object):
     def _match_hosts(self, pattern):
         ''' return all matched hosts fitting a pattern '''
 
-        return [ h for h in self.host_list if self._matches(h, pattern) ]
+        rc = [ h for h in self.host_list if self._matches(h, pattern) ]
+        return rc
 
     # *****************************************************
 
@@ -601,7 +615,7 @@ class Runner(object):
         hosts = self._match_hosts(self.pattern)
         if len(hosts) == 0:
             return dict(contacted={}, dark={})
-
+ 
         hosts = [ (self,x) for x in hosts ]
         if self.forks > 1:
             results = self._parallel_exec(hosts)

lib/ansible/utils.py

@@ -257,7 +257,7 @@ def varReplace(raw, vars):
         # Determine replacement value (if unknown variable then preserve
         # original)
         varname = m.group(1).lower()
-        replacement = vars.get(varname, m.group())
+        replacement = str(vars.get(varname, m.group()))
 
         start, end = m.span()
         done.append(raw[:start])    # Keep stuff leading up to token
@@ -268,7 +268,7 @@ def varReplace(raw, vars):
 
 def template(text, vars):
     ''' run a text buffer through the templating engine '''
-    text = varReplace(text, vars)
+    text = varReplace(str(text), vars)
     template = jinja2.Template(text)
     return template.render(vars)