When playbooks fail, attempt to create an inventory file in the inventory…

When playbooks fail, attempt to create an inventory file in the inventory directory that allows rerunning
of the playbook against only the hosts that failed.

CHANGELOG.md

@@ -13,6 +13,7 @@ Core Features:
 * can set ansible_private_key_file as an inventory variable (similar to ansible_ssh_host, etc)
 * 'when' statement can be affixed to task includes to auto-affix the conditional to each task therein
 * cosmetic: "*****" banners in ansible-playbook output are now constant width
+* attempt to create an inventory file to rerun against failed hosts only, without retrying successful ones
 
 Modules added
 

bin/ansible-playbook

@@ -174,6 +174,7 @@ def main(args):
             print 'Playbook Syntax is fine'
             return 0
             
+        failed_hosts = []
 
         try:
 
@@ -182,6 +183,17 @@ def main(args):
             hosts = sorted(pb.stats.processed.keys())
             print callbacks.banner("PLAY RECAP")
             playbook_cb.on_stats(pb.stats)
+
+            for h in hosts:
+                t = pb.stats.summarize(h)
+                if t['unreachable'] > 0 or t['failures'] > 0:
+                    failed_hosts.append(h)
+
+            if len(failed_hosts) > 0:
+                filename = pb.generate_retry_inventory(failed_hosts)
+                if filename:
+                    print "           to rerun against failed hosts only, use -i %s\n" % filename
+
             for h in hosts:
                 t = pb.stats.summarize(h)
                 print "%s : %s %s %s %s" % (
@@ -190,17 +202,16 @@ def main(args):
                     colorize('changed', t['changed'], 'yellow'),
                     colorize('unreachable', t['unreachable'], 'red'),
                     colorize('failed', t['failures'], 'red'))
-
-            print "\n"
-            for h in hosts:
-                stats = pb.stats.summarize(h)
-                if stats['failures'] != 0 or stats['unreachable'] != 0:
-                    return 2
+ 
+            print ""
+            if len(failed_hosts) > 0:
+                return 2
 
         except errors.AnsibleError, e:
             print >>sys.stderr, "ERROR: %s" % e
             return 1
 
+
     return 0
 
 

lib/ansible/inventory/dir.py

@@ -39,6 +39,10 @@ class InventoryDirectory(object):
         for i in self.names:
             if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"):
                 continue
+            if i.endswith(".retry"):
+                # this file is generated on a failed playbook and should only be
+                # used when run specifically
+                continue
             # These are things inside of an inventory basedir
             if i in ("host_vars", "group_vars", "vars_plugins"):
                 continue

lib/ansible/playbook/__init__.py

@@ -25,6 +25,8 @@ import os
 import shlex
 import collections
 from play import Play
+import StringIO
+import pipes
 
 SETUP_CACHE = collections.defaultdict(dict)
 
@@ -129,6 +131,7 @@ class PlayBook(object):
         vars = {}
         if self.inventory.basedir() is not None:
             vars['inventory_dir'] = self.inventory.basedir()
+        self.filename = playbook
         (self.playbook, self.play_basedirs) = self._load_playbook_from_file(playbook, vars)
 
     # *****************************************************
@@ -415,6 +418,90 @@ class PlayBook(object):
 
     # *****************************************************
 
+
+    def generate_retry_inventory(self, replay_hosts):
+        '''
+        called by /usr/bin/ansible when a playbook run fails. It generates a inventory 
+        that allows re-running on ONLY the failed hosts.  This may duplicate some
+        variable information in group_vars/host_vars but that is ok, and expected.
+        ''' 
+
+        # TODO: move this into an inventory.serialize() method
+
+        buf = StringIO.StringIO()
+
+        buf.write("# dynamically generated inventory file\n")
+        buf.write("# retries previously failed hosts only\n")
+        buf.write("\n")
+
+        inventory = self.inventory
+        basedir = inventory.basedir()
+        filename = ".%s.retry" % os.path.basename(self.filename)
+        filename = os.path.join(basedir, filename)
+
+        def _simple_kv_vars(host_vars):
+            buf = ""
+            for (k, v) in host_vars.items():
+                if type(v) not in [ list, dict ]:
+                   if isinstance(v,basestring):
+                       buf = buf + " %s=%s" % (k, pipes.quote(v))
+                   else:
+                       buf = buf + " %s=%s" % (k, v)
+            return buf
+ 
+        # for all group names
+        for gname in inventory.groups_list():
+
+            # write the group name
+            group = inventory.get_group(gname)
+            group_vars = inventory.get_group_variables(gname)
+
+            # but only contain hosts that we want to replay
+            hostz = [ host.name for host in group.hosts ]
+            hostz = [ hname for hname in hostz if hname in replay_hosts ] 
+            if len(hostz):
+                buf.write("[%s]\n" % group.name)
+                for hostname in hostz:
+                    host = inventory.get_host(hostname)
+                    host_vars = host.vars
+                    hostname_vars = _simple_kv_vars(host_vars)
+                    buf.write("%s %s\n" % (hostname, hostname_vars))
+                buf.write("\n")
+
+            # write out any child groups if present
+            if len(group.child_groups) and group.name not in [ 'all', 'ungrouped' ]:
+                buf.write("\n")
+                buf.write("[%s:children]\n" % gname)
+                for child_group in group.child_groups:
+                    buf.write("%s\n" % child_group.name)
+                buf.write("\n")
+
+            # we do NOT write out group variables because they will have already
+            # been blended with the host
+
+            if len(group_vars.keys()) > 0 and group.name not in [ 'all', 'ungrouped' ]:
+                buf.write("[%s:vars]\n" % gname) 
+                for (k,v) in group_vars.items():
+                    if type(v) not in [list,dict]:
+                        if isinstance(type(k), basestring):
+                            buf.write("%s='%s'\n" % (k,v))  
+                        else:  
+                            buf.write("%s=%s\n" % (k,v))
+                buf.write("\n")
+
+        # if file isn't writeable, don't do anything.
+        # TODO: allow a environment variable to pick a different destination for this file
+
+        try:
+            fd = open(filename, 'w')
+            fd.write(buf.getvalue())
+            fd.close()
+            return filename
+        except Exception, e:
+            return None
+
+    # *****************************************************
+
     def _run_play(self, play):
         ''' run a list of tasks for a given pattern, in order '''