Splitting SETUP_CACHE into two caches, one for host vars and one for setup facts

docsite/rst/playbooks_variables.rst

@@ -837,8 +837,11 @@ If multiple variables of the same name are defined in different places, they win
     * -e variables always win
     * then comes "most everything else"
     * then comes variables defined in inventory
+    * then comes facts discovered about a system
     * then "role defaults", which are the most "defaulty" and lose in priority to everything.
 
+.. note:: In versions prior to 1.5.4, facts discovered about a system were in the "most everything else" category above.
+
 That seems a little theoretical.  Let's show some examples and where you would choose to put what based on the kind of 
 control you might want over values.
 

lib/ansible/playbook/__init__.py

@@ -29,7 +29,11 @@ from play import Play
 import StringIO
 import pipes
 
+# the setup cache stores all variables about a host
+# gathered during the setup step, while the vars cache
+# holds all other variables about a host
 SETUP_CACHE = collections.defaultdict(dict)
+VARS_CACHE  = collections.defaultdict(dict)
 
 class PlayBook(object):
     '''
@@ -95,6 +99,7 @@ class PlayBook(object):
         """
 
         self.SETUP_CACHE = SETUP_CACHE
+        self.VARS_CACHE  = VARS_CACHE
 
         arguments = []
         if playbook is None:
@@ -335,6 +340,7 @@ class PlayBook(object):
             default_vars=task.default_vars,
             private_key_file=self.private_key_file,
             setup_cache=self.SETUP_CACHE,
+            vars_cache=self.VARS_CACHE,
             basedir=task.play.basedir,
             conditional=task.when,
             callbacks=self.runner_callbacks,
@@ -426,8 +432,6 @@ class PlayBook(object):
             else:
                 facts = result.get('ansible_facts', {})
                 self.SETUP_CACHE[host].update(facts)
-            # extra vars need to always trump - so update  again following the facts
-            self.SETUP_CACHE[host].update(self.extra_vars)
             if task.register:
                 if 'stdout' in result and 'stdout_lines' not in result:
                     result['stdout_lines'] = result['stdout'].splitlines()
@@ -500,6 +504,7 @@ class PlayBook(object):
             remote_port=play.remote_port,
             private_key_file=self.private_key_file,
             setup_cache=self.SETUP_CACHE,
+            vars_cache=self.VARS_CACHE,
             callbacks=self.runner_callbacks,
             sudo=play.sudo,
             sudo_user=play.sudo_user,

lib/ansible/playbook/play.py

@@ -700,7 +700,7 @@ class Play(object):
         if host is not None:
             inject = {}
             inject.update(self.playbook.inventory.get_variables(host, vault_password=vault_password))
-            inject.update(self.playbook.SETUP_CACHE[host])
+            inject.update(self.playbook.VARS_CACHE[host])
 
         for filename in self.vars_files:
 
@@ -724,8 +724,9 @@ class Play(object):
                         if host is not None:
                             if self._has_vars_in(filename2) and not self._has_vars_in(filename3):
                                 # this filename has variables in it that were fact specific
-                                # so it needs to be loaded into the per host SETUP_CACHE
-                                self.playbook.SETUP_CACHE[host].update(data)
+                                # so it needs to be loaded into the per host VARS_CACHE
+                                data = utils.combine_vars(inject, data)
+                                self.playbook.VARS_CACHE[host].update(data)
                                 self.playbook.callbacks.on_import_for_host(host, filename4)
                         elif not self._has_vars_in(filename4):
                             # found a non-host specific variable, load into vars and NOT
@@ -757,9 +758,14 @@ class Play(object):
                     if host is not None and self._has_vars_in(filename2) and not self._has_vars_in(filename3):
                         # running a host specific pass and has host specific variables
                         # load into setup cache
-                        self.playbook.SETUP_CACHE[host] = utils.combine_vars(
-                            self.playbook.SETUP_CACHE[host], new_vars)
+                        new_vars = utils.combine_vars(inject, new_vars)
+                        self.playbook.VARS_CACHE[host] = utils.combine_vars(
+                            self.playbook.VARS_CACHE[host], new_vars)
                         self.playbook.callbacks.on_import_for_host(host, filename4)
                     elif host is None:
                         # running a non-host specific pass and we can update the global vars instead
                         self.vars = utils.combine_vars(self.vars, new_vars)
+
+        # finally, update the VARS_CACHE for the host, if it is set
+        if host is not None:
+            self.playbook.VARS_CACHE[host].update(self.playbook.extra_vars)

lib/ansible/runner/__init__.py

@@ -86,18 +86,18 @@ def _executor_hook(job_queue, result_queue, new_stdin):
             traceback.print_exc()
 
 class HostVars(dict):
-    ''' A special view of setup_cache that adds values from the inventory when needed. '''
+    ''' A special view of vars_cache that adds values from the inventory when needed. '''
 
-    def __init__(self, setup_cache, inventory):
-        self.setup_cache = setup_cache
+    def __init__(self, vars_cache, inventory):
+        self.vars_cache = vars_cache
         self.inventory = inventory
         self.lookup = dict()
-        self.update(setup_cache)
+        self.update(vars_cache)
 
     def __getitem__(self, host):
         if host not in self.lookup:
             result = self.inventory.get_variables(host)
-            result.update(self.setup_cache.get(host, {}))
+            result.update(self.vars_cache.get(host, {}))
             self.lookup[host] = result
         return self.lookup[host]
 
@@ -123,6 +123,7 @@ class Runner(object):
         background=0,                       # async poll every X seconds, else 0 for non-async
         basedir=None,                       # directory of playbook, if applicable
         setup_cache=None,                   # used to share fact data w/ other tasks
+        vars_cache=None,                    # used to store variables about hosts
         transport=C.DEFAULT_TRANSPORT,      # 'ssh', 'paramiko', 'local'
         conditional='True',                 # run only if this fact expression evals to true
         callbacks=None,                     # used for output
@@ -160,6 +161,7 @@ class Runner(object):
         self.check            = check
         self.diff             = diff
         self.setup_cache      = utils.default(setup_cache, lambda: collections.defaultdict(dict))
+        self.vars_cache       = utils.default(vars_cache, lambda: collections.defaultdict(dict))
         self.basedir          = utils.default(basedir, lambda: os.getcwd())
         self.callbacks        = utils.default(callbacks, lambda: DefaultRunnerCallbacks())
         self.generated_jid    = str(random.randint(0, 999999999999))
@@ -559,13 +561,19 @@ class Runner(object):
             # fireball, local, etc
             port = self.remote_port
 
+        module_vars = template.template(self.basedir, self.module_vars, host_variables)
+
+        # merge the VARS and SETUP caches for this host
+        combined_cache = self.setup_cache.copy()
+        combined_cache.get(host, {}).update(self.vars_cache.get(host, {}))
+
         inject = {}
         inject = utils.combine_vars(inject, self.default_vars)
         inject = utils.combine_vars(inject, host_variables)
-        inject = utils.combine_vars(inject, self.module_vars)
-        inject = utils.combine_vars(inject, self.setup_cache[host])
+        inject = utils.combine_vars(inject, module_vars)
+        inject = utils.combine_vars(inject, combined_cache.get(host, {}))
         inject.setdefault('ansible_ssh_user', self.remote_user)
-        inject['hostvars'] = HostVars(self.setup_cache, self.inventory)
+        inject['hostvars'] = HostVars(combined_cache, self.inventory)
         inject['group_names'] = host_variables.get('group_names', [])
         inject['groups']      = self.inventory.groups_list()
         inject['vars']        = self.module_vars