Adding v2 task/block iterator and some reorganizing

v2/ansible/executor/HostPlaybookIterator.py

-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-# Make coding more python3-ish
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-class HostPlaybookIterator:
-
-   def __init__(self, host, playbook):
-       pass
-
-   def get_next_task(self):
-       assert False
-
-   def is_blocked(self):
-       # depending on strategy, either 
-       # ‘linear’ -- all prev tasks must be completed for all hosts
-       # ‘free’ -- this host doesn’t have any more work to do
-       assert False
-
-

v2/ansible/executor/playbook_iterator.py

+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+class PlaybookState:
+
+   '''
+   A helper class, which keeps track of the task iteration
+   state for a given playbook. This is used in the PlaybookIterator
+   class on a per-host basis.
+   '''
+   def __init__(self, parent_iterator):
+       self._parent_iterator = parent_iterator
+       self._cur_play        = 0
+       self._task_list       = None
+       self._cur_task_pos    = 0
+
+   def next(self):
+       '''
+       Determines and returns the next available task from the playbook,
+       advancing through the list of plays as it goes.
+       '''
+
+       while True:
+           # when we hit the end of the playbook entries list, we return
+           # None to indicate we're there
+           if self._cur_play > len(self._parent_iterator._playbook._entries) - 1:
+               return None
+
+           # initialize the task list by calling the .compile() method
+           # on the play, which will call compile() for all child objects
+           if self._task_list is None:
+               self._task_list = self._parent_iterator._playbook._entries[self._cur_play].compile()
+
+           # if we've hit the end of this plays task list, move on to the next
+           # and reset the position values for the next iteration
+           if self._cur_task_pos > len(self._task_list) - 1:
+               self._cur_play += 1
+               self._task_list = None
+               self._cur_task_pos = 0
+               continue
+           else:
+               # FIXME: do tag/conditional evaluation here and advance
+               #        the task position if it should be skipped without
+               #        returning a task
+               task = self._task_list[self._cur_task_pos]
+               self._cur_task_pos += 1
+
+               # Skip the task if it is the member of a role which has already
+               # been run, unless the role allows multiple executions
+               if task._role:
+                   # FIXME: this should all be done via member functions
+                   #        instead of direct access to internal variables
+                   if task._role.has_run() and not task._role._metadata._allow_duplicates:
+                       continue
+
+               return task
+
+class PlaybookIterator:
+
+   '''
+   The main iterator class, which keeps the state of the playbook
+   on a per-host basis using the above PlaybookState class.
+   '''
+
+   def __init__(self, inventory, log_manager, playbook):
+       self._playbook     = playbook
+       self._log_manager  = log_manager
+       self._host_entries = dict()
+
+       # build the per-host dictionary of playbook states
+       for host in inventory.get_hosts():
+           self._host_entries[host.get_name()] = PlaybookState(parent_iterator=self)
+
+   def get_next_task_for_host(self, host):
+       ''' fetch the next task for the given host '''
+       if host.get_name() not in self._host_entries:
+           raise AnsibleError("invalid host specified for playbook iteration")
+
+       return self._host_entries[host.get_name()].next()

v2/ansible/parsing/yaml/__init__.py

@@ -148,6 +148,10 @@ class DataLoader():
 
         raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)
 
+    def get_basedir(self):
+        ''' returns the current basedir '''
+        return self._basedir
+
     def set_basedir(self, basedir):
         ''' sets the base directory, used to find files when a relative path is given '''
 

v2/ansible/playbook/__init__.py

@@ -57,6 +57,9 @@ class Playbook:
         basedir = os.path.dirname(file_name)
         self._loader.set_basedir(basedir)
 
+        # also add the basedir to the list of module directories
+        push_basedir(basedir)
+
         ds = self._loader.load_from_file(file_name)
         if not isinstance(ds, list):
             raise AnsibleParserError("playbooks must be a list of plays", obj=ds)
@@ -75,4 +78,5 @@ class Playbook:
 
             self._entries.append(entry_obj)
 
-
+    def get_entries(self):
+        return self._entries[:]

v2/ansible/playbook/block.py

@@ -22,6 +22,7 @@ __metaclass__ = type
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.helpers import load_list_of_tasks
+from ansible.playbook.task_include import TaskInclude
 
 class Block(Base):
 
@@ -35,8 +36,10 @@ class Block(Base):
     # similar to the 'else' clause for exceptions
     #_otherwise = FieldAttribute(isa='list')
 
-    def __init__(self, role=None):
-        self.role = role
+    def __init__(self, parent_block=None, role=None, task_include=None):
+        self._parent_block = parent_block
+        self._role = role
+        self._task_include = task_include
         super(Block, self).__init__()
 
     def get_variables(self):
@@ -45,8 +48,8 @@ class Block(Base):
         return dict()
 
     @staticmethod
-    def load(data, role=None, loader=None):
-        b = Block(role=role)
+    def load(data, parent_block=None, role=None, task_include=None, loader=None):
+        b = Block(parent_block=parent_block, role=role, task_include=task_include)
         return b.load_data(data, loader=loader)
 
     def munge(self, ds):
@@ -79,3 +82,14 @@ class Block(Base):
     #def _load_otherwise(self, attr, ds):
     #    return self._load_list_of_tasks(ds, block=self, loader=self._loader)
 
+    def compile(self):
+        '''
+        Returns the task list for this object
+        '''
+
+        task_list = []
+        for task in self.block:
+            # FIXME: evaulate task tags/conditionals here
+            task_list.extend(task.compile())
+
+        return task_list

v2/ansible/playbook/helpers.py

@@ -15,11 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+
+import os
+
 from types import NoneType
 
 from ansible.errors import AnsibleParserError
+from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
 
-def load_list_of_blocks(ds, role=None, loader=None):
+
+def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, loader=None):
     '''
     Given a list of mixed task/block data (parsed from YAML),
     return a list of Block() objects, where implicit blocks
@@ -34,7 +39,7 @@ def load_list_of_blocks(ds, role=None, loader=None):
     block_list = []
     if ds:
         for block in ds:
-            b = Block.load(block, role=role, loader=loader)
+            b = Block.load(block, parent_block=parent_block, role=role, task_include=task_include, loader=loader)
             block_list.append(b)
 
     return block_list
@@ -58,7 +63,17 @@ def load_list_of_tasks(ds, block=None, role=None, task_include=None, loader=None
             raise AnsibleParserError("task/handler entries must be dictionaries (got a %s)" % type(task), obj=ds)
 
         if 'include' in task:
+            cur_basedir = None
+            if isinstance(task, AnsibleBaseYAMLObject) and loader:
+                pos_info = task.get_position_info()
+                new_basedir = os.path.dirname(pos_info[0])
+                cur_basedir = loader.get_basedir()
+                loader.set_basedir(new_basedir)
+
             t = TaskInclude.load(task, block=block, role=role, task_include=task_include, loader=loader)
+
+            if cur_basedir and loader:
+                loader.set_basedir(cur_basedir)
         else:
             t = Task.load(task, block=block, role=role, task_include=task_include, loader=loader)
 
@@ -85,3 +100,15 @@ def load_list_of_roles(ds, loader=None):
 
     return roles
 
+def compile_block_list(block_list):
+    '''
+    Given a list of blocks, compile them into a flat list of tasks
+    '''
+
+    task_list = []
+
+    for block in block_list:
+        task_list.extend(block.compile())
+
+    return task_list
+

v2/ansible/playbook/play.py

@@ -25,7 +25,8 @@ from ansible.parsing.yaml import DataLoader
 
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.base import Base
-from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles
+from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles, compile_block_list
+from ansible.playbook.role import Role
 
 
 __all__ = ['Play']
@@ -155,3 +156,41 @@ class Play(Base):
         return load_list_of_roles(ds, loader=self._loader)
 
     # FIXME: post_validation needs to ensure that su/sudo are not both set
+
+    def _compile_roles(self):
+        '''
+        Handles the role compilation step, returning a flat list of tasks
+        with the lowest level dependencies first. For example, if a role R
+        has a dependency D1, which also has a dependency D2, the tasks from
+        D2 are merged first, followed by D1, and lastly by the tasks from
+        the parent role R last. This is done for all roles in the Play.
+        '''
+
+        task_list = []
+
+        if len(self.roles) > 0:
+            for ri in self.roles:
+                # The internal list of roles are actualy RoleInclude objects,
+                # so we load the role from that now
+                role = Role.load(ri)
+
+                # FIXME: evauluate conditional of roles here?
+                task_list.extend(role.compile())
+
+        return task_list
+
+    def compile(self):
+        '''
+        Compiles and returns the task list for this play, compiled from the
+        roles (which are themselves compiled recursively) and/or the list of
+        tasks specified in the play.
+        '''
+
+        task_list = []
+
+        task_list.extend(compile_block_list(self.pre_tasks))
+        task_list.extend(self._compile_roles())
+        task_list.extend(compile_block_list(self.tasks))
+        task_list.extend(compile_block_list(self.post_tasks))
+
+        return task_list

v2/ansible/playbook/role/__init__.py

@@ -30,7 +30,7 @@ from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.parsing.yaml import DataLoader
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
-from ansible.playbook.helpers import load_list_of_blocks
+from ansible.playbook.helpers import load_list_of_blocks, compile_block_list
 from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role.metadata import RoleMetadata
 
@@ -87,6 +87,10 @@ class Role:
         if parent_role:
             self.add_parent(parent_role)
 
+        # save the current base directory for the loader and set it to the current role path
+        cur_basedir = self._loader.get_basedir()
+        self._loader.set_basedir(self._role_path)
+
         # load the role's files, if they exist
         metadata = self._load_role_yaml('meta')
         if metadata:
@@ -110,6 +114,9 @@ class Role:
         if not isinstance(self._default_vars, (dict, NoneType)):
             raise AnsibleParserError("The default/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds)
 
+        # and finally restore the previous base directory
+        self._loader.set_basedir(cur_basedir)
+
     def _load_role_yaml(self, subdir):
         file_path = os.path.join(self._role_path, subdir)
         if self._loader.path_exists(file_path) and self._loader.is_directory(file_path):
@@ -186,3 +193,26 @@ class Role:
 
         return direct_deps + child_deps
 
+    def get_task_blocks(self):
+        return self._task_blocks[:]
+
+    def get_handler_blocks(self):
+        return self._handler_blocks[:]
+
+    def compile(self):
+        '''
+        Returns the task list for this role, which is created by first
+        recursively compiling the tasks for all direct dependencies, and
+        then adding on the tasks for this role.
+        '''
+
+        task_list = []
+
+        deps = self.get_direct_dependencies()
+        for dep in deps:
+            task_list.extend(dep.compile())
+
+        task_list.extend(compile_block_list(self._task_blocks))
+
+        return task_list
+

v2/ansible/playbook/role/definition.py

@@ -124,6 +124,7 @@ class RoleDefinition(Base):
         # FIXME: make the parser smart about list/string entries
         #        in the yaml so the error line/file can be reported
         #        here
+
         raise AnsibleError("the role '%s' was not found" % role_name)
 
     def _split_role_params(self, ds):

v2/ansible/playbook/task.py

@@ -60,6 +60,7 @@ class Task(Base):
     _delay                = FieldAttribute(isa='int')
     _delegate_to          = FieldAttribute(isa='string')
     _environment          = FieldAttribute(isa='dict')
+    _failed_when          = FieldAttribute(isa='string')
     _first_available_file = FieldAttribute(isa='list')
     _ignore_errors        = FieldAttribute(isa='bool')
 
@@ -179,3 +180,11 @@ class Task(Base):
 
         return new_ds
 
+    def compile(self):
+        '''
+        For tasks, this is just a dummy method returning an array
+        with 'self' in it, so we don't have to care about task types
+        further up the chain.
+        '''
+
+        return [self]

v2/ansible/playbook/task_include.py

@@ -24,7 +24,7 @@ from ansible.parsing.splitter import split_args, parse_kv
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.base import Base
-from ansible.playbook.helpers import load_list_of_tasks
+from ansible.playbook.helpers import load_list_of_blocks, compile_block_list
 from ansible.plugins import lookup_finder
 
 
@@ -57,11 +57,12 @@ class TaskInclude(Base):
     _when      = FieldAttribute(isa='list', default=[])
 
     def __init__(self, block=None, role=None, task_include=None):
-        self._tasks        = []
         self._block        = block
         self._role         = role
         self._task_include = task_include
 
+        self._task_blocks  = []
+
         super(TaskInclude, self).__init__()
 
     @staticmethod
@@ -136,11 +137,27 @@ class TaskInclude(Base):
 
 
     def _load_include(self, attr, ds):
-        ''' loads the file name specified in the ds and returns a list of tasks '''
+        ''' loads the file name specified in the ds and returns a list of blocks '''
 
         data = self._loader.load_from_file(ds)
         if not isinstance(data, list):
             raise AnsibleParsingError("included task files must contain a list of tasks", obj=ds)
 
-        self._tasks = load_list_of_tasks(data, task_include=self, loader=self._loader)
+        self._task_blocks = load_list_of_blocks(
+                                data,
+                                parent_block=self._block,
+                                task_include=self,
+                                role=self._role,
+                                loader=self._loader
+                            )
         return ds
+
+    def compile(self):
+        '''
+        Returns the task list for the included tasks.
+        '''
+
+        task_list = []
+        task_list.extend(compile_block_list(self._task_blocks))
+        return task_list
+

v2/test/executor/test_playbook_iterator.py

+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.compat.tests import unittest
+from ansible.compat.tests.mock import patch, MagicMock
+
+from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.executor.playbook_iterator import PlaybookIterator
+from ansible.playbook import Playbook
+
+from test.mock.loader import DictDataLoader
+
+class TestPlaybookIterator(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_playbook_iterator(self):
+        fake_loader = DictDataLoader({
+            "test_play.yml": """
+            - hosts: all
+              roles:
+              - test_role
+              pre_tasks:
+              - debug: msg="this is a pre_task"
+              tasks:
+              - debug: msg="this is a regular task"
+              post_tasks:
+              - debug: msg="this is a post_task"
+            """,
+            '/etc/ansible/roles/test_role/tasks/main.yml': """
+            - debug: msg="this is a role task"
+            """,
+        })
+
+        p = Playbook.load('test_play.yml', loader=fake_loader)
+
+        hosts = []
+        for i in range(0, 10):
+            host  = MagicMock()
+            host.get_name.return_value = 'host%02d' % i
+            hosts.append(host)
+
+        inventory = MagicMock()
+        inventory.get_hosts.return_value = hosts
+
+        itr = PlaybookIterator(inventory, None, p)
+        task = itr.get_next_task_for_host(hosts[0])
+        print(task)
+        self.assertIsNotNone(task)
+        task = itr.get_next_task_for_host(hosts[0])
+        print(task)
+        self.assertIsNotNone(task)
+        task = itr.get_next_task_for_host(hosts[0])
+        print(task)
+        self.assertIsNotNone(task)
+        task = itr.get_next_task_for_host(hosts[0])
+        print(task)
+        self.assertIsNotNone(task)
+        task = itr.get_next_task_for_host(hosts[0])
+        print(task)
+        self.assertIsNone(task)

v2/test/playbook/test_block.py

@@ -75,3 +75,9 @@ class TestBlock(unittest.TestCase):
         self.assertEqual(len(b.block), 1)
         assert isinstance(b.block[0], Task)
 
+    def test_block_compile(self):
+        ds = [dict(action='foo')]
+        b = Block.load(ds)
+        tasks = b.compile()
+        self.assertEqual(len(tasks), 1)
+        self.assertIsInstance(tasks[0], Task)

v2/test/playbook/test_play.py

@@ -117,4 +117,16 @@ class TestPlay(unittest.TestCase):
             roles=['foo'],
         ), loader=fake_loader)
 
+        tasks = p.compile()
 
+    def test_play_compile(self):
+        p = Play.load(dict(
+            name="test play",
+            hosts=['foo'],
+            gather_facts=False,
+            tasks=[dict(action='shell echo "hello world"')],
+        ))
+
+        tasks = p.compile()
+        self.assertEqual(len(tasks), 1)
+        self.assertIsInstance(tasks[0], Task)

v2/test/playbook/test_playbook.py

@@ -45,6 +45,7 @@ class TestPlaybook(unittest.TestCase):
             """,
         })
         p = Playbook.load("test_file.yml", loader=fake_loader)
+        entries = p.get_entries()
 
     def test_bad_playbook_files(self):
         fake_loader = DictDataLoader({

v2/test/playbook/test_task_include.py

@@ -45,6 +45,7 @@ class TestTaskInclude(unittest.TestCase):
 
     def test_basic_task_include(self):
         ti = TaskInclude.load(AnsibleMapping(include='foo.yml'), loader=self._fake_loader)
+        tasks = ti.compile()
 
     def test_task_include_with_loop(self):
         ti = TaskInclude.load(AnsibleMapping(include='foo.yml', with_items=['a', 'b', 'c']), loader=self._fake_loader)