Performance improvements

* Speed up serialization of hostvars by simply using the internal
  dictionary used for cached lookups
* Use blocking gets/puts on queues instead of spin locking
* Merge sequential implicitly created blocks

lib/ansible/executor/process/result.py

@@ -72,9 +72,9 @@ class ResultProcess(multiprocessing.Process):
                 self._cur_worker = 0
 
             try:
-                if not rslt_q.empty():
+                if rslt_q.qsize() > 0:
                     debug("worker %d has data to read" % self._cur_worker)
-                    result = rslt_q.get(block=False)
+                    result = rslt_q.get()
                     debug("got a result from worker %d: %s" % (self._cur_worker, result))
                     break
             except queue.Empty:
@@ -102,7 +102,7 @@ class ResultProcess(multiprocessing.Process):
             try:
                 result = self._read_worker_result()
                 if result is None:
-                    time.sleep(0.1)
+                    time.sleep(0.01)
                     continue
 
                 # if this task is registering a result, do it now

lib/ansible/executor/process/worker.py

@@ -92,9 +92,8 @@ class WorkerProcess(multiprocessing.Process):
         while True:
             task = None
             try:
-                if not self._main_q.empty():
+                (host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get()
                 debug("there's work to be done!")
-                    (host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get(block=False)
                 debug("got a task/handler to work on: %s" % task)
 
                 # because the task queue manager starts workers (forks) before the
@@ -121,12 +120,9 @@ class WorkerProcess(multiprocessing.Process):
 
                 # put the result on the result queue
                 debug("sending task result")
-                    self._rslt_q.put(task_result, block=False)
+                self._rslt_q.put(task_result)
                 debug("done sending task result")
 
-                else:
-                    time.sleep(0.1)
-
             except queue.Empty:
                 pass
             except (IOError, EOFError, KeyboardInterrupt):

lib/ansible/playbook/block.py

@@ -37,12 +37,13 @@ class Block(Base, Become, Conditional, Taggable):
     # similar to the 'else' clause for exceptions
     #_otherwise = FieldAttribute(isa='list')
 
-    def __init__(self, play=None, parent_block=None, role=None, task_include=None, use_handlers=False):
+    def __init__(self, play=None, parent_block=None, role=None, task_include=None, use_handlers=False, implicit=False):
         self._play         = play
         self._role         = role
         self._task_include = task_include
         self._parent_block = parent_block
         self._use_handlers = use_handlers
+        self._implicit     = implicit
         self._dep_chain    = []
 
         super(Block, self).__init__()
@@ -66,22 +67,27 @@ class Block(Base, Become, Conditional, Taggable):
 
     @staticmethod
     def load(data, play=None, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
-        b = Block(play=play, parent_block=parent_block, role=role, task_include=task_include, use_handlers=use_handlers)
+        implicit = not Block.is_block(data)
+        b = Block(play=play, parent_block=parent_block, role=role, task_include=task_include, use_handlers=use_handlers, implicit=implicit)
         return b.load_data(data, variable_manager=variable_manager, loader=loader)
 
-    def preprocess_data(self, ds):
-        '''
-        If a simple task is given, an implicit block for that single task
-        is created, which goes in the main portion of the block
-        '''
-
+    @staticmethod
+    def is_block(ds):
         is_block = False
+        if isinstance(ds, dict):
             for attr in ('block', 'rescue', 'always'):
                 if attr in ds:
                     is_block = True
                     break
+        return is_block
+
+    def preprocess_data(self, ds):
+        '''
+        If a simple task is given, an implicit block for that single task
+        is created, which goes in the main portion of the block
+        '''
 
-        if not is_block:
+        if not Block.is_block(ds):
             if isinstance(ds, list):
                 return super(Block, self).preprocess_data(dict(block=ds))
             else:

lib/ansible/playbook/helpers.py

@@ -52,6 +52,12 @@ def load_list_of_blocks(ds, play, parent_block=None, role=None, task_include=Non
                 variable_manager=variable_manager,
                 loader=loader
             )
+            # Implicit blocks are created by bare tasks listed in a play withou
+            # an explicit block statement. If we have two implicit blocks in a row,
+            # squash them down to a single block to save processing time later.
+            if b._implicit and len(block_list) > 0 and block_list[-1]._implicit:
+                block_list[-1].block.extend(b.block)
+            else:
                 block_list.append(b)
 
     return block_list

lib/ansible/vars/hostvars.py

@@ -54,8 +54,15 @@ class HostVars(collections.Mapping):
         if item and item is not j2undefined:
             return True
         return False
+
     def __iter__(self):
         raise NotImplementedError('HostVars does not support iteration as hosts are discovered on an as needed basis.')
 
     def __len__(self):
         raise NotImplementedError('HostVars does not support len.  hosts entries are discovered dynamically as needed')
+
+    def __getstate__(self):
+        return self._lookup
+
+    def __setstate__(self, data):
+        self._lookup = data