abbey.py - python script for creating AMIs in the VPC environment

abbey.py will launch an instance into a VPC, run ansible on it
and monitor it for updates using an SQS queue.  At the end of
the ansible run it will create an AMI and terminate the instance.

playbooks/callback_plugins/sqs.py

+# Copyright 2013 John Jarvis <john@jarv.org>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import sys
+import time
+import json
+try:
+    import boto.sqs
+    from boto.exception import NoAuthHandlerFound
+except ImportError:
+    print "Boto is required for the sqs_notify callback plugin"
+    raise
+
+
+class CallbackModule(object):
+    """
+    This Ansible callback plugin sends task events
+    to SQS.
+
+    The following vars must be set in the environment:
+        ANSIBLE_ENABLE_SQS - enables the callback module
+        SQS_REGION - AWS region to connect to
+        SQS_MSG_PREFIX - Additional data that will be put
+                         on the queue (optional)
+
+    The following events are put on the queue
+        - FAILURE events
+        - OK events
+        - TASK events
+        - START events
+    """
+    def __init__(self):
+
+        self.start_time = time.time()
+
+        if 'ANSIBLE_ENABLE_SQS' in os.environ:
+            self.enable_sqs = True
+            if not 'SQS_REGION' in os.environ:
+                print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
+                      'not defined in environment'
+                sys.exit(1)
+            self.region = os.environ['SQS_REGION']
+            try:
+                self.sqs = boto.sqs.connect_to_region(self.region)
+            except NoAuthHandlerFound:
+                print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
+                      'to AWS due invalid credentials'
+                sys.exit(1)
+            if not 'SQS_NAME' in os.environ:
+                print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
+                      'defined in environment'
+                sys.exit(1)
+            self.name = os.environ['SQS_NAME']
+            self.queue = self.sqs.create_queue(self.name)
+            if 'SQS_MSG_PREFIX' in os.environ:
+                self.prefix = os.environ['SQS_MSG_PREFIX']
+            else:
+                self.prefix = ''
+
+            self.last_seen_ts = {}
+        else:
+            self.enable_sqs = False
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        if self.enable_sqs:
+            if not ignore_errors:
+                self._send_queue_message(res, 'FAILURE')
+
+    def runner_on_ok(self, host, res):
+        if self.enable_sqs:
+            # don't send the setup results
+            if res['invocation']['module_name'] != "setup":
+                self._send_queue_message(res, 'OK')
+
+    def playbook_on_task_start(self, name, is_conditional):
+        if self.enable_sqs:
+            self._send_queue_message(name, 'TASK')
+
+    def playbook_on_play_start(self, pattern):
+        if self.enable_sqs:
+            self._send_queue_message(pattern, 'START')
+
+    def playbook_on_stats(self, stats):
+        if self.enable_sqs:
+            d = {}
+            delta = time.time() - self.start_time
+            d['delta'] = delta
+            for s in ['changed', 'failures', 'ok', 'processed', 'skipped']:
+                d[s] = getattr(stats, s)
+            self._send_queue_message(d, 'STATS')
+
+    def _send_queue_message(self, msg, msg_type):
+        if self.enable_sqs:
+            from_start = time.time() - self.start_time
+            payload = {msg_type: msg}
+            payload['TS'] = from_start
+            payload['PREFIX'] = self.prefix
+            # update the last seen timestamp for
+            # the message type
+            self.last_seen_ts[msg_type] = time.time()
+            if msg_type in ['OK', 'FAILURE']:
+                # report the delta between the OK/FAILURE and
+                # last TASK
+                if 'TASK' in self.last_seen_ts:
+                    from_task = \
+                        self.last_seen_ts[msg_type] - self.last_seen_ts['TASK']
+                    payload['delta'] = from_task
+                for output in ['stderr', 'stdout']:
+                    if output in payload[msg_type]:
+                        # only keep the last 1000 characters
+                        # of stderr and stdout
+                        if len(payload[msg_type][output]) > 1000:
+                            payload[msg_type][output] = "(clipping) ... " \
+                                    + payload[msg_type][output][-1000:]
+
+            self.sqs.send_message(self.queue, json.dumps(payload))

playbooks/edx-east/callback_plugins

+../callback_plugins
\ No newline at end of file

playbooks/edx-east/certs.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - certs

playbooks/edx-east/common.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - common

playbooks/edx-east/devpi.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - devpi

playbooks/edx-east/discern.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - discern

playbooks/edx-east/dummy.yml

+---
+# dummy var file
+# This file is needed as a fall through
+# for vars_files
+dummy_var: True

playbooks/edx-east/edxapp.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - edxapp

playbooks/edx-east/forum.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - forum

playbooks/edx-east/ora.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - ora

playbooks/edx-east/rabbitmq.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: False
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - rabbitmq

playbooks/edx-east/xqueue.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - role: xqueue

playbooks/edx-east/xserver.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - role: xserver

playbooks/library/vpc_lookup

+#!/usr/bin/python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = '''
+---
+module: vpc_lookup
+short_description: returns a list of subnet Ids using tags as criteria
+description:
+     - Returns a list of subnet Ids for a given set of tags that identify one or more VPCs
+version_added: "1.5"
+options:
+  region:
+    description:
+      - The AWS region to use.  Must be specified if ec2_url
+        is not used.  If not specified then the value of the
+        EC2_REGION environment variable, if any, is used.
+    required: false
+    default: null
+    aliases: [ 'aws_region', 'ec2_region' ]
+  aws_secret_key:
+    description:
+      - AWS secret key. If not set then the value of
+        the AWS_SECRET_KEY environment variable is used.
+    required: false
+    default: null
+    aliases: [ 'ec2_secret_key', 'secret_key' ]
+  aws_access_key:
+    description:
+      - AWS access key. If not set then the value of the
+        AWS_ACCESS_KEY environment variable is used.
+    required: false
+    default: null
+    aliases: [ 'ec2_access_key', 'access_key' ]
+  tags:
+    desription:
+      - tags to lookup
+    required: false
+    default: null
+    type: dict
+    aliases: []
+
+requirements: [ "boto" ]
+author: John Jarvis
+'''
+
+EXAMPLES = '''
+# Note: None of these examples set aws_access_key, aws_secret_key, or region.
+# It is assumed that their matching environment variables are set.
+
+# Return all instances that match the tag "Name: foo"
+- local_action:
+    module: vpc_lookup
+    tags:
+        Name: foo
+'''
+
+import sys
+
+AWS_REGIONS = ['ap-northeast-1',
+               'ap-southeast-1',
+               'ap-southeast-2',
+               'eu-west-1',
+               'sa-east-1',
+               'us-east-1',
+               'us-west-1',
+               'us-west-2']
+
+try:
+    from boto.vpc import VPCConnection
+    from boto.vpc import connect_to_region
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+
+def main():
+
+    module=AnsibleModule(
+        argument_spec=dict(
+            region=dict(choices=AWS_REGIONS),
+            aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'],
+                                no_log=True),
+            aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
+            tags=dict(default=None, type='dict'),
+        )
+    )
+
+    tags = module.params.get('tags')
+    aws_secret_key = module.params.get('aws_secret_key')
+    aws_access_key = module.params.get('aws_access_key')
+    region = module.params.get('region')
+
+    # If we have a region specified, connect to its endpoint.
+    if region:
+        try:
+            vpc = connect_to_region(region, aws_access_key_id=aws_access_key,
+                                    aws_secret_access_key=aws_secret_key)
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg=str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+    vpc_conn = VPCConnection()
+    subnet_ids = []
+    for subnet in vpc_conn.get_all_subnets(filters={'tag:' + tag: value
+                                      for tag, value in tags.iteritems()}):
+        subnet_ids.append(subnet.id)
+    vpc_ids = []
+    for vpc in vpc.get_all_vpcs(filters={'tag:' + tag: value
+                                      for tag, value in tags.iteritems()}):
+        vpc_ids.append(vpc.id)
+
+    module.exit_json(changed=False, subnet_ids=subnet_ids, vpc_ids=vpc_ids)
+
+
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+
+main()

playbooks/roles/xqueue/tasks/deploy.yml

@@ -34,7 +34,7 @@
 
 
 # Install the python pre requirements into {{ xqueue_venv_dir }}
-- name : install python pre-requirements
+- name : xqueue | install python pre-requirements
   pip: requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
   sudo_user: "{{ xqueue_user }}"
   notify:
@@ -43,7 +43,7 @@
   - deploy
 
 # Install the python post requirements into {{ xqueue_venv_dir }}
-- name : install python post-requirements
+- name : xqueue | install python post-requirements
   pip: requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
   sudo_user: "{{ xqueue_user }}"
   notify:

requirements.txt

-Fabric==1.5.1
-Jinja2==2.6
+Jinja2==2.7.1
+MarkupSafe==0.18
 PyYAML==3.10
-WebOb==1.2.3
+ansible==1.3.2
 argparse==1.2.1
-beautifulsoup4==4.1.3
 boto==2.10.0
-cloudformation==0.0.0
-decorator==3.4.0
-distribute==0.6.30
-docopt==0.6.1
-dogapi==1.2.3
-ipython==0.13.1
-jenkinsapi==0.1.11
-lxml==3.1beta1
-newrelic==1.10.2.38
-path.py==3.0.1
-pingdom==0.2.0
-pycrypto==2.6
-pyparsing==1.5.6
-pyrelic==0.2.0
-python-dateutil==2.1
-requests==1.1.0
-schema==0.1.1
-simplejson==3.3.0
-simples3==1.0-alpha
-six==1.2.0
--e git+https://github.com/bos/statprof.py.git@a17f7923b102c9039763583be9e377e8422e8f5f#egg=statprof-dev
-ujson==1.30
+distribute==0.6.24
+ecdsa==0.10
+paramiko==1.12.0
+pycrypto==2.6.1
 wsgiref==0.1.2
-ansible==1.3.2

util/vpc-tools/abbey.py

+#!/usr/bin/env python -u
+
+import sys
+from argparse import ArgumentParser
+import time
+import json
+try:
+    import boto.ec2
+    import boto.sqs
+    from boto.vpc import VPCConnection
+    from boto.exception import NoAuthHandlerFound
+    from boto.sqs.message import RawMessage
+except ImportError:
+    print "boto required for script"
+    sys.exit(1)
+
+AMI_TIMEOUT = 600  # time to wait for AMIs to complete
+EC2_RUN_TIMEOUT = 180  # time to wait for ec2 state transition
+EC2_STATUS_TIMEOUT = 300  # time to wait for ec2 system status checks
+NUM_TASKS = 5  # number of tasks for time summary report
+
+
+class Unbuffered:
+    """
+    For unbuffered output, not
+    needed if PYTHONUNBUFFERED is set
+    """
+    def __init__(self, stream):
+        self.stream = stream
+
+    def write(self, data):
+        self.stream.write(data)
+        self.stream.flush()
+
+    def __getattr__(self, attr):
+        return getattr(self.stream, attr)
+
+sys.stdout = Unbuffered(sys.stdout)
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument('--noop', action='store_true',
+                        help="don't actually run the cmds",
+                        default=False)
+    parser.add_argument('--secure-vars', required=False,
+                        metavar="SECURE_VAR_FILE",
+                        help="path to secure-vars, defaults to "
+                        "../../../configuration-secure/ansible/"
+                        "vars/DEPLOYMENT/ENVIRONMENT.yml")
+    parser.add_argument('--stack-name',
+                        help="defaults to DEPLOYMENT-ENVIRONMENT",
+                        metavar="STACK_NAME",
+                        required=False)
+    parser.add_argument('-p', '--play',
+                        help='play name without the yml extension',
+                        metavar="PLAY", required=True)
+    parser.add_argument('-d', '--deployment', metavar="DEPLOYMENT",
+                        required=True)
+    parser.add_argument('-e', '--environment', metavar="ENVIRONMENT",
+                        required=True)
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help="turn on verbosity")
+    parser.add_argument('--no-cleanup', action='store_true',
+                        help="don't cleanup on failures")
+    parser.add_argument('--vars', metavar="EXTRA_VAR_FILE",
+                        help="path to extra var file", required=False)
+    parser.add_argument('-a', '--application', required=False,
+                        help="Application for subnet, defaults to admin",
+                        default="admin")
+    parser.add_argument('--configuration-version', required=False,
+                        help="configuration repo version",
+                        default="master")
+    parser.add_argument('--configuration-secure-version', required=False,
+                        help="configuration-secure repo version",
+                        default="master")
+    parser.add_argument('-j', '--jenkins-build', required=False,
+                        help="jenkins build number to update")
+    parser.add_argument('-b', '--base-ami', required=False,
+                        help="ami to use as a base ami",
+                        default="ami-0568456c")
+    parser.add_argument('-i', '--identity', required=False,
+                        help="path to identity file for pulling "
+                             "down configuration-secure",
+                        default=None)
+    parser.add_argument('-r', '--region', required=False,
+                        default="us-east-1",
+                        help="aws region")
+    parser.add_argument('-k', '--keypair', required=False,
+                        default="deployment",
+                        help="AWS keypair to use for instance")
+    parser.add_argument('-t', '--instance-type', required=False,
+                        default="m1.large",
+                        help="instance type to launch")
+    parser.add_argument("--security-group", required=False,
+                        default="abbey", help="Security group to use")
+    parser.add_argument("--role-name", required=False,
+                        default="abbey",
+                        help="IAM role name to use (must exist)")
+    parser.add_argument("--msg-delay", required=False,
+                        default=5,
+                        help="How long to delay message display from sqs "
+                             "to ensure ordering")
+    return parser.parse_args()
+
+
+def create_instance_args():
+    """
+    Looks up security group, subnet
+    and returns arguments to pass into
+    ec2.run_instances() including
+    user data
+    """
+
+    security_group_id = None
+
+    grp_details = ec2.get_all_security_groups()
+
+    for grp in grp_details:
+        if grp.name == args.security_group:
+            security_group_id = grp.id
+            break
+    if not security_group_id:
+        print "Unable to lookup id for security group {}".format(
+            args.security_group)
+        sys.exit(1)
+
+    vpc = VPCConnection()
+    subnet = vpc.get_all_subnets(
+        filters={
+            'tag:aws:cloudformation:stack-name': stack_name,
+            'tag:Application': args.application}
+    )
+    if len(subnet) != 1:
+        sys.stderr.write("ERROR: Expected 1 admin subnet, got {}\n".format(
+            len(subnet)))
+        sys.exit(1)
+    subnet_id = subnet[0].id
+
+    if args.identity:
+        config_secure = 'true'
+        with open(args.identity) as f:
+            identity_file = f.read()
+    else:
+        config_secure = 'false'
+        identity_file = "dummy"
+
+    user_data = """#!/bin/bash
+set -x
+set -e
+exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+base_dir="/var/tmp/edx-cfg"
+extra_vars="$base_dir/extra-vars-$$.yml"
+secure_identity="$base_dir/secure-identity"
+git_ssh="$base_dir/git_ssh.sh"
+configuration_version="{configuration_version}"
+configuration_secure_version="{configuration_secure_version}"
+environment="{environment}"
+deployment="{deployment}"
+play="{play}"
+config_secure={config_secure}
+secure_vars_file="$base_dir/configuration-secure\\
+/ansible/vars/$environment/$environment-$deployment.yml"
+instance_id=\\
+$(curl http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null)
+instance_ip=\\
+$(curl http://169.254.169.254/latest/meta-data/local-ipv4 2>/dev/null)
+instance_type=\\
+$(curl http://169.254.169.254/latest/meta-data/instance-type 2>/dev/null)
+playbook_dir="$base_dir/configuration/playbooks/edx-east"
+git_repo="https://github.com/edx/configuration"
+git_repo_secure="git@github.com:edx/configuration-secure"
+
+if $config_secure; then
+    git_cmd="env GIT_SSH=$git_ssh git"
+else
+    git_cmd="git"
+fi
+
+ANSIBLE_ENABLE_SQS=true
+SQS_NAME={queue_name}
+SQS_REGION=us-east-1
+SQS_MSG_PREFIX="[ $instance_id $instance_ip $environment-$deployment $play ]"
+PYTHONUNBUFFERED=1
+
+# environment for ansible
+export ANSIBLE_ENABLE_SQS SQS_NAME SQS_REGION SQS_MSG_PREFIX PYTHONUNBUFFERED
+
+if [[ ! -x /usr/bin/git || ! -x /usr/bin/pip ]]; then
+    echo "Installing pkg dependencies"
+    /usr/bin/apt-get update
+    /usr/bin/apt-get install -y git python-pip python-apt \\
+        git-core build-essential python-dev libxml2-dev \\
+        libxslt-dev curl --force-yes
+fi
+
+
+rm -rf $base_dir
+mkdir -p $base_dir
+cd $base_dir
+
+cat << EOF > $git_ssh
+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i "$secure_identity" "\$@"
+EOF
+
+chmod 755 $git_ssh
+
+if $config_secure; then
+    cat << EOF > $secure_identity
+{identity_file}
+EOF
+fi
+
+cat << EOF >> $extra_vars
+{extra_vars_yml}
+secure_vars: $secure_vars_file
+EOF
+
+chmod 400 $secure_identity
+
+$git_cmd clone -b $configuration_version $git_repo
+
+if $config_secure; then
+    $git_cmd clone -b $configuration_secure_version \\
+        $git_repo_secure
+fi
+
+cd $base_dir/configuration
+sudo pip install -r requirements.txt
+
+cd $playbook_dir
+
+ansible-playbook -vvvv -c local -i "localhost," $play.yml -e@$extra_vars
+
+rm -rf $base_dir
+
+    """.format(
+                configuration_version=args.configuration_version,
+                configuration_secure_version=args.configuration_secure_version,
+                environment=args.environment,
+                deployment=args.deployment,
+                play=args.play,
+                config_secure=config_secure,
+                identity_file=identity_file,
+                queue_name=run_id,
+                extra_vars_yml=extra_vars_yml)
+
+    ec2_args = {
+        'security_group_ids': [security_group_id],
+        'subnet_id': subnet_id,
+        'key_name': args.keypair,
+        'image_id': args.base_ami,
+        'instance_type': args.instance_type,
+        'instance_profile_name': args.role_name,
+        'user_data': user_data,
+    }
+
+    return ec2_args
+
+
+def poll_sqs_ansible():
+    """
+    Prints events to the console and
+    blocks until a final STATS ansible
+    event is read off of SQS.
+
+    SQS does not guarantee FIFO, for that
+    reason there is a buffer that will delay
+    messages before they are printed to the
+    console.
+
+    Returns length of the ansible run.
+    """
+    oldest_msg_ts = 0
+    buf = []
+    task_report = []  # list of tasks for reporting
+    last_task = None
+    while True:
+        messages = []
+        while True:
+            # get all available messages on the queue
+            msgs = sqs_queue.get_messages(attributes='All')
+            if not msgs:
+                break
+            messages.extend(msgs)
+
+        for message in messages:
+            recv_ts = float(
+                message.attributes['ApproximateFirstReceiveTimestamp']) * .001
+            sent_ts = float(message.attributes['SentTimestamp']) * .001
+            try:
+                msg_info = {
+                    'msg': json.loads(message.get_body()),
+                    'sent_ts': sent_ts,
+                    'recv_ts': recv_ts,
+                }
+                buf.append(msg_info)
+            except ValueError as e:
+                print "!!! ERROR !!! unable to parse queue message, " \
+                      "expecting valid json: {} : {}".format(
+                          message.get_body(), e)
+            if not oldest_msg_ts or recv_ts < oldest_msg_ts:
+                oldest_msg_ts = recv_ts
+            sqs_queue.delete_message(message)
+
+        now = int(time.time())
+        if buf:
+            if (now - max([msg['recv_ts'] for msg in buf])) > args.msg_delay:
+                # sort by TS instead of recv_ts
+                # because the sqs timestamp is not as
+                # accurate
+                buf.sort(key=lambda k: k['msg']['TS'])
+                to_disp = buf.pop(0)
+                if 'START' in to_disp['msg']:
+                    print '\n{:0>2.0f}:{:0>5.2f} {} : Starting "{}"'.format(
+                        to_disp['msg']['TS'] / 60,
+                        to_disp['msg']['TS'] % 60,
+                        to_disp['msg']['PREFIX'],
+                        to_disp['msg']['START']),
+
+                elif 'TASK' in to_disp['msg']:
+                    print "\n{:0>2.0f}:{:0>5.2f} {} : {}".format(
+                        to_disp['msg']['TS'] / 60,
+                        to_disp['msg']['TS'] % 60,
+                        to_disp['msg']['PREFIX'],
+                        to_disp['msg']['TASK']),
+                    last_task = to_disp['msg']['TASK']
+                elif 'OK' in to_disp['msg']:
+                    if args.verbose:
+                        print "\n"
+                        for key, value in to_disp['msg']['OK'].iteritems():
+                            print "    {:<15}{}".format(key, value)
+                    else:
+                        if to_disp['msg']['OK']['changed']:
+                            changed = "*OK*"
+                        else:
+                            changed = "OK"
+                        print " {}".format(changed),
+                    task_report.append({
+                        'TASK': last_task,
+                        'INVOCATION': to_disp['msg']['OK']['invocation'],
+                        'DELTA': to_disp['msg']['delta'],
+                    })
+                elif 'FAILURE' in to_disp['msg']:
+                    print " !!!! FAILURE !!!!",
+                    for key, value in to_disp['msg']['FAILURE'].iteritems():
+                        print "    {:<15}{}".format(key, value)
+                    raise Exception("Failed Ansible run")
+                elif 'STATS' in to_disp['msg']:
+                    print "\n{:0>2.0f}:{:0>5.2f} {} : COMPLETE".format(
+                        to_disp['msg']['TS'] / 60,
+                        to_disp['msg']['TS'] % 60,
+                        to_disp['msg']['PREFIX'])
+                    return (to_disp['msg']['TS'], task_report)
+
+        if not messages:
+            # wait 1 second between sqs polls
+            time.sleep(1)
+
+
+def create_ami(instance_id, name, description):
+
+    params = {'instance_id': instance_id,
+              'name': name,
+              'description': description,
+              'no_reboot': True}
+
+    image_id = ec2.create_image(**params)
+
+    for _ in xrange(AMI_TIMEOUT):
+        try:
+            img = ec2.get_image(image_id)
+            if img.state == 'available':
+                break
+            else:
+                time.sleep(1)
+        except boto.exception.EC2ResponseError as e:
+            if e.error_code == 'InvalidAMIID.NotFound':
+                time.sleep(1)
+            else:
+                raise Exception("Unexpected error code: {}".format(
+                    e.error_code))
+            time.sleep(1)
+    else:
+        raise Exception("Timeout waiting for AMI to finish")
+
+    return image_id
+
+
+if __name__ == '__main__':
+
+    args = parse_args()
+
+    run_summary = []
+
+    start_time = time.time()
+
+    if args.vars:
+        with open(args.vars) as f:
+            extra_vars_yml = f.read()
+    else:
+        extra_vars_yml = "---\n"
+
+    if args.secure_vars:
+        secure_vars = args.secure_vars
+    else:
+        secure_vars = "../../../configuration-secure/" \
+                      "ansible/vars/{}/{}.yml".format(
+                      args.deployment, args.environment)
+    if args.stack_name:
+        stack_name = args.stack_name
+    else:
+        stack_name = "{}-{}".format(args.environment, args.deployment)
+
+    try:
+        sqs = boto.sqs.connect_to_region(args.region)
+        ec2 = boto.ec2.connect_to_region(args.region)
+    except NoAuthHandlerFound:
+        print 'You must be able to connect to sqs and ec2 to use this script'
+        sys.exit(1)
+
+    try:
+        sqs_queue = None
+        instance_id = None
+
+        run_id = "abbey-{}-{}-{}".format(
+            args.environment, args.deployment, int(time.time() * 100))
+
+        ec2_args = create_instance_args()
+
+        print "{:<40}".format(
+            "Creating SQS queue and launching instance for {}:".format(run_id))
+        print
+        for k, v in ec2_args.iteritems():
+            if k != 'user_data':
+                print "    {:<25}{}".format(k, v)
+        print
+
+        sqs_queue = sqs.create_queue(run_id)
+        sqs_queue.set_message_class(RawMessage)
+        res = ec2.run_instances(**ec2_args)
+        inst = res.instances[0]
+        instance_id = inst.id
+
+        print "{:<40}".format("Waiting for running status:"),
+        status_start = time.time()
+        for _ in xrange(EC2_RUN_TIMEOUT):
+            res = ec2.get_all_instances(instance_ids=[instance_id])
+            if res[0].instances[0].state == 'running':
+                status_delta = time.time() - status_start
+                run_summary.append(('EC2 Launch', status_delta))
+                print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
+                    status_delta / 60,
+                    status_delta % 60)
+                break
+            else:
+                time.sleep(1)
+        else:
+            raise Exception("Timeout waiting for running status: {} ".format(
+                instance_id))
+
+        print "{:<40}".format("Waiting for system status:"),
+        system_start = time.time()
+        for _ in xrange(EC2_STATUS_TIMEOUT):
+            status = ec2.get_all_instance_status(inst.id)
+            if status[0].system_status.status == u'ok':
+                system_delta = time.time() - system_start
+                run_summary.append(('EC2 Status Checks', system_delta))
+                print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
+                    system_delta / 60,
+                    system_delta % 60)
+                break
+            else:
+                time.sleep(1)
+        else:
+            raise Exception("Timeout waiting for status checks: {} ".format(
+                instance_id))
+        user_start = time.time()
+
+        print
+        print "{:<40}".format(
+            "Waiting for user-data, polling sqs for Ansible events:")
+
+        (ansible_delta, task_report) = poll_sqs_ansible()
+        user_pre_ansible = time.time() - user_start - ansible_delta
+        run_summary.append(('Ansible run', ansible_delta))
+        print
+        print "{} longest Ansible tasks (seconds):".format(NUM_TASKS)
+        for task in sorted(
+                task_report, reverse=True,
+                key=lambda k: k['DELTA'])[:NUM_TASKS]:
+            print "{:0>3.0f} {}".format(task['DELTA'], task['TASK'])
+            print "  - {}".format(task['INVOCATION'])
+        print
+
+        print "{:<40}".format("Creating AMI:"),
+        ami_start = time.time()
+        ami = create_ami(instance_id, run_id, run_id)
+        ami_delta = time.time() - ami_start
+        print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
+            ami_delta / 60,
+            ami_delta % 60)
+        run_summary.append(('AMI Build', ami_delta))
+        total_time = time.time() - start_time
+        all_stages = sum(run[1] for run in run_summary)
+        if total_time - all_stages > 0:
+            run_summary.append(('Other', total_time - all_stages))
+        run_summary.append(('Total', total_time))
+
+        print
+        print "Summary:\n"
+
+        for run in run_summary:
+            print "{:<30} {:0>2.0f}:{:0>5.2f}".format(
+                run[0], run[1] / 60, run[1] % 60)
+        print "AMI: {}".format(ami)
+
+    finally:
+        print
+        if not args.no_cleanup:
+            if sqs_queue:
+                print "Cleaning up - Removing SQS queue - {}".format(run_id)
+                sqs.delete_queue(sqs_queue)
+            if instance_id:
+                print "Cleaning up - Terminating instance ID - {}".format(
+                    instance_id)
+            ec2.terminate_instances(instance_ids=[instance_id])