abbey.py - python script for creating AMIs in the VPC environment

abbey.py will launch an instance into a VPC, run ansible on it
and monitor it for updates using an SQS queue.  At the end of
the ansible run it will create an AMI and terminate the instance.

playbooks/callback_plugins/sqs.py

+# Copyright 2013 John Jarvis <john@jarv.org>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import sys
+import time
+import json
+try:
+    import boto.sqs
+    from boto.exception import NoAuthHandlerFound
+except ImportError:
+    print "Boto is required for the sqs_notify callback plugin"
+    raise
+
+
+class CallbackModule(object):
+    """
+    This Ansible callback plugin sends task events
+    to SQS.
+
+    The following vars must be set in the environment:
+        ANSIBLE_ENABLE_SQS - enables the callback module
+        SQS_REGION - AWS region to connect to
+        SQS_MSG_PREFIX - Additional data that will be put
+                         on the queue (optional)
+
+    The following events are put on the queue
+        - FAILURE events
+        - OK events
+        - TASK events
+        - START events
+    """
+    def __init__(self):
+
+        self.start_time = time.time()
+
+        if 'ANSIBLE_ENABLE_SQS' in os.environ:
+            self.enable_sqs = True
+            if not 'SQS_REGION' in os.environ:
+                print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
+                      'not defined in environment'
+                sys.exit(1)
+            self.region = os.environ['SQS_REGION']
+            try:
+                self.sqs = boto.sqs.connect_to_region(self.region)
+            except NoAuthHandlerFound:
+                print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
+                      'to AWS due invalid credentials'
+                sys.exit(1)
+            if not 'SQS_NAME' in os.environ:
+                print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
+                      'defined in environment'
+                sys.exit(1)
+            self.name = os.environ['SQS_NAME']
+            self.queue = self.sqs.create_queue(self.name)
+            if 'SQS_MSG_PREFIX' in os.environ:
+                self.prefix = os.environ['SQS_MSG_PREFIX']
+            else:
+                self.prefix = ''
+
+            self.last_seen_ts = {}
+        else:
+            self.enable_sqs = False
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        if self.enable_sqs:
+            if not ignore_errors:
+                self._send_queue_message(res, 'FAILURE')
+
+    def runner_on_ok(self, host, res):
+        if self.enable_sqs:
+            # don't send the setup results
+            if res['invocation']['module_name'] != "setup":
+                self._send_queue_message(res, 'OK')
+
+    def playbook_on_task_start(self, name, is_conditional):
+        if self.enable_sqs:
+            self._send_queue_message(name, 'TASK')
+
+    def playbook_on_play_start(self, pattern):
+        if self.enable_sqs:
+            self._send_queue_message(pattern, 'START')
+
+    def playbook_on_stats(self, stats):
+        if self.enable_sqs:
+            d = {}
+            delta = time.time() - self.start_time
+            d['delta'] = delta
+            for s in ['changed', 'failures', 'ok', 'processed', 'skipped']:
+                d[s] = getattr(stats, s)
+            self._send_queue_message(d, 'STATS')
+
+    def _send_queue_message(self, msg, msg_type):
+        if self.enable_sqs:
+            from_start = time.time() - self.start_time
+            payload = {msg_type: msg}
+            payload['TS'] = from_start
+            payload['PREFIX'] = self.prefix
+            # update the last seen timestamp for
+            # the message type
+            self.last_seen_ts[msg_type] = time.time()
+            if msg_type in ['OK', 'FAILURE']:
+                # report the delta between the OK/FAILURE and
+                # last TASK
+                if 'TASK' in self.last_seen_ts:
+                    from_task = \
+                        self.last_seen_ts[msg_type] - self.last_seen_ts['TASK']
+                    payload['delta'] = from_task
+                for output in ['stderr', 'stdout']:
+                    if output in payload[msg_type]:
+                        # only keep the last 1000 characters
+                        # of stderr and stdout
+                        if len(payload[msg_type][output]) > 1000:
+                            payload[msg_type][output] = "(clipping) ... " \
+                                    + payload[msg_type][output][-1000:]
+
+            self.sqs.send_message(self.queue, json.dumps(payload))

playbooks/edx-east/callback_plugins

+../callback_plugins
\ No newline at end of file

playbooks/edx-east/certs.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - certs

playbooks/edx-east/common.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - common

playbooks/edx-east/devpi.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - devpi

playbooks/edx-east/discern.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - discern

playbooks/edx-east/dummy.yml

+---
+# dummy var file
+# This file is needed as a fall through
+# for vars_files
+dummy_var: True

playbooks/edx-east/edxapp.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - edxapp

playbooks/edx-east/forum.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - forum

playbooks/edx-east/ora.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - ora

playbooks/edx-east/rabbitmq.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: False
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - rabbitmq

playbooks/edx-east/xqueue.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - role: xqueue

playbooks/edx-east/xserver.yml

@@ -2,5 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
   roles:
     - role: xserver

playbooks/library/vpc_lookup

+#!/usr/bin/python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = '''
+---
+module: vpc_lookup
+short_description: returns a list of subnet Ids using tags as criteria
+description:
+     - Returns a list of subnet Ids for a given set of tags that identify one or more VPCs
+version_added: "1.5"
+options:
+  region:
+    description:
+      - The AWS region to use.  Must be specified if ec2_url
+        is not used.  If not specified then the value of the
+        EC2_REGION environment variable, if any, is used.
+    required: false
+    default: null
+    aliases: [ 'aws_region', 'ec2_region' ]
+  aws_secret_key:
+    description:
+      - AWS secret key. If not set then the value of
+        the AWS_SECRET_KEY environment variable is used.
+    required: false
+    default: null
+    aliases: [ 'ec2_secret_key', 'secret_key' ]
+  aws_access_key:
+    description:
+      - AWS access key. If not set then the value of the
+        AWS_ACCESS_KEY environment variable is used.
+    required: false
+    default: null
+    aliases: [ 'ec2_access_key', 'access_key' ]
+  tags:
+    desription:
+      - tags to lookup
+    required: false
+    default: null
+    type: dict
+    aliases: []
+
+requirements: [ "boto" ]
+author: John Jarvis
+'''
+
+EXAMPLES = '''
+# Note: None of these examples set aws_access_key, aws_secret_key, or region.
+# It is assumed that their matching environment variables are set.
+
+# Return all instances that match the tag "Name: foo"
+- local_action:
+    module: vpc_lookup
+    tags:
+        Name: foo
+'''
+
+import sys
+
+AWS_REGIONS = ['ap-northeast-1',
+               'ap-southeast-1',
+               'ap-southeast-2',
+               'eu-west-1',
+               'sa-east-1',
+               'us-east-1',
+               'us-west-1',
+               'us-west-2']
+
+try:
+    from boto.vpc import VPCConnection
+    from boto.vpc import connect_to_region
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+
+def main():
+
+    module=AnsibleModule(
+        argument_spec=dict(
+            region=dict(choices=AWS_REGIONS),
+            aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'],
+                                no_log=True),
+            aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
+            tags=dict(default=None, type='dict'),
+        )
+    )
+
+    tags = module.params.get('tags')
+    aws_secret_key = module.params.get('aws_secret_key')
+    aws_access_key = module.params.get('aws_access_key')
+    region = module.params.get('region')
+
+    # If we have a region specified, connect to its endpoint.
+    if region:
+        try:
+            vpc = connect_to_region(region, aws_access_key_id=aws_access_key,
+                                    aws_secret_access_key=aws_secret_key)
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg=str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+    vpc_conn = VPCConnection()
+    subnet_ids = []
+    for subnet in vpc_conn.get_all_subnets(filters={'tag:' + tag: value
+                                      for tag, value in tags.iteritems()}):
+        subnet_ids.append(subnet.id)
+    vpc_ids = []
+    for vpc in vpc.get_all_vpcs(filters={'tag:' + tag: value
+                                      for tag, value in tags.iteritems()}):
+        vpc_ids.append(vpc.id)
+
+    module.exit_json(changed=False, subnet_ids=subnet_ids, vpc_ids=vpc_ids)
+
+
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+
+main()

playbooks/roles/xqueue/tasks/deploy.yml

@@ -34,7 +34,7 @@
 
 
 # Install the python pre requirements into {{ xqueue_venv_dir }}
-- name : install python pre-requirements
+- name : xqueue | install python pre-requirements
   pip: requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
   sudo_user: "{{ xqueue_user }}"
   notify:
@@ -43,7 +43,7 @@
   - deploy
 
 # Install the python post requirements into {{ xqueue_venv_dir }}
-- name : install python post-requirements
+- name : xqueue | install python post-requirements
   pip: requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
   sudo_user: "{{ xqueue_user }}"
   notify:

requirements.txt

-Fabric==1.5.1
-Jinja2==2.6
+Jinja2==2.7.1
+MarkupSafe==0.18
 PyYAML==3.10
-WebOb==1.2.3
+ansible==1.3.2
 argparse==1.2.1
-beautifulsoup4==4.1.3
 boto==2.10.0
-cloudformation==0.0.0
-decorator==3.4.0
-distribute==0.6.30
-docopt==0.6.1
-dogapi==1.2.3
-ipython==0.13.1
-jenkinsapi==0.1.11
-lxml==3.1beta1
-newrelic==1.10.2.38
-path.py==3.0.1
-pingdom==0.2.0
-pycrypto==2.6
-pyparsing==1.5.6
-pyrelic==0.2.0
-python-dateutil==2.1
-requests==1.1.0
-schema==0.1.1
-simplejson==3.3.0
-simples3==1.0-alpha
-six==1.2.0
--e git+https://github.com/bos/statprof.py.git@a17f7923b102c9039763583be9e377e8422e8f5f#egg=statprof-dev
-ujson==1.30
+distribute==0.6.24
+ecdsa==0.10
+paramiko==1.12.0
+pycrypto==2.6.1
 wsgiref==0.1.2
-ansible==1.3.2