Merge pull request #1548 from edx/e0d/service-builder

E0d/service builder

playbooks/edx-east/edx_service.yml

+---
+
+- name: Build application artifacts
+  hosts: all
+  connection: local
+  gather_facts: False
+  vars:
+    state: "present"
+    auto_scaling_service: True
+  tasks:
+    - name: Manage IAM Role and Profile
+      ec2_iam_role:
+        profile: "{{ profile }}"
+        state: "{{ state }}"
+        instance_profile_name: "{{ instance_profile_name }}"
+        role_name: "{{ role_name }}"
+        policies: "{{ role_policies }}"
+
+    - name: Manage ELB security group
+      ec2_group_local:
+        profile: "{{ profile }}"
+        description: "{{ elb_security_group.description }}"
+        name: "{{ elb_security_group.name }}"
+        vpc_id: "{{ vpc_id }}"
+        region: "{{ aws_region }}"
+        rules: "{{ elb_security_group.rules }}"
+        tags: "{{ elb_security_group.tags }}"
+      register: elb_sec_group
+
+    - debug: msg="{{ service_security_group.rules }}"
+
+    - name: Manage service security group
+      ec2_group_local:
+        profile: "{{ profile }}"
+        description: "{{ service_security_group.description }}"
+        name: "{{ service_security_group.name }}"
+        vpc_id: "{{ vpc_id }}"
+        region: "{{ aws_region }}"
+        rules: "{{ service_security_group.rules }}"
+        tags: "{{ service_security_group.tags }}"
+      register: service_sec_group
+      
+    - name: Manage ACLs
+      ec2_acl:
+        profile: "{{ profile }}"
+        name: "{{ item.name }}"
+        vpc_id: "{{ vpc_id }}"
+        state: "{{ state }}"
+        region: "{{ aws_region }}"
+        rules: "{{ item.rules }}"
+      with_items: acls
+      register: created_acls
+
+    - name: Apply function to acl_data
+      util_map:
+        function: 'zip_to_dict'
+        input: "{{ created_acls.results }}"
+        args:
+          - "name"
+          - "id"
+      register: acl_data
+         
+    # - name: Manage ELB Subnets
+    #   ec2_subnet:
+    #     state: "{{ state }}"
+    #     region: "{{ aws_region }}"
+    #     name: "{{ item.name }}"
+    #     vpc_id: "{{ vpc_id }}"
+    #     cidr_block: "{{ item.cidr }}"
+    #     az: "{{ item.az }}"
+    #     route_table_id: "{{ item.route_table_id }}"
+    #     tags: "{{ item.tags }}"
+    #   register: created_elb_subnets
+    #   with_items: elb_subnets
+
+      #
+      # Hack alert, this registers a list in the global namespace
+      # of just the subnet ids that were created above
+      #
+    # - debug: msg="{{ created_elb_subnets.results|map(attribute='subnet_id')| list }}"
+    #   register: elb_sn_list
+      
+    - name: Manage Service Subnets
+      ec2_subnet:
+        profile: "{{ profile }}"
+        state: "{{ state }}"
+        region: "{{ aws_region }}"
+        name: "{{ item.name }}"
+        vpc_id: "{{ vpc_id }}"
+        cidr_block: "{{ item.cidr }}"
+        az: "{{ item.az }}"
+        route_table_id: "{{ item.route_table_id }}"
+        tags: "{{ item.tags }}"
+      register: created_service_subnets
+      with_items: service_subnets
+
+    #
+    # Stubbed out
+    # For now we'll be using an existing route table
+    #
+    # - name: Manage Route Table
+    #   ec2_rt:
+    #     state: "{{ state }}"
+    #     region: "{{ aws_region }}"
+    #     name: "{{ rt.name }}"
+    #     vpc_id: "{{ vpc_id }}"
+    #     destination_cidr: "{{ rt.destination_cidr }}"
+    #     target: "local" # simplifying generalization of instnace-id, gateway-id or local
+
+    - name: Manage ELB
+      ec2_elb_lb:
+        profile: "{{ profile }}"
+        region: "{{ aws_region }}"
+        scheme: "{{ elb_scheme }}"
+        name: "{{ elb_name }}"
+        state: "{{ state }}"
+        security_group_ids: "{{ elb_sec_group.group_id }}"
+        subnets: "{{ elb_subnets }}"
+        health_check: "{{ elb_healthcheck }}"
+        listeners: "{{ elb_listeners }}"
+      register: elb
+#
+# Service related components
+#
+    - name: Manage the launch configuration
+      ec2_lc:
+        profile: "{{ profile }}"
+        region: "{{ aws_region }}"
+        name: "{{ service_config.name }}"
+        image_id: "{{ service_config.ami }}"
+        key_name: "{{ service_config.key_name }}"
+        security_groups: "{{ service_sec_group.group_id }}"
+        instance_type: "{{ service_config.instance_type }}"
+        instance_profile_name: "{{ instance_profile_name }}"
+        volumes: "{{ service_config.volumes }}"
+      when: auto_scaling_service
+
+      #
+      # Hack alert, this registers a string in the global namespace
+      # of just the subnet ids for the service that were created above
+      #
+    - debug: msg="{{ created_service_subnets.results|map(attribute='subnet_id')| list | join(',') }}"
+      register: service_vpc_zone_identifier_string
+        
+    - name: Manage ASG
+      ec2_asg:
+        profile: "{{ profile }}"
+        region: "{{ aws_region }}"
+        name: "{{ asg_name }}"
+        launch_config_name: "{{ service_config.name }}"
+        load_balancers: "{{ elb_name }}"
+        availability_zones: "{{ aws_availability_zones }}"
+        min_size: "{{ asg_min_size }}"
+        max_size: "{{ asg_max_size }}"
+        desired_capacity: "{{ asg_desired_capacity }}"
+        vpc_zone_identifier: "{{ service_vpc_zone_identifier_string.msg }}"
+        tags: "{{ asg_instance_tags }}"
+      register: asg
+      when: auto_scaling_service
+
+    - name: Manage scaling policies
+      ec2_scaling_policy:
+        state: "{{ item.state }}"
+        profile: "{{ item.profile }}"
+        region: "{{ item.region }}"
+        name: "{{ item.name }}"
+        adjustment_type: "{{ item.adjustment_type }}"
+        asg_name: "{{ item.asg_name }}"
+        scaling_adjustment: "{{ item.scaling_adjustment }}"
+        min_adjustment_step: "{{ item.min_adjustment_step }}"
+        cooldown: "{{ item.cooldown }}"
+      with_items: scaling_policies
+      register: created_policies
+      when: auto_scaling_service
+
+    - name: Apply function to policy data
+      util_map:
+        function: 'zip_to_dict'
+        input: "{{ created_policies.results }}"
+        args:
+          - "name"
+          - "arn"
+      register: policy_data
+      when: auto_scaling_service
+
+    - name: Manage metric alarms
+      ec2_metric_alarm:
+        profile: "{{ profile }}"
+        state: "{{ item.state }}"
+        region: "{{ aws_region }}"
+        name: "{{ item.name }}"
+        metric: "{{ item.metric }}"
+        namespace: "{{ item.namespace }}"
+        statistic: "{{ item.statistic }}"
+        comparison: "{{ item.comparison }}"
+        threshold: "{{ item.threshold }}"
+        period: "{{ item.period }}"
+        evaluation_periods: "{{ item.evaluation_periods }}"
+        unit: "{{ item.unit }}"
+        description: "{{ item.description }}"
+        dimensions: "{{ item.dimensions }}"
+        alarm_actions: "{{ policy_data.function_output[item.target_policy] }}"
+      with_items: metric_alarms 
+      when: auto_scaling_service
+
+    - name: See if instances already exist
+      local_action:
+        module: "ec2_lookup"
+        region: "{{ aws_region }}"
+        tags: "{{ asg_instance_tags }}"
+      register: potential_existing_instances
+
+    - name: Manage instances
+      ec2:
+        profile: "{{ profile }}"
+        region: "{{ aws_region }}"
+        wait: "yes"
+        group_id: "{{ service_sec_group.group_id }}"
+        key_name: "{{ service_config.key_name }}"
+        vpc_subnet_id: "{{ item.subnet_id }}"
+        instance_type: "{{ service_config.instance_type }}"
+        instance_tags: "{{ asg_instance_tags }}"
+        image: "{{ service_config.ami }}"
+        instance_profile_name: "{{ instance_profile_name }}"
+        volumes: "{{ service_config.volumes }}"
+      with_items: created_service_subnets.results
+      when: not auto_scaling_service and potential_existing_instances.instances|length == 0
+

playbooks/edx-east/edx_vpc.yml

 ---
-- hosts: first_in_tag_role_mongo
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - role: 'mongo'
-      mongo_create_users: yes
-#- hosts: tag_role_mongo:!first_in_tag_role_mongo
-#  sudo: True
-#  vars_files:
-#    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-#    - "{{ secure_dir }}/vars/common/common.yml"
-#  roles:
-#    - user
-#    - mongo
-- hosts: first_in_tag_role_edxapp
-  sudo: True
-  serial: 1
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - datadog
-    - role: nginx
-      nginx_sites:
-      - lms
-      - cms
-      - lms-preview
-      nginx_default_sites:
-      - lms
-    - role: 'edxapp'
-      edxapp_lms_env: 'lms.envs.load_test'
-      migrate_db: '{{ RUN_EDXAPP_MIGRATION }}'
-      openid_workaround: 'yes'
-    - splunkforwarder
-- hosts: tag_role_edxapp:!first_in_tag_role_edxapp
-  sudo: True
-  serial: 1
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - datadog
-    - role: nginx
-      nginx_sites:
-      - lms
-      - cms
-      - lms-preview
-      nginx_default_site:
-      - lms
-    - role: 'edxapp'
-      edxapp_lms_env: 'lms.envs.load_test'
-    - splunkforwarder
-- hosts: tag_role_worker
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - datadog
-    - role: nginx
-      nginx_sites:
-      - lms
-      - cms
-      - lms-preview
-      nginx_default_site:
-      - lms
-    - role: 'edxapp'
-      edxapp_lms_env: 'lms.envs.load_test'
-      celery_worker: True
-    - splunkforwarder
-- hosts: tag_role_xserver
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - role: nginx
-      nginx_sites:
-      - xserver
-    - xserver
-    - splunkforwarder
-- hosts: tag_role_rabbitmq
-  serial: 1
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - rabbitmq
-    - splunkforwarder
-- hosts: first_in_tag_role_xqueue
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - role: nginx
-      nginx_sites:
-      - xqueue
-    - role: xqueue
-      migrate_db: '{{ RUN_XQUEUE_MIGRATION }}'
-    - splunkforwarder
-- hosts: tag_role_xqueue:!first_in_tag_role_xqueue
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - role: nginx
-      nginx_sites:
-      - xqueue
-    - xqueue
-    - splunkforwarder
-- hosts: tag_role_forum
-  sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/common/common.yml"
-  roles:
-    - user
-    - oraclejdk
-    - elasticsearch
-    - forum
+# Sample command: ansible-playbook -c local -i localhost, edx_vpc.yml -e@/Users/feanil/src/edx-secure/cloud_migrations/vpcs/test.yml -vvv
+- name: Create a simple empty vpc
+  hosts: all
+  connection: local
+  gather_facts: False
+  vars:
+    vpc_state: present
+  roles:
+    - edx_vpc

playbooks/library/ec2_acl

+#!/usr/bin/env python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = """
+---
+module: ec2_acl
+short_description: Create or delete AWS Network ACLs.
+description:
+  - Can create or delete AwS Network ACLs.
+version_added: "1.8"
+author: Edward Zarecor
+options:
+  state:
+    description:
+      - create, update or delete the acl
+    required: true
+    choices: ['present', 'absent']
+  name:
+    description:
+      - Unique name for acl
+    required: true
+  vpc_id:
+    description:
+      - The VPC that this acl belongs to
+    required: true
+    default: null
+extends_documentation_fragment: aws
+"""
+
+EXAMPLES = '''
+- ec2_acl:
+    name: public-acls
+    state: present
+    vpc_id: 'vpc-abababab'
+
+'''
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+import sys
+try:
+    import boto.vpc
+except ImportError:
+    print "failed=True msg={0}".format(sys.executable)
+    #print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+from boto.exception import NoAuthHandlerFound
+
+PROTOCOL_NUMBERS = {"ICMP": 1, "TCP": 6, "UDP": 17, "ALL": -1 }
+
+
+class DuplicateAclError(Exception):
+    pass
+
+
+class ACLManager():
+
+    def __init__(self, connection, vpc_id, acl_name, rules, tags=[]):
+        self.connection = connection
+        self.vpc_id = vpc_id
+        self.acl_name = acl_name
+        self.rules = rules
+        self.tags = tags
+        self.acl = None
+
+    def get_acl(self):
+
+        if not self.acl:
+            results = self.connection.get_all_network_acls(filters={"vpc_id": self.vpc_id, "tag:Name": self.acl_name})
+
+            if len(results) == 1:
+                self.acl = results[0]
+            elif len(results) > 1:
+                raise DuplicateAclError("Found multiple network acls name {0} in vpc with id {1}".
+                                        format(self.acl_name, self.vpc_id))
+            else:
+                # Does exist yet
+                pass
+
+        return self.acl
+
+    def create_acl(self):
+        self.acl = self.connection.create_network_acl(self.vpc_id)
+        changed = True
+        self.do_tags()
+        return changed
+
+    def update_acl(self):
+        changed = False
+        self.update_rules()
+        self.do_tags()
+        return changed
+
+    # TODO refactor out repitition
+    def update_rules(self):
+
+        current_ingress = [x.rule_number for x in self.acl.network_acl_entries if x.egress == 'false']
+        current_egress = [x.rule_number for x in self.acl.network_acl_entries if x.egress == 'true']
+
+        modified_ingress = []
+        modified_egress = []
+
+        for rule in self.rules:
+            egress = True if rule['type'] == "egress" else False
+            protocol = PROTOCOL_NUMBERS[rule['protocol'].upper()]
+
+            if not egress:
+                if rule['number'] not in current_ingress:
+                    # new rule
+                    self.connection.create_network_acl_entry(
+                        self.acl.id,
+                        rule['number'],
+                        protocol,
+                        rule['rule_action'],
+                        rule['cidr_block'],
+                        egress=egress,
+                        port_range_from=rule['from_port'],
+                        port_range_to=rule['to_port'])
+                else:
+                    # blindly replace rather than attempting
+                    # to determine in the entry has changed
+                    modified_ingress.append(rule['number'])
+                    self.connection.replace_network_acl_entry (
+                        self.acl.id,
+                        rule['number'],
+                        protocol,
+                        rule['rule_action'],
+                        rule['cidr_block'],
+                        egress=egress,
+                        port_range_from=rule['from_port'],
+                        port_range_to=rule['to_port'])
+            else:
+                if rule['number'] not in current_egress:
+                    # new rule
+                    self.connection.create_network_acl_entry(
+                        self.acl.id,
+                        rule['number'],
+                        protocol,
+                        rule['rule_action'],
+                        rule['cidr_block'],
+                        egress=egress,
+                        port_range_from=rule['from_port'],
+                        port_range_to=rule['to_port'])
+                else:
+                    # blindly replace rather than attempting
+                    # to determine in the entry has changed
+                    modified_egress.append(rule['number'])
+                    self.connection.replace_network_acl_entry (
+                        self.acl.id,
+                        rule['number'],
+                        protocol,
+                        rule['rule_action'],
+                        rule['cidr_block'],
+                        egress=egress,
+                        port_range_from=rule['from_port'],
+                        port_range_to=rule['to_port'])
+
+        removed_ingress_rule_numbers = [ c for c in current_ingress if c not in modified_ingress ]
+        removed_egress_rule_numbers = [ c for c in current_egress if c not in modified_egress ]
+
+        for number in removed_ingress_rule_numbers:
+            n = int(number)
+            # reserved range for AWS
+            if n <  32767:
+                self.connection.delete_network_acl_entry(self.acl.id, n, False)
+
+        for number in removed_egress_rule_numbers:
+            n = int(number)
+            # reserved range for AWS
+            if n <  32767:
+                self.connection.delete_network_acl_entry(self.acl.id, n, True)
+
+
+    def create_rules(self):
+        if self.rules is None:
+            return
+        for rule in self.rules:
+            egress = True if rule['type'] == "egress" else False
+            protocol = PROTOCOL_NUMBERS[rule['protocol'].upper()]
+            self.connection.create_network_acl_entry(
+                self.acl.id,
+                rule['number'],
+                protocol,
+                rule['rule_action'],
+                rule['cidr_block'],
+                egress=egress,
+                port_range_from=rule['from_port'],
+                port_range_to=rule['to_port'])
+
+    def do_tags(self):
+
+        tags = {'Name': self.acl_name}
+        if self.tags:
+            for tag in self.tags:
+                tags[tag['key']] = tag['value']
+        self.get_acl().add_tags(tags)
+
+    def present(self):
+
+        existing = self.get_acl()
+
+        if not existing:
+            changed = self.create_acl()
+            self.create_rules()
+        else:
+            changed = self.update_acl()
+
+        results = dict(changed=changed,
+                       id=self.acl.id,
+                       name=self.acl_name,
+                       entries=self.rules)
+
+        return results
+
+    def absent(self):
+        acl = self.get_acl()
+        changed = False
+
+        if acl:
+            changed = self.connection.delete_network_acl(acl.id)
+        results = dict(changed=changed,
+                       id=self.acl.id,
+                       name=self.acl_name)
+
+        return results
+
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(
+        dict(
+            name=dict(required=True, type='str'),
+            state=dict(default='present', choices=['present', 'absent']),
+            vpc_id=dict(required=True, type='str'),
+            rules=dict(type='list'),
+            tags=dict(type='list'),
+        )
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec)
+
+    ec2_url, aws_access_key, aws_secret_key, region = get_ec2_creds(module)
+    profile = module.params.get('profile')
+    if region:
+        try:
+            connection = boto.vpc.connect_to_region(region, profile_name=profile)
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg=str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+
+    vpc_id = module.params.get('vpc_id')
+    acl_name = module.params.get('name')
+    rules_in = module.params.get('rules')
+    tags = module.params.get('tags')
+
+    manager = ACLManager(connection, vpc_id, acl_name, rules_in, tags)
+
+    state = module.params.get('state')
+
+    results = dict()
+
+    if state == 'present':
+        results = manager.present()
+    elif state == 'absent':
+        results = manager.absent()
+
+    module.exit_json(**results)
+
+main()

playbooks/library/ec2_group_local

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+DOCUMENTATION = '''
+---
+module: ec2_group
+version_added: "1.3"
+short_description: maintain an ec2 VPC security group.
+description:
+    - maintains ec2 security groups. This module has a dependency on python-boto >= 2.5
+options:
+  name:
+    description:
+      - Name of the security group.
+    required: true
+  description:
+    description:
+      - Description of the security group.
+    required: true
+  vpc_id:
+    description:
+      - ID of the VPC to create the group in.
+    required: false
+  rules:
+    description:
+      - List of firewall inbound rules to enforce in this group (see example).
+    required: false
+  rules_egress:
+    description:
+      - List of firewall outbound rules to enforce in this group (see example).
+    required: false
+    version_added: "1.6"
+  tags:
+    description:
+      - List of tags to apply to this security group
+    required: false
+    version_added: "1.8"
+  region:
+    description:
+      - the EC2 region to use
+    required: false
+    default: null
+    aliases: []
+  state:
+    version_added: "1.4"
+    description:
+      - create or delete security group
+    required: false
+    default: 'present'
+    aliases: []
+
+extends_documentation_fragment: aws
+
+notes:
+  - If a rule declares a group_name and that group doesn't exist, it will be
+    automatically created. In that case, group_desc should be provided as well.
+    The module will refuse to create a depended-on group without a description.
+'''
+
+EXAMPLES = '''
+- name: example ec2 group
+  local_action:
+    module: ec2_group
+    name: example
+    description: an example EC2 group
+    vpc_id: 12345
+    region: eu-west-1a
+    aws_secret_key: SECRET
+    aws_access_key: ACCESS
+    rules:
+      - proto: tcp
+        from_port: 80
+        to_port: 80
+        cidr_ip: 0.0.0.0/0
+      - proto: tcp
+        from_port: 22
+        to_port: 22
+        cidr_ip: 10.0.0.0/8
+      - proto: udp
+        from_port: 10050
+        to_port: 10050
+        cidr_ip: 10.0.0.0/8
+      - proto: udp
+        from_port: 10051
+        to_port: 10051
+        group_id: sg-12345678
+      - proto: all
+        # the containing group name may be specified here
+        group_name: example
+    rules_egress:
+      - proto: tcp
+        from_port: 80
+        to_port: 80
+        group_name: example-other
+        # description to use if example-other needs to be created
+        group_desc: other example EC2 group
+    tags:
+      - key: environment
+        value: production
+'''
+
+try:
+    import boto.ec2
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+
+def addRulesToLookup(rules, prefix, dict):
+    for rule in rules:
+        for grant in rule.grants:
+            dict["%s-%s-%s-%s-%s-%s" % (prefix, rule.ip_protocol, rule.from_port, rule.to_port,
+                                        grant.group_id, grant.cidr_ip)] = rule
+
+
+def get_target_from_rule(module, rule, name, group, groups):
+    """
+    Returns tuple of (group_id, ip) after validating rule params.
+
+    rule: Dict describing a rule.
+    name: Name of the security group being managed.
+    groups: Dict of all available security groups.
+
+    AWS accepts an ip range or a security group as target of a rule. This
+    function validate the rule specification and return either a non-None
+    group_id or a non-None ip range.
+    """
+
+    group_id = None
+    group_name = None
+    ip = None
+    target_group_created = False
+    if 'group_id' in rule and 'cidr_ip' in rule:
+        module.fail_json(msg="Specify group_id OR cidr_ip, not both")
+    elif 'group_name' in rule and 'cidr_ip' in rule:
+        module.fail_json(msg="Specify group_name OR cidr_ip, not both")
+    elif 'group_id' in rule and 'group_name' in rule:
+        module.fail_json(msg="Specify group_id OR group_name, not both")
+    elif 'group_id' in rule:
+        group_id = rule['group_id']
+    elif 'group_name' in rule:
+        group_name = rule['group_name']
+        if group_name in groups:
+            group_id = groups[group_name].id
+        elif group_name == name:
+            group_id = group.id
+            groups[group_id] = group
+            groups[group_name] = group
+        else:
+            if not rule.get('group_desc', '').strip():
+                module.fail_json(msg="group %s will be automatically created by rule %s and no description was provided" % (group_name, rule))
+            if not module.check_mode:
+                auto_group = ec2.create_security_group(group_name, rule['group_desc'], vpc_id=vpc_id)
+                group_id = auto_group.id
+                groups[group_id] = auto_group
+                groups[group_name] = auto_group
+            target_group_created = True
+    elif 'cidr_ip' in rule:
+        ip = rule['cidr_ip']
+
+    return group_id, ip, target_group_created
+
+## can be removed if https://github.com/ansible/ansible/pull/9113 is merged upstream
+def is_taggable(object):
+
+    from boto.ec2.ec2object import TaggedEC2Object
+    if not object or not issubclass(object.__class__, TaggedEC2Object):
+        return False
+
+    return True
+
+def do_tags(module, object, tags):
+    """
+    General function for adding tags to objects that are subclasses
+    of boto.ec2.ec2object.TaggedEC2Object.  Currently updates
+    existing tags, as the API overwrites them, but does not remove
+    orphans.
+    :param module:
+    :param object:
+    :param tags:
+    """
+    dry_run = True if module.check_mode else False
+
+    if (is_taggable(object)):
+
+        tag_dict = {}
+
+        for tag in tags:
+            tag_dict[tag['key']] = tag['value']
+
+        object.add_tags(tag_dict, dry_run)
+    else:
+        module.fail_json(msg="Security group object is not a subclass of TaggedEC2Object")
+## end can be removed
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(dict(
+            name=dict(required=True),
+            description=dict(required=True),
+            vpc_id=dict(),
+            rules=dict(),
+            rules_egress=dict(),
+            tags=dict(type='list', default=[]),
+            state = dict(default='present', choices=['present', 'absent']),
+        )
+    )
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    name = module.params['name']
+    description = module.params['description']
+    vpc_id = module.params['vpc_id']
+    rules = module.params['rules']
+    rules_egress = module.params['rules_egress']
+    tags = module.params['tags']
+    state = module.params.get('state')
+
+    changed = False
+
+    ec2 = ec2_connect(module)
+
+    # find the group if present
+    group = None
+    groups = {}
+    for curGroup in ec2.get_all_security_groups():
+        groups[curGroup.id] = curGroup
+        groups[curGroup.name] = curGroup
+
+        if curGroup.name == name and (vpc_id is None or curGroup.vpc_id == vpc_id):
+            group = curGroup
+
+    # Ensure requested group is absent
+    if state == 'absent':
+        if group:
+            '''found a match, delete it'''
+            try:
+                group.delete()
+            except Exception, e:
+                module.fail_json(msg="Unable to delete security group '%s' - %s" % (group, e))
+            else:
+                group = None
+                changed = True
+        else:
+            '''no match found, no changes required'''
+
+    # Ensure requested group is present
+    elif state == 'present':
+        if group:
+            '''existing group found'''
+            # check the group parameters are correct
+            group_in_use = False
+            rs = ec2.get_all_instances()
+            for r in rs:
+                for i in r.instances:
+                    group_in_use |= reduce(lambda x, y: x | (y.name == 'public-ssh'), i.groups, False)
+
+            if group.description != description:
+                if group_in_use:
+                    module.fail_json(msg="Group description does not match, but it is in use so cannot be changed.")
+
+        # if the group doesn't exist, create it now
+        else:
+            '''no match found, create it'''
+            if not module.check_mode:
+                group = ec2.create_security_group(name, description, vpc_id=vpc_id)
+
+                # When a group is created, an egress_rule ALLOW ALL
+                # to 0.0.0.0/0 is added automatically but it's not
+                # reflected in the object returned by the AWS API
+                # call. We re-read the group for getting an updated object
+                # amazon sometimes takes a couple seconds to update the security group so wait till it exists
+                while len(ec2.get_all_security_groups(filters={ 'group_id': group.id, })) == 0:
+                    time.sleep(0.1)
+
+                group = ec2.get_all_security_groups(group_ids=(group.id,))[0]
+            changed = True
+
+        # tag the security group, function imported from ansible.module_utils.ec2
+        do_tags(module, group, tags)
+    else:
+        module.fail_json(msg="Unsupported state requested: %s" % state)
+
+    # create a lookup for all existing rules on the group
+    if group:
+
+        # Manage ingress rules
+        groupRules = {}
+        addRulesToLookup(group.rules, 'in', groupRules)
+
+        # Now, go through all provided rules and ensure they are there.
+        if rules:
+            for rule in rules:
+                group_id, ip, target_group_created = get_target_from_rule(module, rule, name, group, groups)
+                if target_group_created:
+                    changed = True
+
+                if rule['proto'] == 'all':
+                    rule['proto'] = -1
+                    rule['from_port'] = None
+                    rule['to_port'] = None
+
+                # If rule already exists, don't later delete it
+                ruleId = "%s-%s-%s-%s-%s-%s" % ('in', rule['proto'], rule['from_port'], rule['to_port'], group_id, ip)
+                if ruleId in groupRules:
+                    del groupRules[ruleId]
+                # Otherwise, add new rule
+                else:
+                    grantGroup = None
+                    if group_id:
+                        grantGroup = groups[group_id]
+
+                    if not module.check_mode:
+                        group.authorize(rule['proto'], rule['from_port'], rule['to_port'], ip, grantGroup)
+                    changed = True
+
+        # Finally, remove anything left in the groupRules -- these will be defunct rules
+        for rule in groupRules.itervalues():
+            for grant in rule.grants:
+                grantGroup = None
+                if grant.group_id:
+                    grantGroup = groups[grant.group_id]
+                if not module.check_mode:
+                    group.revoke(rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip, grantGroup)
+                changed = True
+
+        # Manage egress rules
+        groupRules = {}
+        addRulesToLookup(group.rules_egress, 'out', groupRules)
+
+        # Now, go through all provided rules and ensure they are there.
+        if rules_egress:
+            for rule in rules_egress:
+                group_id, ip, target_group_created = get_target_from_rule(module, rule, name, group, groups)
+                if target_group_created:
+                    changed = True
+
+                if rule['proto'] == 'all':
+                    rule['proto'] = -1
+                    rule['from_port'] = None
+                    rule['to_port'] = None
+
+                # If rule already exists, don't later delete it
+                ruleId = "%s-%s-%s-%s-%s-%s" % ('out', rule['proto'], rule['from_port'], rule['to_port'], group_id, ip)
+                if ruleId in groupRules:
+                    del groupRules[ruleId]
+                # Otherwise, add new rule
+                else:
+                    grantGroup = None
+                    if group_id:
+                        grantGroup = groups[group_id].id
+
+                    if not module.check_mode:
+                        ec2.authorize_security_group_egress(
+                                group_id=group.id,
+                                ip_protocol=rule['proto'],
+                                from_port=rule['from_port'],
+                                to_port=rule['to_port'],
+                                src_group_id=grantGroup,
+                                cidr_ip=ip)
+                    changed = True
+        elif vpc_id and not module.check_mode:
+            # when using a vpc, but no egress rules are specified, 
+            # we add in a default allow all out rule, which was the
+            # default behavior before egress rules were added
+            default_egress_rule = 'out--1-None-None-None-0.0.0.0/0'
+            if default_egress_rule not in groupRules:
+                ec2.authorize_security_group_egress(
+                    group_id=group.id,
+                    ip_protocol=-1,
+                    from_port=None,
+                    to_port=None,
+                    src_group_id=None,
+                    cidr_ip='0.0.0.0/0'
+                )
+                changed = True
+            else:
+                # make sure the default egress rule is not removed
+                del groupRules[default_egress_rule]
+
+        # Finally, remove anything left in the groupRules -- these will be defunct rules
+        for rule in groupRules.itervalues():
+            for grant in rule.grants:
+                grantGroup = None
+                if grant.group_id:
+                    grantGroup = groups[grant.group_id].id
+                if not module.check_mode:
+                    ec2.revoke_security_group_egress(
+                            group_id=group.id,
+                            ip_protocol=rule.ip_protocol,
+                            from_port=rule.from_port,
+                            to_port=rule.to_port,
+                            src_group_id=grantGroup,
+                            cidr_ip=grant.cidr_ip)
+                changed = True
+
+    if group:
+        module.exit_json(changed=changed, group_id=group.id)
+    else:
+        module.exit_json(changed=changed, group_id=None)
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+
+main()

playbooks/library/ec2_iam_role

+#!/usr/bin/env python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = """
+---
+module: ec2_iam_role
+short_description: Create or delete iam roles.
+description:
+  - Can create or delete AwS iam roles.
+version_added: "1.8"
+author: Edward Zarecor
+options:
+  state:
+    description:
+      - create, update or delete the role
+    required: true
+    choices: ['present', 'absent']
+  name:
+    description:
+      - Name for the role
+    required: true
+  vpc_id:
+    description:
+      - The VPC that this acl belongs to
+    required: true
+    default: null
+extends_documentation_fragment: aws
+"""
+
+EXAMPLES = '''
+- ec2_acl:
+    name: public-acls
+    state: present
+    vpc_id: 'vpc-abababab'
+
+'''
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+import sys
+try:
+    import boto
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+def present(connection, module):
+
+    profile_name = module.params.get('instance_profile_name')
+    role_name =  module.params.get('role_name')
+    policies = module.params.get('policies')
+
+    fetched_profile = None
+    fetched_role = None
+
+    profile_arn = None
+    role_arn = None
+
+    try:
+        fetched_profile = connection.get_instance_profile(profile_name)
+    except boto.exception.BotoServerError as bse:
+        pass
+
+    if not fetched_profile:
+        instance_profile = connection.create_instance_profile(profile_name)
+        profile_arn = instance_profile.arn
+    else:
+        profile_arn = fetched_profile.arn
+
+    try:
+        fetched_role = connection.get_role(role_name)
+    except boto.exception.BotoServerError as bse:
+        pass
+
+    if not fetched_role:
+        role = connection.create_role(role_name)
+        role_arn = role.arn
+    else:
+        role_arn = fetched_role.arn
+
+    if not fetched_profile and not fetched_role:
+        connection.add_role_to_instance_profile(profile_name, role_name)
+
+    for policy in policies:
+
+        fetched_policy = None
+
+        try:
+            fetched_policy = connection.get_role_policy(role_name, policy['name'])
+        except boto.exception.BotoServerError as bse:
+            pass
+
+        if not fetched_policy:
+            connection.put_role_policy(role_name, policy['name'], policy['document'])
+        else:
+            # TODO: idempotent?
+            connection.put_role_policy(role_name, policy['name'], policy['document'])
+
+
+    module.exit_json(changed=True,
+                     instance_profile_arn=profile_arn,
+                     role_arn=role_arn)
+
+
+def absent(connection, module):
+
+    profile_name = module.params.get('instance_profile_name')
+    role_name =  module.params.get('role_name')
+    policies = module.params.get('policies')
+
+    for policy in policies:
+        try:
+            connection.delete_role_policy(role_name,policy['name'])
+        except boto.exception.BotoServerError as bse:
+            # TODO: parse code to verify that this is not found case
+            pass
+
+    connection.remove_role_from_instance_profile(profile_name,role_name)
+    connection.delete_role(role_name)
+    connection.delete_instance_profile(profile_name)
+
+    module.exit_json(changed=True)
+
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(
+        dict(
+            state=dict(default='present', choices=['present', 'absent']),
+            instance_profile_name=dict(required=True, type='str'),
+            role_name=dict(required=True, type='str'),
+            policies=dict(type='list')
+        )
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec)
+    profile = module.params.get('profile')
+
+    try:
+        connection = boto.connect_iam(profile_name=profile)
+    except boto.exception.NoAuthHandlerFound, e:
+        module.fail_json(msg = str(e))
+
+    state = module.params.get('state')
+
+    if state == 'present':
+        present(connection, module)
+    elif state == 'absent':
+        absent(connection, module)
+
+main()

playbooks/library/ec2_lookup

@@ -96,11 +96,15 @@ def main():
             aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'],
                                 no_log=True),
             aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
-            tags=dict(default=None, type='dict'),
+            tags=dict(default=None, type='list'),
         )
     )
 
-    tags = module.params.get('tags')
+    tags = {}
+    for item in module.params.get('tags'):
+        for k,v in item.iteritems():
+            tags[k] = v
+
     aws_secret_key = module.params.get('aws_secret_key')
     aws_access_key = module.params.get('aws_access_key')
     region = module.params.get('region')

playbooks/library/ec2_rt

+#!/usr/bin/env python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = """
+---
+module: ec2_rt
+short_description: Create or delete AWS Route Table
+description:
+  - Can create or delete AwS Subnets
+version_added: "1.8"
+author: Edward Zarecor
+options:
+  state:
+    description:
+      - create, update or delete the subnet
+    required: true
+    choices: ['present', 'absent']
+  name:
+    description:
+      - Unique name for subnet
+    required: true
+  destination_cidr:
+    description:
+      - The cidr block of the subnet
+    aliases: ['cidr']
+  vpc_id:
+    description:
+      - The VPC that this acl belongs to
+    required: true
+    default: null
+extends_documentation_fragment: aws
+"""
+
+EXAMPLES = '''
+'''
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+import sys
+try:
+    import boto.vpc
+except ImportError:
+    print "failed=True msg={0}".format(sys.executable)
+    sys.exit(1)
+
+
+class DuplicateRouteTableError(Exception):
+    pass
+
+class InconsistentRouteError(Exception):
+    pass
+
+class RTManager():
+
+    def __init__(self, connection, vpc_id, route_name, routes, tags):
+        self.connection = connection
+        self.vpc_id = vpc_id
+        self.name = route_name
+        self.routes = routes
+        self.tags = tags
+        self.rt = None
+
+    def get_rt(self):
+        rt_filter = { "vpc_id": self.vpc_id,
+                      "tag:Name": self.name,
+                    }
+        results = self.connection.get_all_route_tables(filters=rt_filter)
+
+        if len(results) == 1:
+            self.rt = results[0]
+        elif len(results) > 1:
+            msg = "Found multiple route tables with name '{}' in vpc with id '{}'."
+            raise DuplicateRouteTableError(msg.format(self.name, self.vpc_id))
+        else:
+            pass
+            # Doesn't exist yet
+
+        return self.rt
+
+    def do_tags(self):
+        tags = { "Name" : self.name }
+        if self.tags:
+            for tag in self.tags:
+                tags[tag['key']] = tag['value']
+        self.rt.add_tags(tags)
+
+    def create_rt(self):
+        self.rt = self.connection.create_route_table(self.vpc_id)
+        changed = True
+        self.do_tags()
+        return changed
+
+    def routes_match(self, new_route, existing_route):
+        # Not the same route
+        if new_route['cidr'] != existing_route.destination_cidr_block:
+            return False
+
+        instance_matches = existing_route.instance_id \
+           and existing_route.instance_id == new_route['instance']
+
+        gateway_matches = existing_route.gateway_id \
+           and existing_route.gateway_id == new_route['gateway']
+
+        return instance_matches or gateway_matches
+
+    def update_routes(self):
+        changed = False
+        existing_routes = { x.destination_cidr_block : x for x in self.rt.routes }
+        
+        for route in self.routes:
+            # Build the args used to call the boto API
+            call_args = {
+                "route_table_id": self.rt.id,
+                "destination_cidr_block": route['cidr'],
+            }
+
+            if "gateway" in route and "instance" in route:
+                msg = "Both gateway and instance specified for route" + \
+                      "with CIDR {}"
+                raise InconsistentRouteError(msg.format(route['cidr']))
+            elif "gateway" in route:
+                call_args['gateway_id'] = route['gateway']
+            elif "instance" in route:
+                call_args['instance_id'] = route['instance']
+            else:
+                msg = "No gateway or instance provided for route with" + \
+                      "CIDR {}"
+                raise InconsistentRouteError(msg.format(route['cidr']))
+
+            if route['cidr'] in existing_routes:
+                # Update the route
+                existing_route = existing_routes[route['cidr']]
+
+                if self.routes_match(route, existing_route):
+                    continue
+
+                self.connection.replace_route(**call_args)
+                changed = True
+            else:
+                # Create a new route
+                self.connection.create_route(**call_args)
+                changed = True
+
+        return changed
+
+    def present(self):
+        changed = False
+        existing = self.get_rt()
+
+        if existing:
+            changed = self.update_routes()
+        else:
+            changed = self.create_rt()
+            self.update_routes()
+
+        results = dict(changed=changed,
+                       id=self.rt.id,
+                       name=self.name,
+                       routes=self.routes,
+                      )
+
+        return results
+
+    def absent(self):
+        rt = self.get_rt()
+        changed = False
+
+        if rt:
+            changed = self.connection.delet_route_table(rt.id)
+
+        results = dict(changed=changed,
+                       id=self.rt.id,
+                       name=self.name,
+                      )
+
+        return results
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(
+        dict(
+            name=dict(required=True, type='str'),
+            state=dict(default='present', choices=['present', 'absent']),
+            vpc_id=dict(required=True, type='str'),
+            routes=dict(required=True, type='list', aliases=['dest_routes']),
+            tags=dict(type='list'),
+        )
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec)
+    ec2_url, aws_access_key, aws_secret_key, region = get_ec2_creds(module)
+    profile = module.params.get('profile')
+    vpc_id = module.params.get('vpc_id')
+    route_name = module.params.get('name')
+    routes = module.params.get('routes')
+    tags = module.params.get('tags')
+
+    if region:
+        try:
+            connection = boto.vpc.connect_to_region(region,profile_name=profile)
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg = str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+
+    manager = RTManager(connection, vpc_id, route_name, routes, tags)
+
+    state = module.params.get('state')
+
+    results = dict()
+    if state == 'present':
+        results = manager.present()
+    elif state == 'absent':
+        results = manager.absent()
+
+    module.exit_json(**results)
+
+main()

playbooks/library/ec2_subnet

+#!/usr/bin/env python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = """
+---
+module: ec2_subnet
+short_description: Create or delete AWS Subnets
+description:
+  - Can create or delete AwS Subnets
+version_added: "1.8"
+author: Edward Zarecor
+options:
+  state:
+    description:
+      - create, update or delete the subnet
+    required: true
+    choices: ['present', 'absent']
+  name:
+    description:
+      - Unique name for subnet
+    required: true
+  cidr_block:
+    description:
+      - The cidr block of the subnet
+    aliases: ['cidr']
+  availability_zone
+    description:
+      - The availability zone of the subnet
+    aliases: ['az']
+  vpc_id:
+    description:
+      - The VPC that this acl belongs to
+    required: true
+    default: null
+extends_documentation_fragment: aws
+"""
+
+EXAMPLES = '''
+'''
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+import sys
+try:
+    import boto.vpc
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+from boto.exception import NoAuthHandlerFound
+
+
+class NonUniqueSubnetSpecification(Exception):
+    pass
+
+
+class SubnetManager:
+
+    def __init__(self, connection, vpc_id, cidr_block, az, name, route_table_id, network_acl_id, tags=[]):
+
+        self.connection = connection
+        self.vpc_id = vpc_id
+        self.cidr_block = cidr_block
+        self.az = az
+        self.name = name
+        self.route_table_id = route_table_id
+        self.network_acl_id = network_acl_id
+        self.tags = tags
+        self.subnet = None
+
+    def get_subnet(self):
+
+        if not self.subnet:
+
+            subnets = self.connection. \
+                get_all_subnets(filters={"vpc_id": self.vpc_id,
+                                         "cidr_block": self.cidr_block,
+                                         "availability_zone": self.az})
+            if len(subnets) > 1:
+                message = "Subnet specifier of vpc_id {vpc_id}, cidr_block {cidr_block} " \
+                          "and az {az}, return more than one result".format(
+                    vpc_id=self.vpc_id, cidr_block=self.cidr_block,az=self.az)
+                raise NonUniqueSubnetSpecification(message)
+            elif len(subnets) == 1:
+                self.subnet = subnets[0]
+
+        return self.subnet
+
+    def present(self):
+
+        if self.get_subnet():
+            changed = self.update_subnet()
+        else:
+            changed = self.create_subnet()
+
+        results = dict(changed=changed,
+                       subnet_id=self.subnet.id,
+                       subnet_name=self.name,
+                       vpc_id=self.vpc_id)
+
+        return results
+
+    def create_subnet(self):
+        changed = True
+        self.subnet = self.connection.create_subnet(self.vpc_id, self.cidr_block, availability_zone=self.az)
+        self.do_tags()
+        self.connection.associate_route_table(self.route_table_id, self.subnet.id)
+        if self.network_acl_id:
+            self.connection.associate_network_acl(self.network_acl_id, self.subnet.id)
+        return changed
+
+    def update_subnet(self):
+        changed = False
+        self.do_tags()
+
+        results = self.connection.get_all_route_tables(
+                    filters={'association.subnet_id': self.subnet.id, 'vpc_id': self.vpc_id})
+
+        if len(results) == 1:
+            route_table = results[0]
+            assoc = self.get_association_from_route_table(route_table, self.subnet)
+            if assoc.route_table_id != self.route_table_id:
+                self.connection.replace_route_table_association_with_assoc(assoc.id, self.route_table_id)
+                changed = True
+        elif len(results) == 0:
+            # unlikely unless manual monkeying around
+            self.connection.associate_route_table(self.route_table_id, self.subnet.id)
+            changed == True
+
+        if self.network_acl_id:
+            self.connection.associate_network_acl(self.network_acl_id, self.subnet.id)
+
+        # acl_results = self.connection.get_all_network_acls(
+        #     filters={'association.subnet_id': self.subnet.id, 'vpc_id': self.vpc_id})
+        #
+        # if len(acl_results) == 1:
+        #     acl = acl_results[0]
+        #
+        #     if acl.id != self.network_acl_id:
+        #         self.connection.disassociate_network_acl
+
+        return changed
+
+    def absent(self):
+        changed = self.connection.delete_subnet(self.subnet.id)
+        return dict(changed=changed)
+
+    def get_association_from_route_table(self, route_table, subnet):
+        target = None
+        for assoc in route_table.associations:
+            if assoc.subnet_id == subnet.id:
+                target = assoc
+                break
+
+        return target
+
+    def do_tags(self):
+        """
+        Utility that creates all tags including the Name tag which is treated
+        as a first class params as a convenience.  Currently updates
+        existing tags, as the API overwrites them, but does not remove
+        orphans.
+        :return: None
+        """
+        tags = {'Name': self.name}
+        if self.tags:
+            for tag in self.tags:
+                tags[tag['key']] = tag['value']
+
+        self.subnet.add_tags(tags)
+
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(
+        dict(
+            name=dict(required=True, type='str'),
+            state=dict(default='present', choices=['present', 'absent']),
+            vpc_id=dict(required=True, type='str'),
+            cidr_block=dict(required=True, type='str', aliases=['cidr']),
+            az=dict(required=True, type='str'),
+            route_table_id=dict(required=True, type='str'),
+            network_acl_id=dict(type='str'),
+            tags=dict(type='list'),
+        )
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec)
+    ec2_url, aws_access_key, aws_secret_key, region = get_ec2_creds(module)
+    profile = module.params.get('profile')
+    if region:
+        try:
+            connection = boto.vpc.connect_to_region(region, profile_name=profile)
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg=str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+
+    vpc_id = module.params.get('vpc_id')
+    cidr_block = module.params.get('cidr_block')
+    az = module.params.get('az')
+    name = module.params.get('name')
+    route_table_id = module.params.get('route_table_id')
+    network_acl_id = module.params.get('network_acl_id')
+    tags = module.params.get('tags')
+
+    manager = SubnetManager(connection, vpc_id, cidr_block, az, name, route_table_id, network_acl_id, tags)
+
+    state = module.params.get('state')
+
+    if state == 'present':
+        results = manager.present()
+    elif state == 'absent':
+        results = manager.absent()
+    else:
+        raise Exception("Unexpected value for state {0}".format(state))
+
+    module.exit_json(**results)
+
+main()

playbooks/library/ec2_vpc_local

+#!/usr/bin/python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Taken from version: 1.9 - PR to push up the changes here:
+# https://github.com/ansible/ansible-modules-core/pull/1323
+
+
+DOCUMENTATION = '''
+---
+module: ec2_vpc
+short_description: configure AWS virtual private clouds
+description:
+    - Create or terminates AWS virtual private clouds.  This module has a dependency on python-boto.
+version_added: "1.4"
+options:
+  cidr_block:
+    description:
+      - "The cidr block representing the VPC, e.g. 10.0.0.0/16"
+    required: false, unless state=present
+  instance_tenancy:
+    description:
+      - "The supported tenancy options for instances launched into the VPC."
+    required: false
+    default: "default"
+    choices: [ "default", "dedicated" ]
+  dns_support:
+    description:
+      - toggles the "Enable DNS resolution" flag
+    required: false
+    default: "yes"
+    choices: [ "yes", "no" ]
+  dns_hostnames:
+    description:
+      - toggles the "Enable DNS hostname support for instances" flag
+    required: false
+    default: "yes"
+    choices: [ "yes", "no" ]
+  subnets:
+    description:
+      - 'A dictionary array of subnets to add of the form: { cidr: ..., az: ... , resource_tags: ... }. Where az is the desired availability zone of the subnet, but it is not required. Tags (i.e.: resource_tags) is also optional and use dictionary form: { "Environment":"Dev", "Tier":"Web", ...}. All VPC subnets not in this list will be removed. As of 1.8, if the subnets parameter is not specified, no existing subnets will be modified.'
+    required: false
+    default: null
+    aliases: []
+  vpc_id:
+    description:
+      - A VPC id to terminate when state=absent
+    required: false
+    default: null
+    aliases: []
+  resource_tags:
+    description:
+      - 'A dictionary array of resource tags of the form: { tag1: value1, tag2: value2 }.  Tags in this list are used in conjunction with CIDR block to uniquely identify a VPC in lieu of vpc_id. Therefore, if CIDR/Tag combination does not exits, a new VPC will be created.  VPC tags not on this list will be ignored. Prior to 1.7, specifying a resource tag was optional.'
+    required: true
+    default: null
+    aliases: []
+    version_added: "1.6"
+  internet_gateway:
+    description:
+      - Toggle whether there should be an Internet gateway attached to the VPC
+    required: false
+    default: "no"
+    choices: [ "yes", "no" ]
+    aliases: []
+  route_tables:
+    description:
+      - 'A dictionary array of route tables to add of the form: { subnets: [172.22.2.0/24, 172.22.3.0/24,], routes: [{ dest: 0.0.0.0/0, gw: igw},] }. Where the subnets list is those subnets the route table should be associated with, and the routes list is a list of routes to be in the table.  The special keyword for the gw of igw specifies that you should the route should go through the internet gateway attached to the VPC. gw also accepts instance-ids in addition igw. This module is currently unable to affect the "main" route table due to some limitations in boto, so you must explicitly define the associated subnets or they will be attached to the main table implicitly. As of 1.8, if the route_tables parameter is not specified, no existing routes will be modified.'
+    required: false
+    default: null
+    aliases: []
+  wait:
+    description:
+      - wait for the VPC to be in state 'available' before returning
+    required: false
+    default: "no"
+    choices: [ "yes", "no" ]
+    aliases: []
+  wait_timeout:
+    description:
+      - how long before wait gives up, in seconds
+    default: 300
+    aliases: []
+  state:
+    description:
+      - Create or terminate the VPC
+    required: true
+    default: present
+    aliases: []
+  region:
+    description:
+      - The AWS region to use. If not specified then the value of the AWS_REGION or EC2_REGION environment variable, if any, is used.
+    required: true
+    default: null
+    aliases: ['aws_region', 'ec2_region']
+author: Carson Gee
+extends_documentation_fragment: aws
+'''
+
+EXAMPLES = '''
+# Note: None of these examples set aws_access_key, aws_secret_key, or region.
+# It is assumed that their matching environment variables are set.
+
+# Basic creation example:
+      ec2_vpc:
+        state: present
+        cidr_block: 172.23.0.0/16
+        resource_tags: { "Environment":"Development" }
+        region: us-west-2
+# Full creation example with subnets and optional availability zones.
+# The absence or presence of subnets deletes or creates them respectively.
+      ec2_vpc:
+        state: present
+        cidr_block: 172.22.0.0/16
+        resource_tags: { "Environment":"Development" }
+        subnets:
+          - cidr: 172.22.1.0/24
+            az: us-west-2c
+            resource_tags: { "Environment":"Dev", "Tier" : "Web" }
+          - cidr: 172.22.2.0/24
+            az: us-west-2b
+            resource_tags: { "Environment":"Dev", "Tier" : "App" }
+          - cidr: 172.22.3.0/24
+            az: us-west-2a
+            resource_tags: { "Environment":"Dev", "Tier" : "DB" }
+        internet_gateway: True
+        route_tables:
+          - subnets:
+              - 172.22.2.0/24
+              - 172.22.3.0/24
+            routes:
+              - dest: 0.0.0.0/0
+                gw: igw
+          - subnets:
+              - 172.22.1.0/24
+            routes:
+              - dest: 0.0.0.0/0
+                gw: igw
+        region: us-west-2
+      register: vpc
+
+# Removal of a VPC by id
+      ec2_vpc:
+        state: absent
+        vpc_id: vpc-aaaaaaa
+        region: us-west-2
+If you have added elements not managed by this module, e.g. instances, NATs, etc then
+the delete will fail until those dependencies are removed.
+'''
+
+
+import time
+
+try:
+    import boto.ec2
+    import boto.vpc
+    from boto.exception import EC2ResponseError
+
+    HAS_BOTO = True
+except ImportError:
+    HAS_BOTO = False
+
+
+def get_vpc_info(vpc):
+    """
+    Retrieves vpc information from an instance
+    ID and returns it as a dictionary
+    """
+
+    return({
+        'id': vpc.id,
+        'cidr_block': vpc.cidr_block,
+        'dhcp_options_id': vpc.dhcp_options_id,
+        'region': vpc.region.name,
+        'state': vpc.state,
+    })
+
+def get_igw_info(igw):
+    """
+    Get info about the internet gateway.
+    """
+    if igw is None:
+        return {}
+
+    return ({
+        'id': igw.id,
+    })
+
+def find_vpc(module, vpc_conn, vpc_id=None, cidr=None):
+    """
+    Finds a VPC that matches a specific id or cidr + tags
+
+    module : AnsibleModule object
+    vpc_conn: authenticated VPCConnection connection object
+
+    Returns:
+        A VPC object that matches either an ID or CIDR and one or more tag values
+    """
+
+    if vpc_id == None and cidr == None:
+        module.fail_json(
+            msg='You must specify either a vpc_id or a cidr block + list of unique tags, aborting'
+        )
+
+    found_vpcs = []
+
+    resource_tags = module.params.get('resource_tags')
+
+    # Check for existing VPC by cidr_block or id
+    if vpc_id is not None:
+        found_vpcs = vpc_conn.get_all_vpcs(None, {'vpc-id': vpc_id, 'state': 'available',})
+
+    else:
+        previous_vpcs = vpc_conn.get_all_vpcs(None, {'cidr': cidr, 'state': 'available'})
+
+        for vpc in previous_vpcs:
+            # Get all tags for each of the found VPCs
+            vpc_tags = dict((t.name, t.value) for t in vpc_conn.get_all_tags(filters={'resource-id': vpc.id}))
+
+            # If the supplied list of ID Tags match a subset of the VPC Tags, we found our VPC
+            if resource_tags and set(resource_tags.items()).issubset(set(vpc_tags.items())):
+                found_vpcs.append(vpc)
+
+    found_vpc = None
+
+    if len(found_vpcs) == 1:
+        found_vpc = found_vpcs[0]
+
+    if len(found_vpcs) > 1:
+        module.fail_json(msg='Found more than one vpc based on the supplied criteria, aborting')
+
+    return (found_vpc)
+
+def routes_match(rt_list=None, rt=None, igw=None):
+
+    """
+    Check if the route table has all routes as in given list
+    
+    rt_list      : A list if routes provided in the module 
+    rt           : The Remote route table object
+    igw          : The internet gateway object for this vpc
+
+    Returns:
+        True when there provided routes and remote routes are the same. 
+        False when provided routes and remote routes are diffrent.
+    """
+
+    local_routes = []
+    remote_routes = []
+    for route in rt_list:
+        route_kwargs = {}
+        if route['gw'] == 'igw':
+            route_kwargs['gateway_id'] = igw.id
+            route_kwargs['instance_id'] = None
+            route_kwargs['state'] = 'active'
+        elif route['gw'].startswith('i-'):
+            route_kwargs['instance_id'] = route['gw']
+            route_kwargs['gateway_id'] = None
+            route_kwargs['state'] = 'active'
+        else:
+            route_kwargs['gateway_id'] = route['gw']
+            route_kwargs['instance_id'] = None
+            route_kwargs['state'] = 'active'
+        route_kwargs['destination_cidr_block'] = route['dest']
+        local_routes.append(route_kwargs)
+    for j in rt.routes:
+        remote_routes.append(j.__dict__)
+    match = []
+    for i in local_routes:
+        change = "false"
+        for j in remote_routes:
+            if set(i.items()).issubset(set(j.items())):
+                change = "true"
+        match.append(change)
+    if 'false' in match:
+        return False
+    else:
+        return True
+
+def rtb_changed(route_tables=None, vpc_conn=None, module=None, vpc=None, igw=None):
+    """
+    Checks if the remote routes match the local routes.
+
+    route_tables : Route_tables parameter in the module
+    vpc_conn     : The VPC conection object
+    module       : The module object
+    vpc          : The vpc object for this route table
+    igw          : The internet gateway object for this vpc
+
+    Returns:
+        True when there is diffrence beween the provided routes and remote routes and if subnet assosications are diffrent.
+        False when both routes and subnet associations matched.
+ 
+    """
+    #We add a one for the main table
+    rtb_len = len(route_tables) + 1
+    remote_rtb_len = len(vpc_conn.get_all_route_tables(filters={'vpc_id': vpc.id}))
+    if remote_rtb_len != rtb_len:
+        return True
+    for rt in route_tables:
+        rt_id = None
+        for sn in rt['subnets']:
+            rsn = vpc_conn.get_all_subnets(filters={'cidr': sn, 'vpc_id': vpc.id })
+            if len(rsn) != 1:
+                module.fail_json(
+                    msg='The subnet {0} to associate with route_table {1} ' \
+                    'does not exist, aborting'.format(sn, rt)
+                )
+            nrt  = vpc_conn.get_all_route_tables(filters={'vpc_id': vpc.id, 'association.subnet-id': rsn[0].id})
+            if not nrt:
+                return True
+            else:
+                nrt = nrt[0]
+                if not rt_id:
+                    rt_id = nrt.id
+                    if not routes_match(rt['routes'], nrt, igw):
+                        return True
+                    continue
+                else:
+                    if rt_id == nrt.id:
+                        continue
+                    else:
+                        return True
+            return True
+    return False
+
+
+def create_vpc(module, vpc_conn):
+    """
+    Creates a new or modifies an existing VPC.
+
+    module : AnsibleModule object
+    vpc_conn: authenticated VPCConnection connection object
+
+    Returns:
+        A dictionary with information
+        about the VPC and subnets that were launched
+    """
+
+    id = module.params.get('vpc_id')
+    cidr_block = module.params.get('cidr_block')
+    instance_tenancy = module.params.get('instance_tenancy')
+    dns_support = module.params.get('dns_support')
+    dns_hostnames = module.params.get('dns_hostnames')
+    subnets = module.params.get('subnets')
+    internet_gateway = module.params.get('internet_gateway')
+    route_tables = module.params.get('route_tables')
+    vpc_spec_tags = module.params.get('resource_tags')
+    wait = module.params.get('wait')
+    wait_timeout = int(module.params.get('wait_timeout'))
+    changed = False
+
+    # Check for existing VPC by cidr_block + tags or id
+    previous_vpc = find_vpc(module, vpc_conn, id, cidr_block)
+
+    if previous_vpc is not None:
+        changed = False
+        vpc = previous_vpc
+    else:
+        changed = True
+        try:
+            vpc = vpc_conn.create_vpc(cidr_block, instance_tenancy)
+            # wait here until the vpc is available
+            pending = True
+            wait_timeout = time.time() + wait_timeout
+            while wait and wait_timeout > time.time() and pending:
+                try:
+                    pvpc = vpc_conn.get_all_vpcs(vpc.id)
+                    if hasattr(pvpc, 'state'):
+                        if pvpc.state == "available":
+                            pending = False
+                    elif hasattr(pvpc[0], 'state'):
+                        if pvpc[0].state == "available":
+                            pending = False
+                # sometimes vpc_conn.create_vpc() will return a vpc that can't be found yet by vpc_conn.get_all_vpcs()
+                # when that happens, just wait a bit longer and try again
+                except boto.exception.BotoServerError, e:
+                    if e.error_code != 'InvalidVpcID.NotFound':
+                        raise
+                if pending:
+                    time.sleep(5)
+            if wait and wait_timeout <= time.time():
+                # waiting took too long
+                module.fail_json(msg = "wait for vpc availability timeout on %s" % time.asctime())
+
+        except boto.exception.BotoServerError, e:
+            module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message))
+
+    # Done with base VPC, now change to attributes and features.
+
+    # Add resource tags
+    vpc_tags = dict((t.name, t.value) for t in vpc_conn.get_all_tags(filters={'resource-id': vpc.id}))
+
+    if not set(vpc_spec_tags.items()).issubset(set(vpc_tags.items())):
+        new_tags = {}
+
+        for (key, value) in set(vpc_spec_tags.items()):
+            if (key, value) not in set(vpc_tags.items()):
+                new_tags[key] = value
+
+        if new_tags:
+            vpc_conn.create_tags(vpc.id, new_tags)
+
+
+    # boto doesn't appear to have a way to determine the existing
+    # value of the dns attributes, so we just set them.
+    # It also must be done one at a time.
+    vpc_conn.modify_vpc_attribute(vpc.id, enable_dns_support=dns_support)
+    vpc_conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=dns_hostnames)
+
+
+    # Process all subnet properties
+    if subnets is not None:
+        if not isinstance(subnets, list):
+            module.fail_json(msg='subnets needs to be a list of cidr blocks')
+
+        current_subnets = vpc_conn.get_all_subnets(filters={ 'vpc_id': vpc.id })
+
+        # First add all new subnets
+        for subnet in subnets:
+            add_subnet = True
+            for csn in current_subnets:
+                if subnet['cidr'] == csn.cidr_block:
+                    add_subnet = False
+            if add_subnet:
+                try:
+                    new_subnet = vpc_conn.create_subnet(vpc.id, subnet['cidr'], subnet.get('az', None))
+                    new_subnet_tags = subnet.get('resource_tags', None)
+                    if new_subnet_tags:
+                        # Sometimes AWS takes its time to create a subnet and so using new subnets's id
+                        # to create tags results in exception.
+                        # boto doesn't seem to refresh 'state' of the newly created subnet, i.e.: it's always 'pending'
+                        # so i resorted to polling vpc_conn.get_all_subnets with the id of the newly added subnet
+                        while len(vpc_conn.get_all_subnets(filters={ 'subnet-id': new_subnet.id })) == 0:
+                            time.sleep(0.1)
+
+                        vpc_conn.create_tags(new_subnet.id, new_subnet_tags)
+
+                    changed = True
+                except EC2ResponseError, e:
+                    module.fail_json(msg='Unable to create subnet {0}, error: {1}'.format(subnet['cidr'], e))
+
+        # Now delete all absent subnets
+        for csubnet in current_subnets:
+            delete_subnet = True
+            for subnet in subnets:
+                if csubnet.cidr_block == subnet['cidr']:
+                    delete_subnet = False
+            if delete_subnet:
+                try:
+                    vpc_conn.delete_subnet(csubnet.id)
+                    changed = True
+                except EC2ResponseError, e:
+                    module.fail_json(msg='Unable to delete subnet {0}, error: {1}'.format(csubnet.cidr_block, e))
+
+    # Handle Internet gateway (create/delete igw)
+    igw = None
+    igws = vpc_conn.get_all_internet_gateways(filters={'attachment.vpc-id': vpc.id})
+    if len(igws) > 1:
+        module.fail_json(msg='EC2 returned more than one Internet Gateway for id %s, aborting' % vpc.id)
+    if internet_gateway:
+        if len(igws) != 1:
+            try:
+                igw = vpc_conn.create_internet_gateway()
+                vpc_conn.attach_internet_gateway(igw.id, vpc.id)
+                changed = True
+            except EC2ResponseError, e:
+                module.fail_json(msg='Unable to create Internet Gateway, error: {0}'.format(e))
+        else:
+            # Set igw variable to the current igw instance for use in route tables.
+            igw = igws[0]
+    else:
+        if len(igws) > 0:
+            try:
+                vpc_conn.detach_internet_gateway(igws[0].id, vpc.id)
+                vpc_conn.delete_internet_gateway(igws[0].id)
+                changed = True
+            except EC2ResponseError, e:
+                module.fail_json(msg='Unable to delete Internet Gateway, error: {0}'.format(e))
+
+    # Handle route tables - this may be worth splitting into a
+    # different module but should work fine here. The strategy to stay
+    # indempotent is to basically build all the route tables as
+    # defined, track the route table ids, and then run through the
+    # remote list of route tables and delete any that we didn't
+    # create.  This shouldn't interrupt traffic in theory, but is the
+    # only way to really work with route tables over time that I can
+    # think of without using painful aws ids.  Hopefully boto will add
+    # the replace-route-table API to make this smoother and
+    # allow control of the 'main' routing table.
+    if route_tables is not None:
+        rtb_needs_change = rtb_changed(route_tables, vpc_conn, module, vpc, igw)
+    if route_tables is not None and rtb_needs_change:
+        if not isinstance(route_tables, list):
+            module.fail_json(msg='route tables need to be a list of dictionaries')
+
+        # Work through each route table and update/create to match dictionary array
+        all_route_tables = []
+        for rt in route_tables:
+            try:
+                new_rt = vpc_conn.create_route_table(vpc.id)
+                for route in rt['routes']:
+                    route_kwargs = {}
+                    if route['gw'] == 'igw':
+                        if not internet_gateway:
+                            module.fail_json(
+                                msg='You asked for an Internet Gateway ' \
+                                '(igw) route, but you have no Internet Gateway'
+                            )
+                        route_kwargs['gateway_id'] = igw.id
+                    elif route['gw'].startswith('i-'):
+                        route_kwargs['instance_id'] = route['gw']
+                    else:
+                        route_kwargs['gateway_id'] = route['gw']
+                    vpc_conn.create_route(new_rt.id, route['dest'], **route_kwargs)
+
+                # Associate with subnets
+                for sn in rt['subnets']:
+                    rsn = vpc_conn.get_all_subnets(filters={'cidr': sn, 'vpc_id': vpc.id })
+                    if len(rsn) != 1:
+                        module.fail_json(
+                            msg='The subnet {0} to associate with route_table {1} ' \
+                            'does not exist, aborting'.format(sn, rt)
+                        )
+                    rsn = rsn[0]
+
+                    # Disassociate then associate since we don't have replace
+                    old_rt = vpc_conn.get_all_route_tables(
+                        filters={'association.subnet_id': rsn.id, 'vpc_id': vpc.id}
+                    )
+                    old_rt = [ x for x in old_rt if x.id != None ]
+                    if len(old_rt) == 1:
+                        old_rt = old_rt[0]
+                        association_id = None
+                        for a in old_rt.associations:
+                            if a.subnet_id == rsn.id:
+                                association_id = a.id
+                        vpc_conn.disassociate_route_table(association_id)
+
+                    vpc_conn.associate_route_table(new_rt.id, rsn.id)
+
+                all_route_tables.append(new_rt)
+                changed = True
+            except EC2ResponseError, e:
+                module.fail_json(
+                    msg='Unable to create and associate route table {0}, error: ' \
+                    '{1}'.format(rt, e)
+                )
+
+        # Now that we are good to go on our new route tables, delete the
+        # old ones except the 'main' route table as boto can't set the main
+        # table yet.
+        all_rts = vpc_conn.get_all_route_tables(filters={'vpc-id': vpc.id})
+        for rt in all_rts:
+            if rt.id is None:
+                continue
+            delete_rt = True
+            for newrt in all_route_tables:
+                if newrt.id == rt.id:
+                    delete_rt = False
+                    break
+            if delete_rt:
+                rta = rt.associations
+                is_main = False
+                for a in rta:
+                    if a.main:
+                        is_main = True
+                        break
+                try:
+                    if not is_main:
+                        vpc_conn.delete_route_table(rt.id)
+                        changed = True
+                except EC2ResponseError, e:
+                    module.fail_json(msg='Unable to delete old route table {0}, error: {1}'.format(rt.id, e))
+
+    vpc_dict = get_vpc_info(vpc)
+    igw_dict = get_igw_info(igw)
+    created_vpc_id = vpc.id
+    returned_subnets = []
+    current_subnets = vpc_conn.get_all_subnets(filters={ 'vpc_id': vpc.id })
+
+    for sn in current_subnets:
+        returned_subnets.append({
+            'resource_tags': dict((t.name, t.value) for t in vpc_conn.get_all_tags(filters={'resource-id': sn.id})),
+            'cidr': sn.cidr_block,
+            'az': sn.availability_zone,
+            'id': sn.id,
+        })
+
+    if subnets is not None:
+        # Sort subnets by the order they were listed in the play
+        order = {}
+        for idx, val in enumerate(subnets):
+            order[val['cidr']] = idx
+
+        # Number of subnets in the play
+        subnets_in_play = len(subnets)
+        returned_subnets.sort(key=lambda x: order.get(x['cidr'], subnets_in_play))
+
+    return (vpc_dict, created_vpc_id, returned_subnets, igw_dict, changed)
+
+def terminate_vpc(module, vpc_conn, vpc_id=None, cidr=None):
+    """
+    Terminates a VPC
+
+    module: Ansible module object
+    vpc_conn: authenticated VPCConnection connection object
+    vpc_id: a vpc id to terminate
+    cidr: The cidr block of the VPC - can be used in lieu of an ID
+
+    Returns a dictionary of VPC information
+    about the VPC terminated.
+
+    If the VPC to be terminated is available
+    "changed" will be set to True.
+
+    """
+    vpc_dict = {}
+    terminated_vpc_id = ''
+    changed = False
+
+    vpc = find_vpc(module, vpc_conn, vpc_id, cidr)
+
+    if vpc is not None:
+        if vpc.state == 'available':
+            terminated_vpc_id=vpc.id
+            vpc_dict=get_vpc_info(vpc)
+            try:
+                subnets = vpc_conn.get_all_subnets(filters={'vpc_id': vpc.id})
+                for sn in subnets:
+                    vpc_conn.delete_subnet(sn.id)
+
+                igws = vpc_conn.get_all_internet_gateways(
+                    filters={'attachment.vpc-id': vpc.id}
+                )
+                for igw in igws:
+                    vpc_conn.detach_internet_gateway(igw.id, vpc.id)
+                    vpc_conn.delete_internet_gateway(igw.id)
+
+                rts = vpc_conn.get_all_route_tables(filters={'vpc_id': vpc.id})
+                for rt in rts:
+                    rta = rt.associations
+                    is_main = False
+                    for a in rta:
+                        if a.main:
+                            is_main = True
+                    if not is_main:
+                        vpc_conn.delete_route_table(rt.id)
+
+                vpc_conn.delete_vpc(vpc.id)
+            except EC2ResponseError, e:
+                module.fail_json(
+                    msg='Unable to delete VPC {0}, error: {1}'.format(vpc.id, e)
+                )
+            changed = True
+
+    return (changed, vpc_dict, terminated_vpc_id)
+
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(dict(
+            cidr_block = dict(),
+            instance_tenancy = dict(choices=['default', 'dedicated'], default='default'),
+            wait = dict(type='bool', default=False),
+            wait_timeout = dict(default=300),
+            dns_support = dict(type='bool', default=True),
+            dns_hostnames = dict(type='bool', default=True),
+            subnets = dict(type='list'),
+            vpc_id = dict(),
+            internet_gateway = dict(type='bool', default=False),
+            resource_tags = dict(type='dict', required=True),
+            route_tables = dict(type='list'),
+            state = dict(choices=['present', 'absent'], default='present'),
+        )
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+    )
+
+    if not HAS_BOTO:
+        module.fail_json(msg='boto required for this module')
+
+    state = module.params.get('state')
+
+    region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)
+
+    # If we have a region specified, connect to its endpoint.
+    if region:
+        try:
+            vpc_conn = boto.vpc.connect_to_region(
+                region,
+                **aws_connect_kwargs
+            )
+        except boto.exception.NoAuthHandlerFound, e:
+            module.fail_json(msg = str(e))
+    else:
+        module.fail_json(msg="region must be specified")
+
+    igw_dict = {}
+    if module.params.get('state') == 'absent':
+        vpc_id = module.params.get('vpc_id')
+        cidr = module.params.get('cidr_block')
+        (changed, vpc_dict, new_vpc_id) = terminate_vpc(module, vpc_conn, vpc_id, cidr)
+        subnets_changed = None
+    elif module.params.get('state') == 'present':
+        # Changed is always set to true when provisioning a new VPC
+        (vpc_dict, new_vpc_id, subnets_changed, igw_dict, changed) = create_vpc(module, vpc_conn)
+
+    module.exit_json(changed=changed, vpc_id=new_vpc_id, vpc=vpc_dict, igw=igw_dict, subnets=subnets_changed)
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+
+main()

playbooks/library/util_map

+#!/usr/bin/env python
+
+DOCUMENTATION = """
+---
+module: util_map
+short_description: Applies a function to input and returns the result with the key function_output
+description:
+  - Applies functions to data structures returning a new data structure.
+version_added: "1.8"
+author: Edward Zarecor
+options:
+  function:
+    description:
+      - The function to apply, currently ['zip_to_dict','flatten']
+    required: true
+  input:
+    description:
+      - The input
+    required: true
+  args
+    description:
+      - Arguments to the function other than the input, varies by function.
+"""
+
+EXAMPLES = '''
+- name: Apply function to results from ec2_scaling_policies
+  util_map:
+    function: 'zip_to_dict'
+    input: "{{ created_policies.results }}"
+    args:
+      - "name"
+      - "arn"
+  register: policy_data
+
+- name: Apply function to policy data
+  util_map:
+    function: 'flatten'
+    input:
+      - 'a'
+      - 'b'
+      - 'c'
+      - ['d','e','f']
+  register: flat_list
+'''
+
+from ansible.module_utils.basic import *
+import ast
+import itertools
+
+class ArgumentError(Exception):
+    pass
+
+def flatten(module, input):
+    """
+    Takes an iterable and returns a flat list
+
+    With the input of
+
+    [['a','b','c'],'d',['e','f','g'],{'a','b'}]
+
+    this function will return
+
+    ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b']
+
+    :param module: The ansible module
+    :param input: An iterable
+    :return: A flat list.
+    """
+
+    try:
+        flat = list(itertools.chain.from_iterable(input))
+    except TypeError as te:
+        raise ArgumentError("Flatten resulted in a type error, {0}.".format(te.message))
+
+    module.exit_json(function_output = flat)
+
+def zip_to_dict(module, input, key_key, value_key):
+    """
+    Takes an array of dicts and flattens it to a single dict by extracting the values
+    of a provided key as the keys in the new dict and the values of the second
+    provided key as the corresponding values.
+
+    For example, the input dict of
+
+    [{'name':'fred', 'id':'123'},{'name':'bill', 'id':'321'}]
+
+    with an args array of ['id','name']
+
+    would return
+
+    {'123':'fred','321':'bill'}
+
+    :param input: an array of dicts, typically the results of an ansible module
+    :param key_key: a key into the input dict returning a value to be used as a key in the flattened dict
+    :param value_key: a key into the input dict returning a value to be used as a value in the flattened dict
+    :return: the flattened dict
+    """
+
+    results = {}
+
+    for item in input:
+        results[item[key_key]]=item[value_key]
+
+    module.exit_json(function_output = results)
+
+def zip_to_list(module, input, key):
+    """
+    Takes an array of dicts and flattens it to a single list by extracting the value
+    of a provided key as an item in the new list.
+
+    For example, the input list of dicts like
+
+    [{'name':'fred', 'id':'123'},{'name':'bill', 'id':'321'}]
+
+    with an args array of ['name']
+
+    would return
+
+    ['fred','bill']
+
+    :param input: an array of dicts, typically the results of an ansible module
+    :param key: a key into the input dict returning a value to be used as an item in the flattend list
+    :return: the flattened list
+    """
+
+    results = []
+
+    for item in input:
+        results.append(item[key])
+
+    module.exit_json(function_output = results)
+
+def main():
+
+    arg_spec = dict(
+        function=dict(required=True, type='str'),
+        input=dict(required=True, type='str'),
+        args=dict(required=False, type='list'),
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=False)
+
+    target = module.params.get('function')
+    # reify input data
+    input = ast.literal_eval(module.params.get('input'))
+    args = module.params.get('args')
+
+    if target == 'zip_to_dict':
+        zip_to_dict(module, input, *args)
+    elif target == 'flatten':
+        flatten(module,input)
+    elif target == 'zip_to_list':
+        zip_to_list(module, input, *args)
+    else:
+        raise NotImplemented("Function {0} is not implemented.".format(target))
+
+main()

playbooks/roles/edx_vpc/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx_vpc
+# 
+
+#
+# vars are namespace with the module name.
+#
+vpc_role_name: vpc
+
+#
+# OS packages
+#
+
+vpc_debian_pkgs: []
+
+vpc_redhat_pkgs: []

playbooks/roles/edx_vpc/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role edx_vpc
+# 
+# Overview:
+# This role creates an opinionated vpc for containing cluster of edx services.
+# 
+# It currently assumes that we will be multi-az, with a single NAT, and all
+# traffic going over that NAT.  A public subnet, and both public and private
+# route tables are created by default that can be used by new services in this
+# vpc.  The public subnet should house ELBs and any newly created private subnets
+# can use the existing private route table to be able to reach the internet from
+# private machines.
+# 
+#
+# Example play:
+#
+# ansible-playbook -c local -i localhost, edx_vpc.yml -e@/Users/feanil/src/edx-secure/cloud_migrations/vpcs/test.yml
+
+# DO NOT use the subnet or route table sections of this command.
+# They will delete any subnets or rts not defined here which is
+# probably not what you want, since other services were added
+# to the vpc whose subnets and rts are not enumerated here.
+- name: create a vpc
+  local_action:
+    module: "ec2_vpc_local"
+    resource_tags: "{{ vpc_tags }}"
+    cidr_block: "{{ vpc_cidr }}"
+    region: "{{ vpc_aws_region }}"
+    state: "{{ vpc_state }}"
+    internet_gateway: yes
+    wait: yes
+  register: created_vpc
+
+# A default network acl is created when a vpc is created so each VPC
+# should have one but we create one here that allows access to the
+# outside world using the internet gateway.
+- name: create public network acl
+  ec2_acl:
+    profile: "{{ vpc_aws_profile }}"
+    name: "{{ vpc_public_acl.name }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    rules: "{{ vpc_public_acl.rules }}"
+  register: created_public_acl
+
+- name: create public route table
+  ec2_rt:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ vpc_name }}-public"
+    routes: "{{ vpc_public_route_table }}"
+  register: created_public_rt
+
+- name: create public subnets
+  ec2_subnet:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ item.name }}"
+    cidr: "{{ item.cidr }}"
+    az: "{{ item.az }}"
+    route_table_id: "{{ created_public_rt.id }}"
+    network_acl_id: "{{ created_public_acl.id }}"
+  with_items: vpc_public_subnets
+  register: created_public_subnets
+
+- name: create NAT security group
+  ec2_group:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    name: "{{ nat_security_group.name }}"
+    rules: "{{ nat_security_group.rules }}"
+    description: "{{ nat_security_group.description }}"
+    rules_egress: "{{ nat_security_group.rules_egress }}"
+  register: created_nat_security_group
+
+- name: check to see if we already have a nat instance
+  local_action:
+    module: "ec2_lookup"
+    region: "{{ vpc_aws_region }}"
+    tags:
+      - Name: "{{ vpc_name }}-nat-instance"
+  register: nat_instance
+
+- name: create nat instance
+  local_action:
+    module: "ec2"
+    state: "present"
+    wait: yes
+    source_dest_check: false
+    region: "{{ vpc_aws_region }}"
+    profile: "{{ vpc_aws_profile }}"
+    group_id: "{{ created_nat_security_group.group_id }}"
+    key_name: "{{ vpc_keypair }}"
+    vpc_subnet_id: "{{ created_public_subnets.results[0].subnet_id }}"
+    instance_type: "{{ vpc_nat_instance_type }}"
+    instance_tags:
+      Name: "{{ vpc_name }}-nat-instance"
+    image: "{{ vpc_nat_ami_id }}"
+  register: new_nat_instance
+  when: nat_instance.instances|length == 0
+
+# We need to do this instead of registering the output of the above
+# command because if the above command get skipped, the output does
+# not contain information about the instance.
+- name: lookup the created nat_instance
+  local_action:
+    module: "ec2_lookup"
+    region: "{{ vpc_aws_region }}"
+    tags:
+      - Name: "{{ vpc_name }}-nat-instance"
+  register: nat_instance
+
+- name: assign eip to nat
+  ec2_eip:
+    profile: "{{ vpc_aws_profile }}"
+    region: "{{ vpc_aws_region }}"
+    instance_id: "{{ nat_instance.instances[0].id }}"
+    in_vpc: true
+    reuse_existing_ip_allowed: true
+  when: new_nat_instance.changed
+
+- name: create private route table
+  ec2_rt:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ vpc_name }}-private"
+    routes: "{{ vpc_private_route_table }}"
+  register: created_private_rt
+
+- name: create db network acl
+  ec2_acl:
+    profile: "{{ vpc_aws_profile }}"
+    name: "{{ vpc_db_acl.name }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    rules: "{{ vpc_db_acl.rules }}"
+  register: created_db_acl
+
+- name: create db subnets
+  ec2_subnet:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ item.name }}"
+    cidr: "{{ item.cidr }}"
+    az: "{{ item.az }}"
+    route_table_id: "{{ created_private_rt.id }}"
+    network_acl_id: "{{ created_db_acl.id }}"
+  with_items: vpc_db_subnets
+  register: created_db_subnets
+
+- name: output a vpc_config for using to build services
+  local_action:
+    module: template
+    src: "vpc_config.yml.j2"
+    dest: "~/{{ e_d }}.yml"

playbooks/roles/edx_vpc/templates/vpc_config.yml.j2

+#
+# Configuration for the environment-deployment
+#
+profile: "{{ vpc_aws_profile }}"
+vpc_id: "{{ created_vpc.vpc_id }}"
+vpc_cidr: "{{ vpc_cidr }}"
+vpc_class_b: "{{ vpc_class_b }}"
+
+env: "{{ vpc_environment }}"
+deployment: "{{ vpc_deployment }}"
+e_d_c: "{{ vpc_environment }}-{{ vpc_deployment }}-{{ '{{' }} cluster {{ '}}' }}"
+aws_region: "{{ vpc_aws_region }}"
+
+
+aws_availability_zones:
+{% for subnet in vpc_public_subnets %}
+  - {{ subnet.az }}
+{% endfor %}
+
+# Should this be service specific
+ssl_cert: "{{ vpc_ssl_cert }}"
+
+# used for ELB
+public_route_table: "{{ created_public_rt.id }}"
+# used for service subnet
+private_route_table: "{{ created_private_rt.id }}"
+instance_key_name: "{{ vpc_keypair }}"
+
+# subject to change #TODO: provide the correct var for the eni
+nat_device: "{{ nat_instance.instances[0].id }}"
+
+public_subnet_1: "{{ vpc_public_subnets[0].cidr }}"
+public_subnet_2: "{{ vpc_public_subnets[1].cidr }}"
+
+# /28 per AZ NEEDE?
+# private_subnet_1: "{{ vpc_class_b }}.110.16/28"
+# private_subnet_2: "{{ vpc_class_b }}.120.16/28"
+
+elb_subnets:
+{% for subnet in created_public_subnets.results %}
+  - "{{ subnet.subnet_id }}"
+{% endfor %}
+
+db_subnets:
+{% for subnet in created_db_subnets %}
+  - "{{ subnet.subnet_id }}"
+{% endfor %}
+
+#
+# Do not use vars in policies :(
+# Should be specific to the service right?
+role_policies: []
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-s3-policy"
+#     document: |
+#       {
+#         "Statement":[
+#           {
+#             "Effect":"Allow",
+#             "Action":["s3:*"],
+#             "Resource":["arn:aws:s3:::edx-stage-edx"]
+#           }
+#         ]
+#         }  
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-create-instance-tags"
+#     document: |
+#       {
+#         "Statement": [
+#           {
+#             "Effect": "Allow",
+#             "Action": ["ec2:CreateTags"],
+#             "Resource": ["arn:aws:ec2:us-east-1:xxxxxxxxxxxx:instance/*"]
+#           }
+#         ]
+#         }        
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-describe-ec2"
+#     document: |
+#         {"Statement":[
+#         {"Resource":"*",
+#         "Action":["ec2:DescribeInstances","ec2:DescribeTags","ec2:DescribeVolumes"],
+#         "Effect":"Allow"}]}