Merge pull request #2006 from edx/feanil/vpc_tool_update

Feanil/vpc tool update

playbooks/edx-east/edx_service.yml

@@ -17,7 +17,7 @@
         policies: "{{ role_policies }}"
 
     - name: Manage ELB security group
-      ec2_group:
+      ec2_group_local:
         profile: "{{ profile }}"
         description: "{{ elb_security_group.description }}"
         name: "{{ elb_security_group.name }}"
@@ -26,9 +26,11 @@
         rules: "{{ elb_security_group.rules }}"
         tags: "{{ elb_security_group.tags }}"
       register: elb_sec_group
-        
+
+    - debug: msg="{{ service_security_group.rules }}"
+
     - name: Manage service security group
-      ec2_group:
+      ec2_group_local:
         profile: "{{ profile }}"
         description: "{{ service_security_group.description }}"
         name: "{{ service_security_group.name }}"

playbooks/edx-east/edx_vpc.yml

 ---
+# Sample command: ansible-playbook -c local -i localhost, edx_vpc.yml -e@/Users/feanil/src/edx-secure/cloud_migrations/vpcs/test.yml -vvv
 - name: Create a simple empty vpc
   hosts: all
   connection: local
   gather_facts: False
   vars:
     vpc_state: present
-  tasks:
-    # DO NOT use the subnet or route table sections of this command.
-    # They will delete any subnets or rts not defined here which is
-    # probably not what you want, since other services were added
-    # to the vpc whose subnets and rts are not enumerated here.
-    - name: create a vpc
-      local_action:
-        module: 'ec2_vpc_local'
-        resource_tags: '{{ vpc_tags }}'
-        cidr_block: '{{ vpc_cidr }}'
-        region: '{{ aws_region }}'
-        state: '{{ vpc_state }}'
-        internet_gateway: yes
-        wait: yes
-      register: created_vpc
-
-    # A default network acl is created when a vpc is created so each VPC
-    # should have one but we create one here that allows access to the
-    # outside world using the internet gateway.
-    - name: create public network acl
-      ec2_acl:
-        profile: "{{ profile }}"
-        name: "{{ vpc_public_acl.name }}"
-        vpc_id: "{{ created_vpc.vpc_id }}"
-        state: "present"
-        region: "{{ aws_region }}"
-        rules: "{{ vpc_public_acl.rules }}"
-      register: created_public_acl
-
-    - name: create public route table
-      ec2_rt:
-        profile: "{{ profile }}"
-        vpc_id: "{{ created_vpc.vpc_id }}"
-        region: "{{ aws_region }}"
-        state: "present"
-        name: "{{ vpc_name }}-public"
-        routes: "{{ vpc_public_route_table }}"
-      register: created_public_rt
-
-    - name: create public subnets
-      ec2_subnet:
-        profile: "{{ profile }}"
-        vpc_id: "{{ created_vpc.vpc_id }}"
-        region: "{{ aws_region }}"
-        state: "present"
-        name: "{{ item.name }}"
-        cidr: "{{ item.cidr }}"
-        az: "{{ item.az }}"
-        route_table_id: "{{ created_public_rt.id }}"
-        network_acl_id: "{{ created_public_acl.id }}"
-      with_items: vpc_public_subnets
-      register: created_public_subnets
-
-    - name: create NAT security group
-      ec2_group:
-        profile: "{{ profile }}"
-        vpc_id: "{{ created_vpc.vpc_id }}"
-        state: "present"
-        region: "{{ aws_region }}"
-        name: "{{ nat_security_group.name }}"
-        rules: "{{ nat_security_group.rules }}"
-        description: "{{ nat_security_group.description }}"
-        rules_egress: "{{ nat_security_group.rules_egress }}"
-      register: created_nat_security_group
-
-    - name: check to see if we already have a nat instance
-      local_action:
-        module: "ec2_lookup"
-        region: "{{ aws_region }}"
-        tags:
-          Name: "{{ vpc_name }}-nat-instance"
-      register: potential_existing_nat
-
-    - debug: msg="{{ potential_existing_nat }}"
-
-    - name: create nat instance
-      local_action:
-        module: 'ec2'
-        state: 'present'
-        wait: "yes"
-        source_dest_check: false
-        region: "{{ aws_region }}"
-        profile: "{{ profile }}"
-        group_id: "{{ created_nat_security_group.group_id }}"
-        key_name: "{{ vpc_nat_keypair }}"
-        vpc_subnet_id: "{{ created_public_subnets.results[0].subnet_id }}"
-        instance_type: "{{ vpc_nat_instance_type }}"
-        instance_tags:
-          Name: "{{ vpc_name }}-nat-instance"
-        image: "{{ vpc_nat_ami_id }}"
-      register: created_nat_instance
-      when: potential_existing_nat.instances|length == 0
-
-    - name: assign eip to nat
-      ec2_eip:
-        profile: "{{ profile }}"
-        region: "{{ aws_region }}"
-        instance_id: "{{ created_nat_instance.instances[0].id }}"
-        in_vpc: true
-        reuse_existing_ip_allowed: true
-      when: potential_existing_nat.instances|length == 0
-
-    - name: create private route table
-      ec2_rt:
-        profile: "{{ profile }}"
-        vpc_id: "{{ created_vpc.vpc_id }}"
-        region: "{{ aws_region }}"
-        state: "present"
-        name: "{{ vpc_name }}-private"
-        routes: "{{ vpc_private_route_table }}"
-      register: created_public_rt
-      when: potential_existing_nat.instances|length == 0
-
+  roles:
+    - edx_vpc

playbooks/library/util_map

@@ -103,10 +103,37 @@ def zip_to_dict(module, input, key_key, value_key):
 
     module.exit_json(function_output = results)
 
+def zip_to_list(module, input, key):
+    """
+    Takes an array of dicts and flattens it to a single list by extracting the value
+    of a provided key as an item in the new list.
+
+    For example, the input list of dicts like
+
+    [{'name':'fred', 'id':'123'},{'name':'bill', 'id':'321'}]
+
+    with an args array of ['name']
+
+    would return
+
+    ['fred','bill']
+
+    :param input: an array of dicts, typically the results of an ansible module
+    :param key: a key into the input dict returning a value to be used as an item in the flattend list
+    :return: the flattened list
+    """
+
+    results = []
+
+    for item in input:
+        results.append(item[key])
+
+    module.exit_json(function_output = results)
+
 def main():
 
     arg_spec = dict(
-        function=dict(required=True,choices=['zip_to_dict','flatten']),
+        function=dict(required=True, type='str'),
         input=dict(required=True, type='str'),
         args=dict(required=False, type='list'),
     )
@@ -122,6 +149,8 @@ def main():
         zip_to_dict(module, input, *args)
     elif target == 'flatten':
         flatten(module,input)
+    elif target == 'zip_to_list':
+        zip_to_list(module, input, *args)
     else:
         raise NotImplemented("Function {0} is not implemented.".format(target))
 

playbooks/roles/edx_vpc/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx_vpc
+# 
+
+#
+# vars are namespace with the module name.
+#
+vpc_role_name: vpc
+
+#
+# OS packages
+#
+
+vpc_debian_pkgs: []
+
+vpc_redhat_pkgs: []

playbooks/roles/edx_vpc/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role edx_vpc
+# 
+# Overview:
+# This role creates an opinionated vpc for containing cluster of edx services.
+# 
+# It currently assumes that we will be multi-az, with a single NAT, and all
+# traffic going over that NAT.  A public subnet, and both public and private
+# route tables are created by default that can be used by new services in this
+# vpc.  The public subnet should house ELBs and any newly created private subnets
+# can use the existing private route table to be able to reach the internet from
+# private machines.
+# 
+#
+# Example play:
+#
+# ansible-playbook -c local -i localhost, edx_vpc.yml -e@/Users/feanil/src/edx-secure/cloud_migrations/vpcs/test.yml
+
+# DO NOT use the subnet or route table sections of this command.
+# They will delete any subnets or rts not defined here which is
+# probably not what you want, since other services were added
+# to the vpc whose subnets and rts are not enumerated here.
+- name: create a vpc
+  local_action:
+    module: "ec2_vpc_local"
+    resource_tags: "{{ vpc_tags }}"
+    cidr_block: "{{ vpc_cidr }}"
+    region: "{{ vpc_aws_region }}"
+    state: "{{ vpc_state }}"
+    internet_gateway: yes
+    wait: yes
+  register: created_vpc
+
+# A default network acl is created when a vpc is created so each VPC
+# should have one but we create one here that allows access to the
+# outside world using the internet gateway.
+- name: create public network acl
+  ec2_acl:
+    profile: "{{ vpc_aws_profile }}"
+    name: "{{ vpc_public_acl.name }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    rules: "{{ vpc_public_acl.rules }}"
+  register: created_public_acl
+
+- name: create public route table
+  ec2_rt:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ vpc_name }}-public"
+    routes: "{{ vpc_public_route_table }}"
+  register: created_public_rt
+
+- name: create public subnets
+  ec2_subnet:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ item.name }}"
+    cidr: "{{ item.cidr }}"
+    az: "{{ item.az }}"
+    route_table_id: "{{ created_public_rt.id }}"
+    network_acl_id: "{{ created_public_acl.id }}"
+  with_items: vpc_public_subnets
+  register: created_public_subnets
+
+- name: create NAT security group
+  ec2_group:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    name: "{{ nat_security_group.name }}"
+    rules: "{{ nat_security_group.rules }}"
+    description: "{{ nat_security_group.description }}"
+    rules_egress: "{{ nat_security_group.rules_egress }}"
+  register: created_nat_security_group
+
+- name: check to see if we already have a nat instance
+  local_action:
+    module: "ec2_lookup"
+    region: "{{ vpc_aws_region }}"
+    tags:
+      - Name: "{{ vpc_name }}-nat-instance"
+  register: nat_instance
+
+- name: create nat instance
+  local_action:
+    module: "ec2"
+    state: "present"
+    wait: yes
+    source_dest_check: false
+    region: "{{ vpc_aws_region }}"
+    profile: "{{ vpc_aws_profile }}"
+    group_id: "{{ created_nat_security_group.group_id }}"
+    key_name: "{{ vpc_keypair }}"
+    vpc_subnet_id: "{{ created_public_subnets.results[0].subnet_id }}"
+    instance_type: "{{ vpc_nat_instance_type }}"
+    instance_tags:
+      Name: "{{ vpc_name }}-nat-instance"
+    image: "{{ vpc_nat_ami_id }}"
+  register: new_nat_instance
+  when: nat_instance.instances|length == 0
+
+# We need to do this instead of registering the output of the above
+# command because if the above command get skipped, the output does
+# not contain information about the instance.
+- name: lookup the created nat_instance
+  local_action:
+    module: "ec2_lookup"
+    region: "{{ vpc_aws_region }}"
+    tags:
+      - Name: "{{ vpc_name }}-nat-instance"
+  register: nat_instance
+
+- name: assign eip to nat
+  ec2_eip:
+    profile: "{{ vpc_aws_profile }}"
+    region: "{{ vpc_aws_region }}"
+    instance_id: "{{ nat_instance.instances[0].id }}"
+    in_vpc: true
+    reuse_existing_ip_allowed: true
+  when: new_nat_instance.changed
+
+- name: create private route table
+  ec2_rt:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ vpc_name }}-private"
+    routes: "{{ vpc_private_route_table }}"
+  register: created_private_rt
+
+- name: create db network acl
+  ec2_acl:
+    profile: "{{ vpc_aws_profile }}"
+    name: "{{ vpc_db_acl.name }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    state: "present"
+    region: "{{ vpc_aws_region }}"
+    rules: "{{ vpc_db_acl.rules }}"
+  register: created_db_acl
+
+- name: create db subnets
+  ec2_subnet:
+    profile: "{{ vpc_aws_profile }}"
+    vpc_id: "{{ created_vpc.vpc_id }}"
+    region: "{{ vpc_aws_region }}"
+    state: "present"
+    name: "{{ item.name }}"
+    cidr: "{{ item.cidr }}"
+    az: "{{ item.az }}"
+    route_table_id: "{{ created_private_rt.id }}"
+    network_acl_id: "{{ created_db_acl.id }}"
+  with_items: vpc_db_subnets
+  register: created_db_subnets
+
+- name: output a vpc_config for using to build services
+  local_action:
+    module: template
+    src: "vpc_config.yml.j2"
+    dest: "~/{{ e_d }}.yml"

playbooks/roles/edx_vpc/templates/vpc_config.yml.j2

+#
+# Configuration for the environment-deployment
+#
+profile: "{{ vpc_aws_profile }}"
+vpc_id: "{{ created_vpc.vpc_id }}"
+vpc_cidr: "{{ vpc_cidr }}"
+vpc_class_b: "{{ vpc_class_b }}"
+
+env: "{{ vpc_environment }}"
+deployment: "{{ vpc_deployment }}"
+e_d_c: "{{ vpc_environment }}-{{ vpc_deployment }}-{{ '{{' }} cluster {{ '}}' }}"
+aws_region: "{{ vpc_aws_region }}"
+
+
+aws_availability_zones:
+{% for subnet in vpc_public_subnets %}
+  - {{ subnet.az }}
+{% endfor %}
+
+# Should this be service specific
+ssl_cert: "{{ vpc_ssl_cert }}"
+
+# used for ELB
+public_route_table: "{{ created_public_rt.id }}"
+# used for service subnet
+private_route_table: "{{ created_private_rt.id }}"
+instance_key_name: "{{ vpc_keypair }}"
+
+# subject to change #TODO: provide the correct var for the eni
+nat_device: "{{ nat_instance.instances[0].id }}"
+
+public_subnet_1: "{{ vpc_public_subnets[0].cidr }}"
+public_subnet_2: "{{ vpc_public_subnets[1].cidr }}"
+
+# /28 per AZ NEEDE?
+# private_subnet_1: "{{ vpc_class_b }}.110.16/28"
+# private_subnet_2: "{{ vpc_class_b }}.120.16/28"
+
+elb_subnets:
+{% for subnet in created_public_subnets.results %}
+  - "{{ subnet.subnet_id }}"
+{% endfor %}
+
+db_subnets:
+{% for subnet in created_db_subnets %}
+  - "{{ subnet.subnet_id }}"
+{% endfor %}
+
+#
+# Do not use vars in policies :(
+# Should be specific to the service right?
+role_policies: []
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-s3-policy"
+#     document: |
+#       {
+#         "Statement":[
+#           {
+#             "Effect":"Allow",
+#             "Action":["s3:*"],
+#             "Resource":["arn:aws:s3:::edx-stage-edx"]
+#           }
+#         ]
+#         }  
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-create-instance-tags"
+#     document: |
+#       {
+#         "Statement": [
+#           {
+#             "Effect": "Allow",
+#             "Action": ["ec2:CreateTags"],
+#             "Resource": ["arn:aws:ec2:us-east-1:xxxxxxxxxxxx:instance/*"]
+#           }
+#         ]
+#         }        
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-describe-ec2"
+#     document: |
+#         {"Statement":[
+#         {"Resource":"*",
+#         "Action":["ec2:DescribeInstances","ec2:DescribeTags","ec2:DescribeVolumes"],
+#         "Effect":"Allow"}]}