Generate a varfile for use with the edx_service.yml playbook.

playbooks/edx-east/edx_service.yml

@@ -17,7 +17,7 @@
         policies: "{{ role_policies }}"
 
     - name: Manage ELB security group
-      ec2_group:
+      ec2_group_local:
         profile: "{{ profile }}"
         description: "{{ elb_security_group.description }}"
         name: "{{ elb_security_group.name }}"
@@ -26,9 +26,11 @@
         rules: "{{ elb_security_group.rules }}"
         tags: "{{ elb_security_group.tags }}"
       register: elb_sec_group
-        
+
+    - debug: msg="{{ service_security_group.rules }}"
+
     - name: Manage service security group
-      ec2_group:
+      ec2_group_local:
         profile: "{{ profile }}"
         description: "{{ service_security_group.description }}"
         name: "{{ service_security_group.name }}"

playbooks/library/ec2_group_local

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+DOCUMENTATION = '''
+---
+module: ec2_group
+version_added: "1.3"
+short_description: maintain an ec2 VPC security group.
+description:
+    - maintains ec2 security groups. This module has a dependency on python-boto >= 2.5
+options:
+  name:
+    description:
+      - Name of the security group.
+    required: true
+  description:
+    description:
+      - Description of the security group.
+    required: true
+  vpc_id:
+    description:
+      - ID of the VPC to create the group in.
+    required: false
+  rules:
+    description:
+      - List of firewall inbound rules to enforce in this group (see example).
+    required: false
+  rules_egress:
+    description:
+      - List of firewall outbound rules to enforce in this group (see example).
+    required: false
+    version_added: "1.6"
+  tags:
+    description:
+      - List of tags to apply to this security group
+    required: false
+    version_added: "1.8"
+  region:
+    description:
+      - the EC2 region to use
+    required: false
+    default: null
+    aliases: []
+  state:
+    version_added: "1.4"
+    description:
+      - create or delete security group
+    required: false
+    default: 'present'
+    aliases: []
+
+extends_documentation_fragment: aws
+
+notes:
+  - If a rule declares a group_name and that group doesn't exist, it will be
+    automatically created. In that case, group_desc should be provided as well.
+    The module will refuse to create a depended-on group without a description.
+'''
+
+EXAMPLES = '''
+- name: example ec2 group
+  local_action:
+    module: ec2_group
+    name: example
+    description: an example EC2 group
+    vpc_id: 12345
+    region: eu-west-1a
+    aws_secret_key: SECRET
+    aws_access_key: ACCESS
+    rules:
+      - proto: tcp
+        from_port: 80
+        to_port: 80
+        cidr_ip: 0.0.0.0/0
+      - proto: tcp
+        from_port: 22
+        to_port: 22
+        cidr_ip: 10.0.0.0/8
+      - proto: udp
+        from_port: 10050
+        to_port: 10050
+        cidr_ip: 10.0.0.0/8
+      - proto: udp
+        from_port: 10051
+        to_port: 10051
+        group_id: sg-12345678
+      - proto: all
+        # the containing group name may be specified here
+        group_name: example
+    rules_egress:
+      - proto: tcp
+        from_port: 80
+        to_port: 80
+        group_name: example-other
+        # description to use if example-other needs to be created
+        group_desc: other example EC2 group
+    tags:
+      - key: environment
+        value: production
+'''
+
+try:
+    import boto.ec2
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+
+def addRulesToLookup(rules, prefix, dict):
+    for rule in rules:
+        for grant in rule.grants:
+            dict["%s-%s-%s-%s-%s-%s" % (prefix, rule.ip_protocol, rule.from_port, rule.to_port,
+                                        grant.group_id, grant.cidr_ip)] = rule
+
+
+def get_target_from_rule(module, rule, name, group, groups):
+    """
+    Returns tuple of (group_id, ip) after validating rule params.
+
+    rule: Dict describing a rule.
+    name: Name of the security group being managed.
+    groups: Dict of all available security groups.
+
+    AWS accepts an ip range or a security group as target of a rule. This
+    function validate the rule specification and return either a non-None
+    group_id or a non-None ip range.
+    """
+
+    group_id = None
+    group_name = None
+    ip = None
+    target_group_created = False
+    if 'group_id' in rule and 'cidr_ip' in rule:
+        module.fail_json(msg="Specify group_id OR cidr_ip, not both")
+    elif 'group_name' in rule and 'cidr_ip' in rule:
+        module.fail_json(msg="Specify group_name OR cidr_ip, not both")
+    elif 'group_id' in rule and 'group_name' in rule:
+        module.fail_json(msg="Specify group_id OR group_name, not both")
+    elif 'group_id' in rule:
+        group_id = rule['group_id']
+    elif 'group_name' in rule:
+        group_name = rule['group_name']
+        if group_name in groups:
+            group_id = groups[group_name].id
+        elif group_name == name:
+            group_id = group.id
+            groups[group_id] = group
+            groups[group_name] = group
+        else:
+            if not rule.get('group_desc', '').strip():
+                module.fail_json(msg="group %s will be automatically created by rule %s and no description was provided" % (group_name, rule))
+            if not module.check_mode:
+                auto_group = ec2.create_security_group(group_name, rule['group_desc'], vpc_id=vpc_id)
+                group_id = auto_group.id
+                groups[group_id] = auto_group
+                groups[group_name] = auto_group
+            target_group_created = True
+    elif 'cidr_ip' in rule:
+        ip = rule['cidr_ip']
+
+    return group_id, ip, target_group_created
+
+## can be removed if https://github.com/ansible/ansible/pull/9113 is merged upstream
+def is_taggable(object):
+
+    from boto.ec2.ec2object import TaggedEC2Object
+    if not object or not issubclass(object.__class__, TaggedEC2Object):
+        return False
+
+    return True
+
+def do_tags(module, object, tags):
+    """
+    General function for adding tags to objects that are subclasses
+    of boto.ec2.ec2object.TaggedEC2Object.  Currently updates
+    existing tags, as the API overwrites them, but does not remove
+    orphans.
+    :param module:
+    :param object:
+    :param tags:
+    """
+    dry_run = True if module.check_mode else False
+
+    if (is_taggable(object)):
+
+        tag_dict = {}
+
+        for tag in tags:
+            tag_dict[tag['key']] = tag['value']
+
+        object.add_tags(tag_dict, dry_run)
+    else:
+        module.fail_json(msg="Security group object is not a subclass of TaggedEC2Object")
+## end can be removed
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(dict(
+            name=dict(required=True),
+            description=dict(required=True),
+            vpc_id=dict(),
+            rules=dict(),
+            rules_egress=dict(),
+            tags=dict(type='list', default=[]),
+            state = dict(default='present', choices=['present', 'absent']),
+        )
+    )
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    name = module.params['name']
+    description = module.params['description']
+    vpc_id = module.params['vpc_id']
+    rules = module.params['rules']
+    rules_egress = module.params['rules_egress']
+    tags = module.params['tags']
+    state = module.params.get('state')
+
+    changed = False
+
+    ec2 = ec2_connect(module)
+
+    # find the group if present
+    group = None
+    groups = {}
+    for curGroup in ec2.get_all_security_groups():
+        groups[curGroup.id] = curGroup
+        groups[curGroup.name] = curGroup
+
+        if curGroup.name == name and (vpc_id is None or curGroup.vpc_id == vpc_id):
+            group = curGroup
+
+    # Ensure requested group is absent
+    if state == 'absent':
+        if group:
+            '''found a match, delete it'''
+            try:
+                group.delete()
+            except Exception, e:
+                module.fail_json(msg="Unable to delete security group '%s' - %s" % (group, e))
+            else:
+                group = None
+                changed = True
+        else:
+            '''no match found, no changes required'''
+
+    # Ensure requested group is present
+    elif state == 'present':
+        if group:
+            '''existing group found'''
+            # check the group parameters are correct
+            group_in_use = False
+            rs = ec2.get_all_instances()
+            for r in rs:
+                for i in r.instances:
+                    group_in_use |= reduce(lambda x, y: x | (y.name == 'public-ssh'), i.groups, False)
+
+            if group.description != description:
+                if group_in_use:
+                    module.fail_json(msg="Group description does not match, but it is in use so cannot be changed.")
+
+        # if the group doesn't exist, create it now
+        else:
+            '''no match found, create it'''
+            if not module.check_mode:
+                group = ec2.create_security_group(name, description, vpc_id=vpc_id)
+
+                # When a group is created, an egress_rule ALLOW ALL
+                # to 0.0.0.0/0 is added automatically but it's not
+                # reflected in the object returned by the AWS API
+                # call. We re-read the group for getting an updated object
+                # amazon sometimes takes a couple seconds to update the security group so wait till it exists
+                while len(ec2.get_all_security_groups(filters={ 'group_id': group.id, })) == 0:
+                    time.sleep(0.1)
+
+                group = ec2.get_all_security_groups(group_ids=(group.id,))[0]
+            changed = True
+
+        # tag the security group, function imported from ansible.module_utils.ec2
+        do_tags(module, group, tags)
+    else:
+        module.fail_json(msg="Unsupported state requested: %s" % state)
+
+    # create a lookup for all existing rules on the group
+    if group:
+
+        # Manage ingress rules
+        groupRules = {}
+        addRulesToLookup(group.rules, 'in', groupRules)
+
+        # Now, go through all provided rules and ensure they are there.
+        if rules:
+            for rule in rules:
+                group_id, ip, target_group_created = get_target_from_rule(module, rule, name, group, groups)
+                if target_group_created:
+                    changed = True
+
+                if rule['proto'] == 'all':
+                    rule['proto'] = -1
+                    rule['from_port'] = None
+                    rule['to_port'] = None
+
+                # If rule already exists, don't later delete it
+                ruleId = "%s-%s-%s-%s-%s-%s" % ('in', rule['proto'], rule['from_port'], rule['to_port'], group_id, ip)
+                if ruleId in groupRules:
+                    del groupRules[ruleId]
+                # Otherwise, add new rule
+                else:
+                    grantGroup = None
+                    if group_id:
+                        grantGroup = groups[group_id]
+
+                    if not module.check_mode:
+                        group.authorize(rule['proto'], rule['from_port'], rule['to_port'], ip, grantGroup)
+                    changed = True
+
+        # Finally, remove anything left in the groupRules -- these will be defunct rules
+        for rule in groupRules.itervalues():
+            for grant in rule.grants:
+                grantGroup = None
+                if grant.group_id:
+                    grantGroup = groups[grant.group_id]
+                if not module.check_mode:
+                    group.revoke(rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip, grantGroup)
+                changed = True
+
+        # Manage egress rules
+        groupRules = {}
+        addRulesToLookup(group.rules_egress, 'out', groupRules)
+
+        # Now, go through all provided rules and ensure they are there.
+        if rules_egress:
+            for rule in rules_egress:
+                group_id, ip, target_group_created = get_target_from_rule(module, rule, name, group, groups)
+                if target_group_created:
+                    changed = True
+
+                if rule['proto'] == 'all':
+                    rule['proto'] = -1
+                    rule['from_port'] = None
+                    rule['to_port'] = None
+
+                # If rule already exists, don't later delete it
+                ruleId = "%s-%s-%s-%s-%s-%s" % ('out', rule['proto'], rule['from_port'], rule['to_port'], group_id, ip)
+                if ruleId in groupRules:
+                    del groupRules[ruleId]
+                # Otherwise, add new rule
+                else:
+                    grantGroup = None
+                    if group_id:
+                        grantGroup = groups[group_id].id
+
+                    if not module.check_mode:
+                        ec2.authorize_security_group_egress(
+                                group_id=group.id,
+                                ip_protocol=rule['proto'],
+                                from_port=rule['from_port'],
+                                to_port=rule['to_port'],
+                                src_group_id=grantGroup,
+                                cidr_ip=ip)
+                    changed = True
+        elif vpc_id and not module.check_mode:
+            # when using a vpc, but no egress rules are specified, 
+            # we add in a default allow all out rule, which was the
+            # default behavior before egress rules were added
+            default_egress_rule = 'out--1-None-None-None-0.0.0.0/0'
+            if default_egress_rule not in groupRules:
+                ec2.authorize_security_group_egress(
+                    group_id=group.id,
+                    ip_protocol=-1,
+                    from_port=None,
+                    to_port=None,
+                    src_group_id=None,
+                    cidr_ip='0.0.0.0/0'
+                )
+                changed = True
+            else:
+                # make sure the default egress rule is not removed
+                del groupRules[default_egress_rule]
+
+        # Finally, remove anything left in the groupRules -- these will be defunct rules
+        for rule in groupRules.itervalues():
+            for grant in rule.grants:
+                grantGroup = None
+                if grant.group_id:
+                    grantGroup = groups[grant.group_id].id
+                if not module.check_mode:
+                    ec2.revoke_security_group_egress(
+                            group_id=group.id,
+                            ip_protocol=rule.ip_protocol,
+                            from_port=rule.from_port,
+                            to_port=rule.to_port,
+                            src_group_id=grantGroup,
+                            cidr_ip=grant.cidr_ip)
+                changed = True
+
+    if group:
+        module.exit_json(changed=changed, group_id=group.id)
+    else:
+        module.exit_json(changed=changed, group_id=None)
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.ec2 import *
+
+main()

playbooks/roles/edx_vpc/tasks/main.yml

@@ -182,4 +182,8 @@
     description: "{{ vpc_db_subnet_group.description }}"
     subnets: "{{ db_subnet_ids.function_output }}"
   
-  
+- name: output a vpc_config for using to build services
+  local_action:
+    module: template
+    src: "vpc_config.yml.j2"
+    dest: "~/{{ e_d }}.yml"

playbooks/roles/edx_vpc/templates/vpc_config.yml.j2

+#
+# Configuration for the environment-deployment
+#
+profile: "{{ vpc_aws_profile }}"
+vpc_id: "{{ created_vpc.vpc_id }}"
+vpc_cidr: "{{ vpc_cidr }}"
+vpc_class_b: "{{ vpc_class_b }}"
+
+env: "{{ vpc_environment }}"
+deployment: "{{ vpc_deployment }}"
+e_d_c: "{{ vpc_environment }}-{{ vpc_deployment }}-{{ '{{' }} cluster {{ '}}' }}"
+aws_region: "{{ vpc_aws_region }}"
+
+
+aws_availability_zones:
+{% for subnet in vpc_public_subnets %}
+  - {{ subnet.az }}
+{% endfor %}
+
+# Should this be service specific
+ssl_cert: "{{ vpc_ssl_cert }}"
+
+# used for ELB
+public_route_table: "{{ created_public_rt.id }}"
+# used for service subnet
+private_route_table: "{{ created_private_rt.id }}"
+instance_key_name: "{{ vpc_keypair }}"
+
+# subject to change #TODO: provide the correct var for the eni
+nat_device: "{{ nat_instance.instances[0].id }}"
+
+public_subnet_1: "{{ vpc_public_subnets[0].cidr }}"
+public_subnet_2: "{{ vpc_public_subnets[1].cidr }}"
+
+# /28 per AZ NEEDE?
+# private_subnet_1: "{{ vpc_class_b }}.110.16/28"
+# private_subnet_2: "{{ vpc_class_b }}.120.16/28"
+
+elb_subnets:
+{% for subnet in created_public_subnets.results %}
+  - "{{ subnet.subnet_id }}"
+{% endfor %}
+
+#
+# Do not use vars in policies :(
+# Should be specific to the service right?
+role_policies: []
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-s3-policy"
+#     document: |
+#       {
+#         "Statement":[
+#           {
+#             "Effect":"Allow",
+#             "Action":["s3:*"],
+#             "Resource":["arn:aws:s3:::edx-stage-edx"]
+#           }
+#         ]
+#         }  
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-create-instance-tags"
+#     document: |
+#       {
+#         "Statement": [
+#           {
+#             "Effect": "Allow",
+#             "Action": ["ec2:CreateTags"],
+#             "Resource": ["arn:aws:ec2:us-east-1:xxxxxxxxxxxx:instance/*"]
+#           }
+#         ]
+#         }        
+#   - name: "{{ '{{ ' + 'e_d_c' + '}}' }}-describe-ec2"
+#     document: |
+#         {"Statement":[
+#         {"Resource":"*",
+#         "Action":["ec2:DescribeInstances","ec2:DescribeTags","ec2:DescribeVolumes"],
+#         "Effect":"Allow"}]}