Merge pull request #361 from edx/feanil/automate_stack_creation

Feanil/automate stack creation

util/vpc-tools/create_stack.py

+import argparse
+import boto
+from os.path import basename
+from time import sleep
+
+FAILURE_STATES = [
+    'CREATE_FAILED',
+    'ROLLBACK_IN_PROGRESS',
+    'ROLLBACK_FAILED',
+    'ROLLBACK_COMPLETE',
+    'DELETE_IN_PROGRESS',
+    'DELETE_FAILED',
+    'DELETE_COMPLETE',
+    ]
+
+def upload_file(file_path, bucket_name, key_name):
+    """
+    Upload a file to the given s3 bucket and return a template url.
+    """
+    conn = boto.connect_s3()
+    try:
+        bucket = conn.get_bucket(bucket_name)
+    except boto.exception.S3ResponseError as e:
+        conn.create_bucket(bucket_name)
+        bucket = conn.get_bucket(bucket_name, validate=False)
+
+    key = boto.s3.key.Key(bucket)
+    key.key = key_name 
+    key.set_contents_from_filename(file_path)
+
+    url = 'https://s3.amazonaws.com/{}/{}'.format(bucket_name, key_name)
+    return url
+
+def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_bucket='edx-sandbox-devops'):
+    cfn = boto.connect_cloudformation()
+
+    # Upload the template to s3
+    key_name = 'cloudformation/auto/{}_{}'.format(stack_name, basename(template))
+    template_url = upload_file(template, temp_bucket, key_name)
+
+    # Reference the stack.
+    try:
+        stack_id = cfn.create_stack(stack_name,
+            template_url=template_url,
+            capabilities=['CAPABILITY_IAM'],
+            tags={'autostack':'true'},
+            parameters=[('KeyName', 'continuous-integration')])
+    except Exception as e:
+        print(e.message)
+        raise e
+
+   
+    status = None 
+    while blocking:
+        sleep(5)
+        stack_instance = cfn.describe_stacks(stack_id)[0]
+        status = stack_instance.stack_status
+        print(status)
+        if 'COMPLETE' in status:
+            break
+
+    if status in FAILURE_STATES:
+        raise Exception('Creation Failed. Stack Status: {}, ID:{}'.format(
+            status, stack_id))
+
+    return stack_id
+
+
+if __name__ == '__main__':
+        description = 'Create a cloudformation stack from a template.'
+        parser = argparse.ArgumentParser(description=description)
+
+        msg = 'Name for the cloudformation stack.'
+        parser.add_argument('-n', '--stackname', required=True, help=msg)
+
+        msg = 'Name of the bucket to use for temporarily uploading the \
+            template.'
+        parser.add_argument('-b', '--bucketname', default="edx-sandbox-devops",
+            help=msg)
+
+        msg = 'The path to the cloudformation template.'
+        parser.add_argument('-t', '--template', required=True, help=msg)
+       
+        msg = 'The AWS region to build this stack in.'
+        parser.add_argument('-r', '--region', default='us-east-1', help=msg)
+       
+        args = parser.parse_args() 
+        stack_name = args.stackname
+        template = args.template
+        region = args.region
+        bucket_name = args.bucketname
+
+        create_stack(stack_name, template, region, bucket_name)
+        print('Stack({}) created.'.format(stack_name))

util/vpc-tools/vpc-dns.py

-"""vpc-dns.py
-
-Usage:
-    vpc-dns.py create-zone (vpc <vpc_id> | stack-name <stack_name>)
-    vpc-dns.py (-h --help)
-    vpc-dns.py (-v --version)
-
-Options:
-    -h --help       Show this screen.
-    -v --version    Show version.
-"""
-import boto
-from boto.route53.record import ResourceRecordSets
-from docopt import docopt
-from vpcutil import vpc_for_stack_name
-
-class VPCDns:
-    BACKEND_ZONE = "Z4AI6ADZTL3HN"
-    DNS_SUFFIX = ".vpc.edx.org"
-    ZONE = "{name}" + DNS_SUFFIX
-
-    def __init__(self,vpc_id=None):
-        self.vpc_id = vpc_id
-        self.elb = boto.connect_elb()
-        self.r53 = boto.connect_route53()
-
-    def create_zone(self, vpc_id):
-        zone_name = self.ZONE.format(name=vpc_id)
-        print zone_name
-        hosted_zone = self.get_or_create_hosted_zone(zone_name)
-
-        elbs = self.elb.get_all_load_balancers()
-
-        for elb in [x for x in elbs if x.vpc_id == self.vpc_id]:
-            self.create_service_dns(elb,self.get_zone_id_from_retval(
-                hosted_zone.Id),self.vpc_id)
-
-    def get_zone_id_from_retval(self,retval):
-        """
-        The data structure returned by the create_hosted_zone call
-        pre-pends the string /hostedzone/ to the Id for some reason.
-        """
-        return retval.replace("/hostedzone/","")
-
-
-    def get_or_create_hosted_zone(self, zone_name):
-
-        hosted_zone = self.r53.get_hosted_zone_by_name(zone_name)
-
-        if not hosted_zone:
-            zone_data = self.r53.create_hosted_zone(zone_name,
-                                        comment="Created by automation.")
-            hosted_zone = self.r53.get_hosted_zone_by_name(zone_name)
-
-        return hosted_zone
-
-
-    def get_elb_service(self, elb):
-
-        services = ["edxapp","rabbit","xqueue","xserver","worker"]
-
-        for service in services:
-            if service in elb.dns_name.lower():
-                return service
-
-        raise Exception("No service mapping for " + elb.dns_name)
-
-
-    def create_service_dns(self, elb, zone, vpc_id):
-        """
-        """
-        records = self.r53.get_all_rrsets(zone)
-
-        old_names = [r.name for r in records]
-
-        HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX
-
-        service = self.get_elb_service(elb)
-
-        dns_name = HOST_TEMPLATE.format(service=service,
-                                        vpc_id=vpc_id)
-
-        change_set = ResourceRecordSets()
-
-        if dns_name + '.' in old_names:
-            print "adding delete"
-            change = change_set.add_change(
-                'DELETE',
-                dns_name,
-                'CNAME',
-                600)
-
-            change.add_value(elb.dns_name)
-
-        change = change_set.add_change(
-            'CREATE',
-            dns_name,
-            'CNAME',
-            600 )
-
-        change.add_value(elb.dns_name)
-
-        print change_set.to_xml()
-
-        self.r53.change_rrsets(zone, change_set.to_xml())
-
-VERSION="0.1"
-
-def dispatch(args):
-    if args.get("vpc"):
-      vpc_id = args.get("<vpc_id>")
-    elif args.get("stack-name"):
-      stack_name = args.get("<stack_name>")
-      vpc_id = vpc_for_stack_name(stack_name)
-    else:
-      raise Exception("No vpc_id or stack_name provided.")
-
-    c = VPCDns(vpc_id=vpc_id)
-
-    if args.get("create-zone"):
-        c.create_zone(vpc_id)
-
-if __name__ == "__main__":
-    args = docopt(__doc__, version=VERSION)
-    dispatch(args)
-

util/vpc-tools/vpc_dns.py

+import argparse
+import boto
+from vpcutil import vpc_for_stack_name
+from pprint import pprint
+
+r53 = boto.connect_route53()
+
+
+# Utility Functions
+def add_or_update_record(zone, record_name, record_type, record_ttl, record_values):
+    zone_id = zone.Id.replace("/hostedzone/","")
+
+    records = r53.get_all_rrsets(zone_id)
+
+    old_records = { r.name[:-1] : r for r in records }
+    pprint(old_records)
+
+    change_set = boto.route53.record.ResourceRecordSets()
+
+    # If the record name already points to something.
+    # Delete the existing connection.
+    if record_name in old_records.keys():
+        print "adding delete"
+        change = change_set.add_change(
+            'DELETE',
+            record_name,
+            record_type,
+            record_ttl)
+
+        for value in old_records[record_name].resource_records:
+            change.add_value(value)
+
+    change = change_set.add_change(
+        'CREATE',
+        record_name,
+        record_type,
+        record_ttl)
+
+    for value in record_values:
+        change.add_value(value)
+     
+    print(change_set.to_xml())
+    r53.change_rrsets(zone_id, change_set.to_xml())
+
+
+def add_zone_to_parent(zone, parent):
+    #Add a reference for the new zone to its parent zone.
+    parent_name = parent.Name[:-1]
+    zone_name = zone.Name[:-1]
+
+    add_or_update_record(parent, zone_name, 'NS', 900, zone.NameServers)
+
+
+def get_or_create_hosted_zone(zone_name):
+    # Get the parent zone.
+    parent_zone_name = ".".join(zone_name.split('.')[1:])
+    parent_zone = r53.get_hosted_zone_by_name(parent_zone_name)
+    if not parent_zone:
+        msg = "Parent zone({}) does not exist."
+        raise Exception(msg.format(parent_zone_name))
+
+    hosted_zone = r53.get_hosted_zone_by_name(zone_name)
+
+    if not hosted_zone:
+        r53.create_hosted_zone(zone_name,
+                               comment="Created by automation.")
+        hosted_zone = r53.get_hosted_zone_by_name(zone_name)
+
+    add_zone_to_parent(hosted_zone, parent_zone)
+
+    return hosted_zone
+
+
+def elbs_for_stack_name(stack_name):
+    vpc_id = vpc_for_stack_name(stack_name)
+    elbs = boto.connect_elb()
+    for elb in elbs.get_all_load_balancers():
+        if elb.vpc_id == vpc_id:
+            yield elb
+
+
+def ensure_service_dns(elb, prefix, zone):
+    dns_template = "{prefix}.{zone_name}"
+
+    # Have to remove the trailing period that is on zone names.
+    zone_name = zone.Name[:-1]
+    dns_name = dns_template.format(prefix=prefix,
+                                   zone_name=zone_name)
+
+    add_or_update_record(zone, dns_name, 'CNAME', 600, [elb.dns_name])
+
+
+if __name__ == "__main__":
+    description = "Give a cloudformation stack name, for an edx stack, setup \
+        DNS names for the ELBs in the stack."
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-n', '--stackname',
+        help="The name of the cloudformation stack.",
+        required=True)
+    args = parser.parse_args()
+    stack_name = args.stackname
+
+    # Create DNS for edxapp and xqueue.
+    dns_settings = {
+        'edxapp': ['courses', 'studio'],
+        'xqueue': ['xqueue'],
+        'rabbit': ['rabbit'],
+        'xserver': ['xserver'],
+        'worker': ['worker'],
+        'forum': ['forum'],
+    }
+
+    # Create a zone for the stack.
+    zone_name = "{}.vpc.edx.org".format(stack_name)
+
+    zone = get_or_create_hosted_zone(zone_name)
+
+    stack_elbs = elbs_for_stack_name(stack_name)
+    for elb in stack_elbs:
+        for role, dns_prefixes in dns_settings.items():
+            #FIXME this breaks when the service name is in the stack name ie. testforumstack.
+            # Get the tags for the instances in this elb and compare the service against the role tag.
+            if role in elb.dns_name.lower():
+                for prefix in dns_prefixes:
+                    ensure_service_dns(elb, prefix, zone)