Merge pull request #2994 from edx/feanil/abbey_update

Feanil/abbey update

playbooks/roles/supervisor/files/pre_supervisor_checks.py

 import argparse
-import boto
-from boto.utils import get_instance_metadata
+import boto.ec2
+from boto.utils import get_instance_metadata, get_instance_identity
 from boto.exception import AWSConnectionError
 import hipchat
 import os
@@ -27,12 +27,14 @@ HIPCHAT_USER = "PreSupervisor"
 MAX_BACKOFF = 120
 INITIAL_BACKOFF = 1
 
+REGION = get_instance_identity()['document']['region']
+
 def services_for_instance(instance_id):
     """
     Get the list of all services named by the services tag in this
     instance's tags.
     """
-    ec2 = boto.connect_ec2()
+    ec2 = boto.ec2.connect_to_region(REGION)
     reservations = ec2.get_all_instances(instance_ids=[instance_id])
     for reservation in reservations:
         for instance in reservation.instances:
@@ -47,7 +49,7 @@ def services_for_instance(instance_id):
                     yield service
 
 def edp_for_instance(instance_id):
-    ec2 = boto.connect_ec2()
+    ec2 = boto.ec2.connect_to_region(REGION)
     reservations = ec2.get_all_instances(instance_ids=[instance_id])
     for reservation in reservations:
         for instance in reservation.instances:
@@ -164,8 +166,7 @@ if __name__ == '__main__':
     instance_id = get_instance_metadata()['instance-id']
     prefix = instance_id
 
-
-    ec2 = boto.connect_ec2()
+    ec2 = boto.ec2.connect_to_region(REGION)
     reservations = ec2.get_all_instances(instance_ids=[instance_id])
     instance = reservations[0].instances[0]
     if instance.instance_profile['arn'].endswith('/abbey'):

util/jenkins/build-ami.sh

@@ -110,6 +110,16 @@ if [[ ! -z "$callback_url" ]]; then
   fi
 fi
 
+region_params=""
+if [[ ! -z "$region" ]]; then
+  region_params="--region $region"
+fi
+
+identity_params="--identity /edx/var/jenkins/.ssh/id_rsa"
+if [[ ! -z "$identity_path" ]]; then
+  identity_params="--identity $identity_path"
+fi
+
 cd configuration
 pip install -r requirements.txt
 
@@ -117,4 +127,4 @@ cd util/vpc-tools/
 
 echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
 cat /var/tmp/$BUILD_ID-extra-vars.yml
-python -u abbey.py -p $play -t m3.large -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params $notification_params $datadog_params
+python -u abbey.py -p $play -t m3.large -d $deployment -e $environment $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params $notification_params $datadog_params $region_params $identity_params

util/vpc-tools/abbey.py

@@ -9,7 +9,7 @@ import requests
 try:
     import boto.ec2
     import boto.sqs
-    from boto.vpc import VPCConnection
+    import boto.vpc
     from boto.exception import NoAuthHandlerFound, EC2ResponseError
     from boto.sqs.message import RawMessage
     from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
@@ -90,7 +90,7 @@ def parse_args():
                         help="configuration-private repo gitref",
                         default="master")
     parser.add_argument('--configuration-private-repo', required=False,
-                        default="git@github.com:edx-ops/ansible-private",
+                        default="",
                         help="repo to use for private playbooks")
     parser.add_argument('-c', '--cache-id', required=True,
                         help="unique id to use as part of cache prefix")
@@ -157,6 +157,18 @@ def get_instance_sec_group(vpc_id):
     )
 
     if len(grp_details) < 1:
+        #
+        # try scheme for non-cloudformation builds
+        #
+
+        grp_details = ec2.get_all_security_groups(
+            filters={
+                'tag:play': args.play,
+                'tag:environment': args.environment,
+                'tag:deployment': args.deployment}
+        )
+
+    if len(grp_details) < 1:
         sys.stderr.write("ERROR: Expected atleast one security group, got {}\n".format(
             len(grp_details)))
 
@@ -188,7 +200,7 @@ def create_instance_args():
     user data
     """
 
-    vpc = VPCConnection()
+    vpc = boto.vpc.connect_to_region(args.region)
     subnet = vpc.get_all_subnets(
         filters={
             'tag:aws:cloudformation:stack-name': stack_name,
@@ -202,14 +214,14 @@ def create_instance_args():
 
         subnet = vpc.get_all_subnets(
             filters={
-                'tag:cluster': args.play,
+                'tag:play': args.play,
                 'tag:environment': args.environment,
                 'tag:deployment': args.deployment}
         )
 
     if len(subnet) < 1:
-        sys.stderr.write("ERROR: Expected at least one subnet, got {}\n".format(
-            len(subnet)))
+        sys.stderr.write("ERROR: Expected at least one subnet, got {} for {}-{}-{}\n".format(
+            len(subnet), args.environment, args.deployment, args.play))
         sys.exit(1)
     subnet_id = subnet[0].id
     vpc_id = subnet[0].vpc_id
@@ -265,7 +277,7 @@ fi
 
 ANSIBLE_ENABLE_SQS=true
 SQS_NAME={queue_name}
-SQS_REGION=us-east-1
+SQS_REGION={region}
 SQS_MSG_PREFIX="[ $instance_id $instance_ip $environment-$deployment $play ]"
 PYTHONUNBUFFERED=1
 HIPCHAT_TOKEN={hipchat_token}
@@ -392,7 +404,8 @@ rm -rf $base_dir
                 extra_vars_yml=extra_vars_yml,
                 secure_vars_file=secure_vars_file,
                 cache_id=args.cache_id,
-                datadog_api_key=args.datadog_api_key)
+                datadog_api_key=args.datadog_api_key,
+                region=args.region)
 
     mapping = BlockDeviceMapping()
     root_vol = BlockDeviceType(size=args.root_vol_size,