Merge pull request #955 from edx/feanil/fix_paramaterized_jobs

Feanil/fix paramaterized jobs

playbooks/roles/jenkins_admin/templates/edx/var/jenkins/jobs/build-ami/config.xml.j2

@@ -33,12 +33,12 @@
         </hudson.model.TextParameterDefinition>
         <hudson.model.StringParameterDefinition>
           <name>configuration</name>
-          <description>The GITREF of configuration to use.</description>
+          <description>The GITREF of configuration to use. Leave blank to default to master.</description>
           <defaultValue></defaultValue>
         </hudson.model.StringParameterDefinition>
         <hudson.model.StringParameterDefinition>
           <name>configuration_secure</name>
-          <description>The GITREF of configuration-secure repository to use.</description>
+          <description>The GITREF of configuration-secure repository to use. Leave blank to default to master.</description>
           <defaultValue></defaultValue>
         </hudson.model.StringParameterDefinition>
         <hudson.model.StringParameterDefinition>
@@ -46,6 +46,11 @@
           <description></description>
           <defaultValue></defaultValue>
         </hudson.model.StringParameterDefinition>
+        <hudson.model.BooleanParameterDefinition>
+          <name>use_blessed</name>
+          <description></description>
+          <defaultValue>true</defaultValue>
+        </hudson.model.BooleanParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
     <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.20">
@@ -151,27 +156,10 @@
       <nature>shell</nature>
       <command>
 #!/bin/bash -x
+export jenkins_admin_ec2_key="{{ JENKINS_ADMIN_EC2_KEY }}"
+export jenkins_admin_configuration_secure_repo="{{ JENKINS_ADMIN_CONFIGURATION_SECURE_REPO }}"
 
-if [[ "$play" == "" ]]; then
-    echo "No Play Specified. Nothing to Do."
-    exit 0
-fi
-
-export PYTHONUNBUFFERED=1
-export PIP_DOWNLOAD_CACHE=$WORKSPACE/pip-cache
-
-cd configuration
-pip install -r requirements.txt
-
-cd util/vpc-tools/
-
-echo "$refs" > /var/tmp/$BUILD_ID-refs.yml
-cat /var/tmp/$BUILD_ID-refs.yml
-
-echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
-cat /var/tmp/$BUILD_ID-extra-vars.yml
-
-python -u abbey.py -p $play -t c1.medium  -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa -b $base_ami --vars /var/tmp/$BUILD_ID-extra-vars.yml --refs /var/tmp/$BUILD_ID-refs.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k {{ JENKINS_ADMIN_EC2_KEY }} --configuration-secure-repo &quot;{{ JENKINS_ADMIN_CONFIGURATION_SECURE_REPO }}&quot;
+configuration/util/jenkins/build-ami.sh
       </command>
       <ignoreExitCode>false</ignoreExitCode>
     </jenkins.plugins.shiningpanda.builders.VirtualenvBuilder>

util/jenkins/build-ami.sh

+#!/bin/bash -x
+# This script is meant to be run from jenkins and expects the
+# following variables to be set:
+#   - BUILD_ID - set by jenkins, Unique ID of build
+#   - BUILD_NUMBER - set by jenkins, Build number
+#   - refs - repo revisions to pass to abbey. This is provided in YAML syntax,
+#            and we put the contents in a file that abbey reads. Refs are
+#            different from 'vars' in that each ref is set as a tag on the
+#            output AMI.
+#   - vars - other vars to pass to abbey. This is provided in YAML syntax,
+#            and we put the contents in a file that abby reads.
+#   - deployment - edx, edge, etc
+#   - environment - stage,prod, etc
+#   - play - forum, edxapp, xqueue, etc
+#   - base_ami - Optional AMI to use as base AMI for abby instance
+#   - configuration - the version of the configuration repo to use
+#   - configuration_secure - the version of the secure repo to use
+#   - jenkins_admin_ec2_key - location of the ec2 key to pass to abbey
+#   - jenkins_admin_configuration_secure_repo - the git repo to use for secure vars
+#   - use_blessed - whether or not to use blessed AMIs
+
+if [[ -z "$BUILD_ID" ]]; then
+  echo "BUILD_ID not specified."
+  exit -1
+fi
+
+if [[ -z "$BUILD_NUMBER" ]]; then
+  echo "BUILD_NUMBER not specified."
+  exit -1
+fi
+
+if [[ -z "$refs" ]]; then
+  echo "refs not specified."
+  exit -1
+fi
+
+if [[ -z "$deployment" ]]; then
+  echo "deployment not specified."
+  exit -1
+fi
+
+if [[ -z "$environment" ]]; then
+  echo "environment not specified."
+  exit -1
+fi
+
+if [[ -z "$play" ]]; then
+  echo "play not specified."
+  exit -1
+fi
+
+if [[ -z "$jenkins_admin_ec2_key" ]]; then
+  echo "jenkins_admin_ec2_key not specified."
+  exit -1
+fi
+
+if [[ -z "$jenkins_admin_configuration_secure_repo" ]]; then
+  echo "jenkins_admin_configuration_secure_repo not specified."
+  exit -1
+fi
+
+export PYTHONUNBUFFERED=1
+
+if [[ -z $configuration ]]; then
+  cd configuration
+  configuration=`git rev-parse HEAD`
+  cd ..
+fi
+
+if [[ -z $configuration_secure ]]; then
+  cd configuration-secure
+  configuration_secure=`git rev-parse HEAD`
+  cd ..
+fi
+
+base_params=""
+if [[ -n "$base_ami" ]]; then
+  base_params="-b $base_ami"
+fi
+
+blessed_params=""
+if [[ "$use_blessed" == "true" ]]; then
+  blessed_params="--blessed"
+fi
+
+cd configuration
+pip install -r requirements.txt
+
+cd util/vpc-tools/
+
+echo "$refs" > /var/tmp/$BUILD_ID-refs.yml
+cat /var/tmp/$BUILD_ID-refs.yml
+
+echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
+cat /var/tmp/$BUILD_ID-extra-vars.yml
+
+python -u abbey.py -p $play -t c1.medium  -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params --vars /var/tmp/$BUILD_ID-extra-vars.yml --refs /var/tmp/$BUILD_ID-refs.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo

util/vpc-tools/abbey.py

@@ -71,9 +71,6 @@ def parse_args():
                         help="path to extra var file", required=False)
     parser.add_argument('--refs', metavar="GIT_REFS_FILE",
                         help="path to a var file with app git refs", required=False)
-    parser.add_argument('-a', '--application', required=False,
-                        help="Application for subnet, defaults to admin",
-                        default="admin")
     parser.add_argument('--configuration-version', required=False,
                         help="configuration repo branch(no hashes)",
                         default="master")
@@ -85,9 +82,6 @@ def parse_args():
                         help="repo to use for the secure files")
     parser.add_argument('-c', '--cache-id', required=True,
                         help="unique id to use as part of cache prefix")
-    parser.add_argument('-b', '--base-ami', required=False,
-                        help="ami to use as a base ami",
-                        default="ami-0568456c")
     parser.add_argument('-i', '--identity', required=False,
                         help="path to identity file for pulling "
                              "down configuration-secure",
@@ -108,6 +102,15 @@ def parse_args():
                         default=5,
                         help="How long to delay message display from sqs "
                              "to ensure ordering")
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('-b', '--base-ami', required=False,
+                        help="ami to use as a base ami",
+                        default="ami-0568456c")
+    group.add_argument('--blessed', action='store_true',
+                        help="Look up blessed ami for env-dep-play.",
+                        default=False)
+
     return parser.parse_args()
 
 
@@ -126,6 +129,21 @@ def get_instance_sec_group(vpc_id):
 
     return grp_details[0].id
 
+def get_blessed_ami():
+    images = ec2.get_all_images(
+        filters={
+            'tag:environment': args.environment,
+            'tag:deployment': args.deployment,
+            'tag:play': args.play,
+            'tag:blessed': True
+        }
+    )
+
+    if len(images) != 1:
+        raise Exception("ERROR: Expected only one blessed ami, got {}\n".format(
+            len(images)))
+
+    return images[0].id
 
 def create_instance_args():
     """
@@ -292,7 +310,7 @@ rm -rf $base_dir
         'security_group_ids': [security_group_id],
         'subnet_id': subnet_id,
         'key_name': args.keypair,
-        'image_id': args.base_ami,
+        'image_id': base_ami,
         'instance_type': args.instance_type,
         'instance_profile_name': args.role_name,
         'user_data': user_data,
@@ -597,6 +615,11 @@ if __name__ == '__main__':
         print 'You must be able to connect to sqs and ec2 to use this script'
         sys.exit(1)
 
+    if args.blessed:
+        base_ami = get_blessed_ami()
+    else:
+        base_ami = args.base_ami
+
     try:
         sqs_queue = None
         instance_id = None