Merge pull request #1262 from edx/feanil/abbey_gitrefs

Feanil/abbey gitrefs

playbooks/roles/alton/tasks/deploy.yml

@@ -3,6 +3,7 @@
     dest="{{ alton_code_dir }}" repo="{{ alton_source_repo }}"
     version="{{ alton_version }}" accept_hostkey=yes
   sudo_user: "{{ alton_user }}"
+  register: alton_checkout
   notify: restart alton
 
 - name: install the requirements
@@ -55,3 +56,5 @@
     state=started
   when: not disable_edx_services
 
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/alton/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+  tags:
+  - deploy
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:alton" : "{{ alton_source_repo }} {{ alton_checkout.after |truncate(7,True,'')}}"
+  when: alton_checkout.after is defined
+  tags:
+  - deploy

playbooks/roles/analytics-api/tasks/deploy.yml

@@ -109,3 +109,6 @@
 
 - name: remove read-only ssh key for the content repo
   file: path={{ analytics_api_git_identity_file }} state=absent
+
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/analytics-api/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:analytics_api" : "{{ analytics_api_source_repo }} {{ analytics_api_code_checkout.after |truncate(7,True,'')}}"
+  when: analytics_api_code_checkout.after is defined

playbooks/roles/certs/tasks/deploy.yml

@@ -52,6 +52,7 @@
   sudo_user: "{{ certs_user }}"
   environment:
     GIT_SSH: "{{ certs_git_ssh }}"
+  register: certs_checkout
   notify: restart certs
 
 - name: remove read-only ssh key for the certs repo
@@ -96,4 +97,7 @@
   - python
   - pip
 
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE
+
 - set_fact: certs_installed=true

playbooks/roles/certs/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:certs" : "{{ CERT_REPO }} {{ certs_checkout.after|truncate(7,True,'') }}"
+  when: certs_checkout.after is defined

playbooks/roles/common/defaults/main.yml

@@ -55,6 +55,7 @@ COMMON_MONGO_READ_ONLY_PASS: !!null
 COMMON_ENABLE_DATADOG: False
 COMMON_ENABLE_SPLUNKFORWARDER: False
 COMMON_ENABLE_NEWRELIC: False
+COMMON_TAG_EC2_INSTANCE: False
 common_debian_pkgs:
   - ntp
   - ack-grep

playbooks/roles/edxapp/tasks/deploy.yml

@@ -29,12 +29,14 @@
 # Do A Checkout
 - name: checkout edx-platform repo into {{edxapp_code_dir}}
   git: >
-    dest={{edxapp_code_dir}} repo={{edx_platform_repo}} version={{edx_platform_version}}
+    dest={{edxapp_code_dir}}
+    repo={{edx_platform_repo}}
+    version={{edx_platform_version}}
     accept_hostkey=yes
-  register: chkout
   sudo_user: "{{ edxapp_user }}"
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
+  register: edxapp_platform_checkout
   notify:
   - "restart edxapp"
   - "restart edxapp_workers"
@@ -48,12 +50,15 @@
 
 - name: checkout theme
   git: >
-    dest={{ edxapp_app_dir }}/themes/{{edxapp_theme_name}} repo={{edxapp_theme_source_repo}} version={{edxapp_theme_version}}
+    dest={{ edxapp_app_dir }}/themes/{{edxapp_theme_name}}
+    repo={{edxapp_theme_source_repo}}
+    version={{edxapp_theme_version}}
     accept_hostkey=yes
   when: edxapp_theme_name != ''
   sudo_user: "{{ edxapp_user }}"
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
+  register: edxapp_theme_checkout
   notify:
   - "restart edxapp"
   - "restart edxapp_workers"
@@ -417,4 +422,7 @@
   file: path={{ edxapp_git_identity }} state=absent
   when: EDXAPP_USE_GIT_IDENTITY
 
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE
+
 - set_fact: edxapp_installed=true

playbooks/roles/edxapp/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance with edx_platform version
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:edx_platform" : "{{ edx_platform_repo }} {{ edxapp_platform_checkout.after|truncate(7,True,'') }}"
+  when: edxapp_platform_checkout.after is defined
+
+- name: tag instance  with edxapp theme version
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:edxapp_theme" : "{{ edxapp_theme_source_repo }} {{ edxapp_theme_checkout.after|truncate(7,True,'') }}"
+  when: edxapp_theme_checkout.after is defined

playbooks/roles/forum/tasks/deploy.yml

@@ -34,6 +34,7 @@
     dest={{ forum_code_dir }} repo={{ forum_source_repo }} version={{ forum_version }}
     accept_hostkey=yes
   sudo_user: "{{ forum_user }}"
+  register: forum_checkout
   notify: restart the forum service
 
 # TODO: This is done as the common_web_user
@@ -66,4 +67,7 @@
 
 - include: test.yml tags=deploy
 
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE
+
 - set_fact: forum_installed=true

playbooks/roles/forum/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:forum" : "{{ forum_source_repo }} {{ forum_checkout.after|truncate(7,True,'') }}"
+  when: forum_checkout.after is defined

playbooks/roles/xqueue/tasks/deploy.yml

@@ -32,6 +32,7 @@
     dest={{ xqueue_code_dir }} repo={{ xqueue_source_repo }} version={{ xqueue_version }}
     accept_hostkey=yes
   sudo_user: "{{ xqueue_user }}"
+  register: xqueue_checkout
   notify:
     - restart xqueue
 
@@ -114,4 +115,7 @@
   - python
   - pip
 
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE
+
 - set_fact: xqueue_installed=true

playbooks/roles/xqueue/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:xqueue" : "{{ xqueue_source_repo }} {{ xqueue_checkout.after|truncate(7,True,'') }}"
+  when: xqueue_checkout.after is defined

playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

@@ -7,6 +7,7 @@
     dest={{ xqwatcher_code_dir }} repo={{ xqwatcher_source_repo }} version={{ XQWATCHER_VERSION }}
     accept_hostkey=yes
     ssh_opts="{{ xqwatcher_git_ssh_opts }}"
+  register: xqwatcher_checkout
 
 - name: install application requirements
   pip: >
@@ -39,4 +40,7 @@
 - name: update supervisor configuration
   shell: "{{ xqwatcher_supervisor_ctl }} -c {{ xqwatcher_supervisor_app_dir }}/supervisord.conf update"
   when: not disable_edx_services
-  notify: restart xqwatcher
\ No newline at end of file
+  notify: restart xqwatcher
+
+- include: tag_ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/xqwatcher/tasks/tag_ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:xqwatcher" : "{{ xqwatcher_source_repo }} {{ xqwatcher_checkout.after|truncate(7,True,'') }}"
+  when: xqwatcher_checkout.after is defined

playbooks/roles/xserver/tasks/deploy.yml

@@ -16,6 +16,7 @@
     dest={{xserver_code_dir}} repo={{xserver_source_repo}} version={{xserver_version}}
     accept_hostkey=yes
   sudo_user: "{{ xserver_user }}"
+  register: xserver_checkout
   notify: restart xserver
 
 - name: install requirements
@@ -56,6 +57,7 @@
   environment:
     GIT_SSH: /tmp/git_ssh.sh
   notify: restart xserver
+  register: xserver_grader_checkout
   sudo_user: "{{ xserver_user }}"
 
 - name: remove read-only ssh key for the content repo
@@ -92,3 +94,6 @@
 
 - name: enforce app-armor rules
   command: aa-enforce {{ xserver_venv_sandbox_dir }}
+
+- include: ec2.yml tags=deploy
+  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/xserver/tasks/ec2.yml

+---
+
+- name: get instance information
+  action: ec2_facts
+
+- name: tag instance for xserver
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:xserver" : "{{ xserver_source_repo }} {{ xserver_checkout.after|truncate(7,True,'') }}"
+  when: xserver_checkout.after is defined
+
+- name: tag instance for xserver grader
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:xserver_grader" : "{{ XSERVER_GRADER_SOURCE }} {{ xserver_grader_checkout.after|truncate(7,True,'') }}"
+  when: xserver_grader_checkout.after is defined

util/jenkins/build-ami.sh

@@ -29,11 +29,6 @@ if [[ -z "$BUILD_NUMBER" ]]; then
   exit -1
 fi
 
-if [[ -z "$refs" ]]; then
-  echo "refs not specified."
-  exit -1
-fi
-
 if [[ -z "$deployment" ]]; then
   echo "deployment not specified."
   exit -1
@@ -65,6 +60,10 @@ cd $WORKSPACE/configuration
 configuration=`git rev-parse --short HEAD`
 cd $WORKSPACE
 
+cd $WORKSPACE/configuration-secure
+configuration_secure=`git rev-parse --short HEAD`
+cd $WORKSPACE
+
 base_params=""
 if [[ -n "$base_ami" ]]; then
   base_params="-b $base_ami"
@@ -103,10 +102,7 @@ pip install -r requirements.txt
 
 cd util/vpc-tools/
 
-echo "$refs" > /var/tmp/$BUILD_ID-refs.yml
-cat /var/tmp/$BUILD_ID-refs.yml
-
 echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
 cat /var/tmp/$BUILD_ID-extra-vars.yml
 
-python -u abbey.py -p $play -t c3.large -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml --refs /var/tmp/$BUILD_ID-refs.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params
+python -u abbey.py -p $play -t c3.large -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params

util/vpc-tools/abbey.py

@@ -76,19 +76,17 @@ def parse_args():
                         help="don't cleanup on failures")
     parser.add_argument('--vars', metavar="EXTRA_VAR_FILE",
                         help="path to extra var file", required=False)
-    parser.add_argument('--refs', metavar="GIT_REFS_FILE",
-                        help="path to a var file with app git refs", required=False)
     parser.add_argument('--configuration-version', required=False,
-                        help="configuration repo branch(no hashes)",
+                        help="configuration repo gitref",
                         default="master")
     parser.add_argument('--configuration-secure-version', required=False,
-                        help="configuration-secure repo branch(no hashes)",
+                        help="configuration-secure repo gitref",
                         default="master")
     parser.add_argument('--configuration-secure-repo', required=False,
                         default="git@github.com:edx-ops/prod-secure",
                         help="repo to use for the secure files")
     parser.add_argument('--configuration-private-version', required=False,
-                        help="configuration-private repo branch(no hashes)",
+                        help="configuration-private repo gitref",
                         default="master")
     parser.add_argument('--configuration-private-repo', required=False,
                         default="git@github.com:edx-ops/ansible-private",
@@ -287,8 +285,6 @@ cat << EOF >> $extra_vars
 # of all the repositories
 {extra_vars_yml}
 
-{git_refs_yml}
-
 # abbey will always run fake migrations
 # this is so that the application can come
 # up healthy
@@ -299,6 +295,7 @@ EDXAPP_UPDATE_STATIC_FILES_KEY: true
 edxapp_dynamic_cache_key: {deployment}-{environment}-{play}-{cache_id}
 
 disable_edx_services: true
+COMMON_TAG_EC2_INSTANCE: true
 
 # abbey should never take instances in
 # and out of elbs
@@ -369,7 +366,6 @@ rm -rf $base_dir
                 identity_contents=identity_contents,
                 queue_name=run_id,
                 extra_vars_yml=extra_vars_yml,
-                git_refs_yml=git_refs_yml,
                 secure_vars_file=secure_vars_file,
                 cache_id=args.cache_id)
 
@@ -528,18 +524,21 @@ def create_ami(instance_id, name, description):
                 time.sleep(AWS_API_WAIT_TIME)
                 img.add_tag("play", args.play)
                 time.sleep(AWS_API_WAIT_TIME)
-                img.add_tag("configuration_ref", args.configuration_version)
-                time.sleep(AWS_API_WAIT_TIME)
-                img.add_tag("configuration_secure_ref", args.configuration_secure_version)
+                conf_tag = "{} {}".format("http://github.com/edx/configuration", args.configuration_version)
+                img.add_tag("version: configuration", conf_tag)
                 time.sleep(AWS_API_WAIT_TIME)
-                img.add_tag("configuration_secure_repo", args.configuration_secure_repo)
+                conf_secure_tag = "{} {}".format(args.configuration_secure_repo, args.configuration_secure_version)
+                img.add_tag("version: configuration_secure", conf_secure_tag)
                 time.sleep(AWS_API_WAIT_TIME)
                 img.add_tag("cache_id", args.cache_id)
                 time.sleep(AWS_API_WAIT_TIME)
-                for repo, ref in git_refs.items():
-                    key = "refs:{}".format(repo)
-                    img.add_tag(key, ref)
-                    time.sleep(AWS_API_WAIT_TIME)
+
+                # Get versions from the instance.
+                tags = ec2.get_all_tags(filters={'resource-id': instance_id})
+                for tag in tags:
+                    if tag.name.startswith('version:'):
+                        img.add_tag(tag.name, tag.value)
+                        time.sleep(AWS_API_WAIT_TIME)
                 break
             else:
                 time.sleep(1)
@@ -674,14 +673,6 @@ if __name__ == '__main__':
         extra_vars_yml = ""
         extra_vars = {}
 
-    if args.refs:
-        with open(args.refs) as f:
-            git_refs_yml = f.read()
-            git_refs = yaml.load(git_refs_yml)
-    else:
-        git_refs_yml = ""
-        git_refs = {}
-
     if args.secure_vars_file:
         # explicit path to a single
         # secure var file