Improving the sandbox update script

This modifies the edx_ansible role to dump a clean var
file on disk that can be used for subsequent update runs
so that config vars will not be lost

also changes the ansible tmp directory to /var/tmp which is a better
location than using $HOME (used for debugging mostly)

playbooks/ansible.cfg

@@ -7,3 +7,4 @@
 jinja2_extensions=jinja2.ext.do
 hash_behaviour=merge
 host_key_checking = False
+remote_tmp=/var/tmp/.ansible

playbooks/edx-east/edx_ansible.yml

-- name: Deploy ansible
+- name: Deploy the edx_ansible role
   hosts: all
   sudo: True
-  gather_facts: True
+  gather_facts: False
   roles:
     - edx_ansible

playbooks/roles/edx_ansible/defaults/main.yml

@@ -32,5 +32,6 @@ edx_ansible_venv_bin: "{{ edx_ansible_venv_dir }}/bin"
 edx_ansible_user: "edx-ansible"
 edx_ansible_source_repo: https://github.com/edx/configuration.git
 edx_ansible_requirements_file: "{{ edx_ansible_code_dir }}/requirements.txt"
+edx_ansible_var_file: "{{ edx_ansible_data_dir }}/server-vars.yml"
 # edX configuration repo
 configuration_version: master

playbooks/roles/edx_ansible/meta/main.yml

@@ -10,4 +10,4 @@
 ##
 # Role includes for role edx_ansible
 dependencies:
-  - supervisor
+  - common

playbooks/roles/edx_ansible/tasks/deploy.yml

@@ -8,3 +8,44 @@
   pip: requirements="{{ edx_ansible_requirements_file }}" virtualenv="{{ edx_ansible_venv_dir }}" state=present
   sudo_user: "{{ edx_ansible_user }}"
   tags: deploy
+
+- name: edx_ansible | create update script
+  template: >
+    dest={{ edx_ansible_app_dir}}/update
+    src=update.j2 owner={{ edx_ansible_user }} group={{ edx_ansible_user }} mode=755
+  tags: deploy
+
+- name: edx_ansible | create a symlink for update.sh
+  file: >
+    src={{ edx_ansible_app_dir }}/update
+    dest={{ COMMON_BIN_DIR }}/update
+    state=link
+  tags: deploy
+
+- name: edx_ansible | dump all vars to yaml
+  template: src=dumpall.yml.j2 dest={{ edx_ansible_var_file }} mode=0600
+  tags: deploy
+
+- name: edx_ansible | clean up var file, removing all version vars
+  shell: sed -i -e "/{{item}}/d" {{ edx_ansible_var_file }}
+  with_items:
+    - edx_platform_version
+    - edx_platform_commit
+    - xqueue_version
+    - forum_version
+    - xserver_version
+    - discern_ease_version
+    - ora_ease_version
+    - discern_version
+    - ora_version
+    - configuration_version
+
+- name: edx_ansible | remove the special _original_file var
+  shell: sed -i -e "/_original_file/d" {{ edx_ansible_var_file }}
+
+- name: edxapp | create a symlink for var file
+  file: >
+    src={{ edx_ansible_var_file }}
+    dest={{ COMMON_CFG_DIR }}/{{ edx_ansible_var_file|basename }}
+    state=link
+  tags: deploy

playbooks/roles/edx_ansible/tasks/main.yml

@@ -38,20 +38,10 @@
     group="{{ common_web_group }}"
   with_items:
     - "{{ edx_ansible_app_dir }}"
+    - "{{ edx_ansible_data_dir }}"
     - "{{ edx_ansible_venvs_dir }}"
 
 - name: edx_ansible | install a bunch of system packages on which edx_ansible relies
   apt: pkg={{','.join(edx_ansible_debian_pkgs)}} state=present
 
 - include: deploy.yml
-
-- name: edx_ansible | create update script
-  template: >
-    dest={{ edx_ansible_app_dir}}/update
-    src=update.j2 owner={{ edx_ansible_user }} group={{ edx_ansible_user }} mode=755
-
-- name: edxapp | create a symlink for update.sh
-  file: >
-    src={{ edx_ansible_app_dir }}/update
-    dest={{ COMMON_BIN_DIR }}/update
-    state=link

playbooks/roles/edx_ansible/templates/update.j2

@@ -19,17 +19,6 @@ EO
 IFS=$SAVE_IFS
 }
 
-declare -A repos_to_cmd
-edx_ansible_cmd="{{ edx_ansible_venv_bin}}/ansible-playbook -i localhost, -c local --tags deploy"
-repos_to_cmd["edx-platform"]="$edx_ansible_cmd edxapp.yml -e 'edx_platform_version=$2'"
-repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'xqueue_version=$2'"
-repos_to_cmd["forums"]="$edx_ansible_cmd forums.yml -e 'forum_version=$2'"
-repos_to_cmd["xserver"]="$edx_ansible_cmd forums.yml -e 'xserver_version=$2'"
-repos_to_cmd["ease"]="$edx_ansible_cmd discern.yml -e 'discern_ease_version=$2' && $edx_ansible_cmd ora.yml -e 'ora_ease_version=$2'"
-repos_to_cmd["discern"]="$edx_ansible_cmd discern.yml -e 'discern_version=$2'"
-repos_to_cmd["edx-ora"]="$edx_ansible_cmd ora.yml -e 'ora_version=$2'"
-repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuration_version=$2'"
-
 PROG=${0##*/}
 while getopts "vh" opt; do
   case $opt in
@@ -45,6 +34,23 @@ while getopts "vh" opt; do
 done
 
 
+if [[ -f {{ edx_ansible_var_file }} ]]; then
+    extra_args="-e@{{ edx_ansible_var_file }}"
+fi
+
+declare -A repos_to_cmd
+edx_ansible_cmd="{{ edx_ansible_venv_bin}}/ansible-playbook -i localhost, -c local --tags deploy $extra_args "
+
+repos_to_cmd["edx-platform"]="$edx_ansible_cmd edxapp.yml -e 'edx_platform_version=$2'"
+repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'xqueue_version=$2'"
+repos_to_cmd["forums"]="$edx_ansible_cmd forums.yml -e 'forum_version=$2'"
+repos_to_cmd["xserver"]="$edx_ansible_cmd forums.yml -e 'xserver_version=$2'"
+repos_to_cmd["ease"]="$edx_ansible_cmd discern.yml -e 'discern_ease_version=$2' && $edx_ansible_cmd ora.yml -e 'ora_ease_version=$2'"
+repos_to_cmd["discern"]="$edx_ansible_cmd discern.yml -e 'discern_version=$2'"
+repos_to_cmd["edx-ora"]="$edx_ansible_cmd ora.yml -e 'ora_version=$2'"
+repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuration_version=$2'"
+
+
 if [[ -z $1 || -z $2 ]]; then
     echo
     echo "ERROR: You must specify a repo and commit"

util/jenkins/ansible-provision.sh

@@ -159,5 +159,7 @@ for i in "${!deploy[@]}"; do
     fi
 done
 
+# deploy the edx_ansible role
+ansible-playbook edx_ansible.yml -i "${deploy_host}," -e "@${extra_vars}" --user ubuntu --tags deploy
 
 rm -f "$extra_vars"

util/jenkins/create-var-file.sh

@@ -15,7 +15,7 @@ EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 EDXAPP_CMS_NGINX_PORT: 80
 EDXAPP_SITE_NAME: ${deploy_host}
 COMMON_PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
-XSERVER_GRADER_DIR: "{{ xserver_data_dir }}/data/content-mit-600x~2012_Fall"
+XSERVER_GRADER_DIR: "/edx/var/xserver/data/content-mit-600x~2012_Fall"
 XSERVER_GRADER_SOURCE: "git@github.com:/MITx/6.00x.git"
 XSERVER_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull
 CERTS_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull