Merge remote-tracking branch 'origin' into jarv/no-deploy

cloudformation_templates/edx-reference-architecture.json

@@ -512,7 +512,13 @@
             { "Ref":"AWS::Region" },
             "AZone0"
           ]
-        }
+        },
+        "Tags":[ 
+          { 
+            "Key":"immutable_metadata", 
+            "Value":"{'purpose':'external','target':'ec2'}" 
+          } 
+        ]
       }
     },
     "PublicSubnet02":{
@@ -534,7 +540,13 @@
             { "Ref":"AWS::Region" },
             "AZone1"
           ]
-        }
+        },
+        "Tags":[ 
+          { 
+            "Key":"immutable_metadata", 
+            "Value":"{'purpose':'external','target':'ec2'}" 
+          } 
+        ]
       }
     },
     "AdminSubnet":{
@@ -597,6 +609,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key":"immutable_metadata", 
+            "Value":"{'purpose':'internal-edxapp','target':'ec2'}" 
           }
         ]
       }
@@ -629,6 +645,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key":"immutable_metadata", 
+            "Value":"{'purpose':'internal-edxapp','target':'ec2'}" 
           }
         ]
       }
@@ -661,6 +681,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-xqueue','target':'ec2'}" 
           }
         ]
       }
@@ -693,6 +717,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-xqueue','target':'ec2'}" 
           }
         ]
       }
@@ -725,6 +753,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-rabbit','target':'ec2'}" 
           }
         ]
       }
@@ -757,6 +789,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-rabbit','target':'ec2'}" 
           }
         ]
       }
@@ -789,6 +825,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-xserver','target':'ec2'}" 
           }
         ]
       }
@@ -821,6 +861,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-xserver','target':'ec2'}" 
           }
         ]
       }
@@ -981,6 +1025,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-worker','target':'ec2'}" 
           }
         ]
       }
@@ -1013,6 +1061,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-worker','target':'ec2'}" 
           }
         ]
       }
@@ -1045,6 +1097,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-elasticsearch','target':'ec2'}" 
           }
         ]
       }
@@ -1077,6 +1133,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-elasticsearch','target':'ec2'}" 
           }
         ]
       }
@@ -1109,6 +1169,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-forum','target':'ec2'}" 
           }
         ]
       }
@@ -1141,6 +1205,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-forum','target':'ec2'}" 
           }
         ]
       }
@@ -1173,6 +1241,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-mongo','target':'ec2'}" 
           }
         ]
       }
@@ -1205,6 +1277,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-mongo','target':'ec2'}" 
           }
         ]
       }
@@ -1237,6 +1313,10 @@
           {
             "Key":"Network",
             "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata", 
+            "Value" : "{'purpose':'internal-mongo','target':'ec2'}" 
           }
         ]
       }

playbooks/edx-east/edx_ansible.yml

-- name: Deploy ansible
+- name: Deploy the edx_ansible role
   hosts: all
   sudo: True
-  gather_facts: True
+  gather_facts: False
   roles:
     - edx_ansible

playbooks/roles/edx_ansible/defaults/main.yml

@@ -32,5 +32,6 @@ edx_ansible_venv_bin: "{{ edx_ansible_venv_dir }}/bin"
 edx_ansible_user: "edx-ansible"
 edx_ansible_source_repo: https://github.com/edx/configuration.git
 edx_ansible_requirements_file: "{{ edx_ansible_code_dir }}/requirements.txt"
+edx_ansible_var_file: "{{ edx_ansible_data_dir }}/server-vars.yml"
 # edX configuration repo
 configuration_version: master

playbooks/roles/edx_ansible/meta/main.yml

@@ -10,4 +10,4 @@
 ##
 # Role includes for role edx_ansible
 dependencies:
-  - supervisor
+  - common

playbooks/roles/edx_ansible/tasks/deploy.yml

@@ -8,3 +8,48 @@
   pip: requirements="{{ edx_ansible_requirements_file }}" virtualenv="{{ edx_ansible_venv_dir }}" state=present
   sudo_user: "{{ edx_ansible_user }}"
   tags: deploy
+
+- name: edx_ansible | create update script
+  template: >
+    dest={{ edx_ansible_app_dir}}/update
+    src=update.j2 owner={{ edx_ansible_user }} group={{ edx_ansible_user }} mode=755
+  tags: deploy
+
+- name: edx_ansible | create a symlink for update.sh
+  file: >
+    src={{ edx_ansible_app_dir }}/update
+    dest={{ COMMON_BIN_DIR }}/update
+    state=link
+  tags: deploy
+
+- name: edx_ansible | dump all vars to yaml
+  template: src=dumpall.yml.j2 dest={{ edx_ansible_var_file }} mode=0600
+  tags: deploy
+
+- name: edx_ansible | clean up var file, removing all version vars
+  shell: sed -i -e "/{{item}}/d" {{ edx_ansible_var_file }}
+  with_items:
+    - edx_platform_version
+    - edx_platform_commit
+    - xqueue_version
+    - forum_version
+    - xserver_version
+    - discern_ease_version
+    - ora_ease_version
+    - discern_version
+    - ora_version
+    - configuration_version
+    - ease_version
+    - certs_version
+  tags: deploy
+
+- name: edx_ansible | remove the special _original_file var
+  shell: sed -i -e "/_original_file/d" {{ edx_ansible_var_file }}
+  tags: deploy
+
+- name: edxapp | create a symlink for var file
+  file: >
+    src={{ edx_ansible_var_file }}
+    dest={{ COMMON_CFG_DIR }}/{{ edx_ansible_var_file|basename }}
+    state=link
+  tags: deploy

playbooks/roles/edx_ansible/tasks/main.yml

@@ -38,20 +38,10 @@
     group="{{ common_web_group }}"
   with_items:
     - "{{ edx_ansible_app_dir }}"
+    - "{{ edx_ansible_data_dir }}"
     - "{{ edx_ansible_venvs_dir }}"
 
 - name: edx_ansible | install a bunch of system packages on which edx_ansible relies
   apt: pkg={{','.join(edx_ansible_debian_pkgs)}} state=present
 
 - include: deploy.yml
-
-- name: edx_ansible | create update script
-  template: >
-    dest={{ edx_ansible_app_dir}}/update
-    src=update.j2 owner={{ edx_ansible_user }} group={{ edx_ansible_user }} mode=755
-
-- name: edxapp | create a symlink for update.sh
-  file: >
-    src={{ edx_ansible_app_dir }}/update
-    dest={{ COMMON_BIN_DIR }}/update
-    state=link

playbooks/roles/edx_ansible/templates/dumpall.yml.j2

+---
+{{ vars | to_nice_yaml }}

playbooks/roles/edx_ansible/templates/update.j2

@@ -19,17 +19,6 @@ EO
 IFS=$SAVE_IFS
 }
 
-declare -A repos_to_cmd
-edx_ansible_cmd="{{ edx_ansible_venv_bin}}/ansible-playbook -i localhost, -c local --tags deploy"
-repos_to_cmd["edx-platform"]="$edx_ansible_cmd edxapp.yml -e 'edx_platform_version=$2'"
-repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'xqueue_version=$2'"
-repos_to_cmd["forums"]="$edx_ansible_cmd forums.yml -e 'forum_version=$2'"
-repos_to_cmd["xserver"]="$edx_ansible_cmd forums.yml -e 'xserver_version=$2'"
-repos_to_cmd["ease"]="$edx_ansible_cmd discern.yml -e 'discern_ease_version=$2' && $edx_ansible_cmd ora.yml -e 'ora_ease_version=$2'"
-repos_to_cmd["discern"]="$edx_ansible_cmd discern.yml -e 'discern_version=$2'"
-repos_to_cmd["edx-ora"]="$edx_ansible_cmd ora.yml -e 'ora_version=$2'"
-repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuration_version=$2'"
-
 PROG=${0##*/}
 while getopts "vh" opt; do
   case $opt in
@@ -45,6 +34,23 @@ while getopts "vh" opt; do
 done
 
 
+if [[ -f {{ edx_ansible_var_file }} ]]; then
+    extra_args="-e@{{ edx_ansible_var_file }}"
+fi
+
+declare -A repos_to_cmd
+edx_ansible_cmd="{{ edx_ansible_venv_bin}}/ansible-playbook -i localhost, -c local --tags deploy $extra_args "
+
+repos_to_cmd["edx-platform"]="$edx_ansible_cmd edxapp.yml -e 'edx_platform_version=$2'"
+repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'xqueue_version=$2'"
+repos_to_cmd["forums"]="$edx_ansible_cmd forums.yml -e 'forum_version=$2'"
+repos_to_cmd["xserver"]="$edx_ansible_cmd forums.yml -e 'xserver_version=$2'"
+repos_to_cmd["ease"]="$edx_ansible_cmd discern.yml -e 'discern_ease_version=$2' && $edx_ansible_cmd ora.yml -e 'ora_ease_version=$2'"
+repos_to_cmd["discern"]="$edx_ansible_cmd discern.yml -e 'discern_version=$2'"
+repos_to_cmd["edx-ora"]="$edx_ansible_cmd ora.yml -e 'ora_version=$2'"
+repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuration_version=$2'"
+
+
 if [[ -z $1 || -z $2 ]]; then
     echo
     echo "ERROR: You must specify a repo and commit"

util/jenkins/ansible-provision.sh

@@ -159,5 +159,7 @@ for i in "${!deploy[@]}"; do
     fi
 done
 
+# deploy the edx_ansible role
+ansible-playbook edx_ansible.yml -i "${deploy_host}," -e "@${extra_vars}" --user ubuntu
 
 rm -f "$extra_vars"

util/jenkins/create-var-file.sh

@@ -15,7 +15,7 @@ EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 EDXAPP_CMS_NGINX_PORT: 80
 EDXAPP_SITE_NAME: ${deploy_host}
 COMMON_PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
-XSERVER_GRADER_DIR: "{{ xserver_data_dir }}/data/content-mit-600x~2012_Fall"
+XSERVER_GRADER_DIR: "/edx/var/xserver/data/content-mit-600x~2012_Fall"
 XSERVER_GRADER_SOURCE: "git@github.com:/MITx/6.00x.git"
 XSERVER_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull
 CERTS_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull