Merge pull request #328 from edx/jarv/deploy

Jarv/deploy

playbooks/edx-east/deploy_edx_platform.yml → playbooks/edx-east/deploy_edxapp.yml

-- name: Configure instance(s)
+- name: Deploy edxapp
   hosts: all
   sudo: True
   gather_facts: True
-  vars:
-    openid_workaround: True
   roles:
     - edxapp

playbooks/edx-east/deploy_forum.yml

+- name: Deploy forum
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - forum

playbooks/edx-east/deploy_ora.yml

+- name: Deploy ora
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - ora

playbooks/edx-east/deploy_xqueue.yml

+- name: Deploy xqueue
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - xqueue

playbooks/edx-east/deploy_xserver.yml

+- name: Deploy xserver
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - xserver

playbooks/edx-east/edx_continuous_integration.yml

@@ -2,6 +2,7 @@
   hosts: all
   sudo: True
   gather_facts: True
+  serial: 10
   vars:
     migrate_db: "yes"
     openid_workaround: True

playbooks/roles/edxapp/tasks/cms.yml

@@ -5,19 +5,19 @@
 ---
 - name: create cms application config
   template: src=cms.env.json.j2 dest=$app_base_dir/cms.env.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - cms-env
   - cms
   - update
+  - deploy
 
 - name: create cms auth file
   template: src=cms.auth.json.j2 dest=$app_base_dir/cms.auth.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - cms-env
   - cms
   - update
+  - deploy
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms
   when: celery_worker is not defined
@@ -32,6 +32,7 @@
   - cms-env
   - logging
   - update
+  - deploy
 
 # Creates CMS upstart file
 - include: upstart.yml basename=cms

playbooks/roles/edxapp/tasks/deploy.yml

@@ -14,9 +14,20 @@
   tags:
   - deploy
 
+# update json configs for the application
+
+- include: lms.yml
+  when: "'lms' in service_variants_enabled"
+- include: cms.yml
+  when: "'cms' in service_variants_enabled"
+- include: lms-preview.yml
+  when: "'lms-preview' in service_variants_enabled"
+
+
 # Do A Checkout
 - name: edxapp | checkout edx-platform repo into {{edx_platform_code_dir}}
   git: dest={{edx_platform_code_dir}} repo={{edx_platform_repo}} version={{edx_platform_commit}}
+  register: edx_platform_checkout
   tags:
   - lms
   - cms
@@ -25,6 +36,7 @@
 
 - name: git clean after checking out edx-platform
   shell: cd {{edx_platform_code_dir}} && git clean -xdf
+  when: edx_platform_checkout.changed
   tags:
   - lms
   - cms
@@ -58,6 +70,7 @@
   file: path={{edx_platform_code_dir}} state=directory owner=www-data group=www-data recurse=yes
   # Post Checkout tasks will get run as handlers when the {{ edx_platform_code_dir }} is ready.
   # Look at the handlers/main.yml in this role for a description of the tasks stated below.
+  when: edx_platform_checkout.changed
   tags:
   - lms
   - cms
@@ -66,14 +79,18 @@
 
 # Ruby plays that need to be run after platform updates.
 - name: gem | gem install bundler
-  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle chdir={{ edx_platform_code_dir }}
+  shell: >
+    RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle 
+      chdir={{ edx_platform_code_dir }}
   tags:
   - ruby
   - deploy
   - install
 
 - name: bundle | bundle install
-  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ gem_home }}/bin/bundle install --binstubs chdir={{ edx_platform_code_dir }}
+  shell: >
+    RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ gem_home }}/bin/bundle install --binstubs 
+      chdir={{ edx_platform_code_dir }}
   tags:
   - ruby
   - deploy

playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -5,17 +5,17 @@
 ---
 - name: create lms application config
   template: src=lms-preview.env.json.j2 dest=$app_base_dir/lms-preview.env.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - lms-preview
   - lms-preview-env
+  - deploy
 
 - name: create lms auth file
   template: src=lms-preview.auth.json.j2 dest=$app_base_dir/lms-preview.auth.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - lms-preview
   - lms-preview-env
+  - deploy
 
 - name: Create lms-preview log target directory
   file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=syslog mode=2750
@@ -24,6 +24,7 @@
   - lms-preview-env
   - logging
   - update
+  - deploy
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-preview
   when: celery_worker is not defined

playbooks/roles/edxapp/tasks/lms.yml

 ---
 - name: create lms application config
   template: src=lms.env.json.j2 dest=$app_base_dir/lms.env.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - lms
   - lms-env
   - update
+  - deploy
 
 - name: create lms auth file
   template: src=lms.auth.json.j2 dest=$app_base_dir/lms.auth.json mode=640 owner=www-data group=adm
-  notify: restart edxapp
   tags:
   - lms
   - lms-env
   - update
+  - deploy
 
 - name: Create lms log target directory
   file: path={{log_base_dir}}/lms state=directory owner=syslog group=syslog mode=2750
@@ -22,6 +22,7 @@
   - lms-env
   - logging
   - update
+  - deploy
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
   when: celery_worker is not defined

playbooks/roles/edxapp/tasks/main.yml

@@ -35,14 +35,6 @@
   - cms
   - install
 
-
-- include: lms.yml
-  when: "'lms' in service_variants_enabled"
-- include: cms.yml
-  when: "'cms' in service_variants_enabled"
-- include: lms-preview.yml
-  when: "'lms-preview' in service_variants_enabled"
-
 - name: creating edxapp upstart script
   sudo: True
   template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root

playbooks/roles/edxapp/tasks/upstart.yml

@@ -6,3 +6,4 @@
   - upstart
   - gunicorn
   - update
+  - deploy

playbooks/roles/nginx/tasks/nginx_site.yml

@@ -15,6 +15,7 @@
   - cms
   - nginx-env
   - update
+  - deploy
 
 - name: nginx | Creating nginx config link {{ site_name }}
   file: src=/etc/nginx/sites-available/{{ site_name }} dest=/etc/nginx/sites-enabled/{{ site_name }} state={{ state }} owner=root group=root
@@ -26,3 +27,4 @@
   - cms
   - nginx-env
   - update
+  - deploy

playbooks/roles/ora/tasks/deploy.yml

@@ -29,6 +29,34 @@
   - ora
   - deploy
 
+- name: ora | create ora application config
+  template: src=ora.env.json.j2 dest={{ora_code_dir}}/../ora.env.json mode=0640 owner={{ ora_user }} group=adm
+  tags:
+  - ora
+  - deploy
+
+- name: ora | create ora auth file
+  template: src=ora.auth.json.j2 dest={{ora_code_dir}}/../ora.auth.json mode=0640 owner={{ ora_user }} group=adm
+  tags:
+  - ora
+  - deploy
+
+- name: ora | create ora upstart script
+  template: src=edx-ora.conf.j2 dest=/etc/init/edx-ora.conf mode=0640 owner=root group=adm
+  tags:
+  - ora
+  - deploy
+
+- name: ora | create ora-celery upstart script
+  template: src=edx-ora-celery.conf.j2 dest=/etc/init/edx-ora-celery.conf mode=0640 owner=root group=adm
+  tags:
+  - ora
+  - deploy
+
+# Install nginx site
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=ora
+
+
 # Do A Checkout
 - name: ora | git checkout ora repo into $app_base_dir
   git: dest={{ora_code_dir}} repo={{ora_source_repo}} version={{ora_version}}

playbooks/roles/ora/tasks/main.yml

@@ -19,34 +19,11 @@
   tags:
   - ora
 
-- name: ora | create ora application config
-  template: src=ora.env.json.j2 dest={{ora_code_dir}}/../ora.env.json mode=0640 owner={{ ora_user }} group=adm
-  tags:
-  - ora
-
-- name: ora | create ora auth file
-  template: src=ora.auth.json.j2 dest={{ora_code_dir}}/../ora.auth.json mode=0640 owner={{ ora_user }} group=adm
-  tags:
-  - ora
-
-- name: ora | create ora upstart script
-  template: src=edx-ora.conf.j2 dest=/etc/init/edx-ora.conf mode=0640 owner=root group=adm
-  tags:
-  - ora
-
-- name: ora | create ora-celery upstart script
-  template: src=edx-ora-celery.conf.j2 dest=/etc/init/edx-ora-celery.conf mode=0640 owner=root group=adm
-  tags:
-  - ora
-
 - name: ora | install debian packages that ora needs
   apt: pkg={{item}} state=present
   with_items: ora_debian_pkgs
   tags:
   - ora
 
-# Install nginx site
-- include: ../../nginx/tasks/nginx_site.yml state=link site_name=ora
-
 - include: ease.yml
 - include: deploy.yml

playbooks/roles/xqueue/tasks/deploy.yml

@@ -11,6 +11,33 @@
   - xqueue
   - deploy
 
+- name: xqueue | create xqueue application config
+  template: src=xqueue.env.json.j2 dest={{app_base_dir}}/xqueue.env.json mode=0640 owner={{ xqueue_user }} group=adm
+  tags:
+  - xqueue
+  - deploy
+
+- name: xqueue | create xqueue auth file
+  template: src=xqueue.auth.json.j2 dest={{app_base_dir}}/xqueue.auth.json mode=0640 owner={{ xqueue_user }} group=adm
+  tags:
+  - xqueue
+  - deploy
+
+- name: xqueue | creating xqueue upstart script
+  template: src=xqueue.conf.j2 dest=/etc/init/xqueue.conf mode=0640 owner=root group=adm
+  tags:
+  - xqueue
+  - deploy
+
+- name: xqueue | create xqueue consumer upstart script
+  template: src=xqueue_consumer.conf.j2 dest=/etc/init/xqueue_consumer.conf mode=0640 owner=root group=adm
+  tags:
+  - xqueue
+  - deploy
+
+# Install nginx site
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=xqueue
+
 # Do A Checkout
 - name: xqueue | git checkout xqueue repo into {{app_base_dir}}
   git: dest={{xqueue_code_dir}} repo={{xqueue_source_repo}} version={{xqueue_version}}

playbooks/roles/xqueue/tasks/main.yml

@@ -55,37 +55,4 @@
     encoding=utf8
   when: xqueue_create_db is defined and xqueue_create_db|lower == "yes"
 
-- name: xqueue | create xqueue application config
-  template: src=xqueue.env.json.j2 dest={{app_base_dir}}/xqueue.env.json mode=0640 owner={{ xqueue_user }} group=adm
-  notify:
-  - xqueue | restart xqueue
-  - xqueue | restart xqueue consumer
-  tags:
-  - xqueue
-
-- name: xqueue | create xqueue auth file
-  template: src=xqueue.auth.json.j2 dest={{app_base_dir}}/xqueue.auth.json mode=0640 owner={{ xqueue_user }} group=adm
-  notify:
-  - xqueue | restart xqueue
-  - xqueue | restart xqueue consumer
-  tags:
-  - xqueue
-
-- name: xqueue | creating xqueue upstart script
-  template: src=xqueue.conf.j2 dest=/etc/init/xqueue.conf mode=0640 owner=root group=adm
-  notify:
-  - xqueue | restart xqueue
-  tags:
-  - xqueue
-
-- name: xqueue | create xqueue consumer upstart script
-  template: src=xqueue_consumer.conf.j2 dest=/etc/init/xqueue_consumer.conf mode=0640 owner=root group=adm
-  notify:
-  - xqueue | restart xqueue consumer
-  tags:
-  - xqueue
-
-# Install nginx site
-- include: ../../nginx/tasks/nginx_site.yml state=link site_name=xqueue
-
 - include: deploy.yml

util/jenkins/ansible-configure.sh

@@ -17,6 +17,11 @@
 
 
 export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto
+
+dir=$(dirname $0)
+source "${dir}/ascii-convert.sh"
+
+
 if [[ -z $dns_name ]]; then
   echo "The hostname is required to know what machine to configure"
   exit 1

util/jenkins/ansible-deploy.sh

+#!/usr/bin/env bash
+
+# Ansible deployment wrapper script that
+# assumes the following parameters set
+# as environment variables
+# 
+# {edxapp,forum,xqueue,xserver,ora} - true/false
+# {edxapp,forum,xqueue,xserver,ora}_version - commit or tag
+ 
+export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto
+if [[ -z $WORKSPACE ]]; then
+    dir=$(dirname $0)
+    source "$dir/ascii-convert.sh"
+else
+    source "$WORKSPACE/util/jenkins/ascii-convert.sh"
+fi
+
+if [[ -z $github_username  ]]; then
+  github_username=$BUILD_USER_ID
+fi
+
+if [[ ! -f $BOTO_CONFIG ]]; then
+  echo "AWS credentials not found for $aws_account"
+  exit 1
+fi
+
+extra_vars="/var/tmp/extra-vars-$$.yml"
+
+if [[ -z $deploy_host ]]; then
+  deploy_host="${github_username}.m.sandbox.edx.org"
+fi
+
+cat << EOF > $extra_vars
+---
+
+EDXAPP_PREVIEW_LMS_BASE: preview.${deploy_host}
+EDXAPP_LMS_BASE: ${deploy_host}
+EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
+EDXAPP_CMS_NGINX_PORT: 80
+XSERVER_GRADER_CHECKOUT: False
+c_skip_grader_checkout: True
+
+edx_platform_commit: $edxapp_version
+forum_version: $forum_version
+xqueue_version: $xqueue_version
+xserver_version: $xserver_version
+ora_version: $ora_version
+ease_version: $ease_version
+
+ansible_ssh_private_key_file: /var/lib/jenkins/${keypair}.pem
+
+EOF
+
+cat $extra_vars
+
+echo "Deploying to $deploy_host"
+
+declare -A deploy
+
+deploy[edxapp]=$edxapp
+deploy[forum]=$forum
+deploy[xqueue]=$xqueue
+deploy[xserver]=$xserver
+deploy[ora]=$ora
+
+ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R "$deploy_host"
+
+cd playbooks/edx-east
+for i in "${!deploy[@]}"; do
+    if [[ ${deploy[$i]} == "true" ]]; then
+        ansible-playbook -vvvv deploy_${i}.yml -i "${deploy_host}," -e "@${extra_vars}" --user ubuntu --tags deploy
+    fi
+done
+rm -f "$extra_vars"
+

util/jenkins/ansible-provision.sh

@@ -19,16 +19,8 @@
 # - name_tag
 
 export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto
-
-function ascii_convert {
-  echo $1 | iconv -f utf8 -t ascii//TRANSLIT//IGNORE
-}
-
-# remove non-ascii chars from build user vars
-BUILD_USER_LAST_NAME=$(ascii_convert $BUILD_USER_LAST_NAME)
-BUILD_USER_FIRST_NAME=$(ascii_convert $BUILD_USER_FIRST_NAME)
-BUILD_USER_ID=$(ascii_convert $BUILD_USER_ID)
-BUILD_USER=$(ascii_convert $BUILD_USER)
+dir=$(dirname $0)
+source "${dir}/ascii-convert.sh"
 
 if [[ -z $github_username  ]]; then
   github_username=$BUILD_USER_ID

util/jenkins/ascii-convert.sh

+#!/usr/bin/env bash
+
+function ascii_convert {
+  echo $1 | iconv -f utf8 -t ascii//TRANSLIT//IGNORE
+}
+
+# remove non-ascii chars from build user vars
+BUILD_USER_LAST_NAME=$(ascii_convert $BUILD_USER_LAST_NAME)
+BUILD_USER_FIRST_NAME=$(ascii_convert $BUILD_USER_FIRST_NAME)
+BUILD_USER_ID=$(ascii_convert $BUILD_USER_ID)
+BUILD_USER=$(ascii_convert $BUILD_USER)
+
+