Merge pull request #2772 from edx/ned/merge-dogwood-to-master

Merge dogwood to master

playbooks/roles/certs/handlers/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Handlers for role certs
-#
-# Overview:
-#
-
-- name: restart certs
-  supervisorctl: >
-    name=certs
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: certs_installed is defined and not disable_edx_services

playbooks/roles/certs/tasks/deploy.yml

@@ -5,14 +5,12 @@
     src=certs.env.json.j2
     dest={{ certs_app_dir }}/env.json
   sudo_user: "{{ certs_user }}"
-  notify: restart certs
 
 - name: create certificate auth file
   template: >
     src=certs.auth.json.j2
     dest={{ certs_app_dir }}/auth.json
   sudo_user: "{{ certs_user }}"
-  notify: restart certs
 
 - name: writing supervisor script for certificates
   template: >
@@ -27,14 +25,12 @@
     state=link
     force=yes
     mode=0644
-  notify: restart certs
   when: not disable_edx_services
 
 - name: create ssh script for git
   template: >
     src={{ certs_git_ssh|basename }}.j2 dest={{ certs_git_ssh }}
     owner={{ certs_user }} mode=750
-  notify: restart certs
 
 # This key is only needed if you are pulling down a private
 # certificates repo
@@ -43,7 +39,6 @@
     content="{{ CERTS_GIT_IDENTITY }}" dest={{ certs_git_identity }}
     force=yes owner={{ certs_user }} mode=0600
   when: CERTS_GIT_IDENTITY != "none"
-  notify: restart certs
 
 - name: checkout certificates repo into {{ certs_code_dir }}
   git: >
@@ -54,7 +49,6 @@
     GIT_SSH: "{{ certs_git_ssh }}"
   register: certs_checkout
   when: CERTS_GIT_IDENTITY != "none"
-  notify: restart certs
 
 - name: checkout certificates repo into {{ certs_code_dir }}
   git: >
@@ -63,19 +57,16 @@
   sudo_user: "{{ certs_user }}"
   register: certs_checkout
   when: CERTS_GIT_IDENTITY == "none"
-  notify: restart certs
 
 - name: remove read-only ssh key for the certs repo
   file: path={{ certs_git_identity }} state=absent
   when: CERTS_GIT_IDENTITY != "none"
-  notify: restart certs
 
 - name : install python requirements
   pip: >
     requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ certs_user }}"
-  notify: restart certs
 
   # call supervisorctl update. this reloads
   # the supervisorctl config and restarts
@@ -110,4 +101,10 @@
 - include: tag_ec2.yml tags=deploy
   when: COMMON_TAG_EC2_INSTANCE
 
-- set_fact: certs_installed=true
+- name: restart certs
+  supervisorctl: >
+    name=certs
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  when: not disable_edx_services

playbooks/roles/certs/tasks/main.yml

@@ -37,7 +37,6 @@
     home="{{ certs_app_dir }}"
     createhome=no
     shell=/bin/false
-  notify: restart certs
 
 - name: create certs app dirs
   file: >
@@ -45,7 +44,6 @@
     state=directory
     owner="{{ certs_user }}"
     group="{{ common_web_group }}"
-  notify: restart certs
   with_items:
     - "{{ certs_app_dir }}"
     # needed for the ansible 1.5 git module
@@ -68,14 +66,12 @@
     path="{{ certs_gpg_dir }}" state=directory
     owner="{{ common_web_user }}"
     mode=0700
-  notify: restart certs
 
 - name: copy the private gpg signing key
   copy: >
     src={{ CERTS_LOCAL_PRIVATE_KEY }}
     dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
     owner={{ common_web_user }} mode=0600
-  notify: restart certs
   register: certs_gpg_key
 
 - name: copy the pgp trust export
@@ -83,20 +79,17 @@
     content="{{ CERTS_OWNER_TRUST }}"
     dest={{ certs_app_dir }}/trust.export
     owner={{ common_web_user }} mode=0600
-  notify: restart certs
 
 - name: load the gpg key
   shell: >
     /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
   sudo_user: "{{ common_web_user }}"
   when: certs_gpg_key.changed
-  notify: restart certs
 
 - name: import the trust export
   shell: >
     /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import-ownertrust {{ certs_app_dir }}/trust.export
   sudo_user: "{{ common_web_user }}"
   when: certs_gpg_key.changed
-  notify: restart certs
 
 - include: deploy.yml tags=deploy

playbooks/roles/xqueue/handlers/main.yml

-- name: restart xqueue
-  supervisorctl: >
-    name={{ item }}
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: xqueue_installed is defined and not disable_edx_services
-  with_items:
-  - xqueue
-  - xqueue_consumer

playbooks/roles/xqueue/tasks/deploy.yml

@@ -23,8 +23,6 @@
   template: >
     src=xqueue_gunicorn.py.j2 dest={{ xqueue_app_dir }}/xqueue_gunicorn.py
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:configuration
@@ -32,8 +30,6 @@
 - name: create xqueue application config
   template: src=xqueue.env.json.j2 dest={{ xqueue_app_dir }}/xqueue.env.json mode=0644
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:configuration
@@ -41,8 +37,6 @@
 - name: create xqueue auth file
   template: src=xqueue.auth.json.j2 dest={{ xqueue_app_dir }}/xqueue.auth.json mode=0644
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:configuration
@@ -54,8 +48,6 @@
     accept_hostkey=yes
   sudo_user: "{{ xqueue_user }}"
   register: xqueue_checkout
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:code
@@ -67,8 +59,6 @@
     requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:app-requirements
@@ -79,8 +69,6 @@
     requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:app-requirements
@@ -95,8 +83,6 @@
     DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
     DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
   when: migrate_db is defined and migrate_db|lower == "yes" and COMMON_MYSQL_MIGRATE_PASS
-  notify:
-    - restart xqueue
   tags:
     - migrate
     - migrate:db
@@ -105,8 +91,6 @@
   shell: >
     SERVICE_VARIANT=xqueue {{ xqueue_venv_bin }}/django-admin.py update_users --settings=xqueue.aws_settings --pythonpath={{ xqueue_code_dir }}
   sudo_user: "{{ xqueue_user }}"
-  notify:
-    - restart xqueue
   tags:
     - manage
     - manage:app-users
@@ -148,4 +132,13 @@
     - install
     - install:app-requirements
 
-- set_fact: xqueue_installed=true
+- name: restart xqueue
+  supervisorctl: >
+    name={{ item }}
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  when: not disable_edx_services
+  with_items:
+  - xqueue
+  - xqueue_consumer

playbooks/roles/xqueue/tasks/main.yml

@@ -12,8 +12,6 @@
     home="{{ xqueue_app_dir }}"
     createhome=no
     shell=/bin/false
-  notify:
-    - restart xqueue
   tags:
     - install
     - install:base
@@ -24,8 +22,6 @@
     state=directory
     owner="{{ xqueue_user }}"
     group="{{ common_web_group }}"
-  notify:
-    - restart xqueue
   with_items:
     - "{{ xqueue_app_dir }}"
     - "{{ xqueue_venvs_dir }}"
@@ -35,8 +31,6 @@
 
 - name: install a bunch of system packages on which xqueue relies
   apt: pkg={{','.join(xqueue_debian_pkgs)}} state=present
-  notify:
-  - restart xqueue
   tags:
     - install
     - install:system-requirements

util/vagrant/migrate.sh

@@ -64,14 +64,8 @@ confirm_proceed () {
 }
 
 # Check we are in the right place, and have the info we need.
-
-if [[ "`whoami`" != "vagrant" ]]; then
-  echo "Run this from the vagrant account in your Open edX machine."
-  exit 1
-fi
-
 if [[ ! -d /edx/app/edxapp ]]; then
-  echo "Run this from the vagrant account in your Open edX machine."
+  echo "Run this on your Open edX machine."
   exit 1
 fi
 
@@ -90,6 +84,11 @@ if [[ $CONFIGURATION == none ]]; then
   exit 1
 fi
 
+APPUSER=edxapp
+if [[ $CONFIGURATION == fullstack ]] ; then
+  APPUSER=www-data
+fi
+
 # Birch details
 
 if [[ $TARGET == *birch* && $INTERACTIVE == true ]] ; then
@@ -173,6 +172,22 @@ make_config_venv
 # Dogwood details
 
 if [[ $TARGET == *dogwood* ]] ; then
+  # Run the forum migrations.
+  cat > migrate-008-context.js <<"EOF"
+    // from: https://github.com/edx/cs_comments_service/blob/master/scripts/db/migrate-008-context.js
+    print ("Add the new indexes for the context field");
+    db.contents.ensureIndex({ _type: 1, course_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true})
+    db.contents.ensureIndex({ _type: 1, commentable_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true})
+
+    print ("Adding context to all comment threads where it does not yet exist\n");
+    var bulk = db.contents.initializeUnorderedBulkOp();
+    bulk.find( {_type: "CommentThread", context: {$exists: false}} ).update(  {$set: {context: "course"}} );
+    bulk.execute();
+    printjson (db.runCommand({ getLastError: 1, w: "majority", wtimeout: 5000 } ));
+EOF
+
+  mongo cs_comments_service migrate-008-context.js
+
   # We are upgrading Python from 2.7.3 to 2.7.10, so remake the venvs.
   sudo rm -rf /edx/app/*/v*envs/*
 
@@ -194,7 +209,7 @@ if [[ $TARGET == *dogwood* ]] ; then
   make_config_venv
 
   # Need to get rid of South from edx-platform, or things won't work.
-  sudo -u edxapp /edx/app/edxapp/venvs/edxapp/bin/pip uninstall -y South
+  sudo -u edxapp /edx/bin/pip.edxapp uninstall -y South
 
   echo "Upgrading to the beginning of Django 1.8"
   cd configuration/playbooks/vagrant
@@ -211,10 +226,8 @@ if [[ $TARGET == *dogwood* ]] ; then
 
   echo "Running the Django 1.8 faked migrations"
   for item in lms cms; do
-    sudo -u edxapp \
-      /edx/app/edxapp/venvs/edxapp/bin/python \
-      /edx/app/edxapp/edx-platform/manage.py $item migrate \
-      --settings=aws --noinput --fake-initial
+    sudo -u $APPUSER -E /edx/bin/python.edxapp \
+      /edx/bin/manage.edxapp $item migrate --settings=aws --noinput --fake-initial
   done
 
   if [[ $CONFIGURATION == fullstack ]] ; then
@@ -226,6 +239,7 @@ if [[ $TARGET == *dogwood* ]] ; then
   fi
 fi
 
+echo "Updating to final version of code"
 cd configuration/playbooks
 echo "edx_platform_version: $TARGET" > vars.yml
 echo "ora2_version: $TARGET" >> vars.yml
@@ -238,6 +252,20 @@ sudo ansible-playbook \
     --extra-vars="@vars.yml" \
     $SERVER_VARS \
     vagrant-$CONFIGURATION.yml
+cd ../..
+
+if [[ $TARGET == *dogwood* ]] ; then
+  echo "Running data fixup management commands"
+  sudo -u $APPUSER -E /edx/bin/python.edxapp \
+    /edx/bin/manage.edxapp lms --settings=aws generate_course_overview --all
+
+  sudo -u $APPUSER -E /edx/bin/python.edxapp \
+    /edx/bin/manage.edxapp lms --settings=aws post_cohort_membership_fix --commit
+
+  # Run the forums migrations again to catch things made while this script
+  # was running.
+  mongo cs_comments_service migrate-008-context.js
+fi
 
 cd /
 sudo rm -rf $TEMPDIR

vagrant/release/devstack/Vagrantfile

@@ -63,15 +63,13 @@ openedx_releases = {
   "named-release/dogwood.rc" => {
     :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
   },
-  "named-release/dogwood.rc2" => {
+  "named-release/dogwood" => {
     :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
   },
-  "named-release/dogwood.rc3" => {
-    :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
-  },
-  "named-release/cypress" => {
-    :name => "cypress-devstack", :file => "cypress-devstack.box",
-  },
+  # Cypress is deprecated and unsupported
+  # "named-release/cypress" => {
+  #   :name => "cypress-devstack", :file => "cypress-devstack.box",
+  # },
   # Birch is deprecated and unsupported
   # "named-release/birch.2" => {
   #   :name => "birch-devstack-2", :file => "birch-2-devstack.box",

vagrant/release/fullstack/Vagrantfile

@@ -9,18 +9,16 @@ CPU_COUNT = 2
 # to a name and a file path, which are used for retrieving
 # a Vagrant box from the internet.
 openedx_releases = {
-  "named-release/dogwood.rc" => {
-    :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
-  },
-  "named-release/dogwood.rc2" => {
+  "named-release/dogwood" => {
     :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
   },
-  "named-release/dogwood.rc3" => {
+  "named-release/dogwood.rc" => {
     :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
   },
-  "named-release/cypress" => {
-    :name => "cypress-fullstack", :file => "cypress-fullstack.box",
-  },
+  # Cypress is deprecated and unsupported
+  # "named-release/cypress" => {
+  #   :name => "cypress-fullstack", :file => "cypress-fullstack.box",
+  # },
   # Birch is deprecated and unsupported
   # "named-release/birch.2" => {
   #   :name => "birch-fullstack-2", :file => "birch-2-fullstack.box",