Merge pull request #2735 from edx/ned/dogwood-merge

Merge Dogwood changes (so far) into master.

playbooks/roles/edxapp/defaults/main.yml

@@ -415,11 +415,11 @@ EDXAPP_REGISTRATION_EXTRA_FIELDS:
 EDXAPP_CELERY_WORKERS:
   - queue: low
     service_variant: cms
-    concurrency: 3
+    concurrency: 1
     monitor: True
   - queue: default
     service_variant: cms
-    concurrency: 4
+    concurrency: 1
     monitor: True
   - queue: high
     service_variant: cms
@@ -431,15 +431,15 @@ EDXAPP_CELERY_WORKERS:
     monitor: True
   - queue: default
     service_variant: lms
-    concurrency: 3
+    concurrency: 1
     monitor: True
   - queue: high
     service_variant: lms
-    concurrency: 4
+    concurrency: 1
     monitor: True
   - queue: high_mem
     service_variant: lms
-    concurrency: 2
+    concurrency: 1
     monitor: False
     max_tasks_per_child: 1
 

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -154,7 +154,7 @@
 - name: gather {{ item }} static assets with paver
   command: "{{ COMMON_BIN_DIR }}/edxapp-update-assets-{{ item }}"
   when: celery_worker is not defined and not devstack and item != "lms-preview"
-  tags: gather_static_assets
   with_items: service_variants_enabled
   tags:
+    - gather_static_assets
     - assets

playbooks/roles/mysql/defaults/main.yml

@@ -2,4 +2,4 @@
 mysql_debian_pkgs:
   - software-properties-common
   - python-mysqldb
-  - mysql-server
+  - "mysql-server=5.6.*"

playbooks/vagrant/ansible.cfg

+[defaults]
+
+jinja2_extensions=jinja2.ext.do
+host_key_checking = False
+roles_path=../roles
+callback_plugins=../callback_plugins
+ansible_managed=This file is created and updated by ansible, edit at your peril

playbooks/vagrant/vagrant-devstack-delta.yml

+- name: Update devstack to a specific intermediate revision
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    devstack: true
+    openid_workaround: true
+    disable_edx_services: true
+    mongo_enable_journal: false
+    COMMON_MOTD_TEMPLATE: 'devstack_motd.tail.j2'
+    COMMON_SSH_PASSWORD_AUTH: "yes"
+    EDXAPP_LMS_BASE: 127.0.0.1:8000
+    EDXAPP_OAUTH_ENFORCE_SECURE: false
+    EDXAPP_LMS_BASE_SCHEME: http
+  roles:
+    - common
+    - vhost
+    - edxapp

playbooks/vagrant/vagrant-fullstack-delta.yml

+- name: Update fullstack to a specific intermediate revision
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    openid_workaround: true
+    disable_edx_services: true
+    mongo_enable_journal: false
+    COMMON_MOTD_TEMPLATE: 'devstack_motd.tail.j2'
+    COMMON_SSH_PASSWORD_AUTH: "yes"
+    EDXAPP_LMS_BASE: 127.0.0.1:8000
+    EDXAPP_OAUTH_ENFORCE_SECURE: false
+    EDXAPP_LMS_BASE_SCHEME: http
+  roles:
+    - common
+    - vhost
+    - edxapp
+    - xqueue

util/vagrant/migrate.sh

 #!/usr/bin/env bash
 
+# Stop if any command fails
+set -e
+
 # defaults
-CONFIGURATION="fullstack"
-TARGET="named-release/cypress"
+CONFIGURATION="none"
+TARGET="none"
 INTERACTIVE=true
 OPENEDX_ROOT="/edx"
 
-read -d '' HELP_TEXT <<- EOM
-Attempts to migrate your Open edX installation to a different release.
+show_help () {
+  cat <<- EOM
+
+Migrates your Open edX installation to a different release.
 
 -c CONFIGURATION
-    Use the given configuration. Either \"devstack\" or \"fullstack\".
-    Defaults to \"$CONFIGURATION\"
+    Use the given configuration. Either \"devstack\" or \"fullstack\". You
+    must specify this.
 -t TARGET
-    Migrate to the given git ref. Defaults to \"$TARGET\"
+    Migrate to the given git ref. You must specify this.  Named releases are
+    called \"named-release/cypress\", \"named-release/dogwood.rc2\", and so on.
 -y
     Run in non-interactive mode (reply \"yes\" to all questions)
 -r OPENEDX_ROOT
@@ -21,13 +27,15 @@ Attempts to migrate your Open edX installation to a different release.
     Defaults to \"$OPENEDX_ROOT\"
 -h
     Show this help and exit.
+
 EOM
+}
 
 # override defaults with options
 while getopts "hc:t:y" opt; do
   case "$opt" in
     h)
-      echo  "$HELP_TEXT"
+      show_help
       exit 0
       ;;
     c)
@@ -45,6 +53,45 @@ while getopts "hc:t:y" opt; do
   esac
 done
 
+# Helper to ask to proceed.
+confirm_proceed () {
+  echo "Do you wish to proceed?"
+  read input
+  if [[ "$input" != "yes" && "$input" != "y" ]]; then
+    echo "Quitting"
+    exit 1
+  fi
+}
+
+# Check we are in the right place, and have the info we need.
+
+if [[ "`whoami`" != "vagrant" ]]; then
+  echo "Run this from the vagrant account in your Open edX machine."
+  exit 1
+fi
+
+if [[ ! -d /edx/app/edxapp ]]; then
+  echo "Run this from the vagrant account in your Open edX machine."
+  exit 1
+fi
+
+if [[ $TARGET == none ]]; then
+  cat <<"EOM"
+You must specify a target. This should be the next named release after the one
+you are currently running.  This script can only move forward one release at
+a time.
+EOM
+  show_help
+  exit 1
+fi
+
+if [[ $CONFIGURATION == none ]]; then
+  echo "You must specify a configuration, either fullstack or devstack."
+  exit 1
+fi
+
+# Birch details
+
 if [[ $TARGET == *birch* && $INTERACTIVE == true ]] ; then
   cat <<"EOM"
           WARNING WARNING WARNING WARNING WARNING
@@ -68,15 +115,12 @@ or you have decided to risk the automatic upgrade process, type "yes"
 followed by enter to continue. Otherwise, press ctrl-c to quit. You can
 also run this script with the -y flag to skip this check.
 
-Do you wish to proceed?
 EOM
-  read input
-  if [ "$input" != "yes" -a "$input" != "y" ]; then
-    echo "Quitting"
-    exit 1
-  fi
+  confirm_proceed
 fi
 
+# Cypress details
+
 if [[ $TARGET == *cypress* && $INTERACTIVE == true ]] ; then
   cat <<"EOM"
           WARNING WARNING WARNING WARNING WARNING
@@ -85,16 +129,12 @@ some problems in this migration. If so, check this webpage for solutions:
 
 https://openedx.atlassian.net/wiki/display/OpenOPS/Potential+Problems+Migrating+from+Birch+to+Cypress
 
-Do you wish to proceed?
 EOM
-  read input
-  if [ "$input" != "yes" -a "$input" != "y" ]; then
-    echo "Quitting"
-    exit 1
-  fi
+  confirm_proceed
 fi
 
 if [[ $TARGET == *cypress* ]] ; then
+  # Needed if transitioning to Cypress.
   echo "Killing all celery worker processes."
   sudo ${OPENEDX_ROOT}/bin/supervisorctl stop edxapp_worker:* &
   sleep 3
@@ -109,34 +149,96 @@ if [[ $TARGET == *cypress* ]] ; then
   sudo -u forum git -C ${OPENEDX_ROOT}/app/forum/.rbenv reset --hard
 fi
 
-if [ -f /edx/app/edx_ansible/server-vars.yml ]; then
+if [[ -f /edx/app/edx_ansible/server-vars.yml ]]; then
   SERVER_VARS="--extra-vars=\"@${OPENEDX_ROOT}/app/edx_ansible/server-vars.yml\""
 fi
 
+make_config_venv () {
+  virtualenv venv
+  source venv/bin/activate
+  pip install -r configuration/pre-requirements.txt
+  pip install -r configuration/requirements.txt
+}
+
 TEMPDIR=`mktemp -d`
+echo "Working in $TEMPDIR"
 chmod 777 $TEMPDIR
 cd $TEMPDIR
-git clone https://github.com/edx/configuration.git --depth=1 --single-branch --branch=$TARGET
-virtualenv venv
-source venv/bin/activate
-pip install -r configuration/requirements.txt
-echo "edx_platform_version: $TARGET" >> vars.yml
+# Set the CONFIGURATION_TARGET environment variable to use a different branch
+# in the configuration repo, defaults to $TARGET.
+git clone https://github.com/edx/configuration.git \
+  --depth=1 --single-branch --branch=${CONFIGURATION_TARGET-$TARGET}
+make_config_venv
+
+# Dogwood details
+
+if [[ $TARGET == *dogwood* ]] ; then
+  # We are upgrading Python from 2.7.3 to 2.7.10, so remake the venvs.
+  sudo rm -rf /edx/app/*/v*envs/*
+
+  echo "Upgrading to the end of Django 1.4"
+  cd configuration/playbooks/vagrant
+  sudo ansible-playbook \
+    --inventory-file=localhost, \
+    --connection=local \
+    $SERVER_VARS \
+    --extra-vars="edx_platform_version=release-2015-11-09" \
+    --extra-vars="xqueue_version=named-release/cypress" \
+    --extra-vars="migrate_db=yes" \
+    --skip-tags="edxapp-sandbox" \
+    vagrant-$CONFIGURATION-delta.yml
+  cd ../../..
+
+  # Remake our own venv because of the Python 2.7.10 upgrade.
+  rm -rf venv
+  make_config_venv
+
+  # Need to get rid of South from edx-platform, or things won't work.
+  sudo -u edxapp /edx/app/edxapp/venvs/edxapp/bin/pip uninstall -y South
+
+  echo "Upgrading to the beginning of Django 1.8"
+  cd configuration/playbooks/vagrant
+  sudo ansible-playbook \
+    --inventory-file=localhost, \
+    --connection=local \
+    $SERVER_VARS \
+    --extra-vars="edx_platform_version=dogwood-first-18" \
+    --extra-vars="xqueue_version=dogwood-first-18" \
+    --extra-vars="migrate_db=no" \
+    --skip-tags="edxapp-sandbox" \
+    vagrant-$CONFIGURATION-delta.yml
+  cd ../../..
+
+  echo "Running the Django 1.8 faked migrations"
+  for item in lms cms; do
+    sudo -u edxapp \
+      /edx/app/edxapp/venvs/edxapp/bin/python \
+      /edx/app/edxapp/edx-platform/manage.py $item migrate \
+      --settings=aws --noinput --fake-initial
+  done
+
+  if [[ $CONFIGURATION == fullstack ]] ; then
+    sudo -u xqueue \
+    SERVICE_VARIANT=xqueue \
+    /edx/app/xqueue/venvs/xqueue/bin/python \
+    /edx/app/xqueue/xqueue/manage.py migrate \
+    --settings=xqueue.aws_settings --noinput --fake-initial
+  fi
+fi
+
+cd configuration/playbooks
+echo "edx_platform_version: $TARGET" > vars.yml
 echo "ora2_version: $TARGET" >> vars.yml
 echo "certs_version: $TARGET" >> vars.yml
 echo "forum_version: $TARGET" >> vars.yml
 echo "xqueue_version: $TARGET" >> vars.yml
-cd configuration/playbooks
 sudo ansible-playbook \
     --inventory-file=localhost, \
     --connection=local \
-    --extra-vars=\"@../../vars.yml\" \
+    --extra-vars="@vars.yml" \
     $SERVER_VARS \
     vagrant-$CONFIGURATION.yml
-# if this failed, bail out early
-exitcode=$?
-if [ $exitcode != 0 ]; then
-  exit $exitcode;
-fi
+
 cd /
 sudo rm -rf $TEMPDIR
 echo "Migration complete. Please reboot your machine."

vagrant/release/devstack/Vagrantfile

@@ -66,6 +66,9 @@ openedx_releases = {
   "named-release/dogwood.rc2" => {
     :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
   },
+  "named-release/dogwood.rc3" => {
+    :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
+  },
   "named-release/cypress" => {
     :name => "cypress-devstack", :file => "cypress-devstack.box",
   },

vagrant/release/fullstack/Vagrantfile

@@ -15,6 +15,9 @@ openedx_releases = {
   "named-release/dogwood.rc2" => {
     :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
   },
+  "named-release/dogwood.rc3" => {
+    :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
+  },
   "named-release/cypress" => {
     :name => "cypress-fullstack", :file => "cypress-fullstack.box",
   },