Merge pull request #3354 from edx/ned/eucalyptus-changes

Merge all Eucalyptus changes to master.

playbooks/edx_sandbox.yml

@@ -48,7 +48,7 @@
       when: SANDBOX_ENABLE_ECOMMERCE
     - analytics_api
     - insights
-    - edx_notes_api
+    # not ready yet: - edx_notes_api
     - demo
     - oauth_client_setup
     - oraclejdk

playbooks/vagrant-fullstack.yml

@@ -13,6 +13,7 @@
     certs_version: '{{ OPENEDX_RELEASE | default("master") }}'
     forum_version: '{{ OPENEDX_RELEASE | default("master") }}'
     xqueue_version: '{{ OPENEDX_RELEASE | default("master") }}'
+    demo_version: '{{ OPENEDX_RELEASE | default("master") }}'
   roles:
     - common
     - vhost

util/install/install_stack.sh

 #!/usr/bin/env bash
 
-# Stop if any command fails
+# Setting OPENEDX_DEBUG makes this more verbose.
+if [[ $OPENEDX_DEBUG ]]; then
+    set -x
+fi
+
+# Stop if any command fails.
 set -e
 
 function usage
 {
     cat << EOM
 
-    --- install_stack.sh ---
+    Usage: $ bash ${0##*/} [-b mount_base] [-v] [-h] STACK [RELEASE]
+
+    Installs the Open edX devstack or fullstack. If you encounter any trouble
+    or have questions, head over to https://open.edx.org/getting-help.
 
-    Usage: $ bash install_stack.sh stack release [-b vagrant_mount_base] [-v] [-h]
+    This script captures a log of all output produced during runtime, and saves
+    it in a .log file within the current directory. If you encounter an error
+    during installation, this is an invaluable tool for edX developers to help
+    discover what went wrong, so please share it if you reach out for support!
 
-    Installs the Open edX devstack or fullstack. If you encounter any trouble or have 
-    questions regarding installation of devstack/fullstack, head over to 
-    https://open.edx.org/getting-help.
+    NOTE: This script assumes you have never installed devstack before.
+    Installing multiple versions of devstack can often cause conflicts that
+    this script is not prepared to handle.
 
-    This script captures a log of all output produced during runtime, and saves it in a .log
-    file within the current directory. If you encounter an error during installation, this is
-    an invaluable tool for edX developers to help discover what went wrong, so please share it
-    if you reach out for support!
+    STACK
+        Either 'fullstack' or 'devstack'. Fullstack mimics a production
+        environment, whereas devstack is useful if you plan on modifying the
+        Open edX code. You must specify this.
 
-    NOTE: This script assumes you have never installed devstack before. Installing multiple 
-    versions of devstack can often cause conflicts that this script is not prepared to handle.
-    
+        If you choose fullstack, 'release' should be the latest Open edX
+        release.
 
-    stack
-        Either 'fullstack' or 'devstack' (no quotes). Full stack mimics a production 
-        environment, whereas devstack is useful if you plan on modifying the Open edX 
-        code. You must specify this. If you choose fullstack, 'release' should be the
-        latest open-release. If you choose devstack, 'release' should be the latest
-        open-release or master.
+        If you choose devstack, 'release' should be the latest Open edX
+        release or master.
 
-    release
-        The release of Open edX you wish to run. Install the given git ref 'release'.
-        You must specify this. Named releases are called "open-release/eucalyptus",
-        "open-release/eucalyptus.2", and so on. We recommend the latest stable open 
-        release for general members of the open source community. Named releases can
-        be found at: https://openedx.atlassian.net/wiki/display/DOC/Open+edX+Releases.
-        If you plan on modifying the code, we recommend the "master" branch. 
+    RELEASE
+        The release of Open edX to install.  Defaults to \$OPENEDX_RELEASE.
+        Open edX releases are called "open-release/eucalyptus.1",
+        "open-release/eucalyptus.2", and so on.
 
-    -b vagrant_mount_base
-        Customize the location of the source code that gets cloned during the 
-        devstack provisioning. The default is the current directory. This option is
-        not valid if installing fullstack.
+        We recommend the latest stable open release for general members of the
+        open source community. Details on available open releases can be found
+        at: https://openedx.atlassian.net/wiki/display/DOC/Open+edX+Releases.
 
-    -v 
+        If you plan on modifying the code, we recommend the "master" branch.
+
+    -b mount_base
+        Customize the location of the source code that gets cloned during the
+        devstack provisioning. The default is the current directory. This
+        option is not valid if installing fullstack.
+
+    -v
         Verbose output from ansible playbooks.
 
     -h
         Show this help and exit.
 
-    ---------------------------
-
 EOM
 }
 
@@ -68,16 +75,6 @@ release=""
 # Vagrant source code provision location
 vagrant_mount_location=""
 
-if [[ $# -lt 2 || ${1:0:1} == '-' || ${2:0:1} == '-' ]]; then
-  usage
-  exit 1
-fi
-
-stack=$1
-shift
-release=$1
-shift
-
 while getopts "b:vh" opt; do
     case "$opt" in
         b)
@@ -102,10 +99,38 @@ while getopts "b:vh" opt; do
     esac
 done
 
+shift "$((OPTIND-1))" # Shift off the options we've already parsed
+
+# STACK is a required positional argument.
+if [[ ! $1 ]]; then
+    echo "STACK is required"
+    usage
+    exit 1
+fi
+stack=$1
+shift
+
+# RELEASE is an optional positional argument, defaulting to OPENEDX_RELEASE.
+if [[ $1 ]]; then
+    release=$1
+    shift
+else
+    release=$OPENEDX_RELEASE
+fi
+
+# If there are positional arguments left, something is wrong.
+if [[ $1 ]]; then
+    echo "Don't understand extra arguments: $*"
+    usage
+    exit 1
+fi
+
 exec > >(tee install-$(date +%Y%m%d-%H%M%S).log) 2>&1
 echo "Capturing output to install-$(date +%Y%m%d-%H%M%S).log."
+echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
 
 export OPENEDX_RELEASE=$release
+echo "Installing release '$OPENEDX_RELEASE'"
 
 # Check if mount location was changed
 if [[ $vagrant_mount_location != "" ]]; then
@@ -152,7 +177,7 @@ fi
 
 vagrant up --provider virtualbox
 
-# Check if preview mode was chosen
+# Set preview host.
 if grep -q '192.168.33.10  preview.localhost' /etc/hosts; then
     echo "Studio preview already enabled, skipping..."
 else
@@ -160,5 +185,6 @@ else
     sudo bash -c "echo '192.168.33.10  preview.localhost' >> /etc/hosts"
 fi
 
+echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
 echo -e "${SUCCESS}Finished installing! You may now 'cd $stack' and login using 'vagrant ssh'"
 echo -e "Refer to the edX wiki ($wiki_link) for more information on using $stack.${NC}"

util/install/sandbox.sh

@@ -49,6 +49,10 @@ if [ -n "$OPENEDX_RELEASE" ]; then
     -e forum_version=$OPENEDX_RELEASE \
     -e xqueue_version=$OPENEDX_RELEASE \
     -e configuration_version=$OPENEDX_RELEASE \
+    -e demo_version=$OPENEDX_RELEASE \
+    -e NOTIFIER_VERSION=$OPENEDX_RELEASE \
+    -e INSIGHTS_VERSION=$OPENEDX_RELEASE \
+    -e ANALYTICS_API_VERSION=$OPENEDX_RELEASE \
   $EXTRA_VARS"
   CONFIG_VER=$OPENEDX_RELEASE
 else

util/jenkins/ansible-provision.sh

@@ -171,6 +171,7 @@ xqueue_version: $xqueue_version
 xserver_version: $xserver_version
 certs_version: $certs_version
 configuration_version: $configuration_version
+demo_version: $demo_version
 
 edx_ansible_source_repo: ${configuration_source_repo}
 edx_platform_repo: ${edx_platform_repo}

util/vagrant/upgrade.sh

 #!/usr/bin/env bash
 
-# Stop if any command fails
+# Setting OPENEDX_DEBUG makes this more verbose.
+if [[ $OPENEDX_DEBUG ]]; then
+    set -x
+fi
+
+# Stop if any command fails.
 set -e
 
 # Logging: write all the output to a timestamped log file.
@@ -9,7 +14,7 @@ exec > >(sudo tee /var/log/edx/upgrade-$(date +%Y%m%d-%H%M%S).log) 2>&1
 
 # defaults
 CONFIGURATION="none"
-TARGET="none"
+TARGET=${OPENEDX_RELEASE-none}
 INTERACTIVE=true
 OPENEDX_ROOT="/edx"
 
@@ -21,6 +26,17 @@ exit_cleanly () {
   exit $@
 }
 
+# check_pip succeeds if its first argument is found in the output of pip freeze.
+PIP_EDXAPP="sudo -u edxapp -H $OPENEDX_ROOT/bin/pip.edxapp --disable-pip-version-check"
+check_pip () {
+  how_many=$($PIP_EDXAPP list 2>&- | grep -c "^$1 ")
+  if (( $how_many > 0 )); then
+    return 0
+  else
+    return 1
+  fi
+}
+
 show_help () {
   cat << EOM
 
@@ -31,8 +47,9 @@ Upgrades your Open edX installation to a newer release.
     must specify this.
 
 -t TARGET
-    Upgrade to the given git ref. You must specify this.  Named releases are
-    called "named-release/cypress", "named-release/dogwood.rc2", and so on.
+    Upgrade to the given git ref. Open edX releases are called
+    "open-release/eucalyptus.1", "open-release/eucalyptus.latest", and so on.
+    Defaults to \$OPENEDX_RELEASE if it is defined.
 
 -y
     Run in non-interactive mode (reply "yes" to all questions)
@@ -87,9 +104,9 @@ fi
 
 if [[ $TARGET == none ]]; then
   cat <<"EOM"
-You must specify a target. This should be the next named release after the one
-you are currently running.  This script can only move forward one release at
-a time.
+You must specify a target. This should be the next Open edX release after the
+one you are currently running.  This script can only move forward one release
+at a time.
 EOM
   show_help
   exit_cleanly 1
@@ -168,6 +185,12 @@ if [[ -f ${OPENEDX_ROOT}/app/edx_ansible/server-vars.yml ]]; then
   SERVER_VARS="--extra-vars=\"@${OPENEDX_ROOT}/app/edx_ansible/server-vars.yml\""
 fi
 
+# When tee'ing to a log, ansible (like many programs) buffers its output. This
+# makes it hard to tell what is actually happening during the upgrade.
+# "stdbuf -oL" will run ansible with line-buffered stdout, which makes the
+# messages scroll in the way people expect.
+ANSIBLE_PLAYBOOK="sudo stdbuf -oL ansible-playbook --inventory-file=localhost, --connection=local "
+
 make_config_venv () {
   virtualenv venv
   source venv/bin/activate
@@ -209,9 +232,7 @@ EOF
 
   echo "Upgrading to the end of Django 1.4"
   cd configuration/playbooks/vagrant
-  sudo ansible-playbook \
-    --inventory-file=localhost, \
-    --connection=local \
+  $ANSIBLE_PLAYBOOK \
     $SERVER_VARS \
     --extra-vars="edx_platform_version=release-2015-11-09" \
     --extra-vars="xqueue_version=named-release/cypress" \
@@ -225,13 +246,11 @@ EOF
   make_config_venv
 
   # Need to get rid of South from edx-platform, or things won't work.
-  sudo -u edxapp ${OPENEDX_ROOT}/bin/pip.edxapp uninstall -y South
+  $PIP_EDXAPP uninstall -y South
 
   echo "Upgrading to the beginning of Django 1.8"
   cd configuration/playbooks/vagrant
-  sudo ansible-playbook \
-    --inventory-file=localhost, \
-    --connection=local \
+  $ANSIBLE_PLAYBOOK \
     $SERVER_VARS \
     --extra-vars="edx_platform_version=dogwood-first-18" \
     --extra-vars="xqueue_version=dogwood-first-18" \
@@ -258,8 +277,25 @@ fi
 # Eucalyptus details
 
 if [[ $TARGET == *eucalyptus* ]] ; then
-  echo "Uninstall edx-oauth2-provider"
-  sudo -u edxapp ${OPENEDX_ROOT}/bin/pip.edxapp uninstall --disable-pip-version-check -y django-oauth2-provider edx-oauth2-provider
+  if check_pip edx-oauth2-provider ; then
+    echo "Uninstall edx-oauth2-provider"
+    $PIP_EDXAPP uninstall -y edx-oauth2-provider
+  fi
+  if check_pip django-oauth2-provider ; then
+    echo "Uninstall django-oauth2-provider"
+    $PIP_EDXAPP uninstall -y django-oauth2-provider
+  fi
+
+  # edx-milestones changed how it was installed, so it is possible to have it
+  # installed twice.  Try to uninstall it twice.
+  if check_pip edx-milestones ; then
+    echo "Uninstall edx-milestones"
+    $PIP_EDXAPP uninstall -y edx-milestones
+  fi
+  if check_pip edx-milestones ; then
+    echo "Uninstall edx-milestones again"
+    $PIP_EDXAPP uninstall -y edx-milestones
+  fi
 
   if [[ $CONFIGURATION == devstack ]] ; then
     echo "Remove old Firefox"
@@ -268,9 +304,7 @@ if [[ $TARGET == *eucalyptus* ]] ; then
 
   echo "Upgrade the code"
   cd configuration/playbooks/vagrant
-  sudo ansible-playbook \
-    --inventory-file=localhost, \
-    --connection=local \
+  $ANSIBLE_PLAYBOOK \
     $SERVER_VARS \
     --extra-vars="edx_platform_version=$TARGET" \
     --extra-vars="xqueue_version=$TARGET" \
@@ -292,13 +326,15 @@ fi
 echo "Updating to final version of code"
 cd configuration/playbooks
 echo "edx_platform_version: $TARGET" > vars.yml
-echo "ora2_version: $TARGET" >> vars.yml
 echo "certs_version: $TARGET" >> vars.yml
 echo "forum_version: $TARGET" >> vars.yml
 echo "xqueue_version: $TARGET" >> vars.yml
-sudo ansible-playbook \
-    --inventory-file=localhost, \
-    --connection=local \
+echo "demo_version: $TARGET" >> vars.yml
+echo "NOTIFIER_VERSION: $TARGET" >> vars.yml
+echo "ECOMMERCE_VERSION: $TARGET" >> vars.yml
+echo "ECOMMERCE_WORKER_VERSION: $TARGET" >> vars.yml
+echo "PROGRAMS_VERSION: $TARGET" >> vars.yml
+$ANSIBLE_PLAYBOOK \
     --extra-vars="@vars.yml" \
     $SERVER_VARS \
     vagrant-$CONFIGURATION.yml

vagrant/base/analyticstack/Vagrantfile

@@ -76,6 +76,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         certs_version: ENV['OPENEDX_RELEASE'],
         forum_version: ENV['OPENEDX_RELEASE'],
         xqueue_version: ENV['OPENEDX_RELEASE'],
+        demo_version: ENV['OPENEDX_RELEASE'],
         ANALYTICS_API_VERSION: ENV['OPENEDX_RELEASE'],
         INSIGHTS_VERSION: ENV['OPENEDX_RELEASE'],
       }

vagrant/base/devstack/Vagrantfile

@@ -77,6 +77,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         certs_version: ENV['OPENEDX_RELEASE'],
         forum_version: ENV['OPENEDX_RELEASE'],
         xqueue_version: ENV['OPENEDX_RELEASE'],
+        demo_version: ENV['OPENEDX_RELEASE'],
+        NOTIFIER_VERSION: ENV['OPENEDX_RELEASE'],
+        ECOMMERCE_VERSION: ENV['OPENEDX_RELEASE'],
+        ECOMMERCE_WORKER_VERSION: ENV['OPENEDX_RELEASE'],
+        PROGRAMS_VERSION: ENV['OPENEDX_RELEASE'],
       }
     end
     if ENV['CONFIGURATION_VERSION']
@@ -88,6 +93,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     if ENV['ECOMMERCE_VERSION']
       ansible.extra_vars['ECOMMERCE_VERSION'] = ENV['ECOMMERCE_VERSION']
     end
+    if ENV['ECOMMERCE_WORKER_VERSION']
+      ansible.extra_vars['ECOMMERCE_WORKER_VERSION'] = ENV['ECOMMERCE_WORKER_VERSION']
+    end
     if ENV['PROGRAMS_VERSION']
       ansible.extra_vars['PROGRAMS_VERSION'] = ENV['PROGRAMS_VERSION']
     end

vagrant/base/fullstack/Vagrantfile

@@ -40,6 +40,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         certs_version: ENV['OPENEDX_RELEASE'],
         forum_version: ENV['OPENEDX_RELEASE'],
         xqueue_version: ENV['OPENEDX_RELEASE'],
+        demo_version: ENV['OPENEDX_RELEASE'],
       }
     end
   end

vagrant/release/analyticstack/Vagrantfile

@@ -24,6 +24,7 @@ if [ -n "$OPENEDX_RELEASE" ]; then
     -e certs_version=$OPENEDX_RELEASE \
     -e forum_version=$OPENEDX_RELEASE \
     -e xqueue_version=$OPENEDX_RELEASE \
+    -e demo_version=$OPENEDX_RELEASE \
     -e ANALYTICS_API_VERSION=$OPENEDX_RELEASE \
     -e INSIGHTS_VERSION=$OPENEDX_RELEASE \
   "

vagrant/release/devstack/Vagrantfile

@@ -24,6 +24,11 @@ if [ -n "$OPENEDX_RELEASE" ]; then
     -e certs_version=$OPENEDX_RELEASE \
     -e forum_version=$OPENEDX_RELEASE \
     -e xqueue_version=$OPENEDX_RELEASE \
+    -e demo_version=$OPENEDX_RELEASE \
+    -e NOTIFIER_VERSION=$OPENEDX_RELEASE \
+    -e ECOMMERCE_VERSION=$OPENEDX_RELEASE \
+    -e ECOMMERCE_WORKER_VERSION=$OPENEDX_RELEASE \
+    -e PROGRAMS_VERSION=$OPENEDX_RELEASE \
   "
   CONFIG_VER=$OPENEDX_RELEASE
 else
@@ -55,6 +60,15 @@ end
 # to a name and a file path, which are used for retrieving
 # a Vagrant box from the internet.
 openedx_releases = {
+  "open-release/eucalyptus.master" => {
+    :name => "eucalyptus-devstack-2016-08-19", :file => "eucalyptus-devstack-2016-08-19.box",
+  },
+  "open-release/eucalyptus.1rc2" => {
+    :name => "eucalyptus-devstack-2016-08-19", :file => "eucalyptus-devstack-2016-08-19.box",
+  },
+  "open-release/eucalyptus.1" => {
+    :name => "eucalyptus-devstack-2016-08-19", :file => "eucalyptus-devstack-2016-08-19.box",
+  },
   # Note: the devstack and fullstack boxes differ, because devstack had an issue
   # that needed fixing, but it didn't affect fullstack.
   "named-release/dogwood.rc" => {
@@ -82,7 +96,7 @@ openedx_releases = {
   # },
 }
 openedx_releases.default = {
-  :name => "devstack-periodic-2016-05-16", :file => "devstack-periodic-2016-05-16.box",
+  :name => "eucalyptus-devstack-2016-08-19", :file => "eucalyptus-devstack-2016-08-19.box",
 }
 rel = ENV['OPENEDX_RELEASE']
 

vagrant/release/fullstack/Vagrantfile

@@ -9,11 +9,17 @@ CPU_COUNT = 2
 # to a name and a file path, which are used for retrieving
 # a Vagrant box from the internet.
 openedx_releases = {
-  "open-release/eucalyptus/master" => {
-    :name => "eucalyptus-fullstack-1rc1", :file => "eucalyptus-fullstack-2016-07-22.box",
+  "open-release/eucalyptus.master" => {
+    :name => "eucalyptus-fullstack-2016-08-25", :file => "eucalyptus-fullstack-2016-08-25.box",
   },
   "open-release/eucalyptus/1rc1" => {
-    :name => "eucalyptus-fullstack-1rc1", :file => "eucalyptus-fullstack-2016-07-22.box",
+    :name => "eucalyptus-fullstack-1rc1", :file => "eucalyptus-fullstack-2016-08-19.box",
+  },
+  "open-release/eucalyptus.1rc2" => {
+    :name => "eucalyptus-fullstack-2016-08-19", :file => "eucalyptus-fullstack-2016-08-19.box",
+  },
+  "open-release/eucalyptus.1" => {
+    :name => "eucalyptus-fullstack-2016-08-25", :file => "eucalyptus-fullstack-2016-08-25.box",
   },
   # Note: the devstack and fullstack boxes differ, because devstack had an issue
   # that needed fixing, but it didn't affect fullstack.
@@ -42,7 +48,7 @@ openedx_releases = {
   # },
 }
 openedx_releases.default = {
-  :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
+  :name => "eucalyptus-fullstack-2016-08-25", :file => "eucalyptus-fullstack-2016-08-25.box",
 }
 rel = ENV['OPENEDX_RELEASE']