Merge remote-tracking branch 'upstream/master' into proversity/NVN-update-script-extra-vars

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,4 +8,4 @@ Make sure that the following steps are done before merging:
     - [ ] Update the appropriate internal repo (be sure to update for all our environments)
     - [ ] If you are updating a secure value rather than an internal one, file a DEVOPS ticket with details.
     - [ ] Add an entry to the CHANGELOG.
-  - [ ] Have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)?
+  - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)?  Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes).

.travis.yml

@@ -18,7 +18,7 @@ addons:
 
 before_install:
   - sudo apt-get -y update
-  - sudo apt-get -y install -o Dpkg::Options::="--force-confold" docker-engine
+  - sudo apt-get -y install -o Dpkg::Options::="--force-confold" docker-ce
 
 install:
   - "pip install --allow-all-external -r requirements.txt"

AUTHORS

@@ -57,3 +57,4 @@ Bill DeRusha <bill@edx.org>
 Jillian Vogel <jill@opencraft.com>
 Zubair Afzal <zubair.afzal@arbisoft.com>
 Kyle McCormick <kylemccor@gmail.com>
+Muzaffar Yousaf <muzaffar@edx.org>

CHANGELOG.md

+- Role: discovery
+  - Added `DISCOVERY_REPOS` to allow configuring discovery repository details.
+
+- Role: edx_django_service
+  - Made the keys `edx_django_service_git_protocol`, `edx_django_service_git_domain`, and `edx_django_service_git_path` of `edx_django_service_repos` all individually configurable.
+
+- Role: discovery
+  - Updated LANGUAGE_CODE to generic english. Added configuration for multilingual language package django-parler.
+
+- Role: edxapp
+  - Added `EDXAPP_EXTRA_MIDDLEWARE_CLASSES` for configuring additional middleware logic.
+
+- Role: discovery
+  - Added `OPENEXCHANGERATES_API_KEY` for retrieving currency exchange rates.
+
+- Role: edxapp
+  - Added `EDXAPP_SCORM_PKG_STORAGE_DIR`, with default value as it was in the server template.
+  - Added `EDXAPP_SCORM_PLAYER_LOCAL_STORAGE_ROOT`, with default value as it was in the server template.
+
+- Role: edxapp
+  - Added `EDXAPP_ENTERPRISE_TAGLINE` for customized header taglines for different enterprises.
+  - Added `EDXAPP_PLATFORM_DESCRIPTION` used to describe the specific Open edX platform.
+
+- Role: edxapp
+  - Added `ENTERPRISE_SUPPORT_URL` variable used by the LMS.
+
+- Role: edxapp
+  - Added OAUTH_DELETE_EXPIRED to enable automatic deletion of edx-django-oauth2-provider grants, access tokens, and refresh tokens as they are consumed. This will not do a bulk delete of existing rows.
+
+- Role: mongo_3_2
+  - Added role for mongo 3.2, not yet in use.
+  - Removed MONGO_CLUSTERED variable. In this role mongo replication is always configured, even if there is only one node.
+
+- Role: edxapp
+  - Added creation of enterprise_worker user to provisioning. This user is used by the edx-enterprise package when making API requests to Open edX IDAs.
+
+- Role: neo4j
+  - Increase heap and page caches sizes for neo4j
+
+- Role: neo4j
+  - Updated neo4j to 3.2.2
+  - Removed authentication requirement for neo4j
+
+- Role: forum
+  - Added `FORUM_REBUILD_INDEX` to rebuild the ElasticSearch index from the database, when enabled.  Default: `False`.
+
+- Role: nginx
+  - Added `NGINX_EDXAPP_CMS_APP_EXTRA`, which makes it possible to add custom settings to the site configuration for Studio.
+  - Added `NGINX_EDXAPP_LMS_APP_EXTRA`, which makes it possible to add custom settings to the site configuration for the LMS.
+
+- Role: edxapp
+  - Let `confirm_email` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`.
+  - Let `terms_of_service` in `EDXAPP_REGISTRATION_EXTRA_FIELDS` default to `"hidden"`.
+
+- Role: ecommerce
+  - Added ECOMMERCE_LANGUAGE_COOKIE_NAME which is the name of the cookie the ecommerce django app looks at for determining the language preference.
+
+- Role: neo4j
+  - Enabled splunk forwarding for neo4j logs.
+  - Increased maximum amount of open files to 40000, as suggested by neo4j.
+  - Updated the java build that neo4j uses to run.
+
+- Role: edxapp
+  - Set the default value for EDXAPP_POLICY_CHANGE_GRADES_ROUTING_KEY to
+ 'edx.lms.core.default'.
+
+- Role: edxapp
+  - Set the default value for EDXAPP_BULK_EMAIL_ROUTING_KEY_SMALL_JOBS to
+ 'edx.lms.core.low'.
+
+- Role: jenkins_master
+  - Update pinned use of JDK7 in Jenkins installs to default JDK version from role `oraclejdk`.
+
+- Role: notifier
+  - Added `NOTIFIER_DATABASE_ENGINE`, `NOTIFIER_DATABASE_NAME`, `NOTIFIER_DATABASE_USER`, `NOTIFIER_DATABASE_PASSWORD`, `NOTIFIER_DATABASE_HOST`, and `NOTIFIER_DATABASE_PORT` to be able to configure the `notifier` service to use a database engine other than sqlite. Defaults to local sqlite.
+  - Deprecated: `NOTIFIER_DB_DIR`: Please use `NOTIFIER_DATABASE_NAME` instead.
+
 - Role: elasticsearch
   - Replaced `elasticsearch_apt_key` and `elastic_search_apt_keyserver` with `elasticsearch_apt_key_url`
   - Updated elasticsearch version to 1.5.0
@@ -20,6 +97,19 @@
   - Added the EDXAPP_ACTIVATION_EMAIL_SUPPORT_LINK URL with default value `''`.
   - Added the EDXAPP_PASSWORD_RESET_SUPPORT_LINK URL with default value `''`.
 
+- Role: nginx
+  - Modified `server-template.j2` to be more accessible and configurable.
+  - The template should contain the `lang` attribute in the HTML tag.
+  - If the image loaded has some meaning, as a logo, it should have the `alt` attribute.
+  - After the header 1 (h1) there is no relevant text content, so next it can not be
+    another header (h2). It was changed to be a paragraph with the header 2 CSS style.
+  - Added `NGINX_SERVER_ERROR_IMG_ALT` with default value as it was in the server template
+  - Added `NGINX_SERVER_ERROR_LANG` with default value `en`
+  - Added `NGINX_SERVER_ERROR_STYLE_H1` with default value as it was in the server template
+  - Added `NGINX_SERVER_ERROR_STYLE_P_H2` with default value as it was in the server template
+  - Added `NGINX_SERVER_ERROR_STYLE_P` with default value as it was in the server template
+  - Added `NGINX_SERVER_ERROR_STYLE_DIV` with default value as it was in the server template
+
 - Role: edxapp
   - Added the EDXAPP_SHOW_HEADER_LANGUAGE_SELECTOR feature flag with default value [false]
   - Added the EDXAPP_SHOW_FOOTER_LANGUAGE_SELECTOR feature flag with default value [false]
@@ -284,3 +374,52 @@
 
 - Role: insights
   - Removed `SUPPORT_EMAIL` setting from `INSIGHTS_CONFIG`, as it is was replaced by `SUPPORT_URL`.
+
+- Role: insights
+  - Added `INSIGHTS_DOMAIN` to configure the domain Insights is deployed on
+  - Added `INSIGHTS_CLOUDFRONT_DOMAIN` to configure the domain static files can be served from
+  - Added `INSIGHTS_CORS_ORIGIN_WHITELIST_EXTRA` to configure allowing CORS on domains other than the `INSIGHTS_DOMAIN`
+
+- Role: edxapp
+  - Added `EDXAPP_VIDEO_IMAGE_SETTINGS` to configure S3-backed video images.
+
+- Role: edxapp
+  - Added `EDXAPP_BASE_COOKIE_DOMAIN` for sharing cookies across edx domains.
+
+- Role: insights
+  - Removed `bower install` task
+  - Replaced r.js build task with webpack build task
+  - Removed `./manage.py compress` task
+
+- Role: insights
+  - Moved `THEME_SCSS` from `INSIGHTS_CONFIG` to `insights_environment`
+
+- Role: analytics_api
+  - Added a number of `ANALYTICS_API_DEFAULT_*` and `ANALYTICS_API_REPORTS_*` variables to allow more selective specification of database parameters (rather than
+      overriding the whole structure).
+
+- Role: edxapp
+  - Remove EDXAPP_ANALYTICS_API_KEY, EDXAPP_ANALYTICS_SERVER_URL, EDXAPP_ANALYTICS_DATA_TOKEN, EDXAPP_ANALYTICS_DATA_URL since they are old and
+  no longer consumed.
+
+- Role: edxapp
+  - Added `PASSWORD_MIN_LENGTH` for password minimum length validation on reset page.
+  - Added `PASSWORD_MAX_LENGTH` for password maximum length validation on reset page.
+
+- Role: credentials
+  - Replaced `CREDENTIALS_OAUTH_URL_ROOT` with `COMMON_OAUTH_URL_ROOT` from `common_vars`
+  - Replaced `CREDENTIALS_OIDC_LOGOUT_URL` with `COMMON_OAUTH_LOGOUT_URL` from `common_vars`
+  - Replaced `CREDENTIALS_JWT_AUDIENCE` with `COMMON_JWT_AUDIENCE` from `common_vars`
+  - Replaced `CREDENTIALS_JWT_ISSUER` with `COMMON_JWT_ISSUER` from `common_vars`
+  - Replaced `CREDENTIALS_JWT_SECRET_KEY` with `COMMON_JWT_SECRET_KEY` from `common_vars`
+  - Replaced `CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_ISSUER` with `COMMON_JWT_ISSUER` from `common_vars`
+
+- Role: ecommerce
+  - Replaced `ECOMMERCE_OAUTH_URL_ROOT` with `COMMON_OAUTH_URL_ROOT` from `common_vars`
+  - Replaced `ECOMMERCE_OIDC_LOGOUT_URL` with `COMMON_OAUTH_LOGOUT_URL` from `common_vars`
+  - Replaced `ECOMMERCE_JWT_SECRET_KEY` with `COMMON_JWT_SECRET_KEY` from `common_vars`
+  - Replaced `ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_ISSUER` with `COMMON_JWT_ISSUER` from `common_vars`
+
+- Role: edxapp
+  - Added `EDXAPP_VIDEO_TRANSCRIPTS_SETTINGS` to configure S3-backed video transcripts.
+  - Removed unused `EDXAPP_BOOK_URL` setting

CONTRIBUTING.md

+Do not use GitHub issues for Open edX support. The mailing list and Slack channels are explained here: http://open.edx.org/getting-help. If it turns out there's a bug in the configuration scripts, we can open an issue or PR here.

docker/build/automated/Dockerfile

-FROM edxops/precise-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 ADD . /edx/app/edx_ansible/edx_ansible

docker/build/chrome/Dockerfile

+FROM selenium/standalone-chrome-debug:3.4.0-einsteinium
+MAINTAINER edxops
+
+USER root
+
+# Install a password generator
+RUN apt-get update -qqy \
+  && apt-get -qqy install \
+  pwgen \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+USER seluser
+
+CMD export VNC_PASSWORD=$(pwgen -s -1 $(shuf -i 10-20 -n 1)) \
+  && x11vnc -storepasswd $VNC_PASSWORD /home/seluser/.vnc/passwd \
+  && echo "Chrome VNC password: $VNC_PASSWORD" \
+  && /opt/bin/entry_point.sh
+
+EXPOSE 4444 5900

docker/build/devstack/ansible_overrides.yml

@@ -2,8 +2,7 @@ EDXAPP_LMS_BASE: 'edx.devstack.lms:18000'
 EDXAPP_LMS_ROOT_URL: 'http://{{ EDXAPP_LMS_BASE }}'
 EDXAPP_LMS_PUBLIC_ROOT_URL: 'http://localhost:18000'
 
-COMMON_OAUTH_LOGOUT_URL: '{{ EDXAPP_LMS_PUBLIC_ROOT_URL }}/logout'
-COMMON_OAUTH_PUBLIC_URL_ROOT: '{{ EDXAPP_LMS_PUBLIC_ROOT_URL }}/oauth2'
+COMMON_OAUTH_BASE_URL: '{{ EDXAPP_LMS_PUBLIC_ROOT_URL }}'
 COMMON_OAUTH_URL_ROOT: '{{ EDXAPP_LMS_ROOT_URL }}/oauth2'
 COMMON_JWT_AUDIENCE: 'lms-key'
 COMMON_JWT_SECRET_KEY: 'lms-secret'

docker/build/ecommerce/ansible_overrides.yml

@@ -12,4 +12,6 @@ ECOMMERCE_DATABASES:
     HOST: 'db.{{ DOCKER_TLD  }}'
     PORT: '3306'
     ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+    CONN_MAX_AGE: 60
\ No newline at end of file
+
+ECOMMERCE_MEMCACHE: ['edx.devstack.memcached:11211']

docker/build/edxapp/Dockerfile

@@ -29,6 +29,7 @@ RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook edxapp.yml
     --extra-vars=edx_platform_version=${OPENEDX_RELEASE} \
     --extra-vars="@/ansible_overrides.yml" \
     --extra-vars="@/devstack.yml" \
-    --extra-vars="@/devstack/ansible_overrides.yml"
+    --extra-vars="@/devstack/ansible_overrides.yml" \
+    && rm -rf /edx/app/edxapp/edx-platform
 
 EXPOSE 18000 18010

docker/build/firefox/Dockerfile

+FROM selenium/standalone-firefox-debug:3.4.0-einsteinium
+MAINTAINER edxops
+
+USER root
+
+# Install a password generator and the codecs needed to support mp4 video in Firefox
+RUN apt-get update -qqy \
+  && apt-get -qqy install \
+  gstreamer1.0-libav \
+  pwgen \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+USER seluser
+
+CMD export VNC_PASSWORD=$(pwgen -s -1 $(shuf -i 10-20 -n 1)) \
+  && x11vnc -storepasswd $VNC_PASSWORD /home/seluser/.vnc/passwd \
+  && echo "Firefox VNC password: $VNC_PASSWORD" \
+  && /opt/bin/entry_point.sh
+
+EXPOSE 4444 5900

docker/build/forum/ansible_overrides.yml

@@ -9,3 +9,5 @@ FORUM_ELASTICSEARCH_HOST: "es.{{ FLOCK_TLD }}"
 FORUM_USE_TCP: "true"
 FORUM_RACK_ENV: "staging"
 FORUM_SINATRA_ENV: "staging"
+
+devstack: "true"

docker/build/go-agent/Dockerfile

 # Build using: docker build -f Dockerfile.gocd-agent -t gocd-agent .
 # https://hub.docker.com/r/gocd/gocd-agent-deprecated/
-FROM gocd/gocd-agent-deprecated:17.1.0
+FROM gocd/gocd-agent-deprecated:17.7.0
 
 LABEL version="0.02" \
       description="This custom go-agent docker file installs additional requirements for the edx pipeline"

docker/build/harstorage/Dockerfile.out

-FROM edxops/precise-common
+FROM edxops/xenial-common
 MAINTAINER edxops
 
 USER root
 
-# Fix selinux issue with useradd on 12.04
-RUN curl http://salilab.org/~ben/libselinux1_2.1.0-5.1ubuntu1_amd64.deb -o /tmp/libselinux1_2.1.0-5.1ubuntu1_amd64.deb
-RUN dpkg -i /tmp/libselinux1_2.1.0-5.1ubuntu1_amd64.deb
-
 RUN apt-get update
 ADD . /edx/app/edx_ansible/edx_ansible
 COPY docker/build/xqwatcher/ansible_overrides.yml /

docker/build/jenkins_analytics/Dockerfile.noci

-FROM edxops/precise-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 USER root

docker/build/mongo/Dockerfile

+FROM edxops/xenial-common:latest
+MAINTAINER edxops
+
+ADD . /edx/app/edx_ansible/edx_ansible
+COPY docker/build/mongo/ansible_overrides.yml /
+
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook mongo.yml \
+    -i '127.0.0.1,' -c local \
+    -t 'install' \
+    -e@/ansible_overrides.yml
+
+WORKDIR /edx/app
+EXPOSE 27017

docker/build/mongo/ansible_overrides.yml

+{}

docker/build/nginx/Dockerfile

-FROM edxops/precise-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 USER root

docker/build/precise-common/Dockerfile

-FROM ubuntu:precise 
-MAINTAINER edxops 
-
-# Set locale to UTF-8 which is not the default for docker.
-# See the links for details:
-# http://jaredmarkell.com/docker-and-locales/
-# https://github.com/docker-library/python/issues/13
-# https://github.com/docker-library/python/pull/14/files
-ENV LANG C.UTF-8
-
-ENV ANSIBLE_REPO="https://github.com/edx/ansible"
-ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
-ENV CONFIGURATION_VERSION="master"
-
-ADD util/install/ansible-bootstrap.sh /tmp/ansible-bootstrap.sh
-RUN chmod +x /tmp/ansible-bootstrap.sh
-RUN /tmp/ansible-bootstrap.sh

docker/build/tools_jenkins/Dockerfile

-FROM edxops/trusty-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 USER root

docker/build/xenial-common/Dockerfile

@@ -6,7 +6,12 @@ MAINTAINER edxops
 # http://jaredmarkell.com/docker-and-locales/
 # https://github.com/docker-library/python/issues/13
 # https://github.com/docker-library/python/pull/14/files
-ENV LANG C.UTF-8
+RUN apt-get update &&\
+    apt-get install -y locales &&\
+    locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 ENV ANSIBLE_REPO="https://github.com/edx/ansible"
 ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"

docker/build/xqueue/Dockerfile

-FROM edxops/precise-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 USER root

docker/compose/edx/docker-compose.yml

-#
-# Single Docker Compose cluster that will eventually start
-# all edX services in a single flock of coordinated containers
-#
-# This work is currently experimental and a number of services
-# are missing entirely.  Containers that are present will not
-# currently work without manual steps.  We are working on
-# addressing that.
-#
-# When running compose you must pass in two environment variables
-#
-# DOCKER_EDX_ROOT which points to the directory into which you checkout
-# your edX source code.  For example, assuming the following directory
-# structure under /home/me
-#
-# |-- edx-src
-# |   |-- discovery
-# |   |-- cs_comments_service
-# |   |-- edx_discovery
-# |   |-- edx-platform
-# |   |-- xqueue
-
-# you would define DOCKER_EDX_ROOT="/home/me/edx-src"
-#
-# DOCKER_DATA_ROOT is the location on your host machine where Docker
-# guests can access your local filesystem for storing persistent data
-# files, say MongoDB or MySQL data files.
-#
-
-db:
-  container_name: db
-  image: mysql:5.6
-  environment:
-    - MYSQL_ROOT_PASSWORD='password'
-    #- MYSQL_DATABASE=''
-    - MYSQL_USER='migrate'
-    - MYSQL_PASSWORD='password'
-  volumes:
-    - ${DOCKER_DATA_ROOT}/mysql/data:/data
-  ports:
-    - 3306:3306
-
-
-mongo:
-  container_name: mongo
-  image: mongo:3.0
-  volumes:
-    - ${DOCKER_DATA_ROOT}/mongo/data:/data
-  ports:
-    - 27017:27017
-
-
-# Need to build our own for ES 0.9
-es:
-  container_name: es
-  image: edxops/elasticsearch:v1
-  volumes:
-    - ${DOCKER_DATA_ROOT}/elasticsearch/data:/data
-  ports:
-    - 9100:9100
-    - 9200:9200
-    - 9300:9300
-
-
-memcache:
-  container_name: memcache
-  image: memcached:1.4.24
-  volumes:
-    - ${DOCKER_DATA_ROOT}/memcache/data:/data
-  ports:
-    - 11211:11211
-
-
-nginx:
-  container_name: nginx
-  image: edxops/nginx:v1
-  ports:
-   - 80:80
-   - 443:443
-
-
-rabbitmq:
-  container_name: rabbitmq
-  image: rabbitmq:3.5.3
-  volumes:
-    - ${DOCKER_DATA_ROOT}/rabbitmq/data:/data
-  ports:
-    - 5672:5672
-
-
-forum:
-  container_name: forum
-  # Image built from the opencraft fork as it fixes
-  # an auth bug.  Update when the change merges
-  # upstream
-  image: edxops/forums:opencraft-v2
-  volumes:
-    - ${DOCKER_EDX_ROOT}/cs_comments_service:/edx/app/forum/cs_comments_service
-  ports:
-    - 4567:4567
-
-
-xqueue:
-  container_name: xqueue
-  image: edxops/xqueue:v1
-  ports:
-    - 8040:8040
-    - 18040:18040
-  volumes:
-    - ${DOCKER_EDX_ROOT}/xqueue:/edx/app/edxapp/xqueue
-
-
-lms:
-  container_name: lms
-  image: edxops/edxapp:v2
-  ports:
-    - 8000:8000
-    - 18000:18000
-  volumes:
-    - ${DOCKER_EDX_ROOT}/edx-platform:/edx/app/edxapp/edx-platform
-
-
-cms:
-  container_name: cms
-  image: edxops/edxapp:v2
-  ports:
-    - 8010:8010
-    - 18010:18010
-  volumes:
-    - ${DOCKER_EDX_ROOT}/edx-platform:/edx/app/edxapp/edx-platform

docker/plays/mongo.yml

+- name: Deploy MongoDB 3.2
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - common_vars
+    - docker
+    - mongo_3_2

playbooks/continuous_delivery/ansible.cfg

@@ -6,10 +6,10 @@
 
 jinja2_extensions=jinja2.ext.do
 host_key_checking=False
-roles_path=../../../ansible-roles/roles:../../../ansible-private/roles:../../../ansible-roles/
+roles_path=../../../ansible-roles/roles:../../../ansible-private/roles:../../../ansible-roles/:../../playbooks/roles
 library=../library/
 ansible_managed=This file is created and updated by ansible, edit at your peril
 
 [ssh_connection]
 ssh_args=-o ControlMaster=auto -o ControlPersist=60s -o ControlPath="~/.ansible/tmp/ansible-ssh-%h-%p-%r" -o ServerAliveInterval=30
-retries=5
+retries=5
\ No newline at end of file

playbooks/continuous_delivery/cleanup.yml

@@ -47,13 +47,3 @@
     file:
       path: "{{ artifact_path }}"
       state: absent
-
-  - name: Send Hipchat notification cleanup has finished
-    hipchat:
-      api: "{{ hipchat_url }}"
-      token: "{{ hipchat_token }}"
-      room: "{{ hipchat_room }}"
-      msg: "Cleanup for run id: {{ keypair_id }} complete."
-    ignore_errors: yes
-    when: hipchat_token is defined
-

playbooks/continuous_delivery/create_ami.yml

@@ -57,7 +57,7 @@
     register: instance_tags
 
   - name: Create AMI
-    ec2_ami_2_0_0_1:
+    ec2_ami:
       instance_id: "{{ instance_id }}"
       name: "{{ edx_environment }} -- {{ deployment }} -- {{ play }} -- {{ extra_name_identifier }} -- {{ app_version[:7] }}"
       region: "{{ ec2_region }}"
@@ -116,7 +116,7 @@
       api: "{{ hipchat_url }}"
       token: "{{ hipchat_token }}"
       room: "{{ hipchat_room }}"
-      msg: "Finished baking AMI for: {{ play }} \n
+      msg: "Finished baking AMI for: {{ edx_environment }}-{{ deployment }}-{{ play }} \n
             AMI-ID: {{ ami_register.image_id }} \n
            "
     ignore_errors: yes

playbooks/continuous_delivery/launch_instance.yml

@@ -70,13 +70,12 @@
       key_name: "{{ automation_prefix }} {{ unique_key_name.stdout }}"
       instance_type: "{{ ec2_instance_type }}"
       image: "{{ launch_ami_id }}"
-      wait: yes
       group_id: "{{ ec2_security_group_id }}"
       count: 1
       vpc_subnet_id: "{{ ec2_vpc_subnet_id }}"
       assign_public_ip: "{{ ec2_assign_public_ip }}"
       volumes:
-        - device_name: /dev/sdf
+        - device_name: /dev/sda1
           volume_type: 'gp2'
           volume_size: "{{ ebs_volume_size }}"
       wait: yes

playbooks/ec2.py

@@ -118,6 +118,7 @@ from boto import ec2
 from boto import rds
 from boto import route53
 import ConfigParser
+import traceback
 
 try:
     import json
@@ -612,5 +613,11 @@ class Ec2Inventory(object):
 
 
 # Run the script
-Ec2Inventory()
-
+RETRIES = 3
+
+for _ in xrange(RETRIES):
+    try:
+        Ec2Inventory()
+        break
+    except Exception:
+        traceback.print_exc()

playbooks/edx-auth-proxy.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - role: python
+      tags:
+        - install
+        - install:system-requirements
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - oauth2_proxy

playbooks/edx-east/analytics_single.yml

@@ -6,7 +6,6 @@
     migrate_db: "yes"
     disable_edx_services: false
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
   roles:
     - aws

playbooks/edx-east/analyticsapi.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
     CLUSTER_NAME: 'analytics-api'
   roles:
@@ -19,3 +18,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/common_with_security.yml

+- name: Deploy common
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    SECURITY_UNATTENDED_UPGRADES: true
+    COMMON_SECURITY_UPDATES: true
+  roles:
+    - common

playbooks/edx-east/create_db_and_users.yml

@@ -4,6 +4,10 @@
 #
 # ansible-playbook -c local -i 'localhost,' create_dbs_and_users.yml -e@./db.yml
 #
+# If running ansible from a python virtualenv you will need a command like the following
+#
+# ansible-playbook -c local -i 'localhost,' create_dbs_and_users.yml -e@./db.yml -e "ansible_python_interpreter=$(which python)"
+#
 # where the content of db.yml contains the following dictionaries
 #
 # database_connection: &default_connection
@@ -67,6 +71,7 @@
     - name: create mysql users and assign privileges
       mysql_user:
         name: "{{ item.name }}"
+        state: "{{ item.state | default('present') }}"
         priv: "{{ '/'.join(item.privileges) }}"
         password: "{{ item.password }}"
         host: "{{ item.host }}"

playbooks/edx-east/create_rds_secondary.yml

@@ -16,7 +16,7 @@
     - name: Validate arguments
       fail:
         msg: "One or more arguments were not set correctly: {{ item }}"
-      when: not {{ item }}
+      when: not item
       with_items:
         - from_db
         - rds_name

playbooks/edx-east/credentials.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
     CLUSTER_NAME: 'credentials'
   roles:
@@ -21,3 +20,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/discovery.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
     CLUSTER_NAME: 'discovery'
   roles:
@@ -19,3 +18,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/ecommerce.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
     CLUSTER_NAME: 'ecommerce'
   roles:
@@ -21,3 +20,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/ecomworker.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
   roles:
     - aws
@@ -15,3 +14,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/edx_continuous_integration.yml

@@ -22,7 +22,7 @@
     - role: edxlocal
       tags: edxlocal
     - memcache
-    - mongo
+    - mongo_3_2
     - { role: 'edxapp', celery_worker: True }
     - edxapp
     - testcourses

playbooks/edx-east/edx_provision.yml

@@ -6,8 +6,8 @@
     keypair: continuous-integration
     instance_type: t2.medium
     security_group: sandbox-vpc
-    # ubuntu 12.04
-    ami: ami-f478849c
+    # ubuntu 16.04 - 20170721
+    ami: ami-cd0f5cb6
     region: us-east-1
     zone: us-east-1c
     instance_tags:
@@ -18,6 +18,7 @@
       owner: temp
     root_ebs_size: 50
     dns_name: temp
+    instance_initiated_shutdown_behavior: stop
     dns_zone: sandbox.edx.org
     name_tag: sandbox-temp
     elb: false
@@ -33,6 +34,7 @@
     - role: launch_ec2
       keypair: "{{ keypair }}"
       instance_type: "{{ instance_type }}"
+      instance_initiated_shutdown_behavior: "{{ instance_initiated_shutdown_behavior }}"
       security_group: "{{ security_group }}"
       ami: "{{ ami }}"
       region: "{{ region }}"
@@ -58,7 +60,7 @@
     - name: Wait for cloud-init to finish
       wait_for:
         path: /var/log/cloud-init.log
-        timeout: 15 
+        timeout: 15
         search_regex: "final-message"
     - name: gather_facts
       setup: ""

playbooks/edx-east/edxapp.yml

@@ -7,7 +7,8 @@
     CLUSTER_NAME: 'edxapp'
   serial: "{{ serial_count }}"
   roles:
-   - aws
+   - role: aws
+     when: COMMON_ENABLE_AWS_ROLE
    - role: automated
      AUTOMATED_USERS: "{{ EDXAPP_AUTOMATED_USERS | default({}) }}"
    - role: nginx
@@ -20,11 +21,15 @@
      nginx_extra_configs: "{{ NGINX_EDXAPP_EXTRA_CONFIGS }}"
      nginx_redirects: "{{ NGINX_EDXAPP_CUSTOM_REDIRECTS }}"
    - edxapp
+   - role: devstack_sqlite_fix
+     when: devstack is defined and devstack
    - role: datadog
      when: COMMON_ENABLE_DATADOG
    - role: splunkforwarder
      when: COMMON_ENABLE_SPLUNKFORWARDER
    - role: newrelic
      when: COMMON_ENABLE_NEWRELIC
+   - role: newrelic_infrastructure
+     when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE
    - role: minos
      when: COMMON_ENABLE_MINOS

playbooks/edx-east/forum.yml

@@ -18,3 +18,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/insights.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: True
     CLUSTER_NAME: 'insights'
   roles:
@@ -19,3 +18,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/jenkins_admin.yml

 # Configure an instance with the admin jenkins.
+- name: install python2
+  hosts: all
+  become: True
+  gather_facts: False
+  roles:
+    - python
 - name: Configure instance(s)
   hosts: all
   become: True
   gather_facts: True
   vars:
-    serial_count: 1
     COMMON_SECURITY_UPDATES: yes
     SECURITY_UPGRADE_ON_ANSIBLE: true
-  serial: "{{ serial_count }}"
   roles:
     - aws
     - jenkins_admin
@@ -20,3 +24,5 @@
     #     crcSalt: <SOURCE>
     - role: splunkforwarder
       when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/edx-east/jenkins_build.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    COMMON_ENABLE_DATADOG: True
+    COMMON_ENABLE_SPLUNKFORWARDER: True
+    COMMON_SECURITY_UPDATES: yes
+    SECURITY_UPGRADE_ON_ANSIBLE: true
+
+    SPLUNKFORWARDER_LOG_ITEMS:
+      - source: '/var/lib/jenkins/jobs/*/builds/*/junitResult.xml'
+        recursive: true
+        index: 'testeng'
+        sourcetype: junit
+        followSymlink: false
+        blacklist: '\.gz$'
+        crcSalt: '<SOURCE>'
+
+      - source: '/var/lib/jenkins/jobs/*/builds/*/build.xml'
+        index: 'testeng'
+        recursive: true
+        sourcetype: build_result
+        followSymlink: false
+        crcSalt: '<SOURCE>'
+        blacklist: '\.gz$'
+
+      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/test_root/log/timing.*.log'
+        index: 'testeng'
+        recursive: true
+        sourcetype: 'json_timing_log'
+        followSymlink: false
+        crcSalt: '<SOURCE>'
+        blacklist: coverage|private|subset|specific|custom|special|\.gz$
+
+      - source: '/var/log/jenkins/jenkins.log'
+        index: 'testeng'
+        recursive: false
+        followSymlink: false
+        blacklist: '\.gz$'
+
+  roles:
+    - aws
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+
+    - jenkins_build
+
+    # run just the splunkforwarder role by using '--tags "splunkonly"'
+    # e.g. ansible-playbook jenkins_testeng_master.yml -i inventory.ini --tags "splunkonly" -vvvv
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+      tags:
+        - splunkonly
+        - jenkins:promote-to-production
+      become: True

playbooks/edx-east/jenkins_testeng_master.yml

@@ -28,6 +28,7 @@
         sourcetype: build_result
         followSymlink: false
         crcSalt: '<SOURCE>'
+        blacklist: '(((\.(gz))|\d)$)|(.*seed.*)'
 
       - source: '/var/lib/jenkins/jobs/*/builds/*/log'
         index: 'testeng'
@@ -35,6 +36,7 @@
         sourcetype: build_log
         followSymlink: false
         crcSalt: '<SOURCE>'
+        blacklist: '(((\.(gz))|\d)$)|(.*seed.*)'
 
       - source: '/var/lib/jenkins/jobs/*/builds/*/archive/test_root/log/timing.*.log'
         index: 'testeng'

playbooks/edx-east/lifecycle_inventory.py

@@ -29,71 +29,68 @@ group and state.
 }
 """
 import argparse
-import boto
-import boto.ec2.autoscale
+import boto3
 import json
 from collections import defaultdict
 from os import environ
 
 class LifecycleInventory():
 
-    profile = None
-
-    def __init__(self, profile):
+    def __init__(self, region):
         parser = argparse.ArgumentParser()
-        self.profile = profile
+        self.region = region
 
     def get_e_d_from_tags(self, group):
 
         environment = "default_environment"
         deployment = "default_deployment"
 
-        for r in group.tags:
-            if r.key == "environment":
-                environment = r.value
-            elif r.key == "deployment":
-                deployment = r.value
+        for r in group['Tags']:
+            if r['Key'] == "environment":
+                environment = r['Value']
+            elif r['Key'] == "deployment":
+                deployment = r['Value']
         return environment,deployment
 
     def get_instance_dict(self):
-        ec2 = boto.ec2.connect_to_region(region,profile_name=self.profile)
-        reservations = ec2.get_all_instances()
+        ec2 = boto3.client('ec2', region_name=self.region)
+        reservations = ec2.describe_instances()['Reservations']
 
         dict = {}
 
-        for instance in [i for r in reservations for i in r.instances]:
-            dict[instance.id] = instance
+        for instance in [i for r in reservations for i in r['Instances']]:
+            dict[instance['InstanceId']] = instance
 
         return dict
 
     def run(self):
-        asg = boto.ec2.autoscale.connect_to_region(region,profile_name=self.profile)
-        groups = asg.get_all_groups()
+        asg = boto3.client('autoscaling', region_name=self.region)
+    
+        groups = asg.describe_auto_scaling_groups()['AutoScalingGroups']
 
         instances = self.get_instance_dict()
         inventory = defaultdict(list)
 
         for group in groups:
 
-            for instance in group.instances:
+            for instance in group['Instances']:
 
-                private_ip_address = instances[instance.instance_id].private_ip_address
+                private_ip_address = instances[instance['InstanceId']]['PrivateIpAddress']
                 if private_ip_address:
                     environment,deployment = self.get_e_d_from_tags(group)
-                    inventory[environment + "_" + deployment + "_" + instance.lifecycle_state.replace(":","_")].append(private_ip_address)
-                    inventory[group.name].append(private_ip_address)
-                    inventory[group.name + "_" + instance.lifecycle_state.replace(":","_")].append(private_ip_address)
-                    inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
+                    inventory[environment + "_" + deployment + "_" + instance['LifecycleState'].replace(":","_")].append(private_ip_address)
+                    inventory[group['AutoScalingGroupName']].append(private_ip_address)
+                    inventory[group['AutoScalingGroupName'] + "_" + instance['LifecycleState'].replace(":","_")].append(private_ip_address)
+                    inventory[instance['LifecycleState'].replace(":","_")].append(private_ip_address)
 
         print json.dumps(inventory, sort_keys=True, indent=2)
 
 if __name__=="__main__":
 
     parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--profile', help='The aws profile to use when connecting.')
+    parser.add_argument('-r', '--region', help='The aws region to use when connecting.', default='us-east-1')
     parser.add_argument('-l', '--list', help='Ansible passes this, we ignore it.', action='store_true', default=True)
     args = parser.parse_args()
 
-    region = environ.get('AWS_REGION','us-east-1')
 
-    LifecycleInventory(args.profile).run()
+    LifecycleInventory(args.region).run()

playbooks/edx-east/manage_edxapp_users_and_groups.yml

@@ -10,7 +10,7 @@
 #
 # Overview:
 # This playbook ensures that the specified users and groups exist in the targeted
-# edxapp cluster. 
+# edxapp cluster.
 #
 # Users have the following properties:
 #   - username (required, str)
@@ -72,7 +72,6 @@
 #   for perm in Permission.objects.all():
 #     print '{}:{}:{}'.format(perm.content_type.app_label, perm.content_type.model, perm.codename)
 #
-
 - hosts: all
   vars:
     python_path: /edx/bin/python.edxapp

playbooks/edx-east/mongo_3_0.yml

@@ -25,3 +25,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/mongo_3_2.yml

+# Manages a mongo cluster.
+# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG
+# as used by mongo_replica_set in the mongo_3_2 role.
+#
+# If you are initializing a cluster, your command might look like:
+# ansible-playbook mongo_3_2.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml
+# If you just want to deploy an updated replica set config, you can run
+# ansible-playbook mongo_3_2.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set
+#
+# ADDING A NEW CLUSTER MEMBER
+# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory
+# ansible-playbook mongo_3_2.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+- name: Deploy MongoDB
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - mongo_3_2
+    - munin_node
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/mongo_mms.yml

+- name: Deploy mongo_mms instance
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - aws
+    - mongo_mms
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG

playbooks/edx-east/neo4j.yml

@@ -13,3 +13,5 @@
         - coursegraph
 #    - aws
     - neo4j
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/edx-east/notes.yml

@@ -4,7 +4,6 @@
   gather_facts: True
   vars:
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: True
   roles:
     - aws

playbooks/edx-east/promote_rds_secondary.yml

@@ -21,7 +21,7 @@
     - name: Validate arguments
       fail:
         msg: "One or more arguments were not set correctly: {{ item }}"
-      when: not {{ item }}
+      when: not item
       with_items:
         - rds_name
         - admin_password
@@ -52,7 +52,7 @@
     - name: Modify edxapp history RDS
       shell: >
         aws rds modify-db-instance
-        --db-instance-identifier {{ rds_name }}        
+        --db-instance-identifier {{ rds_name }}
         --apply-immediately
         --multi-az
         --master-user-password {{ admin_password }}

playbooks/edx-east/rabbitmq.yml

@@ -40,6 +40,8 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE
   post_tasks:
     - debug:
         var: ansible_ec2_instance_id

playbooks/edx-east/restart_supervisor.yml

@@ -10,3 +10,6 @@
       service:
         name: "{{ supervisor_service }}"
         state: restarted
+      register: rc
+      until: rc|success
+      retries: 5

playbooks/edx-east/retire_host.yml

@@ -12,10 +12,6 @@
   hosts: "{{TARGET}}"
   become: True
   gather_facts: True
-  pre_tasks:
-    - set_fact:
-        STOP_ALL_EDX_SERVICES_EXTRA_ARGS: "--no-wait"
-      when: ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty'
   roles:
     - stop_all_edx_services
 

playbooks/edx-east/tools_jenkins.yml

+# Documentation on updating tools-edx-jenkins: https://openedx.atlassian.net/wiki/display/EdxOps/Updating+tools-edx-jenkins
+
+# Updating or creating a new install of tools_jenkins (will restart Jenkins)
+# ansible-playbook -i tools-edx-jenkins.m.edx.org, tools_jenkins.yml -e@/path/to/secure-config/tools-edx.yml
+
+# Update tools_jenkins with new plugins (will not restart Jenkins):
+# ansible-playbook -i tools-edx-jenkins.m.edx.org, tools_jenkins.yml -e@/path/to/secure-config/tools-edx.yml --tags install:plugins
+
 # Configure an instance with the tool jenkins.
 - name: Configure Jenkins instance(s)
   hosts: all
@@ -26,3 +34,5 @@
       when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/edx-east/xqwatcher.yml

@@ -6,7 +6,6 @@
     COMMON_APP_DIR: "/edx/app"
     common_web_group: "www-data"
     ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
     ENABLE_NEWRELIC: False
     serial_count: 1
   serial: "{{ serial_count }}"

playbooks/edx_sandbox.yml

@@ -21,10 +21,7 @@
     edx_platform_version: 'master'
     # Set to false if deployed behind another proxy/load balancer.
     NGINX_SET_X_FORWARDED_HEADERS: True
-    # These should stay false for the public AMI
-    COMMON_ENABLE_DATADOG: False
-    SANDBOX_ENABLE_ECOMMERCE: False
-    COMMON_ENABLE_SPLUNKFORWARDER: False
+    DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}'
   roles:
     - role: swapfile
       SWAPFILE_SIZE: 4GB
@@ -35,28 +32,23 @@
       - lms
       - forum
       - xqueue
+      - ecommerce
       nginx_default_sites:
       - lms
-    - role: nginx
-      nginx_sites:
-      - ecommerce
-      when: SANDBOX_ENABLE_ECOMMERCE
     - role: edxlocal
       when: EDXAPP_MYSQL_HOST == 'localhost'
     - role: memcache
       when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)"
-    - role: mongo
+    - role: mongo_3_2
       when: "'localhost' in EDXAPP_MONGO_HOSTS"
     - role: rabbitmq
       rabbitmq_ip: 127.0.0.1
     - role: edxapp
       celery_worker: True
     - edxapp
-    - role: ecommerce
-      when: SANDBOX_ENABLE_ECOMMERCE
+    - ecommerce
     - role: ecomworker
       ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
-      when: SANDBOX_ENABLE_ECOMMERCE
     - analytics_api
     - insights
     # not ready yet: - edx_notes_api
@@ -66,6 +58,7 @@
     - role: elasticsearch
       when: "'localhost' in EDXAPP_ELASTIC_SEARCH_CONFIG|map(attribute='host')"
     - forum
+    - discovery
     - role: notifier
       NOTIFIER_DIGEST_TASK_INTERVAL: 5
     - role: xqueue

playbooks/library/mongodb_replica_set

@@ -351,6 +351,12 @@ def validate_args():
     if (username and not password) or (password and not username):
         module.fail_json(msg="Must provide both username and password or neither.")
 
+    # Check that if votes is 0 priority is also 0
+    for member in module.params.get('rs_config').get('members'):
+        if member.get('votes') == 0 and member.get('priority') != 0:
+            module.fail_json(msg="Non-voting member {} must have priority 0".
+                             format(member['host']))
+
     return module
 
 

playbooks/library/vpc_lookup

-#!/usr/bin/python
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-DOCUMENTATION = '''
----
-module: vpc_lookup
-short_description: returns a list of subnet Ids using tags as criteria
-description:
-     - Returns a list of subnet Ids for a given set of tags that identify one or more VPCs
-version_added: "1.5"
-options:
-  region:
-    description:
-      - The AWS region to use.  Must be specified if ec2_url
-        is not used.  If not specified then the value of the
-        EC2_REGION environment variable, if any, is used.
-    required: false
-    default: null
-    aliases: [ 'aws_region', 'ec2_region' ]
-  aws_secret_key:
-    description:
-      - AWS secret key. If not set then the value of
-        the AWS_SECRET_KEY environment variable is used.
-    required: false
-    default: null
-    aliases: [ 'ec2_secret_key', 'secret_key' ]
-  aws_access_key:
-    description:
-      - AWS access key. If not set then the value of the
-        AWS_ACCESS_KEY environment variable is used.
-    required: false
-    default: null
-    aliases: [ 'ec2_access_key', 'access_key' ]
-  tags:
-    desription:
-      - tags to lookup
-    required: false
-    default: null
-    type: dict
-    aliases: []
-
-requirements: [ "boto" ]
-author: John Jarvis
-'''
-
-EXAMPLES = '''
-# Note: None of these examples set aws_access_key, aws_secret_key, or region.
-# It is assumed that their matching environment variables are set.
-
-# Return all instances that match the tag "Name: foo"
-- local_action:
-    module: vpc_lookup
-    tags:
-        Name: foo
-'''
-
-import sys
-
-AWS_REGIONS = ['ap-northeast-1',
-               'ap-southeast-1',
-               'ap-southeast-2',
-               'eu-west-1',
-               'sa-east-1',
-               'us-east-1',
-               'us-west-1',
-               'us-west-2']
-
-try:
-    from boto.vpc import VPCConnection
-    from boto.vpc import connect_to_region
-except ImportError:
-    print "failed=True msg='boto required for this module'"
-    sys.exit(1)
-
-
-def main():
-
-    module=AnsibleModule(
-        argument_spec=dict(
-            region=dict(choices=AWS_REGIONS),
-            aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'],
-                                no_log=True),
-            aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
-            tags=dict(default=None, type='dict'),
-        )
-    )
-
-    tags = module.params.get('tags')
-    aws_secret_key = module.params.get('aws_secret_key')
-    aws_access_key = module.params.get('aws_access_key')
-    region = module.params.get('region')
-
-    # If we have a region specified, connect to its endpoint.
-    if region:
-        try:
-            vpc = connect_to_region(region, aws_access_key_id=aws_access_key,
-                                    aws_secret_access_key=aws_secret_key)
-        except boto.exception.NoAuthHandlerFound, e:
-            module.fail_json(msg=str(e))
-    else:
-        module.fail_json(msg="region must be specified")
-    vpc_conn = VPCConnection()
-    subnet_ids = []
-    for subnet in vpc_conn.get_all_subnets(filters={'tag:' + tag: value
-                                      for tag, value in tags.iteritems()}):
-        subnet_ids.append(subnet.id)
-    vpc_ids = []
-    for vpc in vpc.get_all_vpcs(filters={'tag:' + tag: value
-                                      for tag, value in tags.iteritems()}):
-        vpc_ids.append(vpc.id)
-
-    module.exit_json(changed=False, subnet_ids=subnet_ids, vpc_ids=vpc_ids)
-
-
-# this is magic, see lib/ansible/module_common.py
-#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
-
-main()

playbooks/roles/analytics_api/defaults/main.yml

@@ -20,9 +20,18 @@ ANALYTICS_API_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 ANALYTICS_API_NGINX_PORT: "18100"
 
 ANALYTICS_API_DEFAULT_DB_NAME: 'analytics-api'
+ANALYTICS_API_DEFAULT_USER: 'api001'
+ANALYTICS_API_DEFAULT_PASSWORD: 'password'
+ANALYTICS_API_DEFAULT_HOST: 'localhost'
+ANALYTICS_API_DEFAULT_PORT: '3306'
 ANALYTICS_API_DEFAULT_MYSQL_OPTIONS:
   connect_timeout: 10
+
 ANALYTICS_API_REPORTS_DB_NAME: 'reports'
+ANALYTICS_API_REPORTS_USER: 'reports001'
+ANALYTICS_API_REPORTS_PASSWORD: 'password'
+ANALYTICS_API_REPORTS_HOST: 'localhost'
+ANALYTICS_API_REPORTS_PORT: '3306'
 ANALYTICS_API_REPORTS_MYSQL_OPTIONS:
   connect_timeout: 10
 
@@ -31,19 +40,19 @@ ANALYTICS_API_DATABASES:
   default:
     ENGINE: 'django.db.backends.mysql'
     NAME: '{{ ANALYTICS_API_DEFAULT_DB_NAME }}'
-    USER: 'api001'
-    PASSWORD: 'password'
-    HOST: 'localhost'
-    PORT: '3306'
+    USER: '{{ ANALYTICS_API_DEFAULT_USER }}'
+    PASSWORD: '{{ ANALYTICS_API_DEFAULT_PASSWORD }}'
+    HOST: '{{ ANALYTICS_API_DEFAULT_HOST }}'
+    PORT: '{{ ANALYTICS_API_DEFAULT_PORT }}'
     OPTIONS: "{{ ANALYTICS_API_DEFAULT_MYSQL_OPTIONS }}"
   # read-only user
   reports:
     ENGINE: 'django.db.backends.mysql'
     NAME: '{{ ANALYTICS_API_REPORTS_DB_NAME }}'
-    USER: 'reports001'
-    PASSWORD: 'password'
-    HOST: 'localhost'
-    PORT: '3306'
+    USER: '{{ ANALYTICS_API_REPORTS_USER }}'
+    PASSWORD: '{{ ANALYTICS_API_REPORTS_PASSWORD }}'
+    HOST: '{{ ANALYTICS_API_REPORTS_HOST }}'
+    PORT: '{{ ANALYTICS_API_REPORTS_PORT }}'
     OPTIONS: "{{ ANALYTICS_API_REPORTS_MYSQL_OPTIONS }}"
 
 ANALYTICS_API_VERSION: "master"
@@ -54,10 +63,6 @@ ANALYTICS_API_USERS:
 ANALYTICS_API_SECRET_KEY: 'Your secret key here'
 ANALYTICS_API_TIME_ZONE: 'UTC'
 ANALYTICS_API_LANGUAGE_CODE: 'en-us'
-ANALYTICS_API_EMAIL_HOST: 'localhost'
-ANALYTICS_API_EMAIL_HOST_USER: 'mail_user'
-ANALYTICS_API_EMAIL_HOST_PASSWORD: 'mail_password'
-ANALYTICS_API_EMAIL_PORT: 587
 ANALYTICS_API_AUTH_TOKEN: 'put-your-api-token-here'
 
 
@@ -107,11 +112,6 @@ ANALYTICS_API_SERVICE_CONFIG:
   SECRET_KEY: '{{ ANALYTICS_API_SECRET_KEY }}'
   TIME_ZONE: '{{ ANALYTICS_API_TIME_ZONE }}'
   LANGUAGE_CODE: '{{ANALYTICS_API_LANGUAGE_CODE }}'
-  # email config
-  EMAIL_HOST: '{{ ANALYTICS_API_EMAIL_HOST }}'
-  EMAIL_HOST_PASSWORD: '{{ ANALYTICS_API_EMAIL_HOST_PASSWORD }}'
-  EMAIL_HOST_USER: '{{ ANALYTICS_API_EMAIL_HOST_USER }}'
-  EMAIL_PORT: '{{ ANALYTICS_API_EMAIL_PORT }}'
   API_AUTH_TOKEN: '{{ ANALYTICS_API_AUTH_TOKEN }}'
   STATICFILES_DIRS: ['static']
   STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ analytics_api_service_name }}/staticfiles"

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

@@ -98,6 +98,8 @@
 
 {{ role_name|upper }}_HOSTNAME: '~^((stage|prod)-)?{{ role_name|replace('_', '-') }}.*'
 
+{{ role_name|upper }}_DEBIAN_EXTRA_PKGS: []
+
 nginx_{{ role_name }}_gunicorn_hosts:
   - 127.0.0.1
 

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/Dockerfile.j2

@@ -7,7 +7,7 @@
 # This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
 # with the currently checked-out configuration repo.
 
-FROM edxops/trusty-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
 ARG {{ role_name|upper }}_VERSION=master

playbooks/roles/ansible-role-django-ida/templates/meta/main.yml.j2

@@ -21,5 +21,5 @@ dependencies:
     edx_service_user: "{{ '{{' }} {{ role_name }}_user }}"
     edx_service_home: "{{ '{{' }} {{ role_name }}_home }}"
     edx_service_packages:
-      debian: "{{ '{{' }} {{ role_name }}_debian_pkgs }}"
+      debian: "{{ '{{' }} {{ role_name }}_debian_pkgs + {{ role_name|upper }}_DEBIAN_EXTRA_PKGS }}"
       redhat: "{{ '{{' }} {{ role_name }}_redhat_pkgs }}"

playbooks/roles/automated/tasks/main.yml

@@ -61,7 +61,6 @@
     group: "root"
     mode: "0440"
     validate: 'visudo -cf %s'
-  when: automated_sudoers_template
   with_dict: "{{ AUTOMATED_USERS }}"
   
 - name: Create .ssh directory

playbooks/roles/browsers/defaults/main.yml

@@ -32,10 +32,10 @@ browser_s3_deb_pkgs:
     url: https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_55.0.2883.87-1_amd64.deb
 
 trusty_browser_s3_deb_pkgs:
-  - name: google-chrome-stable_30.0.1599.114-1_amd64.deb
-    url: https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_30.0.1599.114-1_amd64.deb
   - name: firefox-mozilla-build_42.0-0ubuntu1_amd64.deb
     url: https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox-mozilla-build_42.0-0ubuntu1_amd64.deb
+  - name: google-chrome-stable_59.0.3071.115-1_amd64.deb
+    url: https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_59.0.3071.115-1_amd64.deb
 
 # ChromeDriver
 chromedriver_version: 2.27

playbooks/roles/browsers/tasks/main.yml

@@ -44,7 +44,7 @@
   get_url:
     dest: /tmp/{{ item.name }}
     url: "{{ item.url }}"
-  register: download_deb
+  register: download_trusty_deb
   with_items: "{{ trusty_browser_s3_deb_pkgs }}"
   when: ansible_distribution_release == 'trusty'
   tags:
@@ -55,7 +55,7 @@
   get_url:
     dest: /tmp/{{ item.name }}
     url: "{{ item.url }}"
-  register: download_deb
+  register: download_xenial_deb
   with_items: "{{ browser_s3_deb_pkgs }}"
   when: ansible_distribution_release == 'xenial'
   tags:
@@ -65,7 +65,7 @@
 - name: install trusty browser packages
   shell: gdebi -nq /tmp/{{ item.name }}
   with_items: "{{ trusty_browser_s3_deb_pkgs }}"
-  when: download_deb.changed and
+  when: download_trusty_deb.changed and
         ansible_distribution_release == 'trusty'
   tags:
     - install
@@ -74,7 +74,7 @@
 - name: install xenial browser packages
   shell: gdebi -nq /tmp/{{ item.name }}
   with_items: "{{ browser_s3_deb_pkgs }}"
-  when: download_deb.changed and
+  when: download_xenial_deb.changed and
         ansible_distribution_release == 'xenial'
   tags:
     - install

playbooks/roles/common_vars/defaults/main.yml

@@ -92,13 +92,14 @@ COMMON_ENABLE_DATADOG: False
 COMMON_ENABLE_NGINXTRA: False
 COMMON_ENABLE_SPLUNKFORWARDER: False
 COMMON_ENABLE_NEWRELIC: False
+COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: False
 # enables app reporting, you must enable newrelic
 # as well
 COMMON_ENABLE_NEWRELIC_APP: False
 COMMON_ENABLE_MINOS: False
 COMMON_TAG_EC2_INSTANCE: False
 common_boto_version: '2.34.0'
-common_node_version: '6.9.4'
+common_node_version: '6.11.1'
 common_redhat_pkgs:
   - ntp
   - lynx
@@ -156,7 +157,6 @@ common_debian_variants:
 
 # We only have to install old Python for these releases:
 old_python_ppa_releases:
-  - precise
   - trusty
 
 common_redhat_variants:
@@ -209,12 +209,18 @@ COMMON_TRACKING_LOG_ROTATION:
 COMMON_EXTRA_CONFIGURATION_SOURCES_CHECKING: false
 COMMON_EXTRA_CONFIGURATION_SOURCES: []
 
-COMMON_OAUTH_PUBLIC_URL_ROOT: 'http://127.0.0.1:8000/oauth2'
+COMMON_OAUTH_BASE_URL: 'http://127.0.0.1:8000'
+COMMON_OAUTH_PUBLIC_URL_ROOT: '{{ COMMON_OAUTH_BASE_URL }}/oauth2'
 COMMON_OAUTH_URL_ROOT: '{{ COMMON_OAUTH_PUBLIC_URL_ROOT }}'
-COMMON_OAUTH_LOGOUT_URL: '{{ COMMON_OAUTH_PUBLIC_URL_ROOT }}/logout'
+COMMON_OAUTH_LOGOUT_URL: '{{ COMMON_OAUTH_BASE_URL }}/logout'
 
 COMMON_OIDC_ISSUER: '{{ COMMON_OAUTH_URL_ROOT }}'
 
 COMMON_JWT_AUDIENCE: 'SET-ME-PLEASE'
 COMMON_JWT_ISSUER: '{{ COMMON_OIDC_ISSUER }}'
 COMMON_JWT_SECRET_KEY: 'SET-ME-PLEASE'
+
+# Set worker user default
+CREATE_SERVICE_WORKER_USERS: True
+
+COMMON_ENABLE_AWS_ROLE: true

playbooks/roles/credentials/defaults/main.yml

@@ -53,8 +53,6 @@ CREDENTIALS_DJANGO_SETTINGS_MODULE: "credentials.settings.production"
 CREDENTIALS_DOMAIN: 'credentials'
 CREDENTIALS_URL_ROOT: 'http://{{ CREDENTIALS_DOMAIN }}:18150'
 CREDENTIALS_LOGOUT_URL: '{{ CREDENTIALS_URL_ROOT }}/logout/'
-CREDENTIALS_OAUTH_URL_ROOT: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/oauth2'
-CREDENTIALS_OIDC_LOGOUT_URL: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/logout'
 
 CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 
@@ -66,7 +64,6 @@ CREDENTIALS_LANGUAGE_CODE: 'en_US.UTF-8'
 CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
 CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
 CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
-CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ CREDENTIALS_OAUTH_URL_ROOT }}'
 
 CREDENTIALS_SERVICE_USER: 'credentials_service_user'
 
@@ -146,17 +143,13 @@ NGINX_CREDENTIALS_GUNICORN_HOSTS:
 
 CREDENTIALS_EXTRA_APPS: []
 
-CREDENTIALS_JWT_AUDIENCE: '{{ EDXAPP_JWT_AUDIENCE | default("SET-ME-PLEASE") }}'
-CREDENTIALS_JWT_ISSUER: '{{ CREDENTIALS_OAUTH_URL_ROOT }}'
-CREDENTIALS_JWT_SECRET_KEY: '{{ EDXAPP_JWT_SECRET_KEY | default("lms-secret") }}'
-
 CREDENTIALS_JWT_AUTH:
   JWT_ISSUERS:
-    - AUDIENCE: '{{ CREDENTIALS_JWT_AUDIENCE }}'
-      ISSUER: '{{ CREDENTIALS_JWT_ISSUER }}'
-      SECRET_KEY: '{{ CREDENTIALS_JWT_SECRET_KEY }}'
+    - AUDIENCE: '{{ COMMON_JWT_AUDIENCE }}'
+      ISSUER: '{{ COMMON_JWT_ISSUER }}'
+      SECRET_KEY: '{{ COMMON_JWT_SECRET_KEY }}'
     - AUDIENCE: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-      ISSUER: '{{ CREDENTIALS_JWT_ISSUER }}'
+      ISSUER: '{{ COMMON_JWT_ISSUER }}'
       SECRET_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
 
 CREDENTIALS_SERVICE_CONFIG:
@@ -166,14 +159,14 @@ CREDENTIALS_SERVICE_CONFIG:
   TIME_ZONE: '{{ CREDENTIALS_TIME_ZONE }}'
   LANGUAGE_CODE: '{{ CREDENTIALS_LANGUAGE_CODE }}'
 
-  OAUTH2_PROVIDER_URL: '{{ CREDENTIALS_OAUTH_URL_ROOT }}'
+  OAUTH2_PROVIDER_URL: '{{ COMMON_OAUTH_URL_ROOT }}'
   SOCIAL_AUTH_EDX_OIDC_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
   SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
   SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ CREDENTIALS_OAUTH_URL_ROOT }}'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ COMMON_OAUTH_URL_ROOT }}'
   SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ CREDENTIALS_OIDC_LOGOUT_URL }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_ISSUER }}'
+  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ COMMON_OAUTH_LOGOUT_URL }}'
+  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ COMMON_JWT_ISSUER }}'
 
   EXTRA_APPS: '{{ CREDENTIALS_EXTRA_APPS }}'
 

playbooks/roles/demo/defaults/main.yml

@@ -24,19 +24,24 @@ demo_test_users:
     username: honor
     hashed_password: "{{ demo_hashed_password }}"
     is_staff: false
+    is_superuser: false
   - email: 'audit@example.com'
     username: audit
     hashed_password: "{{ demo_hashed_password }}"
     is_staff: false
+    is_superuser: false
   - email: 'verified@example.com'
     username: verified
     hashed_password: "{{ demo_hashed_password }}"
     is_staff: false
+    is_superuser: false
 demo_staff_user:
   email: 'staff@example.com'
   username: staff
   hashed_password: "{{ demo_hashed_password }}"
   is_staff: true
+  is_superuser: false
+SANDBOX_EDXAPP_USERS: []
 demo_edxapp_user: 'edxapp'
 demo_edxapp_settings: '{{ COMMON_EDXAPP_SETTINGS }}'
 demo_edxapp_venv_bin: '{{ COMMON_APP_DIR }}/{{ demo_edxapp_user }}/venvs/{{demo_edxapp_user}}/bin'

playbooks/roles/demo/tasks/deploy.yml

@@ -26,12 +26,16 @@
     demo_test_and_staff_users: "{{ demo_test_users }}"
   when: not DEMO_CREATE_STAFF_USER
 
+- name: build staff, admin, and test user list
+  set_fact:
+    demo_test_admin_and_staff_users: "{{ demo_test_and_staff_users + SANDBOX_EDXAPP_USERS }}"
+
 - name: create some test users
-  shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} --service-variant lms manage_user {{ item.username}} {{ item.email }} --initial-password-hash {{ item.hashed_password | quote }}{% if item.is_staff %} --staff{% endif %}"
+  shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} --service-variant lms manage_user {{ item.username}} {{ item.email }} --initial-password-hash {{ item.hashed_password | quote }}{% if item.is_staff %} --staff{% endif %}{% if item.is_superuser %} --superuser{% endif %}"
   args:
     chdir: "{{ demo_edxapp_code_dir }}"
   become_user: "{{ common_web_user }}"
-  with_items: "{{ demo_test_and_staff_users }}"
+  with_items: "{{ demo_test_admin_and_staff_users }}"
   when: demo_checkout.changed
 
 - name: enroll test users in the demo course

playbooks/roles/devstack_sqlite_fix/defaults/main.yml

+---
+
+SQLITE_FIX_TMP_DIR: "/var/tmp/sqlite_fix"
+
+PYSQLITE_URL: "https://codeload.github.com/ghaering/pysqlite/tar.gz/2.8.3"
+PYSQLITE_CREATED_PATH: "pysqlite-2.8.3"
+PYSQLITE_TMP_PATH: "{{ SQLITE_FIX_TMP_DIR }}/{{ PYSQLITE_CREATED_PATH }}"
+
+SQLITE_AUTOCONF_URL: "https://www.sqlite.org/2016/sqlite-autoconf-3140100.tar.gz"
+SQLITE_AUTOCONF_CREATED_PATH: "sqlite-autoconf-3140100"
+SQLITE_TMP_PATH: "{{ SQLITE_FIX_TMP_DIR }}/{{ SQLITE_AUTOCONF_CREATED_PATH }}"

playbooks/roles/devstack_sqlite_fix/tasks/main.yml

+---
+- name: Creates directory
+  file:
+    path: "{{ SQLITE_FIX_TMP_DIR }}"
+    state: directory
+    mode: 0775
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+
+# Tasks to download and upgrade pysqlite to prevent segfaults when testing in devstack
+- name: Download and unzip sqlite autoconf update
+  unarchive:
+    src: "{{ SQLITE_AUTOCONF_URL }}"
+    dest: "{{ SQLITE_FIX_TMP_DIR }}"
+    remote_src: yes
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+- name: Download and unzip pysqlite update
+  unarchive:
+    src: "{{ PYSQLITE_URL }}"
+    dest: "{{ SQLITE_FIX_TMP_DIR }}"
+    remote_src: yes
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+# Copy module doesn't support recursive dir copies for remote_src: yes
+- name: Copy pysqlite autoconf into pyslite update dir
+  command: "cp -av . {{ PYSQLITE_TMP_PATH }}/"
+  args:
+    chdir: "{{ SQLITE_TMP_PATH }}"
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+- name: Build and install pysqlite update
+  command: "python setup.py build_static install"
+  args:
+    chdir: "{{ PYSQLITE_TMP_PATH }}"
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+- name: Clean up pysqlite install artifacts
+  file:
+    state: absent
+    path: "{{ SQLITE_FIX_TMP_DIR }}/"
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install

playbooks/roles/discovery/defaults/main.yml

@@ -11,6 +11,7 @@
 # Defaults for role discovery
 #
 
+DISCOVERY_GIT_IDENTITY: !!null
 
 #
 # vars are namespace with the module name.
@@ -21,6 +22,9 @@ discovery_gunicorn_port: 8381
 discovery_environment:
   DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ discovery_service_name }}.yml"
 
+discovery_user: "{{ discovery_service_name }}"
+discovery_home: "{{ COMMON_APP_DIR }}/{{ discovery_service_name }}"
+discovery_code_dir: "{{ discovery_home }}/{{ discovery_service_name }}"
 
 #
 # OS packages
@@ -55,7 +59,20 @@ DISCOVERY_URL_ROOT: 'http://discovery:{{ DISCOVERY_NGINX_PORT }}'
 DISCOVERY_LOGOUT_URL: '{{ DISCOVERY_URL_ROOT }}/logout/'
 
 DISCOVERY_SECRET_KEY: 'Your secret key here'
-DISCOVERY_LANGUAGE_CODE: 'en-us'
+
+DISCOVERY_LANGUAGE_CODE: 'en'
+
+## Configuration for django-parler package. For more information visit
+## https://django-parler.readthedocs.io/en/latest/configuration.html#parler-languages
+DISCOVERY_PARLER_DEFAULT_LANGUAGE_CODE: '{{DISCOVERY_LANGUAGE_CODE}}'
+DISCOVERY_PARLER_LANGUAGES :
+  1:
+    - code: 'en'
+  default:
+    fallbacks:
+      - '{{DISCOVERY_PARLER_DEFAULT_LANGUAGE_CODE}}'
+    hide_untranslated: 'False'
+
 DISCOVERY_DEFAULT_PARTNER_ID: 1
 DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 
@@ -94,10 +111,21 @@ DISCOVERY_EMAIL_HOST_PASSWORD: ''
 
 DISCOVERY_PUBLISHER_FROM_EMAIL: !!null
 
+DISCOVERY_OPENEXCHANGERATES_API_KEY: ''
+
 DISCOVERY_GUNICORN_EXTRA: ''
 
 DISCOVERY_EXTRA_APPS: []
 
+DISCOVERY_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: 'course-discovery.git'
+    VERSION: "{{ DISCOVERY_VERSION }}"
+    DESTINATION: "{{ discovery_code_dir }}"
+    SSH_KEY: "{{ DISCOVERY_GIT_IDENTITY }}"
+
 discovery_service_config_overrides:
   ELASTICSEARCH_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}'
   ELASTICSEARCH_INDEX_NAME: '{{ DISCOVERY_ELASTICSEARCH_INDEX_NAME }}'
@@ -121,5 +149,11 @@ discovery_service_config_overrides:
 
   PUBLISHER_FROM_EMAIL: '{{ DISCOVERY_PUBLISHER_FROM_EMAIL }}'
 
+  OPENEXCHANGERATES_API_KEY: '{{ DISCOVERY_OPENEXCHANGERATES_API_KEY }}'
+
+  LANGUAGE_CODE: '{{DISCOVERY_LANGUAGE_CODE}}'
+  PARLER_DEFAULT_LANGUAGE_CODE: '{{DISCOVERY_PARLER_DEFAULT_LANGUAGE_CODE}}'
+  PARLER_LANGUAGES : '{{DISCOVERY_PARLER_LANGUAGES}}'
+
 # See edx_django_service_automated_users for an example of what this should be
 DISCOVERY_AUTOMATED_USERS: {}

playbooks/roles/discovery/meta/main.yml

@@ -20,9 +20,10 @@
 # }
 dependencies:
   - role: edx_django_service
-    edx_django_service_repo: 'course-discovery'
-    edx_django_service_version: '{{ DISCOVERY_VERSION }}'
+    edx_django_service_repos: '{{ DISCOVERY_REPOS }}'
     edx_django_service_name: '{{ discovery_service_name }}'
+    edx_django_service_user: '{{ discovery_user }}'
+    edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ discovery_service_name }}'
     edx_django_service_config_overrides: '{{ discovery_service_config_overrides }}'
     edx_django_service_debian_pkgs_extra: '{{ discovery_debian_pkgs }}'
     edx_django_service_gunicorn_port: '{{ discovery_gunicorn_port }}'

playbooks/roles/docker-tools/defaults/main.yml

@@ -5,9 +5,10 @@ docker_tools_deps_deb_pkgs:
     - ca-certificates
     - python-pip
 
-docker_apt_keyserver: "hkp://ha.pool.sks-keyservers.net:80"
-docker_apt_key_id: "58118E89F3A912897C070ADBF76221572C52609D"
-docker_repo: "deb https://apt.dockerproject.org/repo ubuntu-xenial main"
+docker_apt_key_url: "https://download.docker.com/linux/ubuntu/gpg"
+docker_repos:
+    - "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    - "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} edge"
 
 docker_group: "docker"
 docker_users: []

playbooks/roles/docker-tools/tasks/main.yml

@@ -29,24 +29,23 @@
 
 - name: add docker apt key
   apt_key:
-    keyserver: "{{ docker_apt_keyserver }}"
-    id: "{{ docker_apt_key_id }}"
+    url: "{{ docker_apt_key_url }}"
   tags:
     - install
     - install:configuration
 
 - name: add docker repo
   apt_repository:
-    repo: "{{ docker_repo }}"
+    repo: "{{ item }}"
+  with_items: "{{ docker_repos }}"
   tags:
     - install
     - install:configuration
 
 - name: install docker-engine
   apt:
-    name: "docker-engine"
+    name: "docker-ce"
     update_cache: yes
-    cache_valid_time: "{{ cache_valid_time }}"
   tags:
     - install
     - install:system-requirements

playbooks/roles/ecommerce/defaults/main.yml

@@ -20,6 +20,8 @@ ECOMMERCE_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 ECOMMERCE_NGINX_PORT: "18130"
 ECOMMERCE_SSL_NGINX_PORT: 48130
 
+ECOMMERCE_MEMCACHE: [ 'localhost:11211' ]
+
 ECOMMERCE_DEFAULT_DB_NAME: 'ecommerce'
 ECOMMERCE_DATABASE_USER: "ecomm001"
 ECOMMERCE_DATABASE_PASSWORD: "password"
@@ -44,65 +46,81 @@ ECOMMERCE_DATABASES:
 ECOMMERCE_VERSION: "master"
 ECOMMERCE_DJANGO_SETTINGS_MODULE: "ecommerce.settings.production"
 
-ECOMMERCE_OAUTH_URL_ROOT: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/oauth2'
-ECOMMERCE_OIDC_LOGOUT_URL: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/logout'
-
 ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 ECOMMERCE_SECRET_KEY: 'Your secret key here'
 ECOMMERCE_TIME_ZONE: 'UTC'
-ECOMMERCE_LANGUAGE_CODE: 'en-us'
+ECOMMERCE_LANGUAGE_CODE: 'en'
+ECOMMERCE_LANGUAGE_COOKIE_NAME: 'openedx-language-preference'
 ECOMMERCE_EDX_API_KEY: 'PUT_YOUR_API_KEY_HERE'  # This should match the value set for edxapp
 ECOMMERCE_ECOMMERCE_URL_ROOT: 'http://localhost:8002'
 ECOMMERCE_LOGOUT_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}/logout/'
 ECOMMERCE_LMS_URL_ROOT: 'http://127.0.0.1:8000'
-ECOMMERCE_JWT_SECRET_KEY: '{{ EDXAPP_JWT_SECRET_KEY | default("lms-secret") }}'
 ECOMMERCE_JWT_ALGORITHM: 'HS256'
 ECOMMERCE_JWT_VERIFY_EXPIRATION: true
 ECOMMERCE_JWT_DECODE_HANDLER: 'ecommerce.extensions.api.handlers.jwt_decode_handler'
 ECOMMERCE_JWT_ISSUERS:
-  - '{{ ECOMMERCE_OAUTH_URL_ROOT }}'
+  - '{{ COMMON_JWT_ISSUER }}'
   - 'ecommerce_worker'  # Must match the value of JWT_ISSUER configured for the ecommerce worker.
 ECOMMERCE_JWT_LEEWAY: 1
 # NOTE: We have an array of keys to allow for support of multiple when, for example,
 # we change keys. This will ensure we continue to operate with JWTs issued signed with the old key
 # while migrating to the new key.
 ECOMMERCE_JWT_SECRET_KEYS:
-  - '{{ ECOMMERCE_JWT_SECRET_KEY }}'
+  - '{{ COMMON_JWT_SECRET_KEY }}'
 
 # Used to automatically configure OAuth2 Client
 ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY : 'ecommerce-key'
 ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET : 'ecommerce-secret'
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
-ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ ECOMMERCE_OAUTH_URL_ROOT }}'
 
 # Settings for affiliate cookie tracking
 ECOMMERCE_AFFILIATE_COOKIE_NAME: '{{ EDXAPP_AFFILIATE_COOKIE_NAME | default("dev_affiliate_id") }}'
 
+ECOMMERCE_OSCAR_FROM_EMAIL: 'oscar@example.com'
+
+# NOTE: The contents of the certificates should be set in private configuration
+ecommerce_apple_pay_merchant_certificate_directory: '/edx/etc/ssl'
+ecommerce_apple_pay_merchant_certificate_filename: 'apple_pay_merchant.pem'
+ecommerce_apple_pay_merchant_certificate_path: '{{ ecommerce_apple_pay_merchant_certificate_directory }}/{{ ecommerce_apple_pay_merchant_certificate_filename }}'
+ECOMMERCE_APPLE_PAY_MERCHANT_CERTIFICATE: |
+  Your PEM file, containing a public and private key,
+  should be set in private configuration. This is how you
+  implement a multi-line string in YAML.
+ECOMMERCE_APPLE_PAY_MERCHANT_ID_DOMAIN_ASSOCIATION: |
+  This value should also be in private configuration. It, too,
+  will span multiple lines.
+ECOMMERCE_APPLE_PAY_MERCHANT_IDENTIFIER: 'merchant.com.example'
+ECOMMERCE_APPLE_PAY_COUNTRY_CODE: 'US'
+
 # CyberSource related
 ECOMMERCE_CYBERSOURCE_PROFILE_ID: 'SET-ME-PLEASE'
 ECOMMERCE_CYBERSOURCE_MERCHANT_ID: 'SET-ME-PLEASE'
 ECOMMERCE_CYBERSOURCE_ACCESS_KEY: 'SET-ME-PLEASE'
 ECOMMERCE_CYBERSOURCE_SECRET_KEY: 'SET-ME-PLEASE'
+ECOMMERCE_CYBERSOURCE_SOP_ACCESS_KEY: 'SET-ME-PLEASE'
+ECOMMERCE_CYBERSOURCE_SOP_PROFILE_ID: 'SET-ME-PLEASE'
+ECOMMERCE_CYBERSOURCE_SOP_SECRET_KEY: 'SET-ME-PLEASE'
+ECOMMERCE_CYBERSOURCE_SOP_PAYMENT_PAGE_URL: 'https://testsecureacceptance.cybersource.com/silent/pay'
 ECOMMERCE_CYBERSOURCE_TRANSACTION_KEY: 'SET-ME-PLEASE'
-ECOMMERCE_CYBERSOURCE_PAYMENT_PAGE_URL: 'https://set-me-please'
-ECOMMERCE_CYBERSOURCE_RECEIPT_PAGE_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/commerce/checkout/receipt/'
-ECOMMERCE_CYBERSOURCE_CANCEL_PAGE_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/commerce/checkout/cancel/'
-ECOMMERCE_CYBERSOURCE_SOAP_API_URL: 'https://set-me-please'
-ECOMMERCE_OSCAR_FROM_EMAIL: 'oscar@example.com'
-
-# PayPal related
-ECOMMERCE_PAYPAL_MODE: 'SET-ME-PLEASE'
+ECOMMERCE_CYBERSOURCE_PAYMENT_PAGE_URL: 'https://testsecureacceptance.cybersource.com/pay'
+ECOMMERCE_CYBERSOURCE_RECEIPT_PAGE_URL: '/checkout/receipt/'
+ECOMMERCE_CYBERSOURCE_CANCEL_PAGE_URL: '/checkout/cancel-checkout/'
+ECOMMERCE_CYBERSOURCE_SEND_LEVEL_2_3_DETAILS: true
+ECOMMERCE_CYBERSOURCE_SOAP_API_URL: 'https://ics2wstest.ic3.com/commerce/1.x/transactionProcessor/CyberSourceTransaction_1.140.wsdl'
+
+# PayPal
+ECOMMERCE_PAYPAL_MODE: 'sandbox'
 ECOMMERCE_PAYPAL_CLIENT_ID: 'SET-ME-PLEASE'
 ECOMMERCE_PAYPAL_CLIENT_SECRET: 'SET-ME-PLEASE'
-ECOMMERCE_PAYPAL_RECEIPT_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/commerce/checkout/receipt/'
-ECOMMERCE_PAYPAL_CANCEL_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/commerce/checkout/cancel/'
-ECOMMERCE_PAYPAL_ERROR_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/commerce/checkout/error/'
+ECOMMERCE_PAYPAL_RECEIPT_URL: '/checkout/receipt/'
+ECOMMERCE_PAYPAL_CANCEL_URL: '/checkout/cancel-checkout/'
+ECOMMERCE_PAYPAL_ERROR_URL: '/checkout/error/'
 
 ECOMMERCE_PAYMENT_PROCESSOR_CONFIG:
   edx:
     cybersource:
-      profile_id: '{{ ECOMMERCE_CYBERSOURCE_PROFILE_ID }}'
       merchant_id: '{{ ECOMMERCE_CYBERSOURCE_MERCHANT_ID }}'
+      profile_id: '{{ ECOMMERCE_CYBERSOURCE_PROFILE_ID }}'
       access_key: '{{ ECOMMERCE_CYBERSOURCE_ACCESS_KEY }}'
       secret_key: '{{ ECOMMERCE_CYBERSOURCE_SECRET_KEY }}'
       transaction_key: '{{ ECOMMERCE_CYBERSOURCE_TRANSACTION_KEY }}'
@@ -110,6 +128,17 @@ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG:
       receipt_page_url: '{{ ECOMMERCE_CYBERSOURCE_RECEIPT_PAGE_URL }}'
       cancel_page_url: '{{ ECOMMERCE_CYBERSOURCE_CANCEL_PAGE_URL }}'
       soap_api_url: '{{ ECOMMERCE_CYBERSOURCE_SOAP_API_URL }}'
+      send_level_2_3_details: '{{ ECOMMERCE_CYBERSOURCE_SEND_LEVEL_2_3_DETAILS }}'
+      sop_profile_id: '{{ ECOMMERCE_CYBERSOURCE_SOP_PROFILE_ID }}'
+      sop_access_key: '{{ ECOMMERCE_CYBERSOURCE_SOP_ACCESS_KEY }}'
+      sop_secret_key: '{{ ECOMMERCE_CYBERSOURCE_SOP_SECRET_KEY }}'
+      sop_payment_page_url: '{{ ECOMMERCE_CYBERSOURCE_SOP_PAYMENT_PAGE_URL }}'
+      # NOTE: These are simple placeholders meant to show what keys are needed for Apple Pay. These values
+      # should be overwritten in private configuration.
+      apple_pay_merchant_identifier: '{{ ECOMMERCE_APPLE_PAY_MERCHANT_IDENTIFIER }}'
+      apple_pay_merchant_id_domain_association: '{{ ECOMMERCE_APPLE_PAY_MERCHANT_ID_DOMAIN_ASSOCIATION }}'
+      apple_pay_merchant_id_certificate_path: '{{ ecommerce_apple_pay_merchant_certificate_path }}'
+      apple_pay_country_code: '{{ ECOMMERCE_APPLE_PAY_COUNTRY_CODE }}'
     paypal:
       mode: '{{ ECOMMERCE_PAYPAL_MODE }}'
       client_id: '{{ ECOMMERCE_PAYPAL_CLIENT_ID }}'
@@ -146,6 +175,7 @@ ECOMMERCE_SERVICE_CONFIG:
 
   SECRET_KEY: '{{ ECOMMERCE_SECRET_KEY }}'
   TIME_ZONE: '{{ ECOMMERCE_TIME_ZONE }}'
+  LANGUAGE_COOKIE_NAME: '{{ ECOMMERCE_LANGUAGE_COOKIE_NAME }}'
   LANGUAGE_CODE: '{{ ECOMMERCE_LANGUAGE_CODE }}'
   EDX_API_KEY: '{{ ECOMMERCE_EDX_API_KEY }}'
   OSCAR_FROM_EMAIL: '{{ ECOMMERCE_OSCAR_FROM_EMAIL }}'
@@ -159,7 +189,7 @@ ECOMMERCE_SERVICE_CONFIG:
   COMMERCE_API_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/api/commerce/v1/'
   LMS_DASHBOARD_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/dashboard'
   JWT_AUTH:
-    JWT_SECRET_KEY: '{{ ECOMMERCE_JWT_SECRET_KEY }}'
+    JWT_SECRET_KEY: '{{ COMMON_JWT_SECRET_KEY }}'
     JWT_ALGORITHM: '{{ ECOMMERCE_JWT_ALGORITHM }}'
     JWT_VERIFY_EXPIRATION: '{{ ECOMMERCE_JWT_VERIFY_EXPIRATION }}'
     JWT_LEEWAY: '{{ ECOMMERCE_JWT_LEEWAY }}'
@@ -169,10 +199,10 @@ ECOMMERCE_SERVICE_CONFIG:
   SOCIAL_AUTH_EDX_OIDC_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
   SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
   SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ ECOMMERCE_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ ECOMMERCE_OIDC_LOGOUT_URL }}'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ COMMON_OAUTH_URL_ROOT }}'
+  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ COMMON_OAUTH_LOGOUT_URL }}'
   SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_ISSUER }}'
+  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ COMMON_JWT_ISSUER }}'
   AFFILIATE_COOKIE_KEY: '{{ ECOMMERCE_AFFILIATE_COOKIE_NAME }}'
 
   STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ ecommerce_service_name }}/staticfiles"
@@ -192,6 +222,11 @@ ECOMMERCE_SERVICE_CONFIG:
   ENABLE_COMPREHENSIVE_THEMING: "{{ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING }}"
   DEFAULT_SITE_THEME: "{{ ECOMMERCE_DEFAULT_SITE_THEME }}"
 
+  CACHES:
+    default:
+      BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
+      KEY_PREFIX: 'ecommerce'
+      LOCATION: '{{ ECOMMERCE_MEMCACHE }}'
 
 ECOMMERCE_REPOS:
   - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"

playbooks/roles/ecommerce/meta/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Role includes for role ecommerce
-# 
+#
 dependencies:
   - common
   - supervisor
@@ -25,6 +25,6 @@ dependencies:
   - role: edx_themes
     theme_users:
       - "{{ ecommerce_user }}"
-    when: "{{ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING }}"
+    when: ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING
 
   - oraclejdk

playbooks/roles/ecommerce/tasks/main.yml

@@ -68,24 +68,6 @@
     - install
     - install:app-requirements
 
-# This is a hacked fix for the fact that the table `thumbnail_kvstore` exists in
-# some environments, which won't need the 3rd party newly introduced migration
-# to create this table, so we fake the migration.
-# This is required for the Ginkgo release.
-# TODO: Delete this task for the Hawthorn release.
-- name: fake thumbnails
-  shell: >
-    table_exists=`mysql -uroot -ss -e "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_schema = '{{ ECOMMERCE_DEFAULT_DB_NAME }}' AND table_name = 'thumbnail_kvstore')"`;
-    if [ "$table_exists" -eq "1" ]; then {{ ecommerce_venv_dir }}/bin/python ./manage.py migrate thumbnail 0001 --fake; fi;
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
 - name: Migrate
   shell: >
     DB_MIGRATION_USER='{{ COMMON_MYSQL_MIGRATE_USER }}'
@@ -170,6 +152,28 @@
     - install
     - install:configuration
 
+- name: Create Apple Pay certificates directory
+  file:
+    path: "{{ ecommerce_apple_pay_merchant_certificate_directory }}"
+    state: directory
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: Write Apple Pay merchant certificates
+  copy:
+    content: "{{ ECOMMERCE_APPLE_PAY_MERCHANT_CERTIFICATE }}"
+    dest: "{{ ecommerce_apple_pay_merchant_certificate_path }}"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: "0644"
+  no_log: true
+  tags:
+    - install
+    - install:configuration
+
 - name: Setup the ecommence env file
   template:
     src: "./{{ ecommerce_home }}/{{ ecommerce_service_name }}_env.j2"

playbooks/roles/edx_django_service/defaults/main.yml

@@ -16,7 +16,9 @@ edx_django_service_name_devstack_logs:
   - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stdout.log'
   - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stderr.log'
 
-
+edx_django_service_git_protocol: '{{ COMMON_GIT_PROTOCOL }}'
+edx_django_service_git_domain: '{{ COMMON_GIT_MIRROR }}'
+edx_django_service_git_path: '{{ COMMON_GIT_PATH }}'
 edx_django_service_version: 'master'
 edx_django_service_git_identity: null
 edx_django_service_django_settings_module: null
@@ -76,9 +78,9 @@ edx_django_service_basic_auth_exempted_paths: '{{ edx_django_service_basic_auth_
 edx_django_service_newrelic_appname: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_django_service_name }}'
 
 edx_django_service_repos:
-  - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}'
-    DOMAIN: '{{ COMMON_GIT_MIRROR }}'
-    PATH: '{{ COMMON_GIT_PATH }}'
+  - PROTOCOL: '{{ edx_django_service_git_protocol }}'
+    DOMAIN: '{{ edx_django_service_git_domain }}'
+    PATH: '{{ edx_django_service_git_path }}'
     REPO: '{{ edx_django_service_repo }}.git'
     VERSION: '{{ edx_django_service_version }}'
     DESTINATION: '{{ edx_django_service_code_dir }}'

playbooks/roles/edx_django_service/tasks/main.yml

@@ -232,6 +232,7 @@
     owner: root
     group: "{{ common_web_user }}"
     mode: 0640
+  when: nginx_app_dir is defined
   notify: reload nginx
   tags:
     - install
@@ -244,6 +245,7 @@
     state: link
     owner: root
     group: root
+  when: nginx_app_dir is defined
   notify: reload nginx
   tags:
     - install

playbooks/roles/edx_service/defaults/main.yml

@@ -17,6 +17,12 @@
 edx_service_name: edx_service
 
 edx_service_repos: []
+
+# A few roles meta this role but don't need a config file written
+# this allows them to not pass a config and the tasks will skip
+# and not write out a config at all.
+edx_service_config: {}
+
 #
 # OS packages
 #

playbooks/roles/edx_service/tasks/main.yml

@@ -95,7 +95,7 @@
     src: "config.yml.j2"
     dest: "{{ COMMON_CFG_DIR  }}/{{ edx_service_name }}.yml"
     mode: "0644"
-  when: edx_service_config is defined
+  when: edx_service_config
   tags:
     - install
     - install:configuration

playbooks/roles/edxapp/meta/main.yml

@@ -7,4 +7,4 @@ dependencies:
   - role: edx_themes
     theme_users:
       - "{{ edxapp_user }}"
-    when: "{{ EDXAPP_ENABLE_COMPREHENSIVE_THEMING }}"
+    when: EDXAPP_ENABLE_COMPREHENSIVE_THEMING

playbooks/roles/edxapp/tasks/deploy.yml

@@ -300,7 +300,7 @@
     - install:app-requirements
 
 - name: compiling all py files in the edx-platform repo
-  shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x .git/.* {{ edxapp_code_dir }}"
+  shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}"
   become_user: "{{ edxapp_user }}"
   tags:
     - install
@@ -417,3 +417,14 @@
   become_user: "{{ common_web_user }}"
   tags:
     - manage
+
+- name: create service worker users
+  shell: "{{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} --service-variant lms manage_user {{ item.username}} {{ item.email }} --unusable-password {% if item.is_staff %} --staff{% endif %}"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ common_web_user }}"
+  with_items: "{{ SERVICE_WORKER_USERS }}"
+  when: CREATE_SERVICE_WORKER_USERS
+  tags:
+    - manage
+    - manage:db

playbooks/roles/edxapp/tasks/main.yml

@@ -109,16 +109,6 @@
     - install
     - install:base
 
-# adding chris-lea nodejs repo
-# TODO: 16.04
-- name: add ppas for current versions of nodejs
-  apt_repository:
-    repo: "{{ edxapp_chrislea_ppa }}"
-  when: ansible_distribution_release == 'precise'
-  tags:
-    - install
-    - install:base
-
 - name: install system packages on which LMS and CMS rely
   apt:
     name: "{{ item }}"

playbooks/roles/edxapp/vars/devstack.yml

@@ -11,6 +11,7 @@ edxapp_requirements_files:
   - "{{ custom_requirements_file }}"
   - "{{ local_requirements_file }}"
   - "{{ base_requirements_file }}"
+  - "{{ django_requirements_file }}"
   - "{{ post_requirements_file }}"
   - "{{ paver_requirements_file }}"
   - "{{ development_requirements_file }}"

playbooks/roles/forum/defaults/main.yml

@@ -45,7 +45,10 @@ FORUM_USE_TCP: false
 # wait this long before attempting to restart it
 FORUM_RESTART_DELAY: 60
 
-forum_environment:
+# Set to rebuild the forum ElasticSearch index from the database.
+FORUM_REBUILD_INDEX: false
+
+forum_base_env: &forum_base_env
   RBENV_ROOT: "{{ forum_rbenv_root }}"
   GEM_HOME: "{{ forum_gem_root }}"
   GEM_PATH: "{{ forum_gem_root }}"
@@ -65,8 +68,18 @@ forum_environment:
   LISTEN_HOST: "{{ FORUM_LISTEN_HOST }}"
   LISTEN_PORT: "{{ FORUM_LISTEN_PORT }}"
 
+
+forum_env:
+  <<: *forum_base_env
+devstack_forum_env:
+  <<: *forum_base_env
+  RACK_ENV: "development"
+  SINATRA_ENV: "development"
+  SEARCH_SERVER: "http://edx.devstack.elasticsearch:9200/"
+  MONGOHQ_URL: "mongodb://cs_comments_service:password@edx.devstack.mongo:27017/cs_comments_service"
+
 forum_user: "forum"
-forum_ruby_version: "1.9.3-p551"
+forum_ruby_version: "2.4.1"
 forum_source_repo: "https://github.com/edx/cs_comments_service.git"
 forum_version: "master"
 

playbooks/roles/forum/tasks/deploy.yml

@@ -54,7 +54,7 @@
 - name: install comments service bundle
   shell: "bundle install --deployment --path {{ forum_gem_root }} chdir={{ forum_code_dir }}"
   become_user: "{{ forum_user }}"
-  environment: "{{ forum_environment }}"
+  environment: "{{ forum_base_env }}"
   notify: restart the forum service
   tags:
     - install
@@ -65,12 +65,23 @@
   args:
     chdir: "{{ forum_code_dir }}"
   become_user: "{{ forum_user }}"
-  environment: "{{ forum_environment }}"
+  environment: "{{ forum_base_env }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
   tags:
     - migrate
     - migrate:db
 
+- name: rebuild elasticsearch indexes
+  command: "{{ forum_code_dir }}/bin/rake search:rebuild_index"
+  args:
+    chdir: "{{ forum_code_dir }}"
+  become_user: "{{ forum_user }}"
+  environment: "{{ forum_base_env }}"
+  when: migrate_db is defined and migrate_db|lower == "yes" and FORUM_REBUILD_INDEX|bool
+  tags:
+    - migrate
+    - migrate:db
+
   # call supervisorctl update. this reloads
   # the supervisorctl config and restarts
   # the services if any of the configurations

playbooks/roles/forum/tasks/main.yml

@@ -44,11 +44,27 @@
     - install
     - install:base
 
-- name: setup the forum env
+- name: setup the forum env for stage/prod
   template:
     src: forum_env.j2
     dest: "{{ forum_app_dir }}/forum_env"
-    owner: "{{ forum_user }}" 
+    owner: "{{ forum_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+  notify:
+    - restart the forum service
+  tags:
+    - install
+    - install:base
+    - install:configuration
+  with_items:
+    - "{{ forum_env }}"
+
+- name: setup the forum env for devstack
+  template:
+    src: forum_env.j2
+    dest: "{{ forum_app_dir }}/devstack_forum_env"
+    owner: "{{ forum_user }}"
     group: "{{ common_web_user }}"
     mode: 0644
   notify:
@@ -56,6 +72,9 @@
   tags:
     - install
     - install:base
+  when: devstack is defined and devstack
+  with_items:
+    - "{{ devstack_forum_env }}"
 
 - name: create {{ forum_data_dir }}
   file:
@@ -67,7 +86,7 @@
   tags:
     - install
     - install:base
-    
+
 - include: deploy.yml
-  tags: 
+  tags:
     - deploy

playbooks/roles/forum/templates/forum_env.j2

 # {{ ansible_managed }}
 
-{% for name,value in forum_environment.items() -%}
+{% for name,value in item.items() -%}
 {%- if value -%}
 export {{ name }}="{{ value }}"
-{% endif %}                 
+{% endif %}
 {%- endfor %}
 
 eval "$(rbenv init -)"

playbooks/roles/git_clone/tasks/main.yml

@@ -96,3 +96,11 @@
   tags:
     - install
     - install:code
+
+- name: Run git clean after checking out code
+  shell: cd {{ item.DESTINATION }} && git clean -xdf
+  become: true
+  with_items: "{{ GIT_REPOS }}"
+  tags:
+    - install
+    - install:code

playbooks/roles/go-server/defaults/main.yml

@@ -13,16 +13,13 @@
 GO_SERVER_SERVICE_NAME: "go-server"
 GO_SERVER_USER: "go"
 GO_SERVER_GROUP: "{{ GO_SERVER_USER }}"
-GO_SERVER_VERSION: "17.1.0-4511"
+GO_SERVER_VERSION: "17.10.0-5380"
 GO_SERVER_HOME: "/var/lib/go-server"
 GO_SERVER_CONF_HOME: "/etc/go"
 GO_SERVER_PLUGIN_DIR: "{{ GO_SERVER_HOME }}/plugins/external/"
 
-# Java version settings
-GO_SERVER_ORACLEJDK_VERSION: "8u65"
-GO_SERVER_ORACLEJDK_BASE: "jdk1.8.0_65"
-GO_SERVER_ORACLEJDK_BUILD: "b17"
-GO_SERVER_ORACLEJDK_LINK: "/usr/lib/jvm/java-8-oracle"
+#Openjdk PPA Apt source
+openjdk_apt_source: "ppa:openjdk-r/ppa"
 
 # java tuning
 GO_SERVER_JAVA_HOME: "{{ GO_SERVER_ORACLEJDK_LINK }}"
@@ -31,7 +28,7 @@ GO_SERVER_JAVA_HOME: "{{ GO_SERVER_ORACLEJDK_LINK }}"
 GO_SERVER_APT_SOURCE: "deb https://download.gocd.io /"
 GO_SERVER_APT_KEY_URL: "https://download.gocd.io/GOCD-GPG-KEY.asc"
 GO_SERVER_APT_NAME: "go-server"
-GO_SERVER_APT_PKGS: ["apache2-utils"]
+GO_SERVER_APT_PKGS: ["apache2-utils","openjdk-8-jdk"]
 
 # gocd-oauth-login
 GO_SERVER_OAUTH_LOGIN_VERSION: "1.2"