Merge branch 'master' into aed/notifier-fake-migration

CHANGELOG.md

 - Role: notifier
   - Updated notifier Syncdb to do `migrate --fake-initial` instead of `syncdb`.
 
+- Role: ecommerce
+  - This role is now dependent on the edx_django_service role. Settings are all the same, but nearly all of the tasks are performed by the edx_django_service role.
+
 - Role: discovery
   - Added `DISCOVERY_REPOS` to allow configuring discovery repository details.
 
@@ -25,6 +28,9 @@
   - Added `EDXAPP_PLATFORM_DESCRIPTION` used to describe the specific Open edX platform.
 
 - Role: edxapp
+  - Added `EDXAPP_REINDEX_ALL_COURSES` to rebuild the course index on deploy. Disabled by default.
+
+- Role: edxapp
   - Added `ENTERPRISE_SUPPORT_URL` variable used by the LMS.
 
 - Role: edxapp

docker/build/ecommerce/Dockerfile

-# docker build -f docker/build/ecommerce/Dockerfile . -t edxops/ecommerce
+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/ecommerce/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
 FROM edxops/xenial-common:latest
 MAINTAINER edxops
+USER root
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 
 ADD . /edx/app/edx_ansible/edx_ansible
-
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
-RUN echo '{ "allow_root": true }' > /root/.bowerrc
-
-RUN apt-get update
-RUN apt install -y xvfb firefox gettext
-
 COPY docker/build/ecommerce/ansible_overrides.yml /
-RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook ecommerce.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml
-COPY docker/build/ecommerce/docker-run.sh /
+COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook ecommerce.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets,devstack' \
+    --extra-vars="@/ansible_overrides.yml" \
+    --extra-vars="@/devstack/ansible_overrides.yml"
 
-CMD ["/docker-run.sh"]
-EXPOSE 8130
+EXPOSE 18130

docker/build/ecommerce/ansible_overrides.yml

 ---
+COMMON_GIT_PATH: 'edx'
+ECOMMERCE_VERSION: 'master'
 
-DOCKER_TLD: "edx"
+COMMON_MYSQL_MIGRATE_USER: '{{ ECOMMERCE_MYSQL_USER }}'
+COMMON_MYSQL_MIGRATE_PASS: '{{ ECOMMERCE_MYSQL_PASSWORD }}'
 
-ECOMMERCE_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ ECOMMERCE_DEFAULT_DB_NAME }}'
-    USER: 'ecomm001'
-    PASSWORD: 'password'
-    HOST: 'db.{{ DOCKER_TLD  }}'
-    PORT: '3306'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+# NOTE: Theming requires downloading a theme from a separate Git repo. This repo (edx/edx-themes) is private for
+# edX.org. In order to build an image with these themes, you must update COMMON_GIT_IDENTITY to an SSH key with access
+# to the private repo. Otherwise, the sample-themes repository, which has no ecommerce themes, will be downloaded if
+# comprehensive theming is enabled.
+ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false
+#THEMES_GIT_IDENTITY: "{{ COMMON_GIT_IDENTITY }}"
+#THEMES_GIT_PROTOCOL: "ssh"
+#THEMES_GIT_MIRROR: "github.com"
+#THEMES_GIT_PATH: "edx"
+#THEMES_REPO: "edx-themes.git"
 
+ECOMMERCE_DATABASE_HOST: 'edx.devstack.mysql'
+ECOMMERCE_DJANGO_SETTINGS_MODULE: 'ecommerce.settings.devstack'
+ECOMMERCE_GUNICORN_EXTRA: '--reload'
 ECOMMERCE_MEMCACHE: ['edx.devstack.memcached:11211']
+ECOMMERCE_ECOMMERCE_URL_ROOT: 'http://localhost:18130'
+ECOMMERCE_LMS_URL_ROOT: 'http://edx.devstack.lms:18000'
+ECOMMERCE_DISCOVERY_SERVICE_URL: 'http://edx.devstack.discovery:18381'
+
+edx_django_service_is_devstack: true
+
+# NOTE: The creation of demo data requires database access,
+# which we don't have when making new images.
+ecommerce_create_demo_data: false

docker/build/ecommerce/docker-run.sh

-#!/bin/bash
-set -e
-
-/usr/sbin/rsyslogd
-/edx/app/supervisor/venvs/supervisor/bin/supervisord --nodaemon --configuration /edx/app/supervisor/supervisord.conf

docker/plays/discovery.yml

@@ -6,7 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - nginx
-    - role: discovery
+    - role: nginx
       nginx_default_sites:
-        - discovery
\ No newline at end of file
+        - discovery
+    - discovery

docker/plays/ecommerce.yml

@@ -6,6 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - common_vars
-    - docker
+    - role: nginx
+      nginx_default_sites:
+        - ecommerce
     - ecommerce

playbooks/edx-east/ecommerce.yml

@@ -9,8 +9,6 @@
   roles:
     - aws
     - role: nginx
-      nginx_sites:
-        - ecommerce
       nginx_default_sites:
         - ecommerce
     - ecommerce

playbooks/edx-east/edx_continuous_integration.yml

@@ -14,7 +14,6 @@
       - xqueue
       - xserver
       - analytics_api
-      - ecommerce
       - credentials
       nginx_default_sites:
       - lms
@@ -30,7 +29,9 @@
     - oraclejdk
     - elasticsearch
     - forum
-    - { role: notifier, NOTIFIER_DIGEST_TASK_INTERVAL: "5" }
+# Removing until Notifier is fully fixed.
+# Can be uncommented once EDUCATOR-1594 has been resolved.
+#    - { role: notifier, NOTIFIER_DIGEST_TASK_INTERVAL: "5" }
     - { role: "xqueue", update_users: True }
     - role: xserver
       when: XSERVER_GIT_IDENTITY|length > 0

playbooks/edx-east/jenkins_build.yml

@@ -13,6 +13,7 @@
   vars:
     COMMON_ENABLE_DATADOG: True
     COMMON_ENABLE_SPLUNKFORWARDER: True
+    COMMON_ENABLE_NEWRELIC: True
     COMMON_SECURITY_UPDATES: yes
     SECURITY_UPGRADE_ON_ANSIBLE: true
 
@@ -33,7 +34,7 @@
         crcSalt: '<SOURCE>'
         blacklist: '\.gz$'
 
-      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/test_root/log/timing.*.log'
+      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/.../test_root/log/timing.*.log'
         index: 'testeng'
         recursive: true
         sourcetype: 'json_timing_log'
@@ -62,3 +63,8 @@
         - splunkonly
         - jenkins:promote-to-production
       become: True
+
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+      tags:
+        - newreliconly

playbooks/edx-east/jenkins_worker.yml

@@ -24,7 +24,7 @@
     - mysql
     - edxlocal
     - memcache
-    - mongo
+    - mongo_3_2
     - browsers
     - browsermob-proxy
     - jenkins_worker

playbooks/edx-east/veda_delivery_worker.yml

+- name: Deploy edX VEDA delivery Worker
+  hosts: all
+  gather_facts: True
+  roles:
+    - veda_delivery_worker

playbooks/edx-east/veda_encode_worker.yml

+- name: Deploy edX VEDA Encode Worker
+  hosts: all
+  gather_facts: True
+  roles:
+    - veda_encode_worker

playbooks/edx-east/veda_intake_worker.yml

+- name: Deploy edX VEDA Intake Worker
+  hosts: all
+  gather_facts: True
+  roles:
+    - veda_intake_worker

playbooks/edx-east/veda_web_frontend.yml

+- name: Deploy edX Video Pipeline Web Frontend
+  hosts: all
+  gather_facts: True
+  roles:
+    - veda_web_frontend

playbooks/edx_sandbox.yml

@@ -22,6 +22,7 @@
     # Set to false if deployed behind another proxy/load balancer.
     NGINX_SET_X_FORWARDED_HEADERS: True
     DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}'
+    ecommerce_create_demo_data: true
   roles:
     - role: swapfile
       SWAPFILE_SIZE: 4GB
@@ -32,7 +33,6 @@
       - lms
       - forum
       - xqueue
-      - ecommerce
       nginx_default_sites:
       - lms
     - role: edxlocal

playbooks/roles/analytics_api/defaults/main.yml

@@ -26,6 +26,7 @@ ANALYTICS_API_DEFAULT_HOST: 'localhost'
 ANALYTICS_API_DEFAULT_PORT: '3306'
 ANALYTICS_API_DEFAULT_MYSQL_OPTIONS:
   connect_timeout: 10
+  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
 
 ANALYTICS_API_REPORTS_DB_NAME: 'reports'
 ANALYTICS_API_REPORTS_USER: 'reports001'
@@ -34,6 +35,7 @@ ANALYTICS_API_REPORTS_HOST: 'localhost'
 ANALYTICS_API_REPORTS_PORT: '3306'
 ANALYTICS_API_REPORTS_MYSQL_OPTIONS:
   connect_timeout: 10
+  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
 
 ANALYTICS_API_DATABASES:
   # rw user

playbooks/roles/credentials/defaults/main.yml

@@ -27,6 +27,7 @@ CREDENTIALS_MYSQL_PASSWORD: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
 CREDENTIALS_DATABASE_PORT: '3306'
 CREDENTIALS_MYSQL_OPTIONS:
   connect_timeout: 10
+  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
 
 CREDENTIALS_DATABASES:
   # rw user
@@ -120,6 +121,10 @@ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT:
 CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA: []
 CREDENTIALS_CORS_ORIGIN_WHITELIST: "{{ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT + CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA }}"
 
+CREDENTIALS_CERTIFICATE_LANGUAGES:
+  'en': 'English'
+  'es_419': 'Spanish'
+
 CREDENTIALS_VERSION: "master"
 CREDENTIALS_REPOS:
   - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
@@ -184,6 +189,8 @@ CREDENTIALS_SERVICE_CONFIG:
   CORS_ORIGIN_WHITELIST: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST }}'
   CORS_ORIGIN_ALLOW_ALL: '{{ CREDENTIALS_CORS_ORIGIN_ALLOW_ALL }}'
 
+  CERTIFICATE_LANGUAGES: '{{ CREDENTIALS_CERTIFICATE_LANGUAGES }}'
+
 #
 # vars are namespace with the module name.
 #

playbooks/roles/ecommerce/defaults/main.yml

@@ -17,38 +17,22 @@ ECOMMERCE_GIT_IDENTITY: !!null
 # and a key being provided via NEWRELIC_LICENSE_KEY
 ECOMMERCE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ ecommerce_service_name }}"
 ECOMMERCE_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-ECOMMERCE_NGINX_PORT: "18130"
+ECOMMERCE_NGINX_PORT: 18130
 ECOMMERCE_SSL_NGINX_PORT: 48130
 
-ECOMMERCE_MEMCACHE: [ 'localhost:11211' ]
-
-ECOMMERCE_DEFAULT_DB_NAME: 'ecommerce'
-ECOMMERCE_DATABASE_USER: "ecomm001"
-ECOMMERCE_DATABASE_PASSWORD: "password"
-ECOMMERCE_DATABASE_HOST: "localhost"
-ECOMMERCE_DATABASE_PORT: 3306
-ECOMMERCE_MYSQL_OPTIONS:
-  connect_timeout: 10
-
-ECOMMERCE_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ ECOMMERCE_DEFAULT_DB_NAME }}'
-    USER: '{{ ECOMMERCE_DATABASE_USER }}'
-    PASSWORD: '{{ ECOMMERCE_DATABASE_PASSWORD }}'
-    HOST: '{{ ECOMMERCE_DATABASE_HOST }}'
-    PORT: '{{ ECOMMERCE_DATABASE_PORT }}'
-    OPTIONS: '{{ ECOMMERCE_MYSQL_OPTIONS }}'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+ECOMMERCE_MEMCACHE:
+  - localhost:11211
+
+ECOMMERCE_DATABASE_NAME: ecommerce
+ECOMMERCE_DATABASE_USER: ecomm001
+ECOMMERCE_DATABASE_PASSWORD: password
+ECOMMERCE_DATABASE_HOST: localhost
 
 ECOMMERCE_VERSION: "master"
 ECOMMERCE_DJANGO_SETTINGS_MODULE: "ecommerce.settings.production"
 
 ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 ECOMMERCE_SECRET_KEY: 'Your secret key here'
-ECOMMERCE_TIME_ZONE: 'UTC'
 ECOMMERCE_LANGUAGE_CODE: 'en'
 ECOMMERCE_LANGUAGE_COOKIE_NAME: 'openedx-language-preference'
 ECOMMERCE_EDX_API_KEY: 'PUT_YOUR_API_KEY_HERE'  # This should match the value set for edxapp
@@ -69,8 +53,8 @@ ECOMMERCE_JWT_SECRET_KEYS:
   - '{{ COMMON_JWT_SECRET_KEY }}'
 
 # Used to automatically configure OAuth2 Client
-ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY : 'ecommerce-key'
-ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET : 'ecommerce-secret'
+ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY: 'ecommerce-key'
+ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET: 'ecommerce-secret'
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
 
 # Settings for affiliate cookie tracking
@@ -150,11 +134,10 @@ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG:
 # Theming
 ECOMMERCE_PLATFORM_NAME: 'Your Platform Name Here'
 ECOMMERCE_THEME_SCSS: 'sass/themes/default.scss'
+ECOMMERCE_COMPREHENSIVE_THEME_DIRS:
+  - '{{ THEMES_CODE_DIR }}'
+  - '{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}/{{ ecommerce_service_name }}/ecommerce/themes'
 
-# Directory name inside edx-themes repo that contain ecommerce themes
-ECOMMERCE_THEMES_DIR_NAME: 'ecommerce'
-
-ECOMMERCE_COMPREHENSIVE_THEME_DIRS: !!null
 ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false
 ECOMMERCE_DEFAULT_SITE_THEME: !!null
 
@@ -167,27 +150,18 @@ ECOMMERCE_BROKER_HOST: '{{ ansible_default_ipv4.address }}'
 ECOMMERCE_BROKER_PORT: 5672
 ECOMMERCE_BROKER_URL: 'amqp://{{ ECOMMERCE_BROKER_USERNAME }}:{{ ECOMMERCE_BROKER_PASSWORD }}@{{ ECOMMERCE_BROKER_HOST }}:{{ ECOMMERCE_BROKER_PORT }}'
 
-ECOMMERCE_COURSE_CATALOG_URL: 'http://localhost:8008'
+ECOMMERCE_DISCOVERY_SERVICE_URL: 'http://localhost:8008'
 ECOMMERCE_ENTERPRISE_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}'
 
-ECOMMERCE_SERVICE_CONFIG:
-  SESSION_EXPIRE_AT_BROWSER_CLOSE: '{{ ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
-
-  SECRET_KEY: '{{ ECOMMERCE_SECRET_KEY }}'
-  TIME_ZONE: '{{ ECOMMERCE_TIME_ZONE }}'
+ecommerce_service_config_overrides:
   LANGUAGE_COOKIE_NAME: '{{ ECOMMERCE_LANGUAGE_COOKIE_NAME }}'
-  LANGUAGE_CODE: '{{ ECOMMERCE_LANGUAGE_CODE }}'
   EDX_API_KEY: '{{ ECOMMERCE_EDX_API_KEY }}'
   OSCAR_FROM_EMAIL: '{{ ECOMMERCE_OSCAR_FROM_EMAIL }}'
 
-  COURSE_CATALOG_API_URL: '{{ ECOMMERCE_COURSE_CATALOG_URL }}/api/v1/'
   ENTERPRISE_SERVICE_URL: '{{ ECOMMERCE_ENTERPRISE_URL }}/enterprise/'
   ECOMMERCE_URL_ROOT: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}'
-  LMS_URL_ROOT: '{{ ECOMMERCE_LMS_URL_ROOT }}'
-  LMS_HEARTBEAT_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/heartbeat'
-  ENROLLMENT_API_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/api/enrollment/v1/enrollment'
-  COMMERCE_API_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/api/commerce/v1/'
-  LMS_DASHBOARD_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/dashboard'
+
+  # TODO LEARNER-3041: Update this service and ecomworker to only use the central JWT access token issuer
   JWT_AUTH:
     JWT_SECRET_KEY: '{{ COMMON_JWT_SECRET_KEY }}'
     JWT_ALGORITHM: '{{ ECOMMERCE_JWT_ALGORITHM }}'
@@ -196,21 +170,10 @@ ECOMMERCE_SERVICE_CONFIG:
     JWT_DECODE_HANDLER: '{{ ECOMMERCE_JWT_DECODE_HANDLER }}'
     JWT_ISSUERS: '{{ ECOMMERCE_JWT_ISSUERS }}'
     JWT_SECRET_KEYS: '{{ ECOMMERCE_JWT_SECRET_KEYS }}'
-  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ COMMON_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ COMMON_OAUTH_LOGOUT_URL }}'
-  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ COMMON_JWT_ISSUER }}'
-  AFFILIATE_COOKIE_KEY: '{{ ECOMMERCE_AFFILIATE_COOKIE_NAME }}'
 
-  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ ecommerce_service_name }}/staticfiles"
-  # db config
-  DATABASES: '{{ ECOMMERCE_DATABASES }}'
+  AFFILIATE_COOKIE_KEY: '{{ ECOMMERCE_AFFILIATE_COOKIE_NAME }}'
 
   PAYMENT_PROCESSOR_CONFIG: '{{ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG }}'
-  OAUTH2_PROVIDER_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/oauth2'
 
   PLATFORM_NAME: '{{ ECOMMERCE_PLATFORM_NAME }}'
   THEME_SCSS: '{{ ECOMMERCE_THEME_SCSS }}'
@@ -222,26 +185,8 @@ ECOMMERCE_SERVICE_CONFIG:
   ENABLE_COMPREHENSIVE_THEMING: "{{ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING }}"
   DEFAULT_SITE_THEME: "{{ ECOMMERCE_DEFAULT_SITE_THEME }}"
 
-  CACHES:
-    default:
-      BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
-      KEY_PREFIX: 'ecommerce'
-      LOCATION: '{{ ECOMMERCE_MEMCACHE }}'
-
-ECOMMERCE_REPOS:
-  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
-    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
-    PATH: "{{ COMMON_GIT_PATH }}"
-    REPO: ecommerce.git
-    VERSION: "{{ ECOMMERCE_VERSION }}"
-    DESTINATION: "{{ ecommerce_code_dir }}"
-    SSH_KEY: "{{ ECOMMERCE_GIT_IDENTITY }}"
-
-ECOMMERCE_GUNICORN_WORKERS: "2"
+
 ECOMMERCE_GUNICORN_EXTRA: ""
-ECOMMERCE_GUNICORN_EXTRA_CONF: ""
-ECOMMERCE_GUNICORN_WORKER_CLASS: "gevent"
-ECOMMERCE_GUNICORN_MAX_REQUESTS: !!null
 
 #
 # vars are namespace with the module name.
@@ -252,27 +197,12 @@ ecommerce_home: "{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}"
 ecommerce_code_dir: "{{ ecommerce_home }}/{{ ecommerce_service_name }}"
 ecommerce_venv_dir: "{{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}"
 
-ecommerce_nodeenv_dir: "{{ ecommerce_home }}/nodeenvs/{{ ecommerce_service_name }}"
-ecommerce_nodeenv_bin: "{{ ecommerce_nodeenv_dir }}/bin"
-ecommerce_node_modules_dir: "{{ ecommerce_code_dir }}/node_modules"
-ecommerce_node_bin: "{{ ecommerce_node_modules_dir }}/.bin"
-ecommerce_node_version: "{{ common_node_version }}"
-
-ecommerce_gunicorn_host: "127.0.0.1"
 ecommerce_gunicorn_port: "8130"
-ecommerce_gunicorn_timeout: "300"
-
-ecommerce_log_dir: "{{ COMMON_LOG_DIR }}/{{ ecommerce_service_name }}"
-
-ecommerce_requirements_base: "{{ ecommerce_code_dir }}/requirements"
-ecommerce_requirements:
-  - production.txt
-  - optional.txt
 
 ecommerce_environment:
-  DJANGO_SETTINGS_MODULE: "{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}"
   ECOMMERCE_CFG: "{{ COMMON_CFG_DIR }}/{{ ecommerce_service_name }}.yml"
-  PATH: "{{ ecommerce_nodeenv_bin }}:{{ ecommerce_venv_dir }}/bin:{{ ansible_env.PATH }}"
+
+ecommerce_create_demo_data: false
 
 #
 # OS packages
@@ -285,3 +215,11 @@ ecommerce_debian_pkgs:
   - libffi-dev
 
 ecommerce_redhat_pkgs: []
+
+ecommerce_post_migrate_commands:
+  - command: './manage.py oscar_populate_countries --initial-only'
+    when: true
+  - command: './manage.py create_or_update_site --site-id=1 --site-domain={{ ECOMMERCE_ECOMMERCE_URL_ROOT.split("://")[1] }} --partner-code=edX --partner-name="Open edX" --lms-url-root={{ ECOMMERCE_LMS_URL_ROOT }} --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id={{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }} --client-secret={{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }} --from-email staff@example.com --discovery_api_url={{ ECOMMERCE_DISCOVERY_SERVICE_URL }}/api/v1/'
+    when: '{{ ecommerce_create_demo_data }}'
+  - command: './manage.py create_demo_data --partner=edX'
+    when: '{{ ecommerce_create_demo_data }}'

playbooks/roles/ecommerce/meta/main.yml

@@ -11,20 +11,35 @@
 # Role includes for role ecommerce
 #
 dependencies:
-  - common
-  - supervisor
-  - role: edx_service
-    edx_service_name: "{{ ecommerce_service_name }}"
-    edx_service_config: "{{ ECOMMERCE_SERVICE_CONFIG }}"
-    edx_service_repos: "{{ ECOMMERCE_REPOS }}"
-    edx_service_user: "{{ ecommerce_user }}"
-    edx_service_home: "{{ ecommerce_home }}"
-    edx_service_packages:
-      debian: "{{ ecommerce_debian_pkgs }}"
-      redhat: "{{ ecommerce_redhat_pkgs }}"
+  - role: edx_django_service
+    edx_django_service_version: '{{ ECOMMERCE_VERSION }}'
+    edx_django_service_name: '{{ ecommerce_service_name }}'
+    edx_django_service_config_overrides: '{{ ecommerce_service_config_overrides }}'
+    edx_django_service_debian_pkgs_extra: '{{ ecommerce_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ ecommerce_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ ecommerce_environment }}'
+    edx_django_service_gunicorn_extra: '{{ ECOMMERCE_GUNICORN_EXTRA }}'
+    edx_django_service_nginx_port: '{{ ECOMMERCE_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ ECOMMERCE_SSL_NGINX_PORT }}'
+    edx_django_service_use_python3: false
+    edx_django_service_language_code: '{{ ECOMMERCE_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ ECOMMERCE_SECRET_KEY }}'
+    edx_django_service_memcache: '{{ ECOMMERCE_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ ECOMMERCE_DATABASE_HOST }}'
+    edx_django_service_default_db_name: '{{ ECOMMERCE_DATABASE_NAME }}'
+    edx_django_service_default_db_atomic_requests: true
+    edx_django_service_db_user: '{{ ECOMMERCE_DATABASE_USER }}'
+    edx_django_service_db_password: '{{ ECOMMERCE_DATABASE_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_session_expire_at_browser_close: '{{ ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_post_migrate_commands: '{{ ecommerce_post_migrate_commands }}'
+    edx_django_service_basic_auth_exempted_paths_extra:
+      - payment
+      - \.well-known/apple-developer-merchantid-domain-association
   - role: edx_themes
     theme_users:
-      - "{{ ecommerce_user }}"
+      - '{{ ecommerce_user }}'
     when: ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING
-
-  - oraclejdk

playbooks/roles/ecommerce/tasks/main.yml

 ---
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role ecommerce
-#
-# Overview:
-#
-#
-# Dependencies:
-#
-#
-# Example play:
-#
-#
-- name: Add gunicorn configuration file
-  template:
-    src: "edx/app/ecommerce/ecommerce_gunicorn.py.j2"
-    dest: "{{ ecommerce_home }}/ecommerce_gunicorn.py"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-      - install
-      - install:configuration
-
-- name: Install application requirements
-  pip:
-    requirements: "{{ ecommerce_requirements_base }}/{{ item }}"
-    virtualenv: "{{ ecommerce_venv_dir }}"
-    state: present
-  become_user: "{{ ecommerce_user }}"
-  with_items: "{{ ecommerce_requirements }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Create nodeenv
-  shell: "{{ ecommerce_venv_dir }}/bin/nodeenv {{ ecommerce_nodeenv_dir }} --node={{ ecommerce_node_version }} --prebuilt --force"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: Install node dependencies
-  npm:
-    executable: "{{ ecommerce_nodeenv_bin }}/npm"
-    path: "{{ ecommerce_code_dir }}"
-    production: yes
-    state: latest
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Install bower dependencies
-  shell: ". {{ ecommerce_nodeenv_bin }}/activate && {{ ecommerce_node_bin }}/bower install --production --config.interactive=false"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Migrate
-  shell: >
-    DB_MIGRATION_USER='{{ COMMON_MYSQL_MIGRATE_USER }}'
-    DB_MIGRATION_PASS='{{ COMMON_MYSQL_MIGRATE_PASS }}'
-    {{ ecommerce_venv_dir }}/bin/python ./manage.py migrate --noinput
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: Populate countries
-  shell: "DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }} DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }} {{ ecommerce_venv_dir }}/bin/python ./manage.py oscar_populate_countries --initial-only"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: compile sass
-  shell: "{{ ecommerce_venv_dir }}/bin/python manage.py {{ item }}"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  with_items:
-    - "update_assets --skip-collect"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Run r.js optimizer
-  shell: ". {{ ecommerce_nodeenv_bin }}/activate && {{ ecommerce_node_bin }}/r.js -o build.js"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Run collectstatic
-  shell: "{{ ecommerce_venv_dir }}/bin/python manage.py {{ item }}"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  with_items:
-    - "collectstatic --noinput"
-    - "compress"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Write out the supervisor wrapper
-  template:
-    src: "edx/app/ecommerce/ecommerce.sh.j2"
-    dest: "{{ ecommerce_home }}/{{ ecommerce_service_name }}.sh"
-    mode: "0650"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: Write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/ecommerce.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ ecommerce_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: "0644"
-  tags:
-    - install
-    - install:configuration
-
 - name: Create Apple Pay certificates directory
   file:
     path: "{{ ecommerce_apple_pay_merchant_certificate_directory }}"
@@ -173,68 +20,3 @@
   tags:
     - install
     - install:configuration
-
-- name: Setup the ecommence env file
-  template:
-    src: "./{{ ecommerce_home }}/{{ ecommerce_service_name }}_env.j2"
-    dest: "{{ ecommerce_home }}/ecommerce_env"
-    owner: "{{ ecommerce_user }}"
-    group: "{{ ecommerce_user }}"
-    mode: "0644"
-  tags:
-    - install
-    - install:configuration
-
-- name: Enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ ecommerce_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ ecommerce_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-  tags:
-    - install
-    - install:configuration
-
-- name: Update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-  tags:
-    - manage
-    - manage:start
-
-- name: Create symlinks from the-er venv bin dir
-  file:
-    src: "{{ ecommerce_venv_dir }}/bin/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.', 1) | first }}.ecommerce"
-    state: link
-  with_items:
-    - python
-    - pip
-    - django-admin.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Create symlinks from the repo dir
-  file:
-    src: "{{ ecommerce_code_dir }}/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.', 1) | first }}.ecommerce"
-    state: link
-  with_items:
-    - manage.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Restart the applicaton
-  supervisorctl:
-    name: "{{ ecommerce_service_name }}"
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
-  tags:
-    - manage
-    - manage:start

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% set ecommerce_venv_bin = ecommerce_home + "/venvs/" + ecommerce_service_name + "/bin" %}
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = ecommerce_venv_bin + '/newrelic-admin run-program ' + ecommerce_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = ecommerce_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-export NEW_RELIC_APP_NAME="{{ ECOMMERCE_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-source {{ ecommerce_home }}/ecommerce_env
-{{ executable }} -c {{ ecommerce_home }}/ecommerce_gunicorn.py {{ ECOMMERCE_GUNICORN_EXTRA }} ecommerce.wsgi:application

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in ecommerce_environment.items() -%}
-{%- if value -%}
-export {{ name }}="{{ value }}"
-{% endif %}
-{%- endfor %}

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_gunicorn.py.j2

-"""
-gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
-{{ ansible_managed }}
-"""
-
-timeout = {{ ecommerce_gunicorn_timeout }}
-bind = "{{ ecommerce_gunicorn_host }}:{{ ecommerce_gunicorn_port }}"
-pythonpath = "{{ ecommerce_code_dir }}"
-workers = {{ ECOMMERCE_GUNICORN_WORKERS }}
-worker_class = "{{ ECOMMERCE_GUNICORN_WORKER_CLASS }}"
-
-{% if ECOMMERCE_GUNICORN_MAX_REQUESTS %}
-max_requests = {{ ECOMMERCE_GUNICORN_MAX_REQUESTS }}
-{% endif %}
-
-{{ ECOMMERCE_GUNICORN_EXTRA_CONF }}

playbooks/roles/ecommerce/templates/edx/app/supervisor/conf.d.available/ecommerce.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ ecommerce_service_name }}]
-
-command={{ ecommerce_home }}/{{ ecommerce_service_name }}.sh
-user={{ common_web_user }}
-directory={{ ecommerce_code_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/edx_django_service/defaults/main.yml

@@ -116,6 +116,7 @@ edx_django_service_db_user: 'REPLACE-ME'
 edx_django_service_db_password: 'password'
 edx_django_service_db_options:
   connect_timeout: 10
+  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
 
 edx_django_service_databases:
   default:
@@ -194,3 +195,22 @@ edx_django_service_automated_users:
         sudo_user: '{{ edx_django_service_user }}'
     authorized_keys:
       - 'SSH authorized key'
+
+# This array contains commands that should be run after migration.
+#
+# The commands will be executed from the code directory with the application's virtualenv activated. The migration
+# environment (e.g. migration DB username/password) will NOT be used, so commands should not rely on these values being
+# set. In other words, don't try to sneak in another run of the migrate management command.
+#
+# Example:
+#   edx_django_service_post_migrate_management_commands:
+#     - command: './manage.py conditional_command'
+#       when: '{{ foo }}'
+#     - command: './manage.py always_command'
+#       when: True
+#
+# In this example, the "conditional_command" will only be run when the variable `foo` is set to `True`. The
+# "always_command" will always be run because its conditional is set to `True`. To minimize surprises, the `when`
+# key *MUST* be supplied for all commands.
+#
+edx_django_service_post_migrate_commands: []

playbooks/roles/edx_django_service/tasks/main.yml

@@ -115,6 +115,30 @@
     - migrate
     - migrate:db
 
+- name: run post-migrate commands
+  command: "{{ item.command }}"
+  args:
+    chdir: "{{ edx_django_service_code_dir }}"
+  become_user: "{{ edx_django_service_user }}"
+  environment: "{{ edx_django_service_environment }}"
+  with_items: '{{ edx_django_service_post_migrate_commands }}'
+  when: migrate_db is defined and migrate_db|lower == "yes" and item.when | bool
+  tags:
+    - migrate
+    - migrate:db
+    - migrate:post
+
+- name: ensure log files exist for tailing
+  file:
+    path: "{{ item }}"
+    state: touch
+    owner: "{{ common_web_user }}"
+    group: "{{ common_web_user }}"
+  with_items: '{{ edx_django_service_name_devstack_logs }}'
+  tags:
+    - install
+    - install:configuration
+
 - name: write out the supervisor wrapper
   template:
     src: "edx/app/app/app.sh.j2"

playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2

@@ -68,8 +68,6 @@ server {
     try_files $uri @proxy_to_app;
   }
 
-  # API endpoints have their own authentication and authorization
-  # schemes, so we bypass basic auth.
   location ~ ^/({{ edx_django_service_basic_auth_exempted_paths | join('|') }})/ {
     try_files $uri @proxy_to_app;
   }

playbooks/roles/edxapp/defaults/main.yml

@@ -465,6 +465,9 @@ EDXAPP_COURSES_WITH_UNSAFE_CODE: []
 EDXAPP_SESSION_COOKIE_DOMAIN: ""
 EDXAPP_SESSION_COOKIE_NAME: "sessionid"
 
+# Whether to run reindex_course on deploy
+EDXAPP_REINDEX_ALL_COURSES: false
+
 # XML Course related flags
 EDXAPP_XML_FROM_GIT: false
 EDXAPP_XML_S3_BUCKET: !!null

playbooks/roles/edxapp/tasks/deploy.yml

@@ -428,3 +428,13 @@
   tags:
     - manage
     - manage:db
+
+- name: reindex all courses
+  shell: "{{ edxapp_venv_bin }}/python ./manage.py cms reindex_course --setup --settings={{ edxapp_settings }}"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ common_web_user }}"
+  when: EDXAPP_REINDEX_ALL_COURSES
+  tags:
+    - install
+    - install:base

playbooks/roles/edxlocal/defaults/main.yml

@@ -4,7 +4,7 @@ edxlocal_debian_pkgs:
   - libjpeg-dev
 
 edxlocal_databases:
-  - "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ ECOMMERCE_DATABASE_NAME | default(None) }}"
   - "{{ INSIGHTS_DATABASE_NAME | default(None) }}"
   - "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}"
   - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}"
@@ -17,7 +17,7 @@ edxlocal_databases:
 
 edxlocal_database_users:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
       user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
       pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
     }

playbooks/roles/git_clone/tasks/main.yml

@@ -96,3 +96,11 @@
   tags:
     - install
     - install:code
+
+- name: Run git clean after checking out code
+  shell: cd {{ item.DESTINATION }} && git clean -xdf
+  become: true
+  with_items: "{{ GIT_REPOS }}"
+  tags:
+    - install
+    - install:code

playbooks/roles/insights/defaults/main.yml

@@ -65,6 +65,7 @@ INSIGHTS_DATABASE_HOST: 127.0.0.1
 INSIGHTS_DATABASE_PORT: 3306
 INSIGHTS_MYSQL_OPTIONS:
   connect_timeout: 10
+  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
 
 INSIGHTS_DATABASES:
   # rw user

playbooks/roles/jenkins_admin/defaults/main.yml

@@ -70,8 +70,6 @@ jenkins_admin_debian_pkgs:
   # Needed by the CMS to manipulate images.
   - libjpeg8-dev
   - libpng12-dev
-  # for status.edx.org
-  - ruby
   # for check-migrations
   - mysql-client
   # for aws cli scripting
@@ -80,10 +78,6 @@ jenkins_admin_debian_pkgs:
   # Need by python script that check SSL expiration
   - libffi-dev
 
-jenkins_admin_gem_pkgs:
-  # for generating status.edx.org
-  - { name: sass, version: "3.2.4" }
-
 jenkins_admin_redhat_pkgs: []
 
 jenkins_admin_plugins: [] # Plugins installed manually, not tracked here.

playbooks/roles/jenkins_admin/tasks/main.yml

@@ -135,14 +135,6 @@
     mode: 0440
     validate: "visudo -cf %s"
 
-- name: install global gem dependencies
-  gem:
-    name: "{{ item.name }}"
-    state: present
-    version: "{{ item.version }}"
-    user_install: no
-  with_items: "{{ jenkins_admin_gem_pkgs }}"
-
 - name: get s3 one time url
   s3:
     bucket: "{{ JENKINS_ADMIN_BACKUP_BUCKET }}"

playbooks/roles/jenkins_common/tasks/main.yml

@@ -50,6 +50,21 @@
     - install
     - install:system-requirements
 
+- name: Delete any existing jenkins-configuration folders to avoid unwanted configuration
+  file:
+    path: '{{ item }}'
+    owner: '{{ jenkins_common_user }}'
+    group: '{{ jenkins_common_group }}'
+    state: absent
+  with_items:
+    - '{{ jenkins_common_home }}/init.groovy.d'
+    - '{{ jenkins_common_config_path }}'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+    - jenkins:local-dev
+
 - name: Create necessary folders
   file:
     path: '{{ item }}'
@@ -68,6 +83,8 @@
   tags:
     - install
     - install:base
+    - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Download Jenkins war file
   get_url:
@@ -118,6 +135,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Run gradle libs
   shell: './gradlew libs'
@@ -132,17 +150,28 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy init scripts into init.groovy.d
   command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/'
   with_items: '{{ jenkins_common_configuration_scripts }}'
   become: true
   become_user: '{{ jenkins_common_user }}'
+  register: init_scripts_copied
   tags:
     - install
     - install:base
     - install:jenkins-configuration
 
+- name: Copy all init scripts other than oauth for local dev
+  command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/'
+  with_items: '{{ jenkins_common_configuration_scripts }}'
+  become: true
+  become_user: '{{ jenkins_common_user }}'
+  when: 'item != "4configureGHOAuth.groovy" and init_scripts_copied is not defined'
+  tags:
+    - jenkins:local-dev
+
 - name: Create jenkins config sub folders
   file:
     path: '{{ item }}'
@@ -156,6 +185,8 @@
   tags:
      - install
      - install:base
+     - install:jenkins-configuration
+     - jenkins:local-dev
 
 - name: Copy non plugins template files
   template:
@@ -170,6 +201,17 @@
     - install:base
     - install:jenkins-configuration
 
+- name: For local dev, copy any config files other than oauth
+  template:
+    src: '{{ role_path }}/templates/config/{{ item }}.yml.j2'
+    dest: '{{ jenkins_common_config_path }}/{{ item }}.yml'
+    owner: '{{ jenkins_common_user }}'
+    group: '{{ jenkins_common_group }}'
+  with_items: '{{ jenkins_common_non_plugin_template_files }}'
+  when: 'item != "security" and templates_copied is not defined'
+  tags:
+    - jenkins:local-dev
+
 - name: Update Github OAUTH settings when promoting jenkins instance to production
   template:
     src: '{{ role_path }}/templates/config/security.yml.j2'
@@ -191,6 +233,7 @@
     - install:base
     - install:plugins
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy ec2 config files
   template:
@@ -204,6 +247,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy xml config files
   template:
@@ -217,6 +261,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Run plugins.gradle
   shell: './gradlew -b plugins.gradle plugins'
@@ -232,6 +277,7 @@
     - install:base
     - install:plugins
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy secret file credentials
   copy:
@@ -243,6 +289,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy ssh key credentials
   copy:
@@ -256,6 +303,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Copy ec2 key
   copy:
@@ -268,6 +316,7 @@
     - install
     - install:base
     - install:jenkins-configuration
+    - jenkins:local-dev
 
 - name: Start Jenkins Service
   systemd:

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -11,6 +11,9 @@ jenkins_debian_pkgs:
     - libffi-dev
     - python-dev
     - libsqlite3-dev
+    - libfreetype6-dev
 
 # packer direct download URL
 packer_url: "https://releases.hashicorp.com/packer/0.8.6/packer_0.8.6_linux_amd64.zip"
+
+jenkins_worker_key_url: null

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -16,8 +16,12 @@
     owner={{ jenkins_user }} group={{ jenkins_group }}
   ignore_errors: yes
 
-- name: Copy ssh keys for jenkins
-  command: cp /home/ubuntu/.ssh/authorized_keys /home/{{ jenkins_user }}/.ssh/authorized_keys
+- name: Get the authorized key that should be used for this machine.
+  authorized_key:
+    user: "{{ jenkins_user }}"
+    state: present
+    key: "{{ jenkins_worker_key_url }}"
+  when: jenkins_worker_key_url
   ignore_errors: yes
 
 - name: Set key permissions

playbooks/roles/mariadb/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role mariadb
-# 
+#
 MARIADB_APT_KEY_XENIAL_ID: '0xF1656F24C74CD1D8'
 MARIADB_APT_KEY_ID: '0xcbcb082a1bb943db'
 
@@ -23,7 +23,7 @@ MARIADB_CLUSTER_PASSWORD_ADMIN: "password"
 MARIADB_HOST_PRIV: '%'
 
 MARIADB_HAPROXY_USER: 'haproxy'
-MARIADB_HAPROXY_HOSTS: 
+MARIADB_HAPROXY_HOSTS:
   - '192.168.33.100'
   - '192.168.33.110'
   - '192.168.33.120'
@@ -32,57 +32,57 @@ MARIADB_LISTEN_ALL: false
 
 MARIADB_DATABASES:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
+      encoding: "utf8"
     }
   - {
       db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
-      encoding: "latin1" 
+      encoding: "latin1"
     }
 
 MARIADB_USERS:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
       user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
       pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
     }

playbooks/roles/mongo_3_2/defaults/main.yml

@@ -26,6 +26,9 @@ mongodb_debian_pkgs:
   - "mongodb-org-tools={{ mongo_version }}"
 
 
+
+mongo_configure_replica_set: true
+
 # Vars Meant to be overridden
 MONGO_ADMIN_USER: 'admin'
 MONGO_ADMIN_PASSWORD: 'password'

playbooks/roles/mongo_3_2/tasks/main.yml

@@ -280,6 +280,7 @@
     rs_config: "{{ MONGO_RS_CONFIG }}"
   run_once: true
   register: replset_status
+  when: mongo_configure_replica_set
   tags:
     - "manage"
     - "manage:db"
@@ -297,6 +298,7 @@
       password: "{{ MONGO_ADMIN_PASSWORD }}"
   register: status
   until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list
+  when: mongo_configure_replica_set
   retries: 5
   delay: 2
   run_once: true
@@ -318,6 +320,7 @@
     replica_set: "{{ MONGO_REPL_SET }}"
   with_items: "{{ MONGO_USERS }}"
   run_once: true
+  when: mongo_configure_replica_set
   tags:
     - "manage"
     - "manage:db"

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/ecommerce.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if "ecommerce" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream ecommerce_app_server {
-{% for host in nginx_ecommerce_gunicorn_hosts %}
-    server {{ host }}:{{ ecommerce_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-server {
-  server_name {{ ECOMMERCE_HOSTNAME }};
-
-  listen {{ ECOMMERCE_NGINX_PORT }} {{ default_site }};
-
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ ECOMMERCE_SSL_NGINX_PORT }} ssl;
-
-  {% include "common-settings.j2" %}
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-  {% endif %}
-
-  # Prevent invalid display courseware in IE 10+ with high privacy settings
-  add_header P3P '{{ NGINX_P3P_MESSAGE }}';
-
-  # Nginx does not support nested condition or or conditions so
-  # there is an unfortunate mix of conditonals here.
-  {% if NGINX_REDIRECT_TO_HTTPS %}
-     {% if NGINX_HTTPS_REDIRECT_STRATEGY == "scheme" %}
-  # Redirect http to https over single instance
-  if ($scheme != "https")
-  {
-   set $do_redirect_to_https "true";
-  }
-
-     {% elif NGINX_HTTPS_REDIRECT_STRATEGY == "forward_for_proto" %}
-  # Forward to HTTPS if we're an HTTP request... and the server is behind ELB
-  if ($http_x_forwarded_proto = "http")
-  {
-   set $do_redirect_to_https "true";
-  }
-     {% endif %}
-  # Execute the actual redirect
-  if ($do_redirect_to_https = "true")
-  {
-  return 301 https://$host$request_uri;
-  }
-  {% endif %}
-
-  location ~ ^/static/(?P<file>.*) {
-    root {{ COMMON_DATA_DIR }}/{{ ecommerce_service_name }};
-    try_files /staticfiles/$file =404;
-  }
-
-  location / {
-    {% if ECOMMERCE_ENABLE_BASIC_AUTH|bool %}
-      {% include "basic-auth.j2" %}
-    {% endif %}
-    try_files $uri @proxy_to_app;
-  }
-
-  # The API should be secured with OAuth 2.0 or or JWT.
-  location /api {
-    try_files $uri @proxy_to_app;
-  }
-
-  # Allow access to this API for POST back from payment processors.
-  location /payment {
-    try_files $uri @proxy_to_app;
-  }
-
-  # Allow access for Apple Pay domain validation
-  location /.well-known/apple-developer-merchantid-domain-association {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% include "robots.j2" %}
-
-location @proxy_to_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
-
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://ecommerce_app_server;
-  }
-
-}
-

playbooks/roles/veda_delivery_worker/defaults/main.yml

+---

playbooks/roles/veda_delivery_worker/meta/main.yml

+---
+
+dependencies:
+  - video_pipeline_base

playbooks/roles/veda_delivery_worker/tasks/main.yml

+---

playbooks/roles/veda_encode_worker/defaults/main.yml

+---

playbooks/roles/veda_encode_worker/meta/main.yml

+---

playbooks/roles/veda_encode_worker/tasks/main.yml

+---

playbooks/roles/veda_intake_worker/defaults/main.yml

+---

playbooks/roles/veda_intake_worker/meta/main.yml

+---
+
+dependencies:
+  - video_pipeline_base

playbooks/roles/veda_intake_worker/tasks/main.yml

+---

playbooks/roles/veda_web_frontend/defaults/main.yml

+---

playbooks/roles/veda_web_frontend/meta/main.yml

+---
+
+dependencies:
+  - video_pipeline_base

playbooks/roles/veda_web_frontend/tasks/main.yml

+---

playbooks/roles/video_pipeline_base/defaults/main.yml

+---

playbooks/roles/video_pipeline_base/meta/main.yml

+---

playbooks/roles/video_pipeline_base/tasks/main.yml

+---

util/jenkins/ansible-provision.sh

@@ -127,9 +127,9 @@ fi
 
 if [[ -z $ami ]]; then
   if [[ $server_type == "full_edx_installation" ]]; then
-    ami="ami-dd9d81a6"
+    ami="ami-8609a6fc"
   elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
-    ami="ami-1d4e7a66"
+    ami="ami-da05a4a0"
   fi
 fi
 
@@ -288,9 +288,12 @@ EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
 EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
 EDXAPP_COURSE_CATALOG_API_URL: "https://catalog-${deploy_host}/api/v1"
 
+# NOTE: This is the same as DISCOVERY_URL_ROOT below
+ECOMMERCE_DISCOVERY_SERVICE_URL: "https://discovery-${deploy_host}"
 ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
 ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
+ecommerce_create_demo_data: true
 
 CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
 CREDENTIALS_DOMAIN: "credentials-${deploy_host}"

util/packer/jenkins_worker.json

@@ -9,7 +9,8 @@
     "test_platform_version": "{{env `TEST_PLATFORM_VERSION`}}",
     "security_group": "{{env `AWS_SECURITY_GROUP`}}",
     "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}",
-    "remote_branch": "{{env `REMOTE_BRANCH`}}"
+    "remote_branch": "{{env `REMOTE_BRANCH`}}",
+    "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}"
   },
   "builders": [{
     "type": "amazon-ebs",
@@ -25,7 +26,13 @@
     "security_group_id": "{{user `security_group`}}",
     "tags": {
       "delete_or_keep": "{{user `delete_or_keep`}}"
-    }
+    },
+    "launch_block_device_mappings": [{
+      "delete_on_termination": true,
+      "device_name": "/dev/sda1",
+      "volume_size": "40",
+      "volume_type": "gp2"
+    }]
   }],
   "provisioners": [{
     "type": "shell",
@@ -52,7 +59,7 @@
     "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook",
     "inventory_groups": "jenkins_worker",
     "extra_arguments": [
-      "-e \"jenkins_edx_platform_version={{user `test_platform_version`}} NEWRELIC_LICENSE_KEY={{user `new_relic_key`}}\"",
+      "-e \"jenkins_edx_platform_version={{user `test_platform_version`}} NEWRELIC_LICENSE_KEY={{user `new_relic_key`}} initialize_replica_set=false mongo_configure_replica_set=false jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}'\"",
       "-vvv"
       ]
   }, {