Updated ecommerce play to use edx_django_service play

- This removes the duplication across the various IDA configurations
- Updated Dockerfile for Docker-based devstack

LEARNER-818

docker/build/ecommerce/Dockerfile

-# docker build -f docker/build/ecommerce/Dockerfile . -t edxops/ecommerce
+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/ecommerce/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
 FROM edxops/xenial-common:latest
 MAINTAINER edxops
+USER root
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 
 ADD . /edx/app/edx_ansible/edx_ansible
-
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
-RUN echo '{ "allow_root": true }' > /root/.bowerrc
-
-RUN apt-get update
-RUN apt install -y xvfb firefox gettext
-
 COPY docker/build/ecommerce/ansible_overrides.yml /
-RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook ecommerce.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml
-COPY docker/build/ecommerce/docker-run.sh /
+COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook ecommerce.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets,devstack' \
+    --extra-vars="@/ansible_overrides.yml" \
+    --extra-vars="@/devstack/ansible_overrides.yml"
 
-CMD ["/docker-run.sh"]
-EXPOSE 8130
+EXPOSE 18130

docker/build/ecommerce/ansible_overrides.yml

 ---
+COMMON_GIT_PATH: 'edx'
+ECOMMERCE_VERSION: 'master'
 
-DOCKER_TLD: "edx"
+COMMON_MYSQL_MIGRATE_USER: '{{ ECOMMERCE_MYSQL_USER }}'
+COMMON_MYSQL_MIGRATE_PASS: '{{ ECOMMERCE_MYSQL_PASSWORD }}'
 
-ECOMMERCE_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ ECOMMERCE_DEFAULT_DB_NAME }}'
-    USER: 'ecomm001'
-    PASSWORD: 'password'
-    HOST: 'db.{{ DOCKER_TLD  }}'
-    PORT: '3306'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+# NOTE: Theming requires downloading a theme from a separate Git repo. This repo (edx/edx-themes) is private for
+# edX.org. In order to build an image with these themes, you must update COMMON_GIT_IDENTITY to an SSH key with access
+# to the private repo. Otherwise, the sample-themes repository, which has no ecommerce themes, will be downloaded if
+# comprehensive theming is enabled.
+ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false
+#THEMES_GIT_IDENTITY: "{{ COMMON_GIT_IDENTITY }}"
+#THEMES_GIT_PROTOCOL: "ssh"
+#THEMES_GIT_MIRROR: "github.com"
+#THEMES_GIT_PATH: "edx"
+#THEMES_REPO: "edx-themes.git"
 
+ECOMMERCE_DATABASE_HOST: 'edx.devstack.mysql'
+ECOMMERCE_DJANGO_SETTINGS_MODULE: 'ecommerce.settings.devstack'
+ECOMMERCE_GUNICORN_EXTRA: '--reload'
 ECOMMERCE_MEMCACHE: ['edx.devstack.memcached:11211']
+ECOMMERCE_ECOMMERCE_URL_ROOT: 'http://localhost:18130'
+
+edx_django_service_is_devstack: true

docker/build/ecommerce/docker-run.sh

-#!/bin/bash
-set -e
-
-/usr/sbin/rsyslogd
-/edx/app/supervisor/venvs/supervisor/bin/supervisord --nodaemon --configuration /edx/app/supervisor/supervisord.conf

docker/plays/discovery.yml

@@ -6,7 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - nginx
-    - role: discovery
+    - role: nginx
       nginx_default_sites:
-        - discovery
\ No newline at end of file
+        - discovery
+    - discovery

docker/plays/ecommerce.yml

@@ -6,6 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - common_vars
-    - docker
+    - role: nginx
+      nginx_default_sites:
+        - ecommerce
     - ecommerce

playbooks/edx-east/ecommerce.yml

@@ -9,8 +9,6 @@
   roles:
     - aws
     - role: nginx
-      nginx_sites:
-        - ecommerce
       nginx_default_sites:
         - ecommerce
     - ecommerce

playbooks/edx-east/edx_continuous_integration.yml

@@ -14,7 +14,6 @@
       - xqueue
       - xserver
       - analytics_api
-      - ecommerce
       - credentials
       nginx_default_sites:
       - lms

playbooks/roles/ecommerce/defaults/main.yml

@@ -17,39 +17,22 @@ ECOMMERCE_GIT_IDENTITY: !!null
 # and a key being provided via NEWRELIC_LICENSE_KEY
 ECOMMERCE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ ecommerce_service_name }}"
 ECOMMERCE_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-ECOMMERCE_NGINX_PORT: "18130"
+ECOMMERCE_NGINX_PORT: 18130
 ECOMMERCE_SSL_NGINX_PORT: 48130
 
-ECOMMERCE_MEMCACHE: [ 'localhost:11211' ]
-
-ECOMMERCE_DEFAULT_DB_NAME: 'ecommerce'
-ECOMMERCE_DATABASE_USER: "ecomm001"
-ECOMMERCE_DATABASE_PASSWORD: "password"
-ECOMMERCE_DATABASE_HOST: "localhost"
-ECOMMERCE_DATABASE_PORT: 3306
-ECOMMERCE_MYSQL_OPTIONS:
-  connect_timeout: 10
-  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
-
-ECOMMERCE_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ ECOMMERCE_DEFAULT_DB_NAME }}'
-    USER: '{{ ECOMMERCE_DATABASE_USER }}'
-    PASSWORD: '{{ ECOMMERCE_DATABASE_PASSWORD }}'
-    HOST: '{{ ECOMMERCE_DATABASE_HOST }}'
-    PORT: '{{ ECOMMERCE_DATABASE_PORT }}'
-    OPTIONS: '{{ ECOMMERCE_MYSQL_OPTIONS }}'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+ECOMMERCE_MEMCACHE:
+  - localhost:11211
+
+ECOMMERCE_DATABASE_NAME: ecommerce
+ECOMMERCE_DATABASE_USER: ecomm001
+ECOMMERCE_DATABASE_PASSWORD: password
+ECOMMERCE_DATABASE_HOST: localhost
 
 ECOMMERCE_VERSION: "master"
 ECOMMERCE_DJANGO_SETTINGS_MODULE: "ecommerce.settings.production"
 
 ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 ECOMMERCE_SECRET_KEY: 'Your secret key here'
-ECOMMERCE_TIME_ZONE: 'UTC'
 ECOMMERCE_LANGUAGE_CODE: 'en'
 ECOMMERCE_LANGUAGE_COOKIE_NAME: 'openedx-language-preference'
 ECOMMERCE_EDX_API_KEY: 'PUT_YOUR_API_KEY_HERE'  # This should match the value set for edxapp
@@ -70,8 +53,8 @@ ECOMMERCE_JWT_SECRET_KEYS:
   - '{{ COMMON_JWT_SECRET_KEY }}'
 
 # Used to automatically configure OAuth2 Client
-ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY : 'ecommerce-key'
-ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET : 'ecommerce-secret'
+ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY: 'ecommerce-key'
+ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET: 'ecommerce-secret'
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
 
 # Settings for affiliate cookie tracking
@@ -151,11 +134,10 @@ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG:
 # Theming
 ECOMMERCE_PLATFORM_NAME: 'Your Platform Name Here'
 ECOMMERCE_THEME_SCSS: 'sass/themes/default.scss'
+ECOMMERCE_COMPREHENSIVE_THEME_DIRS:
+  - '{{ THEMES_CODE_DIR }}'
+  - '{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}/{{ ecommerce_service_name }}/ecommerce/themes'
 
-# Directory name inside edx-themes repo that contain ecommerce themes
-ECOMMERCE_THEMES_DIR_NAME: 'ecommerce'
-
-ECOMMERCE_COMPREHENSIVE_THEME_DIRS: !!null
 ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false
 ECOMMERCE_DEFAULT_SITE_THEME: !!null
 
@@ -168,27 +150,17 @@ ECOMMERCE_BROKER_HOST: '{{ ansible_default_ipv4.address }}'
 ECOMMERCE_BROKER_PORT: 5672
 ECOMMERCE_BROKER_URL: 'amqp://{{ ECOMMERCE_BROKER_USERNAME }}:{{ ECOMMERCE_BROKER_PASSWORD }}@{{ ECOMMERCE_BROKER_HOST }}:{{ ECOMMERCE_BROKER_PORT }}'
 
-ECOMMERCE_COURSE_CATALOG_URL: 'http://localhost:8008'
 ECOMMERCE_ENTERPRISE_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}'
 
-ECOMMERCE_SERVICE_CONFIG:
-  SESSION_EXPIRE_AT_BROWSER_CLOSE: '{{ ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
-
-  SECRET_KEY: '{{ ECOMMERCE_SECRET_KEY }}'
-  TIME_ZONE: '{{ ECOMMERCE_TIME_ZONE }}'
+ecommerce_service_config_overrides:
   LANGUAGE_COOKIE_NAME: '{{ ECOMMERCE_LANGUAGE_COOKIE_NAME }}'
-  LANGUAGE_CODE: '{{ ECOMMERCE_LANGUAGE_CODE }}'
   EDX_API_KEY: '{{ ECOMMERCE_EDX_API_KEY }}'
   OSCAR_FROM_EMAIL: '{{ ECOMMERCE_OSCAR_FROM_EMAIL }}'
 
-  COURSE_CATALOG_API_URL: '{{ ECOMMERCE_COURSE_CATALOG_URL }}/api/v1/'
   ENTERPRISE_SERVICE_URL: '{{ ECOMMERCE_ENTERPRISE_URL }}/enterprise/'
   ECOMMERCE_URL_ROOT: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}'
-  LMS_URL_ROOT: '{{ ECOMMERCE_LMS_URL_ROOT }}'
-  LMS_HEARTBEAT_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/heartbeat'
-  ENROLLMENT_API_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/api/enrollment/v1/enrollment'
-  COMMERCE_API_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/api/commerce/v1/'
-  LMS_DASHBOARD_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/dashboard'
+
+  # TODO LEARNER-3041: Update this service and ecomworker to only use the central JWT access token issuer
   JWT_AUTH:
     JWT_SECRET_KEY: '{{ COMMON_JWT_SECRET_KEY }}'
     JWT_ALGORITHM: '{{ ECOMMERCE_JWT_ALGORITHM }}'
@@ -197,21 +169,10 @@ ECOMMERCE_SERVICE_CONFIG:
     JWT_DECODE_HANDLER: '{{ ECOMMERCE_JWT_DECODE_HANDLER }}'
     JWT_ISSUERS: '{{ ECOMMERCE_JWT_ISSUERS }}'
     JWT_SECRET_KEYS: '{{ ECOMMERCE_JWT_SECRET_KEYS }}'
-  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ COMMON_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ COMMON_OAUTH_LOGOUT_URL }}'
-  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ COMMON_JWT_ISSUER }}'
-  AFFILIATE_COOKIE_KEY: '{{ ECOMMERCE_AFFILIATE_COOKIE_NAME }}'
 
-  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ ecommerce_service_name }}/staticfiles"
-  # db config
-  DATABASES: '{{ ECOMMERCE_DATABASES }}'
+  AFFILIATE_COOKIE_KEY: '{{ ECOMMERCE_AFFILIATE_COOKIE_NAME }}'
 
   PAYMENT_PROCESSOR_CONFIG: '{{ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG }}'
-  OAUTH2_PROVIDER_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/oauth2'
 
   PLATFORM_NAME: '{{ ECOMMERCE_PLATFORM_NAME }}'
   THEME_SCSS: '{{ ECOMMERCE_THEME_SCSS }}'
@@ -223,26 +184,8 @@ ECOMMERCE_SERVICE_CONFIG:
   ENABLE_COMPREHENSIVE_THEMING: "{{ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING }}"
   DEFAULT_SITE_THEME: "{{ ECOMMERCE_DEFAULT_SITE_THEME }}"
 
-  CACHES:
-    default:
-      BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
-      KEY_PREFIX: 'ecommerce'
-      LOCATION: '{{ ECOMMERCE_MEMCACHE }}'
-
-ECOMMERCE_REPOS:
-  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
-    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
-    PATH: "{{ COMMON_GIT_PATH }}"
-    REPO: ecommerce.git
-    VERSION: "{{ ECOMMERCE_VERSION }}"
-    DESTINATION: "{{ ecommerce_code_dir }}"
-    SSH_KEY: "{{ ECOMMERCE_GIT_IDENTITY }}"
-
-ECOMMERCE_GUNICORN_WORKERS: "2"
+
 ECOMMERCE_GUNICORN_EXTRA: ""
-ECOMMERCE_GUNICORN_EXTRA_CONF: ""
-ECOMMERCE_GUNICORN_WORKER_CLASS: "gevent"
-ECOMMERCE_GUNICORN_MAX_REQUESTS: !!null
 
 #
 # vars are namespace with the module name.
@@ -253,27 +196,10 @@ ecommerce_home: "{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}"
 ecommerce_code_dir: "{{ ecommerce_home }}/{{ ecommerce_service_name }}"
 ecommerce_venv_dir: "{{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}"
 
-ecommerce_nodeenv_dir: "{{ ecommerce_home }}/nodeenvs/{{ ecommerce_service_name }}"
-ecommerce_nodeenv_bin: "{{ ecommerce_nodeenv_dir }}/bin"
-ecommerce_node_modules_dir: "{{ ecommerce_code_dir }}/node_modules"
-ecommerce_node_bin: "{{ ecommerce_node_modules_dir }}/.bin"
-ecommerce_node_version: "{{ common_node_version }}"
-
-ecommerce_gunicorn_host: "127.0.0.1"
 ecommerce_gunicorn_port: "8130"
-ecommerce_gunicorn_timeout: "300"
-
-ecommerce_log_dir: "{{ COMMON_LOG_DIR }}/{{ ecommerce_service_name }}"
-
-ecommerce_requirements_base: "{{ ecommerce_code_dir }}/requirements"
-ecommerce_requirements:
-  - production.txt
-  - optional.txt
 
 ecommerce_environment:
-  DJANGO_SETTINGS_MODULE: "{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}"
   ECOMMERCE_CFG: "{{ COMMON_CFG_DIR }}/{{ ecommerce_service_name }}.yml"
-  PATH: "{{ ecommerce_nodeenv_bin }}:{{ ecommerce_venv_dir }}/bin:{{ ansible_env.PATH }}"
 
 #
 # OS packages
@@ -286,3 +212,7 @@ ecommerce_debian_pkgs:
   - libffi-dev
 
 ecommerce_redhat_pkgs: []
+
+ecommerce_post_migrate_commands:
+  - command: './manage.py oscar_populate_countries --initial-only'
+    when: true

playbooks/roles/ecommerce/meta/main.yml

@@ -11,20 +11,35 @@
 # Role includes for role ecommerce
 #
 dependencies:
-  - common
-  - supervisor
-  - role: edx_service
-    edx_service_name: "{{ ecommerce_service_name }}"
-    edx_service_config: "{{ ECOMMERCE_SERVICE_CONFIG }}"
-    edx_service_repos: "{{ ECOMMERCE_REPOS }}"
-    edx_service_user: "{{ ecommerce_user }}"
-    edx_service_home: "{{ ecommerce_home }}"
-    edx_service_packages:
-      debian: "{{ ecommerce_debian_pkgs }}"
-      redhat: "{{ ecommerce_redhat_pkgs }}"
+  - role: edx_django_service
+    edx_django_service_version: '{{ ECOMMERCE_VERSION }}'
+    edx_django_service_name: '{{ ecommerce_service_name }}'
+    edx_django_service_config_overrides: '{{ ecommerce_service_config_overrides }}'
+    edx_django_service_debian_pkgs_extra: '{{ ecommerce_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ ecommerce_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ ecommerce_environment }}'
+    edx_django_service_gunicorn_extra: '{{ ECOMMERCE_GUNICORN_EXTRA }}'
+    edx_django_service_nginx_port: '{{ ECOMMERCE_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ ECOMMERCE_SSL_NGINX_PORT }}'
+    edx_django_service_use_python3: false
+    edx_django_service_language_code: '{{ ECOMMERCE_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ ECOMMERCE_SECRET_KEY }}'
+    edx_django_service_memcache: '{{ ECOMMERCE_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ ECOMMERCE_DATABASE_HOST }}'
+    edx_django_service_default_db_name: '{{ ECOMMERCE_DATABASE_NAME }}'
+    edx_django_service_default_db_atomic_requests: true
+    edx_django_service_db_user: '{{ ECOMMERCE_DATABASE_USER }}'
+    edx_django_service_db_password: '{{ ECOMMERCE_DATABASE_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_session_expire_at_browser_close: '{{ ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_post_migrate_commands: '{{ ecommerce_post_migrate_commands }}'
+    edx_django_service_basic_auth_exempted_paths_extra:
+      - payment
+      - \.well-known/apple-developer-merchantid-domain-association
   - role: edx_themes
     theme_users:
-      - "{{ ecommerce_user }}"
+      - '{{ ecommerce_user }}'
     when: ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING
-
-  - oraclejdk

playbooks/roles/ecommerce/tasks/main.yml

 ---
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role ecommerce
-#
-# Overview:
-#
-#
-# Dependencies:
-#
-#
-# Example play:
-#
-#
-- name: Add gunicorn configuration file
-  template:
-    src: "edx/app/ecommerce/ecommerce_gunicorn.py.j2"
-    dest: "{{ ecommerce_home }}/ecommerce_gunicorn.py"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-      - install
-      - install:configuration
-
-- name: Install application requirements
-  pip:
-    requirements: "{{ ecommerce_requirements_base }}/{{ item }}"
-    virtualenv: "{{ ecommerce_venv_dir }}"
-    state: present
-  become_user: "{{ ecommerce_user }}"
-  with_items: "{{ ecommerce_requirements }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Create nodeenv
-  shell: "{{ ecommerce_venv_dir }}/bin/nodeenv {{ ecommerce_nodeenv_dir }} --node={{ ecommerce_node_version }} --prebuilt --force"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: Install node dependencies
-  npm:
-    executable: "{{ ecommerce_nodeenv_bin }}/npm"
-    path: "{{ ecommerce_code_dir }}"
-    production: yes
-    state: latest
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Install bower dependencies
-  shell: ". {{ ecommerce_nodeenv_bin }}/activate && {{ ecommerce_node_bin }}/bower install --production --config.interactive=false"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Migrate
-  shell: >
-    DB_MIGRATION_USER='{{ COMMON_MYSQL_MIGRATE_USER }}'
-    DB_MIGRATION_PASS='{{ COMMON_MYSQL_MIGRATE_PASS }}'
-    {{ ecommerce_venv_dir }}/bin/python ./manage.py migrate --noinput
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: Populate countries
-  shell: "DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }} DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }} {{ ecommerce_venv_dir }}/bin/python ./manage.py oscar_populate_countries --initial-only"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: compile sass
-  shell: "{{ ecommerce_venv_dir }}/bin/python manage.py {{ item }}"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  with_items:
-    - "update_assets --skip-collect"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Run r.js optimizer
-  shell: ". {{ ecommerce_nodeenv_bin }}/activate && {{ ecommerce_node_bin }}/r.js -o build.js"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Run collectstatic
-  shell: "{{ ecommerce_venv_dir }}/bin/python manage.py {{ item }}"
-  args:
-    chdir: "{{ ecommerce_code_dir }}"
-  become_user: "{{ ecommerce_user }}"
-  environment: "{{ ecommerce_environment }}"
-  with_items:
-    - "collectstatic --noinput"
-    - "compress"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-- name: Write out the supervisor wrapper
-  template:
-    src: "edx/app/ecommerce/ecommerce.sh.j2"
-    dest: "{{ ecommerce_home }}/{{ ecommerce_service_name }}.sh"
-    mode: "0650"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: Write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/ecommerce.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ ecommerce_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: "0644"
-  tags:
-    - install
-    - install:configuration
-
 - name: Create Apple Pay certificates directory
   file:
     path: "{{ ecommerce_apple_pay_merchant_certificate_directory }}"
@@ -173,68 +20,3 @@
   tags:
     - install
     - install:configuration
-
-- name: Setup the ecommence env file
-  template:
-    src: "./{{ ecommerce_home }}/{{ ecommerce_service_name }}_env.j2"
-    dest: "{{ ecommerce_home }}/ecommerce_env"
-    owner: "{{ ecommerce_user }}"
-    group: "{{ ecommerce_user }}"
-    mode: "0644"
-  tags:
-    - install
-    - install:configuration
-
-- name: Enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ ecommerce_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ ecommerce_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-  tags:
-    - install
-    - install:configuration
-
-- name: Update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-  tags:
-    - manage
-    - manage:start
-
-- name: Create symlinks from the-er venv bin dir
-  file:
-    src: "{{ ecommerce_venv_dir }}/bin/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.', 1) | first }}.ecommerce"
-    state: link
-  with_items:
-    - python
-    - pip
-    - django-admin.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Create symlinks from the repo dir
-  file:
-    src: "{{ ecommerce_code_dir }}/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.', 1) | first }}.ecommerce"
-    state: link
-  with_items:
-    - manage.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: Restart the applicaton
-  supervisorctl:
-    name: "{{ ecommerce_service_name }}"
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
-  tags:
-    - manage
-    - manage:start

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% set ecommerce_venv_bin = ecommerce_home + "/venvs/" + ecommerce_service_name + "/bin" %}
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = ecommerce_venv_bin + '/newrelic-admin run-program ' + ecommerce_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = ecommerce_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-export NEW_RELIC_APP_NAME="{{ ECOMMERCE_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-source {{ ecommerce_home }}/ecommerce_env
-{{ executable }} -c {{ ecommerce_home }}/ecommerce_gunicorn.py {{ ECOMMERCE_GUNICORN_EXTRA }} ecommerce.wsgi:application

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in ecommerce_environment.items() -%}
-{%- if value -%}
-export {{ name }}="{{ value }}"
-{% endif %}
-{%- endfor %}

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_gunicorn.py.j2

-"""
-gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
-{{ ansible_managed }}
-"""
-
-timeout = {{ ecommerce_gunicorn_timeout }}
-bind = "{{ ecommerce_gunicorn_host }}:{{ ecommerce_gunicorn_port }}"
-pythonpath = "{{ ecommerce_code_dir }}"
-workers = {{ ECOMMERCE_GUNICORN_WORKERS }}
-worker_class = "{{ ECOMMERCE_GUNICORN_WORKER_CLASS }}"
-
-{% if ECOMMERCE_GUNICORN_MAX_REQUESTS %}
-max_requests = {{ ECOMMERCE_GUNICORN_MAX_REQUESTS }}
-{% endif %}
-
-{{ ECOMMERCE_GUNICORN_EXTRA_CONF }}

playbooks/roles/ecommerce/templates/edx/app/supervisor/conf.d.available/ecommerce.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ ecommerce_service_name }}]
-
-command={{ ecommerce_home }}/{{ ecommerce_service_name }}.sh
-user={{ common_web_user }}
-directory={{ ecommerce_code_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/edx_django_service/defaults/main.yml

@@ -195,3 +195,22 @@ edx_django_service_automated_users:
         sudo_user: '{{ edx_django_service_user }}'
     authorized_keys:
       - 'SSH authorized key'
+
+# This array contains commands that should be run after migration.
+#
+# The commands will be executed from the code directory with the application's virtualenv activated. The migration
+# environment (e.g. migration DB username/password) will NOT be used, so commands should not rely on these values being
+# set. In other words, don't try to sneak in another run of the migrate management command.
+#
+# Example:
+#   edx_django_service_post_migrate_management_commands:
+#     - command: './manage.py conditional_command'
+#       when: '{{ foo }}'
+#     - command: './manage.py always_command'
+#       when: True
+#
+# In this example, the "conditional_command" will only be run when the variable `foo` is set to `True`. The
+# "always_command" will always be run because its conditional is set to `True`. To minimize surprises, the `when`
+# key *MUST* be supplied for all commands.
+#
+edx_django_service_post_migrate_commands: []

playbooks/roles/edx_django_service/tasks/main.yml

@@ -115,6 +115,30 @@
     - migrate
     - migrate:db
 
+- name: run post-migrate commands
+  command: "{{ item.command }}"
+  args:
+    chdir: "{{ edx_django_service_code_dir }}"
+  become_user: "{{ edx_django_service_user }}"
+  environment: "{{ edx_django_service_environment }}"
+  with_items: '{{ edx_django_service_post_migrate_commands }}'
+  when: migrate_db is defined and migrate_db|lower == "yes" and item.when | bool
+  tags:
+    - migrate
+    - migrate:db
+    - migrate:post
+
+- name: ensure log files exist for tailing
+  file:
+    path: "{{ item }}"
+    state: touch
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+  with_items: '{{ edx_django_service_name_devstack_logs }}'
+  tags:
+    - install
+    - install:configuration
+
 - name: write out the supervisor wrapper
   template:
     src: "edx/app/app/app.sh.j2"

playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2

@@ -68,8 +68,6 @@ server {
     try_files $uri @proxy_to_app;
   }
 
-  # API endpoints have their own authentication and authorization
-  # schemes, so we bypass basic auth.
   location ~ ^/({{ edx_django_service_basic_auth_exempted_paths | join('|') }})/ {
     try_files $uri @proxy_to_app;
   }

playbooks/roles/edxlocal/defaults/main.yml

@@ -4,7 +4,7 @@ edxlocal_debian_pkgs:
   - libjpeg-dev
 
 edxlocal_databases:
-  - "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ ECOMMERCE_DATABASE_NAME | default(None) }}"
   - "{{ INSIGHTS_DATABASE_NAME | default(None) }}"
   - "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}"
   - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}"
@@ -17,7 +17,7 @@ edxlocal_databases:
 
 edxlocal_database_users:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
       user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
       pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
     }

playbooks/roles/mariadb/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role mariadb
-# 
+#
 MARIADB_APT_KEY_XENIAL_ID: '0xF1656F24C74CD1D8'
 MARIADB_APT_KEY_ID: '0xcbcb082a1bb943db'
 
@@ -23,7 +23,7 @@ MARIADB_CLUSTER_PASSWORD_ADMIN: "password"
 MARIADB_HOST_PRIV: '%'
 
 MARIADB_HAPROXY_USER: 'haproxy'
-MARIADB_HAPROXY_HOSTS: 
+MARIADB_HAPROXY_HOSTS:
   - '192.168.33.100'
   - '192.168.33.110'
   - '192.168.33.120'
@@ -32,57 +32,57 @@ MARIADB_LISTEN_ALL: false
 
 MARIADB_DATABASES:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
+      encoding: "utf8"
     }
   - {
       db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}",
-      encoding: "utf8" 
+      encoding: "utf8"
     }
   - {
       db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
-      encoding: "latin1" 
+      encoding: "latin1"
     }
 
 MARIADB_USERS:
   - {
-      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      db: "{{ ECOMMERCE_DATABASE_NAME | default(None) }}",
       user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
       pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
     }

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/ecommerce.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if "ecommerce" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream ecommerce_app_server {
-{% for host in nginx_ecommerce_gunicorn_hosts %}
-    server {{ host }}:{{ ecommerce_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-server {
-  server_name {{ ECOMMERCE_HOSTNAME }};
-
-  listen {{ ECOMMERCE_NGINX_PORT }} {{ default_site }};
-
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ ECOMMERCE_SSL_NGINX_PORT }} ssl;
-
-  {% include "common-settings.j2" %}
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-  {% endif %}
-
-  # Prevent invalid display courseware in IE 10+ with high privacy settings
-  add_header P3P '{{ NGINX_P3P_MESSAGE }}';
-
-  # Nginx does not support nested condition or or conditions so
-  # there is an unfortunate mix of conditonals here.
-  {% if NGINX_REDIRECT_TO_HTTPS %}
-     {% if NGINX_HTTPS_REDIRECT_STRATEGY == "scheme" %}
-  # Redirect http to https over single instance
-  if ($scheme != "https")
-  {
-   set $do_redirect_to_https "true";
-  }
-
-     {% elif NGINX_HTTPS_REDIRECT_STRATEGY == "forward_for_proto" %}
-  # Forward to HTTPS if we're an HTTP request... and the server is behind ELB
-  if ($http_x_forwarded_proto = "http")
-  {
-   set $do_redirect_to_https "true";
-  }
-     {% endif %}
-  # Execute the actual redirect
-  if ($do_redirect_to_https = "true")
-  {
-  return 301 https://$host$request_uri;
-  }
-  {% endif %}
-
-  location ~ ^/static/(?P<file>.*) {
-    root {{ COMMON_DATA_DIR }}/{{ ecommerce_service_name }};
-    try_files /staticfiles/$file =404;
-  }
-
-  location / {
-    {% if ECOMMERCE_ENABLE_BASIC_AUTH|bool %}
-      {% include "basic-auth.j2" %}
-    {% endif %}
-    try_files $uri @proxy_to_app;
-  }
-
-  # The API should be secured with OAuth 2.0 or or JWT.
-  location /api {
-    try_files $uri @proxy_to_app;
-  }
-
-  # Allow access to this API for POST back from payment processors.
-  location /payment {
-    try_files $uri @proxy_to_app;
-  }
-
-  # Allow access for Apple Pay domain validation
-  location /.well-known/apple-developer-merchantid-domain-association {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% include "robots.j2" %}
-
-location @proxy_to_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
-
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://ecommerce_app_server;
-  }
-
-}
-