Updated discovery play to use edx_django_service play

- This removes the duplication across the various IDA configurations
- Updated Dockerfile for Docker-based devstack

LEARNER-817

docker/build/discovery/Dockerfile

@@ -9,26 +9,17 @@
 
 FROM edxops/xenial-common:latest
 MAINTAINER edxops
-
-ENV DISCOVERY_VERSION=master
-ENV REPO_OWNER=edx
+USER root
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 
 ADD . /edx/app/edx_ansible/edx_ansible
-
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
-RUN echo '{ "allow_root": true }' > /root/.bowerrc
-
-RUN apt-get update
-RUN apt install -y xvfb firefox gettext
-
 COPY docker/build/discovery/ansible_overrides.yml /
+
 RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook discovery.yml \
     -c local -i '127.0.0.1,' \
-    -t 'install,assets,devstack:install' \
-    --extra-vars="@/ansible_overrides.yml" \
-    --extra-vars="DISCOVERY_VERSION=$DISCOVERY_VERSION" \
-    --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
+    -t 'install,assets,devstack' \
+    --extra-vars="@/ansible_overrides.yml"
 
-USER root
-CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
+EXPOSE 18381

docker/build/discovery/ansible_overrides.yml

 ---
-discovery_gunicorn_host: 0.0.0.0
-DISCOVERY_MYSQL: 'db'
-DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack'
-DISCOVERY_ELASTICSEARCH_HOST: 'es'
-DISCOVERY_GUNICORN_EXTRA: '--reload'
+COMMON_GIT_PATH: 'edx'
+DISCOVERY_VERSION: 'master'
+
 COMMON_MYSQL_MIGRATE_USER: '{{ DISCOVERY_MYSQL_USER }}'
 COMMON_MYSQL_MIGRATE_PASS: '{{ DISCOVERY_MYSQL_PASSWORD }}'
+
+EDXAPP_LMS_BASE: 'edx.devstack.lms:18000'
+EDXAPP_LMS_ROOT_URL: 'http://{{ EDXAPP_LMS_BASE }}'
+EDXAPP_LMS_PUBLIC_ROOT_URL: 'http://localhost:18000'
+EDXAPP_JWT_AUDIENCE: 'lms-key'
+
+DISCOVERY_MYSQL: 'edx.devstack.mysql'
+DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack'
+DISCOVERY_ELASTICSEARCH_HOST: 'edx.devstack.elasticsearch'
+DISCOVERY_GUNICORN_EXTRA: '--reload'
+DISCOVERY_MEMCACHE: ['edx.devstack.memcached:11211']
+DISCOVERY_EXTRA_APPS: ['course_discovery.apps.edx_catalog_extensions']
+
+edx_django_service_is_devstack: true

docker/build/discovery/inventory

-[local]
-localhost

playbooks/edx-east/discovery.yml

@@ -9,8 +9,6 @@
     CLUSTER_NAME: 'discovery'
   roles:
     - aws
-    - role: automated
-      AUTOMATED_USERS: "{{ DISCOVERY_AUTOMATED_USERS | default({}) }}"
     - role: nginx
       nginx_default_sites:
         - discovery

playbooks/roles/common_vars/defaults/main.yml

@@ -208,3 +208,13 @@ COMMON_TRACKING_LOG_ROTATION:
 # COMMON_USING_SECURE_REPO: true
 COMMON_EXTRA_CONFIGURATION_SOURCES_CHECKING: false
 COMMON_EXTRA_CONFIGURATION_SOURCES: []
+
+COMMON_OAUTH_PUBLIC_URL_ROOT: 'http://127.0.0.1:8000/oauth2'
+COMMON_OAUTH_URL_ROOT: '{{ COMMON_OAUTH_PUBLIC_URL_ROOT }}'
+COMMON_OAUTH_LOGOUT_URL: '{{ COMMON_OAUTH_PUBLIC_URL_ROOT }}/logout'
+
+COMMON_OIDC_ISSUER: '{{ COMMON_OAUTH_URL_ROOT }}'
+
+COMMON_JWT_AUDIENCE: 'SET-ME-PLEASE'
+COMMON_JWT_ISSUER: '{{ COMMON_OIDC_ISSUER }}'
+COMMON_JWT_SECRET_KEY: 'SET-ME-PLEASE'

playbooks/roles/discovery/defaults/main.yml

@@ -10,36 +10,36 @@
 ##
 # Defaults for role discovery
 #
-DISCOVERY_GIT_IDENTITY: !!null
 
-# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
-# and a key being provided via NEWRELIC_LICENSE_KEY
-DISCOVERY_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ discovery_service_name }}"
-DISCOVERY_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-DISCOVERY_NGINX_PORT: 18381
-DISCOVERY_SSL_NGINX_PORT: 48381
+
+#
+# vars are namespace with the module name.
+#
+discovery_service_name: "discovery"
+discovery_gunicorn_port: 8381
+
+discovery_environment:
+  DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ discovery_service_name }}.yml"
+
+
+#
+# OS packages
+#
+
+discovery_debian_pkgs:
+  - libxml2-dev
+  - libxslt-dev
+  - libjpeg-dev
+
+
+DISCOVERY_NGINX_PORT: "1{{ discovery_gunicorn_port }}"
+DISCOVERY_SSL_NGINX_PORT: "4{{ discovery_gunicorn_port }}"
 
 DISCOVERY_DEFAULT_DB_NAME: 'discovery'
 DISCOVERY_MYSQL: 'localhost'
 # MySQL usernames are limited to 16 characters
 DISCOVERY_MYSQL_USER: 'discov001'
 DISCOVERY_MYSQL_PASSWORD: 'password'
-DISCOVERY_MYSQL_PORT: '3306'
-DISCOVERY_MYSQL_OPTIONS:
-  connect_timeout: 10
-
-DISCOVERY_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ DISCOVERY_DEFAULT_DB_NAME }}'
-    USER: '{{ DISCOVERY_MYSQL_USER }}'
-    PASSWORD: '{{ DISCOVERY_MYSQL_PASSWORD }}'
-    HOST: '{{ DISCOVERY_MYSQL }}'
-    PORT: '{{ DISCOVERY_MYSQL_PORT }}'
-    OPTIONS: '{{ DISCOVERY_MYSQL_OPTIONS }}'
-    ATOMIC_REQUESTS: false
-    CONN_MAX_AGE: 60
 
 # Using SSL? See https://www.elastic.co/guide/en/shield/current/ssl-tls.html.
 # Using AWS? Use the AWS-provided host (e.g. https://search-test-abc123.us-east-1.es.amazonaws.com/).
@@ -49,53 +49,23 @@ DISCOVERY_ELASTICSEARCH_INDEX_NAME: 'catalog'
 
 DISCOVERY_MEMCACHE: [ 'memcache' ]
 
-DISCOVERY_CACHES:
-  default:
-    BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
-    KEY_PREFIX: '{{ discovery_service_name }}'
-    LOCATION: '{{ DISCOVERY_MEMCACHE }}'
-
 DISCOVERY_VERSION: "master"
 DISCOVERY_DJANGO_SETTINGS_MODULE: "course_discovery.settings.production"
-DISCOVERY_URL_ROOT: 'http://discovery:18381'
+DISCOVERY_URL_ROOT: 'http://discovery:{{ DISCOVERY_NGINX_PORT }}'
 DISCOVERY_LOGOUT_URL: '{{ DISCOVERY_URL_ROOT }}/logout/'
-DISCOVERY_OAUTH_URL_ROOT: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/oauth2'
-DISCOVERY_OIDC_LOGOUT_URL: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/logout'
-
-DISCOVERY_EDX_DRF_EXTENSIONS:
-  OAUTH2_USER_INFO_URL: '{{ DISCOVERY_OAUTH_URL_ROOT }}/user_info'
 
-DISCOVERY_JWT_AUDIENCE: '{{ EDXAPP_JWT_AUDIENCE | default("SET-ME-PLEASE") }}'
-DISCOVERY_JWT_ISSUER: '{{ DISCOVERY_OAUTH_URL_ROOT }}'
-DISCOVERY_JWT_SECRET_KEY: '{{ EDXAPP_JWT_SECRET_KEY | default("lms-secret") }}'
-
-DISCOVERY_JWT_AUTH:
-  JWT_ISSUERS:
-    - AUDIENCE: '{{ DISCOVERY_JWT_AUDIENCE }}'
-      ISSUER: '{{ DISCOVERY_JWT_ISSUER }}'
-      SECRET_KEY: '{{ DISCOVERY_JWT_SECRET_KEY }}'
-
-DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 DISCOVERY_SECRET_KEY: 'Your secret key here'
-DISCOVERY_TIME_ZONE: 'UTC'
 DISCOVERY_LANGUAGE_CODE: 'en-us'
 DISCOVERY_DEFAULT_PARTNER_ID: 1
+DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 
 # Used to automatically configure OAuth2 Client
 DISCOVERY_SOCIAL_AUTH_EDX_OIDC_KEY : 'discovery-key'
 DISCOVERY_SOCIAL_AUTH_EDX_OIDC_SECRET : 'discovery-secret'
 DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
-DISCOVERY_SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ DISCOVERY_OAUTH_URL_ROOT }}'
 
 DISCOVERY_PLATFORM_NAME: 'Your Platform Name Here'
 
-DISCOVERY_LMS_ROOT_URL: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}'
-DISCOVERY_ECOMMERCE_API_URL: 'https://localhost:8002/api/v2/'
-DISCOVERY_COURSES_API_URL: '{{ DISCOVERY_LMS_ROOT_URL }}/api/courses/v1/'
-DISCOVERY_ORGANIZATIONS_API_URL: '{{ DISCOVERY_LMS_ROOT_URL }}/api/organizations/v0/'
-DISCOVERY_MARKETING_API_URL: 'https://example.org/api/catalog/v2/'
-DISCOVERY_MARKETING_URL_ROOT: 'https://example.org/'
-
 DISCOVERY_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ discovery_service_name }}'
 DISCOVERY_MEDIA_ROOT: '{{ DISCOVERY_DATA_DIR }}/media'
 DISCOVERY_MEDIA_URL: '/media/'
@@ -122,52 +92,20 @@ DISCOVERY_EMAIL_USE_TLS: False
 DISCOVERY_EMAIL_HOST_USER: ''
 DISCOVERY_EMAIL_HOST_PASSWORD: ''
 
-DISCOVERY_PUBLISHER_FROM_EMAIL: 'None'
-
-DISCOVERY_EXTRA_APPS: []
-
-DISCOVERY_SERVICE_CONFIG:
-  SESSION_EXPIRE_AT_BROWSER_CLOSE: '{{ DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+DISCOVERY_PUBLISHER_FROM_EMAIL: !!null
 
-  SECRET_KEY: '{{ DISCOVERY_SECRET_KEY }}'
-  TIME_ZONE: '{{ DISCOVERY_TIME_ZONE }}'
-  LANGUAGE_CODE: '{{ DISCOVERY_LANGUAGE_CODE }}'
+DISCOVERY_GUNICORN_EXTRA: ''
 
-  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ DISCOVERY_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ DISCOVERY_OIDC_LOGOUT_URL }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_ISSUER }}'
-
-  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ discovery_service_name }}/staticfiles"
-  # db config
-  DATABASES: '{{ DISCOVERY_DATABASES }}'
-  CACHES: '{{ DISCOVERY_CACHES }}'
+DISCOVERY_EXTRA_APPS: []
 
+discovery_service_config_overrides:
   ELASTICSEARCH_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}'
   ELASTICSEARCH_INDEX_NAME: '{{ DISCOVERY_ELASTICSEARCH_INDEX_NAME }}'
 
   PLATFORM_NAME: '{{ DISCOVERY_PLATFORM_NAME }}'
 
-  ECOMMERCE_API_URL: '{{ DISCOVERY_ECOMMERCE_API_URL }}'
-  COURSES_API_URL: '{{ DISCOVERY_COURSES_API_URL }}'
-  ORGANIZATIONS_API_URL: '{{ DISCOVERY_ORGANIZATIONS_API_URL }}'
-  MARKETING_API_URL: '{{ DISCOVERY_MARKETING_API_URL }}'
-  MARKETING_URL_ROOT: '{{ DISCOVERY_MARKETING_URL_ROOT }}'
-
-  EDX_DRF_EXTENSIONS: '{{ DISCOVERY_EDX_DRF_EXTENSIONS }}'
-
-  JWT_AUTH: '{{ DISCOVERY_JWT_AUTH }}'
-
   DEFAULT_PARTNER_ID: '{{ DISCOVERY_DEFAULT_PARTNER_ID }}'
 
-  EXTRA_APPS: '{{ DISCOVERY_EXTRA_APPS }}'
-
-  MEDIA_STORAGE_BACKEND: '{{ DISCOVERY_MEDIA_STORAGE_BACKEND }}'
-  STATICFILES_STORAGE: '{{ DISCOVERY_STATICFILES_STORAGE }}'
-
   EMAIL_BACKEND: '{{ DISCOVERY_EMAIL_BACKEND }}'
 
   # Settings for django-ses email backend
@@ -183,88 +121,5 @@ DISCOVERY_SERVICE_CONFIG:
 
   PUBLISHER_FROM_EMAIL: '{{ DISCOVERY_PUBLISHER_FROM_EMAIL }}'
 
-DISCOVERY_REPOS:
-  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
-    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
-    PATH: "{{ COMMON_GIT_PATH }}"
-    REPO: course-discovery.git
-    VERSION: "{{ DISCOVERY_VERSION }}"
-    DESTINATION: "{{ discovery_code_dir }}"
-    SSH_KEY: "{{ DISCOVERY_GIT_IDENTITY }}"
-
-
-DISCOVERY_GUNICORN_WORKERS: "2"
-DISCOVERY_GUNICORN_EXTRA: ""
-DISCOVERY_GUNICORN_EXTRA_CONF: ""
-DISCOVERY_GUNICORN_WORKER_CLASS: "sync"
-
-DISCOVERY_HOSTNAME: '~^((stage|prod)-)?discovery.*'
-
-nginx_discovery_gunicorn_hosts:
-  - 127.0.0.1
-
-# Vars that are used when the automated role is "mixed-in" via the deploying play
-# This data structure specifies all the users with access to run command remotely
-# over SSH and the fully qualified command that they can run via sudo to the
-# application user
-
-DISCOVERY_AUTOMATED_USERS:
-  automated_user:
-    sudo_commands:
-      - command: "{{ discovery_venv_dir }}/python {{ discovery_home }}/{{ discovery_service_name }}/manage.py migrate --list"
-        sudo_user: "discovery"
-    authorized_keys:
-      - "SSH authorized key"
-
-#
-# vars are namespace with the module name.
-#
-discovery_role_name: discovery
-discovery_venv_dir: "{{ discovery_home }}/venvs/{{ discovery_service_name }}"
-
-discovery_environment:
-  DJANGO_SETTINGS_MODULE: "{{ DISCOVERY_DJANGO_SETTINGS_MODULE }}"
-  # rename should synch with app
-  COURSE_DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ discovery_service_name }}.yml"
-  PATH: "{{ discovery_nodeenv_bin }}:{{ discovery_venv_dir }}/bin:{{ ansible_env.PATH }}"
-
-
-discovery_migration_environment:
-  DJANGO_SETTINGS_MODULE: "{{ DISCOVERY_DJANGO_SETTINGS_MODULE }}"
-  # rename should synch with app
-  COURSE_DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ discovery_service_name }}.yml"
-  PATH: "{{ discovery_venv_dir }}/bin:{{ ansible_env.PATH }}"
-  DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
-  DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
-
-discovery_service_name: "discovery"
-discovery_user: "{{ discovery_service_name }}"
-discovery_home: "{{ COMMON_APP_DIR }}/{{ discovery_service_name }}"
-discovery_code_dir: "{{ discovery_home }}/{{ discovery_service_name }}"
-
-discovery_nodeenv_dir: "{{ discovery_home }}/nodeenvs/{{ discovery_service_name }}"
-discovery_nodeenv_bin: "{{ discovery_nodeenv_dir }}/bin"
-discovery_node_modules_dir: "{{ discovery_code_dir }}/node_modules"
-discovery_node_bin: "{{ discovery_node_modules_dir }}/.bin"
-discovery_node_version: "{{ common_node_version }}"
-
-discovery_gunicorn_host: "127.0.0.1"
-discovery_gunicorn_port: 8381
-discovery_gunicorn_timeout: 300
-
-discovery_log_dir: "{{ COMMON_LOG_DIR }}/{{ discovery_service_name }}"
-
-#
-# OS packages
-#
-
-discovery_debian_pkgs:
-  - libmysqlclient-dev
-  - libssl-dev
-  - libffi-dev  # Needed to install the Python cryptography library for asymmetric JWT signing
-  - libmemcached-dev # Needed for memcache
-  - libxml2-dev
-  - libxslt-dev
-  - libjpeg-dev
-
-discovery_redhat_pkgs: []
+# See edx_django_service_automated_users for an example of what this should be
+DISCOVERY_AUTOMATED_USERS: {}

playbooks/roles/discovery/meta/main.yml

@@ -9,24 +9,42 @@
 #
 ##
 # Role includes for role discovery
-# 
+#
 # Example:
 #
 # dependencies:
 #   - {
-#   role: my_role 
-#   my_role_var0: "foo"
-#   my_role_var1: "bar"
+#   role: my_role
+#   my_role_var0: 'foo'
+#   my_role_var1: 'bar'
 # }
 dependencies:
-  - common
-  - supervisor
-  - role: edx_service
-    edx_service_name: "{{ discovery_service_name }}"
-    edx_service_config: "{{ DISCOVERY_SERVICE_CONFIG }}"
-    edx_service_repos: "{{ DISCOVERY_REPOS }}"
-    edx_service_user: "{{ discovery_user }}"
-    edx_service_home: "{{ discovery_home }}"
-    edx_service_packages:
-      debian: "{{ discovery_debian_pkgs }}"
-      redhat: "{{ discovery_redhat_pkgs }}"
+  - role: edx_django_service
+    edx_django_service_repo: 'course-discovery'
+    edx_django_service_version: '{{ DISCOVERY_VERSION }}'
+    edx_django_service_name: '{{ discovery_service_name }}'
+    edx_django_service_config_overrides: '{{ discovery_service_config_overrides }}'
+    edx_django_service_debian_pkgs_extra: '{{ discovery_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ discovery_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ DISCOVERY_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ discovery_environment }}'
+    edx_django_service_gunicorn_extra: '{{ DISCOVERY_GUNICORN_EXTRA }}'
+    edx_django_service_wsgi_name: 'course_discovery'
+    edx_django_service_nginx_port: '{{ DISCOVERY_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ DISCOVERY_SSL_NGINX_PORT }}'
+    edx_django_service_language_code: '{{ DISCOVERY_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ DISCOVERY_SECRET_KEY }}'
+    edx_django_service_staticfiles_storage: '{{ DISCOVERY_STATICFILES_STORAGE }}'
+    edx_django_service_media_storage_backend: '{{ DISCOVERY_MEDIA_STORAGE_BACKEND }}'
+    edx_django_service_memcache: '{{ DISCOVERY_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ DISCOVERY_MYSQL }}'
+    edx_django_service_default_db_name: '{{ DISCOVERY_DEFAULT_DB_NAME }}'
+    edx_django_service_default_db_atomic_requests: false
+    edx_django_service_db_user: '{{ DISCOVERY_MYSQL_USER }}'
+    edx_django_service_db_password: '{{ DISCOVERY_MYSQL_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_extra_apps: '{{ DISCOVERY_EXTRA_APPS }}'
+    edx_django_service_session_expire_at_browser_close: '{{ DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_automated_users: '{{ DISCOVERY_AUTOMATED_USERS }}'

playbooks/roles/discovery/tasks/main.yml

@@ -11,7 +11,7 @@
 #
 # Tasks for role discovery
 #
-# Overview:
+# Overview: This role's tasks come from edx_django_service.
 #
 #
 # Dependencies:
@@ -20,219 +20,3 @@
 # Example play:
 #
 #
-
-- name: add gunicorn configuration file
-  template:
-    src: edx/app/discovery/discovery_gunicorn.py.j2
-    dest: "{{ discovery_home }}/discovery_gunicorn.py"
-  become_user: "{{ discovery_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: add deadsnakes repository
-  apt_repository:
-    repo: "ppa:fkrull/deadsnakes"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install python3.5
-  apt:
-    name: "{{ item }}"
-  with_items:
-    - python3.5
-    - python3.5-dev
-  tags:
-    - install
-    - install:system-requirements
-
-- name: build virtualenv
-  command: "virtualenv --python=python3.5 {{ discovery_venv_dir }}"
-  args:
-    creates: "{{ discovery_venv_dir }}/bin/pip"
-  become_user: "{{ discovery_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install nodenv
-  pip:
-    name: "nodeenv"
-    version: "1.1.1"
-    # NOTE (CCB): Using the "virtualenv" option here doesn't seem to work.
-    executable: "{{ discovery_venv_dir }}/bin/pip"
-  become_user: "{{ discovery_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: create nodeenv
-  shell: "{{ discovery_venv_dir }}/bin/nodeenv {{ discovery_nodeenv_dir }} --node={{ discovery_node_version }} --prebuilt --force"
-  become_user: "{{ discovery_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install application requirements
-  command: make production-requirements
-  args:
-    chdir: "{{ discovery_code_dir }}"
-  become_user: "{{ discovery_user }}"
-  environment: "{{ discovery_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: install development requirements
-  command: make requirements
-  args:
-    chdir: "{{ discovery_code_dir }}"
-  become_user: "{{ discovery_user }}"
-  environment: "{{ discovery_environment }}"
-  tags:
-    - devstack
-    - devstack:install
-
-- name: migrate database
-  command: make migrate
-  args:
-    chdir: "{{ discovery_code_dir }}"
-  become_user: "{{ discovery_user }}"
-  environment: "{{ discovery_migration_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: write out the supervisor wrapper
-  template:
-    src: "edx/app/discovery/discovery.sh.j2"
-    dest: "{{ discovery_home }}/{{ discovery_service_name }}.sh"
-    mode: 0650
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/discovery.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ discovery_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: write devstack script
-  template:
-    src: "edx/app/discovery/devstack.sh.j2"
-    dest: "{{ discovery_home }}/devstack.sh"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: 0744
-  tags:
-    - devstack
-    - devstack:install
-
-- name: setup the discovery env file
-  template:
-    src: "./{{ discovery_home }}/{{ discovery_service_name }}_env.j2"
-    dest: "{{ discovery_home }}/discovery_env"
-    owner: "{{ discovery_user }}"
-    group: "{{ discovery_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ discovery_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ discovery_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-  tags:
-    - install
-    - install:configuration
-
-- name: update supervisor configuration
-  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-  tags:
-    - manage
-    - manage:start
-
-- name: create symlinks from the venv bin dir
-  file:
-    src: "{{ discovery_venv_dir }}/bin/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.discovery"
-    state: link
-  with_items:
-    - python
-    - pip
-    - django-admin.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: create symlinks from the repo dir
-  file:
-    src: "{{ discovery_code_dir }}/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.discovery"
-    state: link
-  with_items:
-    - manage.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: run collectstatic
-  command: make static
-  args:
-    chdir: "{{ discovery_code_dir }}"
-  become_user: "{{ discovery_user }}"
-  environment: "{{ discovery_environment }}"
-  tags:
-    - assets
-    - assets:gather
-
-- name: restart the application
-  supervisorctl:
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-    name: "{{ discovery_service_name }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
-  tags:
-    - manage
-    - manage:start
-
-- name: Copying nginx configs for discovery
-  template:
-    src: "edx/app/nginx/sites-available/discovery.j2"
-    dest: "{{ nginx_sites_available_dir }}/discovery"
-    owner: root
-    group: "{{ common_web_user }}"
-    mode: 0640
-  notify: reload nginx
-  tags:
-    - install
-    - install:vhosts
-
-- name: Creating nginx config links for discovery
-  file:
-    src: "{{ nginx_sites_available_dir }}/discovery"
-    dest: "{{ nginx_sites_enabled_dir }}/discovery"
-    state: link
-    owner: root
-    group: root
-  notify: reload nginx
-  tags:
-    - install
-    - install:vhosts

playbooks/roles/discovery/templates/edx/app/discovery/devstack.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-source {{ discovery_home }}/discovery_env
-COMMAND=$1
-
-case $COMMAND in
-    start)
-        {% set discovery_venv_bin = discovery_venv_dir + "/bin" %}
-
-        {{ supervisor_venv_bin }}/supervisord --configuration {{ supervisor_cfg }}
-
-	# Needed to run bower as root. See explaination around 'discovery_user=root'
-	echo '{ "allow_root": true }' > /root/.bowerrc
-
-        cd /edx/app/edx_ansible/edx_ansible/docker/plays
-        /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook discovery.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack,migrate' \
-            --extra-vars="migrate_db=yes" \
-            --extra-vars="@/ansible_overrides.yml" \
-	    --extra-vars="discovery_user=root"  # Needed when sharing the volume with the host machine because node/bower drops
-						# everything in the code directory by default.  So we get issues with permissions
-						# on folders owned by the developer.
-
-
-	# Need to start supervisord and nginx manually because systemd is hard to run on docker
-	# http://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
-	# Both daemon by default
-	nginx
-	/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
-
-        # Docker requires an active foreground task. Tail the logs to appease Docker and
-        # provide useful output for development.
-        cd {{ supervisor_log_dir }}
-        tail -f {{ discovery_service_name }}-stderr.log -f {{ discovery_service_name }}-stdout.log
-        ;;
-    open)
-        cd {{ discovery_code_dir }}
-        . {{ discovery_venv_bin }}/activate
-        /bin/bash
-        ;;
-esac

playbooks/roles/discovery/templates/edx/app/discovery/discovery.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% set discovery_venv_bin = discovery_home + "/venvs/" + discovery_service_name + "/bin" %}
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = discovery_venv_bin + '/newrelic-admin run-program ' + discovery_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = discovery_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-export NEW_RELIC_APP_NAME="{{ DISCOVERY_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-source {{ discovery_home }}/discovery_env
-{{ executable }} -c {{ discovery_home }}/discovery_gunicorn.py {{ DISCOVERY_GUNICORN_EXTRA }} course_discovery.wsgi:application

playbooks/roles/discovery/templates/edx/app/discovery/discovery_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in discovery_environment.items() -%}
-{%- if value -%}
-export {{ name }}="{{ value }}"
-{% endif %}
-{%- endfor %}

playbooks/roles/discovery/templates/edx/app/discovery/discovery_gunicorn.py.j2

-"""
-gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
-{{ ansible_managed }}
-"""
-
-timeout = {{ discovery_gunicorn_timeout }}
-bind = "{{ discovery_gunicorn_host }}:{{ discovery_gunicorn_port }}"
-pythonpath = "{{ discovery_code_dir }}"
-workers = {{ DISCOVERY_GUNICORN_WORKERS }}
-worker_class = "{{ DISCOVERY_GUNICORN_WORKER_CLASS }}"
-
-{{ DISCOVERY_GUNICORN_EXTRA_CONF }}

playbooks/roles/discovery/templates/edx/app/nginx/sites-available/discovery.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if nginx_default_sites is defined and "discovery" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream discovery_app_server {
-{% for host in nginx_discovery_gunicorn_hosts %}
-    server {{ host }}:{{ discovery_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-server {
-  server_name {{ DISCOVERY_HOSTNAME }};
-
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ DISCOVERY_NGINX_PORT }} {{ default_site }};
-  listen {{ DISCOVERY_SSL_NGINX_PORT }} ssl;
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-
-  {% else %}
-  listen {{ DISCOVERY_NGINX_PORT }} {{ default_site }};
-  {% endif %}
-
-  location ~ ^/static/(?P<file>.*) {
-    root {{ COMMON_DATA_DIR }}/{{ discovery_service_name }};
-    try_files /staticfiles/$file =404;
-  }
-
-  location / {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% if NGINX_ROBOT_RULES|length > 0 %}
-  location /robots.txt {
-      root {{ nginx_app_dir }};
-      try_files $uri /robots.txt =404;
-  }
-  {% endif %}
-
-  location @proxy_to_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://discovery_app_server;
-  }
-
-  # Forward to HTTPS if we're an HTTP request...
-  if ($http_x_forwarded_proto = "http") {
-    set $do_redirect "true";
-  }
-
-  # Run our actual redirect...
-  if ($do_redirect = "true") {
-    rewrite ^ https://$host$request_uri? permanent;
-  }
-}

playbooks/roles/discovery/templates/edx/app/supervisor/conf.d.available/discovery.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ discovery_service_name }}]
-
-command={{ discovery_home }}/{{ discovery_service_name }}.sh
-user={{ common_web_user }}
-directory={{ discovery_code_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/edx_django_service/defaults/main.yml

@@ -112,6 +112,8 @@ edx_django_service_default_db_name: '{{ edx_django_service_name }}'
 edx_django_service_default_db_atomic_requests: false
 edx_django_service_db_user: 'REPLACE-ME'
 edx_django_service_db_password: 'password'
+edx_django_service_db_options:
+  connect_timeout: 10
 
 edx_django_service_databases:
   default:
@@ -123,19 +125,20 @@ edx_django_service_databases:
     PORT: '3306'
     ATOMIC_REQUESTS: '{{ edx_django_service_default_db_atomic_requests }}'
     CONN_MAX_AGE: 60
+    OPTIONS: '{{ edx_django_service_db_options }}'
 
 edx_django_service_social_auth_edx_oidc_key: '{{ edx_django_service_name }}-key'
 edx_django_service_social_auth_edx_oidc_secret: '{{ edx_django_service_name }}-secret'
 edx_django_service_social_auth_redirect_is_https: false
 
-edx_django_service_oauth_public_url_root: '{{ EDXAPP_LMS_PUBLIC_ROOT_URL | default("http://127.0.0.1:8000") }}/oauth2'
-edx_django_service_oauth_url_root: '{{ EDXAPP_LMS_ROOT_URL | default("http://127.0.0.1:8000") }}/oauth2'
-edx_django_service_oidc_logout_url: '{{ EDXAPP_LMS_PUBLIC_ROOT_URL | default("http://127.0.0.1:8000") }}/logout'
-edx_django_service_oidc_issuer: '{{ edx_django_service_oauth_url_root }}'
+edx_django_service_oauth_public_url_root: '{{ COMMON_OAUTH_PUBLIC_URL_ROOT }}'
+edx_django_service_oauth_url_root: '{{COMMON_OAUTH_URL_ROOT }}'
+edx_django_service_oidc_logout_url: '{{ COMMON_OAUTH_LOGOUT_URL }}'
+edx_django_service_oidc_issuer: '{{ COMMON_OIDC_ISSUER }}'
 
-edx_django_service_jwt_audience: '{{ EDXAPP_JWT_AUDIENCE | default("SET-ME-PLEASE") }}'
-edx_django_service_jwt_issuer: '{{ edx_django_service_oauth_url_root }}'
-edx_django_service_jwt_secret_key: '{{ EDXAPP_JWT_SECRET_KEY | default("lms-secret") }}'
+edx_django_service_jwt_audience: '{{ COMMON_JWT_AUDIENCE }}'
+edx_django_service_jwt_issuer: '{{ COMMON_JWT_ISSUER }}'
+edx_django_service_jwt_secret_key: '{{ COMMON_JWT_SECRET_KEY }}'
 
 edx_django_service_session_expire_at_browser_close: false
 
@@ -181,3 +184,11 @@ edx_django_service_config_default:
 # NOTE: This should be overridden by inheriting service-specific role.
 edx_django_service_config_overrides: {}
 edx_django_service_config: '{{ edx_django_service_config_default|combine(edx_django_service_config_overrides) }}'
+
+edx_django_service_automated_users:
+  automated_user:
+    sudo_commands:
+      - command: '{{ edx_django_service_venv_dir }}/python {{ edx_django_service_code_dir }}/manage.py migrate --list'
+        sudo_user: '{{ edx_django_service_user }}'
+    authorized_keys:
+      - 'SSH authorized key'

playbooks/roles/edx_django_service/meta/main.yml

@@ -2,6 +2,8 @@
 dependencies:
   - common
   - supervisor
+  - role: automated
+    AUTOMATED_USERS: "{{ edx_django_service_automated_users }}"
   - role: edx_service
     edx_service_name: "{{ edx_django_service_name }}"
     edx_service_config: "{{ edx_django_service_config }}"

playbooks/sample_vars/server_vars.yml

@@ -36,7 +36,6 @@
 #CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 #COURSE_DISCOVERY_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
 #
-#DISCOVERY_OAUTH_URL_ROOT: "https://${deploy_host}"
 #DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
 #DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true