Updated credentials play to use edx_django_service play

- This removes the duplication across the various IDA configurations
- Updated Dockerfile for Docker-based devstack

LEARNER-816

docker/build/credentials/Dockerfile

@@ -9,26 +9,19 @@
 
 FROM edxops/xenial-common:latest
 MAINTAINER edxops
-
-ARG CREDENTIALS_VERSION=master
-ARG REPO_OWNER=edx
+USER root
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 
 ADD . /edx/app/edx_ansible/edx_ansible
-
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
-RUN echo '{ "allow_root": true }' > /root/.bowerrc
-
-RUN apt-get update
-RUN apt install -y xvfb firefox gettext
-
 COPY docker/build/credentials/ansible_overrides.yml /
+COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+
 RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook credentials.yml \
-    -c local -i '127.0.0.1,' \
-    -t 'install,assets,devstack:install' \
+    -c local -i "127.0.0.1," \
+    -t "install,assets,devstack" \
     --extra-vars="@/ansible_overrides.yml" \
-    --extra-vars="CREDENTIALS_VERSION=$CREDENTIALS_VERSION" \
-    --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
+    --extra-vars="@/devstack/ansible_overrides.yml"
 
-USER root
-CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
+EXPOSE 18150

docker/build/credentials/ansible_overrides.yml

 ---
-credentials_gunicorn_host: 0.0.0.0
-CREDENTIALS_MYSQL: 'db'
-CREDENTIALS_DJANGO_SETTINGS_MODULE: 'credentials.settings.devstack'
-CREDENTIALS_GUNICORN_EXTRA: '--reload'
-CREDENTIALS_MYSQL_MATCHER: '%'
-CREDENTIALS_MYSQL_HOST: 'db'
-CREDENTIALS_MYSQL_PASSWORD: 'password'
+COMMON_GIT_PATH: 'edx'
+CREDENTIALS_VERSION: 'master'
+
 COMMON_MYSQL_MIGRATE_USER: '{{ CREDENTIALS_MYSQL_USER }}'
 COMMON_MYSQL_MIGRATE_PASS: '{{ CREDENTIALS_MYSQL_PASSWORD }}'
+
+CREDENTIALS_MYSQL_HOST: 'edx.devstack.mysql'
+CREDENTIALS_DJANGO_SETTINGS_MODULE: 'credentials.settings.devstack'
+CREDENTIALS_GUNICORN_EXTRA: '--reload'
+CREDENTIALS_MEMCACHE: ['edx.devstack.memcached:11211']
+CREDENTIALS_EXTRA_APPS: ['credentials.apps.edx_credentials_extensions']
+CREDENTIALS_URL_ROOT: 'http://localhost:18150'
+
+edx_django_service_is_devstack: true

docker/plays/credentials.yml

-- name: Deploy Credentials
+- name: Deploy credentials
   hosts: all
   become: True
   gather_facts: True
@@ -6,8 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - nginx
-    - docker
-    - role: credentials
+    - role: nginx
       nginx_default_sites:
         - credentials
+    - credentials

playbooks/edx-east/credentials.yml

@@ -7,12 +7,10 @@
     ENABLE_NEWRELIC: False
     CLUSTER_NAME: 'credentials'
   roles:
+    - aws
     - role: nginx
-      nginx_sites:
-        - credentials
       nginx_default_sites:
         - credentials
-    - aws
     - credentials
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx-east/edx_continuous_integration.yml

@@ -14,7 +14,6 @@
       - xqueue
       - xserver
       - analytics_api
-      - credentials
       nginx_default_sites:
       - lms
     - mysql

playbooks/roles/credentials/defaults/main.yml

@@ -10,72 +10,65 @@
 ##
 # Defaults for role credentials
 #
-CREDENTIALS_GIT_IDENTITY: !!null
 
-# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
-# and a key being provided via NEWRELIC_LICENSE_KEY
-CREDENTIALS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ credentials_service_name }}"
-CREDENTIALS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-CREDENTIALS_NGINX_PORT: 18150
-CREDENTIALS_SSL_NGINX_PORT: 48150
+
+#
+# vars are namespace with the module name.
+#
+credentials_service_name: 'credentials'
+
+credentials_environment:
+  CREDENTIALS_CFG: '{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml'
+
+credentials_gunicorn_port: 8150
+
+#
+# OS packages
+#
+
+credentials_debian_pkgs:
+  # Needed to manipulate images.
+  - libjpeg8-dev
+  - libpng12-dev
+
+credentials_redhat_pkgs: []
+
+CREDENTIALS_NGINX_PORT: '1{{ credentials_gunicorn_port }}'
+CREDENTIALS_SSL_NGINX_PORT: '4{{ credentials_gunicorn_port }}'
 
 CREDENTIALS_DEFAULT_DB_NAME: 'credentials'
 CREDENTIALS_MYSQL_HOST: 'localhost'
 # MySQL usernames are limited to 16 characters
 CREDENTIALS_MYSQL_USER: 'credentials001'
-CREDENTIALS_MYSQL_PASSWORD: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
-CREDENTIALS_DATABASE_PORT: '3306'
-CREDENTIALS_MYSQL_OPTIONS:
-  connect_timeout: 10
-  init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
-
-CREDENTIALS_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ CREDENTIALS_DEFAULT_DB_NAME }}'
-    USER: '{{ CREDENTIALS_MYSQL_USER }}'
-    PASSWORD: '{{ CREDENTIALS_MYSQL_PASSWORD }}'
-    HOST: '{{ CREDENTIALS_MYSQL_HOST }}'
-    PORT: '{{ CREDENTIALS_DATABASE_PORT }}'
-    OPTIONS: '{{ CREDENTIALS_MYSQL_OPTIONS }}'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
+CREDENTIALS_MYSQL_PASSWORD: 'password'
 
 CREDENTIALS_MEMCACHE: [ 'memcache' ]
 
-CREDENTIALS_CACHES:
-  default:
-    BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
-    KEY_PREFIX: '{{ credentials_service_name }}'
-    LOCATION: '{{ CREDENTIALS_MEMCACHE }}'
-
-CREDENTIALS_DJANGO_SETTINGS_MODULE: "credentials.settings.production"
+CREDENTIALS_DJANGO_SETTINGS_MODULE: 'credentials.settings.production'
 CREDENTIALS_DOMAIN: 'credentials'
-CREDENTIALS_URL_ROOT: 'http://{{ CREDENTIALS_DOMAIN }}:18150'
+CREDENTIALS_URL_ROOT: 'http://{{ CREDENTIALS_DOMAIN }}:{{ CREDENTIALS_NGINX_PORT }}'
 CREDENTIALS_LOGOUT_URL: '{{ CREDENTIALS_URL_ROOT }}/logout/'
 
-CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
-
 CREDENTIALS_SECRET_KEY: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
-CREDENTIALS_TIME_ZONE: 'UTC'
 CREDENTIALS_LANGUAGE_CODE: 'en_US.UTF-8'
 
 # Used to automatically configure OAuth2 Client
-CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
-CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET: 'SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING'
+CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY: 'credentials-key'
+CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET: 'credentials-secret'
 CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
 
 CREDENTIALS_SERVICE_USER: 'credentials_service_user'
 
-CREDENTIALS_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ credentials_service_name }}'
-CREDENTIALS_MEDIA_ROOT: '{{ CREDENTIALS_DATA_DIR }}/media'
-CREDENTIALS_STATIC_ROOT: '{{ CREDENTIALS_DATA_DIR }}/staticfiles'
-CREDENTIALS_MEDIA_URL: '/media/'
-CREDENTIALS_STATIC_URL: '/static/'
-
-# Example settings to use Amazon S3 as a storage backend with django storages:
-# https://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html#amazon-s3
+# NOTE: The Credentials Service reads the FILE_STORAGE_BACKEND setting, stored in the CREDENTIALS_FILE_STORAGE_BACKEND
+# as a dict variable, and adds all of its keys to the root of the settings variable space. If the dict contains a key,
+# foo, the application can reference it as `settings.foo`.
+#
+# In non-development environments, all static and uploaded files should be stored in long-term storage that is
+# accessible to multiple machines running the service and not lost when an instance is destroyed. For edx.org, this
+# means storing on S3, which is achieved using django-storages. This library also supports other providers.
+#
+# Below are example variables that show the variables one might use to store data on S3. An exhaustive list of settings
+# is available at https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html.
 #
 # Note, AWS_S3_CUSTOM_DOMAIN is required, otherwise boto will generate non-working
 # querystring URLs for assets (see https://github.com/boto/boto/issues/1477)
@@ -84,7 +77,8 @@ CREDENTIALS_STATIC_URL: '/static/'
 # credentials_s3_domain: s3.amazonaws.com
 # CREDENTIALS_MEDIA_ROOT: 'media'
 # CREDENTIALS_STATIC_ROOT: 'static'
-#
+# ## This will be passed as FILE_STORAGE_BACKEND, and all keys will be added to Django's `settings` object, hence the
+# ## nesting of STATIC_ROOT and other variables that aren't related to S3.
 # CREDENTIALS_FILE_STORAGE_BACKEND:
 #  AWS_STORAGE_BUCKET_NAME: '{{ CREDENTIALS_BUCKET }}'
 #  AWS_S3_CUSTOM_DOMAIN: '{{ CREDENTIALS_BUCKET }}.{{ credentials_s3_domain }}'
@@ -105,135 +99,54 @@ CREDENTIALS_STATIC_URL: '/static/'
 #  STATICFILES_STORAGE:  'credentials.apps.core.s3utils.StaticS3BotoStorage'
 #  DEFAULT_FILE_STORAGE: 'credentials.apps.core.s3utils.MediaS3BotoStorage'
 
-CREDENTIALS_FILE_STORAGE_BACKEND:
-  MEDIA_ROOT: '{{ CREDENTIALS_MEDIA_ROOT }}'
-  STATIC_ROOT: '{{ CREDENTIALS_STATIC_ROOT }}'
-  MEDIA_URL: '{{ CREDENTIALS_MEDIA_URL }}'
-  STATIC_URL: '{{ CREDENTIALS_STATIC_URL }}'
-  STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
-  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
 
+CREDENTIALS_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ credentials_service_name }}'
 
-CREDENTIALS_CORS_ORIGIN_ALLOW_ALL: false
-CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT:
- - "{{ CREDENTIALS_DOMAIN }}"
+# TODO: Let edx_django_service manage CREDENTIALS_STATIC_ROOT in phase 2.
+CREDENTIALS_STATIC_ROOT: '{{ CREDENTIALS_DATA_DIR }}/staticfiles'
 
-CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA: []
-CREDENTIALS_CORS_ORIGIN_WHITELIST: "{{ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT + CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA }}"
+CREDENTIALS_MEDIA_ROOT: '{{ CREDENTIALS_DATA_DIR }}/media'
+CREDENTIALS_MEDIA_URL: '/media/'
+CREDENTIALS_STATIC_URL: '/static/'
 
-CREDENTIALS_CERTIFICATE_LANGUAGES:
-  'en': 'English'
-  'es_419': 'Spanish'
+CREDENTIALS_MEDIA_STORAGE_BACKEND:
+  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
+  MEDIA_ROOT: '{{ CREDENTIALS_MEDIA_ROOT }}'
+  MEDIA_URL: '{{ CREDENTIALS_MEDIA_URL }}'
 
-CREDENTIALS_VERSION: "master"
-CREDENTIALS_REPOS:
-  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
-    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
-    PATH: "{{ COMMON_GIT_PATH }}"
-    REPO: credentials.git
-    VERSION: "{{ CREDENTIALS_VERSION }}"
-    DESTINATION: "{{ credentials_code_dir }}"
-    SSH_KEY: "{{ CREDENTIALS_GIT_IDENTITY }}"
+# NOTE: This service is one of the few that stores its static files on S3. We use a backend that adds a hash to the
+# filename to avoid overwriting older files, which may be in use, with newer files during deployments. See
+# https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#manifeststaticfilesstorage for more information.
+CREDENTIALS_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
 
+# NOTE: This only needs to be overridden when using non-local storage. Otherwise, the edx_django_service play will
+# properly configure local storage of media and static files.
+CREDENTIALS_FILE_STORAGE_BACKEND: {}
 
-CREDENTIALS_GUNICORN_WORKERS: "2"
-CREDENTIALS_GUNICORN_EXTRA: ""
-CREDENTIALS_GUNICORN_EXTRA_CONF: ""
-CREDENTIALS_GUNICORN_WORKER_CLASS: "gevent"
 
-CREDENTIALS_HOSTNAME: '~^((stage|prod)-)?credentials.*'
+CREDENTIALS_CORS_ORIGIN_ALLOW_ALL: false
+CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT:
+ - '{{ CREDENTIALS_DOMAIN }}'
 
-NGINX_CREDENTIALS_GUNICORN_HOSTS:
-  - 127.0.0.1
+CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA: []
+CREDENTIALS_CORS_ORIGIN_WHITELIST: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT + CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA }}'
 
-CREDENTIALS_EXTRA_APPS: []
+CREDENTIALS_VERSION: 'master'
 
-CREDENTIALS_JWT_AUTH:
-  JWT_ISSUERS:
-    - AUDIENCE: '{{ COMMON_JWT_AUDIENCE }}'
-      ISSUER: '{{ COMMON_JWT_ISSUER }}'
-      SECRET_KEY: '{{ COMMON_JWT_SECRET_KEY }}'
-    - AUDIENCE: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-      ISSUER: '{{ COMMON_JWT_ISSUER }}'
-      SECRET_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-
-CREDENTIALS_SERVICE_CONFIG:
-  SESSION_EXPIRE_AT_BROWSER_CLOSE: '{{ CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
-
-  SECRET_KEY: '{{ CREDENTIALS_SECRET_KEY }}'
-  TIME_ZONE: '{{ CREDENTIALS_TIME_ZONE }}'
-  LANGUAGE_CODE: '{{ CREDENTIALS_LANGUAGE_CODE }}'
-
-  OAUTH2_PROVIDER_URL: '{{ COMMON_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ COMMON_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ COMMON_OAUTH_LOGOUT_URL }}'
-  SOCIAL_AUTH_EDX_OIDC_ISSUER: '{{ COMMON_JWT_ISSUER }}'
-
-  EXTRA_APPS: '{{ CREDENTIALS_EXTRA_APPS }}'
-
-  # db config
-  DATABASES: '{{ CREDENTIALS_DATABASES }}'
-  CACHES: '{{ CREDENTIALS_CACHES }}'
-
-  # Set credentials files storage backend
-  FILE_STORAGE_BACKEND: '{{ CREDENTIALS_FILE_STORAGE_BACKEND }}'
+CREDENTIALS_GUNICORN_EXTRA: ''
 
-  CREDENTIALS_SERVICE_USER: '{{ CREDENTIALS_SERVICE_USER }}'
+CREDENTIALS_EXTRA_APPS: []
 
-  JWT_AUTH: '{{ CREDENTIALS_JWT_AUTH }}'
+CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
 
-  CORS_ORIGIN_WHITELIST: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST }}'
-  CORS_ORIGIN_ALLOW_ALL: '{{ CREDENTIALS_CORS_ORIGIN_ALLOW_ALL }}'
+CREDENTIALS_CERTIFICATE_LANGUAGES:
+  'en': 'English'
+  'es_419': 'Spanish'
 
+credentials_service_config_overrides:
   CERTIFICATE_LANGUAGES: '{{ CREDENTIALS_CERTIFICATE_LANGUAGES }}'
+  CREDENTIALS_SERVICE_USER: '{{ CREDENTIALS_SERVICE_USER }}'
+  FILE_STORAGE_BACKEND: '{{ CREDENTIALS_FILE_STORAGE_BACKEND }}'
 
-#
-# vars are namespace with the module name.
-#
-credentials_service_name: "credentials"
-credentials_venv_dir: "{{ credentials_home }}/venvs/{{ credentials_service_name }}"
-
-credentials_migration_environment:
-  DJANGO_SETTINGS_MODULE: "{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}"
-  CREDENTIALS_CFG: "{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml"
-  PATH: "{{ credentials_venv_dir }}/bin:{{ ansible_env.PATH }}"
-  DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
-  DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
-
-credentials_user: "{{ credentials_service_name }}"
-credentials_home: "{{ COMMON_APP_DIR }}/{{ credentials_service_name }}"
-credentials_code_dir: "{{ credentials_home }}/{{ credentials_service_name }}"
-
-credentials_nodeenv_dir: "{{ credentials_home }}/nodeenvs/{{ credentials_service_name }}"
-credentials_nodeenv_bin: "{{ credentials_nodeenv_dir }}/bin"
-credentials_node_modules_dir: "{{ credentials_code_dir }}/node_modules"
-credentials_node_bin: "{{ credentials_node_modules_dir }}/.bin"
-credentials_node_version: "{{ common_node_version }}"
-
-credentials_environment:
-  DJANGO_SETTINGS_MODULE: "{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}"
-  CREDENTIALS_CFG: "{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml"
-  PATH: "{{ credentials_venv_dir }}/bin:{{ ansible_env.PATH }}:{{ credentials_nodeenv_bin }}:{{ credentials_node_bin }}"
-
-credentials_gunicorn_host: "127.0.0.1"
-credentials_gunicorn_port: 8150
-credentials_gunicorn_timeout: 300
-
-credentials_log_dir: "{{ COMMON_LOG_DIR }}/{{ credentials_service_name }}"
-
-#
-# OS packages
-#
-
-credentials_debian_pkgs:
-  - libmysqlclient-dev
-  - libssl-dev
-  # Needed to manipulate images.
-  - libjpeg8-dev
-  - libpng12-dev
-
-credentials_redhat_pkgs: []
+# See edx_django_service_automated_users for an example of what this should be
+CREDENTIALS_AUTOMATED_USERS: {}

playbooks/roles/credentials/meta/main.yml

@@ -9,24 +9,33 @@
 #
 ##
 # Role includes for role credentials
-# 
-# Example:
 #
-# dependencies:
-#   - {
-#   role: my_role 
-#   my_role_var0: "foo"
-#   my_role_var1: "bar"
-# }
 dependencies:
-  - common
-  - supervisor
-  - role: edx_service
-    edx_service_name: "{{ credentials_service_name }}"
-    edx_service_config: "{{ CREDENTIALS_SERVICE_CONFIG }}"
-    edx_service_repos: "{{ CREDENTIALS_REPOS }}"
-    edx_service_user: "{{ credentials_user }}"
-    edx_service_home: "{{ credentials_home }}"
-    edx_service_packages:
-      debian: "{{ credentials_debian_pkgs }}"
-      redhat: "{{ credentials_redhat_pkgs }}"
+  - role: edx_django_service
+    edx_django_service_version: '{{ CREDENTIALS_VERSION }}'
+    edx_django_service_name: '{{ credentials_service_name }}'
+    edx_django_service_config_overrides: '{{ credentials_service_config_overrides }}'
+    edx_django_service_debian_pkgs_extra: '{{ credentials_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ credentials_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ credentials_environment }}'
+    edx_django_service_gunicorn_extra: '{{ CREDENTIALS_GUNICORN_EXTRA }}'
+    edx_django_service_nginx_port: '{{ CREDENTIALS_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ CREDENTIALS_SSL_NGINX_PORT }}'
+    edx_django_service_language_code: '{{ CREDENTIALS_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ CREDENTIALS_SECRET_KEY }}'
+    edx_django_service_staticfiles_storage: '{{ CREDENTIALS_STATICFILES_STORAGE }}'
+    edx_django_service_media_storage_backend: '{{ CREDENTIALS_MEDIA_STORAGE_BACKEND }}'
+    edx_django_service_memcache: '{{ CREDENTIALS_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ CREDENTIALS_MYSQL_HOST }}'
+    edx_django_service_default_db_name: '{{ CREDENTIALS_DEFAULT_DB_NAME }}'
+    edx_django_service_default_db_atomic_requests: false
+    edx_django_service_db_user: '{{ CREDENTIALS_MYSQL_USER }}'
+    edx_django_service_db_password: '{{ CREDENTIALS_MYSQL_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_extra_apps: '{{ CREDENTIALS_EXTRA_APPS }}'
+    edx_django_service_session_expire_at_browser_close: '{{ CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_automated_users: '{{ CREDENTIALS_AUTOMATED_USERS }}'
+    edx_django_service_cors_whitelist: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST }}'

playbooks/roles/credentials/tasks/main.yml

@@ -11,7 +11,7 @@
 #
 # Tasks for role credentials
 #
-# Overview:
+# Overview: This role's tasks come from edx_django_service.
 #
 #
 # Dependencies:
@@ -20,233 +20,3 @@
 # Example play:
 #
 #
-
-- name: add gunicorn configuration file
-  template:
-    src: edx/app/credentials/credentials_gunicorn.py.j2
-    dest: "{{ credentials_home }}/credentials_gunicorn.py"
-  become_user: "{{ credentials_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: add deadsnakes repository
-  apt_repository:
-    repo: "ppa:fkrull/deadsnakes"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install python3.5
-  apt:
-    name: "{{ item }}"
-  with_items:
-    - python3.5
-    - python3.5-dev
-  tags:
-    - install
-    - install:system-requirements
-
-- name: build virtualenv
-  command: "virtualenv --python=python3.5 {{ credentials_venv_dir }}"
-  args:
-    creates: "{{ credentials_venv_dir }}/bin/pip"
-  become_user: "{{ credentials_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install nodenv
-  pip:
-    name: "nodeenv"
-    version: "1.1.2"
-    # NOTE (CCB): Using the "virtualenv" option here doesn't seem to work.
-    executable: "{{ credentials_venv_dir }}/bin/pip"
-  become_user: "{{ credentials_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: create nodeenv
-  shell: "{{ credentials_venv_dir }}/bin/nodeenv {{ credentials_nodeenv_dir }} --node={{ credentials_node_version }} --prebuilt --force"
-  become_user: "{{ credentials_user }}"
-  tags:
-    - install
-    - install:system-requirements
-
-- name: install application requirements
-  command: make production-requirements
-  args:
-    chdir: "{{ credentials_code_dir }}"
-  become_user: "{{ credentials_user }}"
-  environment: "{{ credentials_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: install development requirements
-  command: make requirements
-  args:
-    chdir: "{{ credentials_code_dir }}"
-  become_user: "{{ credentials_user }}"
-  environment: "{{ credentials_environment }}"
-  tags:
-    - devstack
-    - devstack:install
-
-- name: migrate database
-  command: make migrate
-  args:
-    chdir: "{{ credentials_code_dir }}"
-  become_user: "{{ credentials_user }}"
-  environment: "{{ credentials_migration_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-# var should have more permissive permissions than the rest
-- name: create credentials var dirs
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0775
-    owner: "{{ credentials_user }}"
-    group: "{{ common_web_group }}"
-  with_items:
-    - "{{ CREDENTIALS_MEDIA_ROOT }}"
-  tags:
-    - install
-    - install:base
-
-- name: write out the supervisor wrapper
-  template:
-    src: "edx/app/credentials/credentials.sh.j2"
-    dest: "{{ credentials_home }}/{{ credentials_service_name }}.sh"
-    mode: 0650
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/credentials.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ credentials_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: write devstack script
-  template:
-    src: "edx/app/credentials/devstack.sh.j2"
-    dest: "{{ credentials_home }}/devstack.sh"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: 0744
-  tags:
-    - devstack
-    - devstack:install
-
-- name: setup the credentials env file
-  template:
-    src: "./{{ credentials_home }}/{{ credentials_service_name }}_env.j2"
-    dest: "{{ credentials_home }}/credentials_env"
-    owner: "{{ credentials_user }}"
-    group: "{{ credentials_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ credentials_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ credentials_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-  tags:
-    - install
-    - install:configuration
-
-- name: update supervisor configuration
-  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-  tags:
-    - manage
-    - manage:start
-
-- name: create symlinks from the venv bin dir
-  file:
-    src: "{{ credentials_venv_dir }}/bin/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.credentials"
-    state: link
-  with_items:
-    - python
-    - pip
-    - django-admin.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: create symlinks from the repo dir
-  file:
-    src: "{{ credentials_code_dir }}/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.credentials"
-    state: link
-  with_items:
-    - manage.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: run collectstatic
-  command: make static
-  args:
-    chdir: "{{ credentials_code_dir }}"
-  become_user: "{{ credentials_user }}"
-  environment: "{{ credentials_environment }}"
-  tags:
-    - assets
-    - assets:gather
-
-- name: restart the application
-  supervisorctl:
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-    name: "{{ credentials_service_name }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
-  tags:
-    - manage
-    - manage:start
-
-- name: Copying nginx configs for credentials
-  template:
-    src: edx/app/nginx/sites-available/credentials.j2
-    dest: "{{ nginx_sites_available_dir }}/credentials"
-    owner: root
-    group: "{{ common_web_user }}"
-    mode: 0640
-  notify: reload nginx
-  tags:
-    - install
-    - install:vhosts
-
-- name: Creating nginx config links for credentials
-  file:
-    src: "{{ nginx_sites_available_dir }}/credentials"
-    dest: "{{ nginx_sites_enabled_dir }}/credentials"
-    state: link
-    owner: root
-    group: root
-  notify: reload nginx
-  tags:
-    - install
-    - install:vhosts

playbooks/roles/credentials/templates/edx/app/credentials/credentials.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% set credentials_venv_bin = credentials_home + "/venvs/" + credentials_service_name + "/bin" %}
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = credentials_venv_bin + '/newrelic-admin run-program ' + credentials_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = credentials_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-export NEW_RELIC_APP_NAME="{{ CREDENTIALS_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-source {{ credentials_home }}/credentials_env
-{{ executable }} -c {{ credentials_home }}/credentials_gunicorn.py {{ CREDENTIALS_GUNICORN_EXTRA }} credentials.wsgi:application

playbooks/roles/credentials/templates/edx/app/credentials/credentials_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in credentials_environment.items() -%}
-{%- if value -%}
-export {{ name }}="{{ value }}"
-{% endif %}
-{%- endfor %}

playbooks/roles/credentials/templates/edx/app/credentials/credentials_gunicorn.py.j2

-"""
-gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
-{{ ansible_managed }}
-"""
-
-timeout = {{ credentials_gunicorn_timeout }}
-bind = "{{ credentials_gunicorn_host }}:{{ credentials_gunicorn_port }}"
-pythonpath = "{{ credentials_code_dir }}"
-workers = {{ CREDENTIALS_GUNICORN_WORKERS }}
-worker_class = "{{ CREDENTIALS_GUNICORN_WORKER_CLASS }}"
-
-{{ CREDENTIALS_GUNICORN_EXTRA_CONF }}

playbooks/roles/credentials/templates/edx/app/credentials/devstack.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-source {{ credentials_home }}/credentials_env
-COMMAND=$1
-
-case $COMMAND in
-    start)
-        {% set credentials_venv_bin = credentials_home + "/venvs/" + credentials_service_name + "/bin" %}
-
-        {{ supervisor_venv_bin }}/supervisord --configuration {{ supervisor_cfg }}
-
-	# Needed to run bower as root. See explaination around 'credentials_user=root'
-	echo '{ "allow_root": true }' > /root/.bowerrc
-
-        cd /edx/app/edx_ansible/edx_ansible/docker/plays
-        /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook credentials.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack,migrate' \
-            --extra-vars="migrate_db=yes" \
-            --extra-vars="@/ansible_overrides.yml" \
-	    --extra-vars="credentials_user=root"  # Needed when sharing the volume with the host machine because node/bower drops
-						  # everything in the code directory by default.  So we get issues with permissions
-						  # on folders owned by the developer.
-
-
-	# Need to start supervisord and nginx manually because systemd is hard to run on docker
-	# http://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
-	# Both daemon by default
-	nginx
-	/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
-
-
-        # Docker requires an active foreground task. Tail the logs to appease Docker and
-        # provide useful output for development.
-        cd {{ supervisor_log_dir }}
-        tail -f {{ credentials_service_name }}-stderr.log -f {{ credentials_service_name }}-stdout.log
-
-        ;;
-    open)
-        cd {{ credentials_code_dir }}/
-        . {{ credentials_venv_bin }}/activate
-        /bin/bash
-        ;;
-esac

playbooks/roles/credentials/templates/edx/app/nginx/sites-available/credentials.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if nginx_default_sites is defined and "credentials" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream credentials_app_server {
-{% for host in NGINX_CREDENTIALS_GUNICORN_HOSTS %}
-    server {{ host }}:{{ credentials_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-# The Origin request header indicates where a fetch originates from. It doesn't include any path information,
-# but only the server name (e.g. https://www.example.com).
-# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin for details.
-#
-# Here we set the value that is included in the Access-Control-Allow-Origin response header. If the origin is one
-# of our known hosts--served via HTTP or HTTPS--we allow for CORS. Otherwise, we set the "null" value, disallowing CORS.
-map $http_origin $cors_origin {
-  default "null";
-{% for host in CREDENTIALS_CORS_ORIGIN_WHITELIST %}
-  "~*^https?:\/\/{{ host|replace('.', '\.') }}$" $http_origin;
-{% endfor %}
-}
-
-server {
-  server_name {{ CREDENTIALS_HOSTNAME }};
-
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
-  listen {{ CREDENTIALS_SSL_NGINX_PORT }} ssl;
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-
-  {% else %}
-  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
-  {% endif %}
-
-  location ~ ^{{ CREDENTIALS_MEDIA_URL }}(?P<file>.*) {
-    root {{ CREDENTIALS_MEDIA_ROOT }};
-    try_files /$file =404;
-  }
-
-  location ~ ^{{ CREDENTIALS_STATIC_URL }}(?P<file>.*) {
-    root {{ CREDENTIALS_STATIC_ROOT }};
-    add_header Cache-Control "max-age=31536000";
-    add_header 'Access-Control-Allow-Origin' $cors_origin;
-
-    # Inform downstream caches to take certain headers into account when reading/writing to cache.
-    add_header 'Vary' 'Accept-Encoding,Origin';
-
-    try_files /$file =404;
-  }
-
-  location / {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% if NGINX_ROBOT_RULES|length > 0 %}
-  location /robots.txt {
-      root {{ nginx_app_dir }};
-      try_files $uri /robots.txt =404;
-  }
-  {% endif %}
-
-  location @proxy_to_app {
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://credentials_app_server;
-  }
-
-  # Forward to HTTPS if we're an HTTP request...
-  if ($http_x_forwarded_proto = "http") {
-    set $do_redirect "true";
-  }
-
-  # Run our actual redirect...
-  if ($do_redirect = "true") {
-    rewrite ^ https://$host$request_uri? permanent;
-  }
-}

playbooks/roles/credentials/templates/edx/app/supervisor/conf.d.available/credentials.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ credentials_service_name }}]
-
-command={{ credentials_home }}/{{ credentials_service_name }}.sh
-user={{ common_web_user }}
-directory={{ credentials_code_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/nginx/defaults/main.yml

@@ -49,7 +49,7 @@ NGINX_LOG_FORMAT_NAME: 'p_combined'
 # headers to reflect the properties of the incoming request.
 NGINX_SET_X_FORWARDED_HEADERS: False
 
-# Increasing these values allows studio to process more complex operations.  
+# Increasing these values allows studio to process more complex operations.
 # Default timeouts limit CMS connections to 60 seconds.
 
 NGINX_CMS_PROXY_CONNECT_TIMEOUT: !!null
@@ -111,8 +111,6 @@ NGINX_EDXAPP_ERROR_PAGES:
   "504": "{{ nginx_default_error_page }}"
 
 CMS_HOSTNAME: '~^((stage|prod)-)?studio.*'
-ECOMMERCE_HOSTNAME: '~^((stage|prod)-)?ecommerce.*'
-CREDENTIALS_HOSTNAME: '~^((stage|prod)-)?credentials.*'
 
 nginx_template_dir: "edx/app/nginx/sites-available"
 

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/credentials.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if "credentials" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream credentials_app_server {
-{% for host in NGINX_CREDENTIALS_GUNICORN_HOSTS %}
-    server {{ host }}:{{ credentials_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-server {
-  server_name {{ CREDENTIALS_HOSTNAME }};
-
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
-  listen {{ CREDENTIALS_SSL_NGINX_PORT }} ssl;
-
-  {% include "common-settings.j2" %}
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-
-  {% else %}
-  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
-  {% endif %}
-  
-  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
-  {% endif %}
-
-  # Prevent invalid display courseware in IE 10+ with high privacy settings
-  add_header P3P '{{ NGINX_P3P_MESSAGE }}';
-
-  location ~ ^/static/(?P<file>.*) {
-    root {{ COMMON_DATA_DIR }}/{{ credentials_service_name }};
-    try_files /staticfiles/$file =404;
-  }
-
-  location / {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% include "robots.j2" %}
-
-location @proxy_to_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
-
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://credentials_app_server;
-  }
-  # Nginx does not support nested condition or or conditions so
-  # there is an unfortunate mix of conditonals here.
-  {% if NGINX_REDIRECT_TO_HTTPS %}
-     {% if NGINX_HTTPS_REDIRECT_STRATEGY == "scheme" %}
-  # Redirect http to https over single instance
-  if ($scheme != "https") 
-  { 
-   set $do_redirect_to_https "true";
-  }
- 
-    {% elif NGINX_HTTPS_REDIRECT_STRATEGY == "forward_for_proto" %}
-
-  # Forward to HTTPS if we're an HTTP request... and the server is behind ELB 
-  if ($http_x_forwarded_proto = "http") 
-  {
-   set $do_redirect_to_https "true";
-  }
-     {% endif %}
-
-  # Execute the actual redirect
-  if ($do_redirect_to_https = "true")
-  {
-  return 301 https://$host$request_uri;
-  }
-  {% endif %}
-}