Merge pull request #2603 from edx/zub/add-credentials-role

add credentials configuration role

.travis.yml

@@ -17,6 +17,10 @@ addons:
       - nodejs
       - python-demjson
 
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install docker-engine
+
 install:
   - "pip install --allow-all-external -r requirements.txt"
   - "pip install --allow-all-external demjson"

docker/build/course-discovery/ansible_overrides.yml

@@ -3,4 +3,5 @@ course_discovery_gunicorn_host: 0.0.0.0
 COURSE_DISCOVERY_MYSQL: 'db'
 COURSE_DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack'
 COURSE_DISCOVERY_ELASTICSEARCH_HOST: 'es'
+COURSE_DISCOVERY_MYSQL_MATCHER: '%'
 

docker/build/credentials/Dockerfile

+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/credentials/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
+FROM edxops/trusty-common:latest
+MAINTAINER edxops
+
+ARG CREDENTIALS_VERSION=master
+ARG REPO_OWNER=edx
+
+ADD . /edx/app/edx_ansible/edx_ansible
+
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+COPY docker/build/credentials/ansible_overrides.yml /
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook credentials.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets,devstack:install' \
+    --extra-vars="@/ansible_overrides.yml" \
+    --extra-vars="CREDENTIALS_VERSION=$CREDENTIALS_VERSION" \
+    --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
+
+USER root 
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]

docker/build/credentials/ansible_overrides.yml

+---
+credentials_gunicorn_host: 127.0.0.1
+CREDENTIALS_MYSQL: 'db'
+CREDENTIALS_DJANGO_SETTINGS_MODULE: 'credentials.settings.devstack'
+CREDENTIALS_MYSQL_MATCHER: '%'
+

docker/plays/credentials.yml

+- name: Deploy Credentials
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - nginx
+    - docker
+    - role: credentials
+      nginx_default_sites:
+        - credentials

playbooks/edx-east/credentials.yml

+- name: Deploy edX Credentials Service
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: False
+    CLUSTER_NAME: 'credentials'
+  roles:
+    - role: nginx
+      nginx_sites:
+        - credentials
+      nginx_default_sites:
+        - credentials
+    - aws
+    - credentials
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/edx-east/edx_continuous_integration.yml

@@ -18,6 +18,7 @@
       - analytics_api
       - ecommerce
       - programs
+      - credentials
       nginx_default_sites:
       - lms
     - mysql
@@ -40,6 +41,7 @@
     - analytics_api
     - ecommerce
     - programs
+    - credentials
     - oauth_client_setup
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx-east/oauth_client_setup.yml

@@ -8,5 +8,6 @@
     - "roles/insights/defaults/main.yml"
     - "roles/ecommerce/defaults/main.yml"
     - "roles/programs/defaults/main.yml"
+    - "roles/credentials/defaults/main.yml"
   roles:
     - oauth_client_setup

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

@@ -32,6 +32,7 @@
 
 {{ role_name|upper }}_DB_ADMIN_USER: 'root'
 {{ role_name|upper }}_DB_ADMIN_PASSWORD: ''
+{{ role_name|upper }}_MYSQL_MATCHER: 'localhost'
 
 {{ role_name|upper }}_MEMCACHE: [ 'memcache' ]
 

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/Dockerfile.j2

@@ -10,8 +10,8 @@
 FROM edxops/trusty-common:latest
 MAINTAINER edxops
 
-ENV {{ role_name|upper }}_VERSION=master
-ENV REPO_OWNER=edx
+ARG {{ role_name|upper }}_VERSION=master
+ARG REPO_OWNER=edx
 
 ADD . /edx/app/edx_ansible/edx_ansible
 

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/ansible_overrides.yml.j2

 ---
-{{ role_name }}_gunicorn_host: 0.0.0.0
+{{ role_name }}_gunicorn_host: 127.0.0.1
 {{ role_name|upper }}_MYSQL: 'db'
 {{ role_name|upper }}_DJANGO_SETTINGS_MODULE: '{{ role_name }}.settings.devstack'
+{{ role_name|upper }}_MYSQL_MATCHER: '%'
 

playbooks/roles/ansible-role-django-ida/templates/tasks/main.yml.j2

@@ -23,6 +23,16 @@
     - install
     - install:configuration
 
+- name: build virtualenv
+  command: "virtualenv {{ '{{' }} {{ role_name }}_venv_dir }}"
+  args:
+    creates: "{{ '{{' }} {{ role_name }}_venv_dir }}/bin/pip"
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  environment: "{{ '{{' }} {{ role_name }}_environment }}"
+  tags:
+    - install
+    - install:app-requirements
+
 - name: install application requirements
   command: make requirements
   args:
@@ -49,8 +59,8 @@
     port: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.PORT }}"
     delay: 2
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: create databases
   mysql_db:
@@ -61,8 +71,8 @@
     state: present
     encoding: utf8
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: create database users
   mysql_user:
@@ -70,7 +80,7 @@
     login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_USER }}"
     login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_PASSWORD }}"
     name: "{{ '{{' }} item.name }}"
-    host: "%"
+    host: "{{ '{{' }} {{ role_name|upper }}_MYSQL_MATCHER }}"
     password: "{{ '{{' }} item.password }}"
     priv: "{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}.*:ALL"
   with_items:
@@ -79,8 +89,8 @@
     - name: "{{ '{{' }} COMMON_MYSQL_MIGRATE_USER }}"
       password: "{{ '{{' }} COMMON_MYSQL_MIGRATE_PASS }}"
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: migrate database
   command: make migrate
@@ -92,8 +102,6 @@
   tags:
     - migrate
     - migrate:db
-    - devstack
-    - devstack:migrate
 
 - name: write out the supervisor wrapper
   template:

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/devstack.sh.j2

@@ -11,8 +11,8 @@ case $COMMAND in
         {{ '{%' }} set executable = {{ role_name }}_venv_bin + '/gunicorn' %}
 
         cd /edx/app/edx_ansible/edx_ansible/docker/plays
-        sudo ansible-playbook {{ role_name }}.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack' \
+        sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook {{ role_name }}.yml -c local -i '127.0.0.1,' \
+            -t 'install:app-requirements,assets:gather,devstack,migrate' \
             --extra-vars="migrate_db=yes" \
             --extra-vars="@/ansible_overrides.yml"
 

playbooks/roles/course_discovery/defaults/main.yml

@@ -46,6 +46,7 @@ COURSE_DISCOVERY_ELASTICSEARCH:
 
 COURSE_DISCOVERY_DB_ADMIN_USER: 'root'
 COURSE_DISCOVERY_DB_ADMIN_PASSWORD: ''
+COURSE_DISCOVERY_MYSQL_MATCHER: 'localhost'
 
 COURSE_DISCOVERY_MEMCACHE: [ 'memcache' ]
 

playbooks/roles/course_discovery/tasks/main.yml

@@ -81,8 +81,8 @@
     port: "{{ COURSE_DISCOVERY_DATABASES.default.PORT }}"
     delay: 2
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: create databases
   mysql_db:
@@ -93,8 +93,8 @@
     state: present
     encoding: utf8
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: create database users
   mysql_user:
@@ -102,7 +102,7 @@
     login_user: "{{ COURSE_DISCOVERY_DB_ADMIN_USER }}"
     login_password: "{{ COURSE_DISCOVERY_DB_ADMIN_PASSWORD }}"
     name: "{{ item.name }}"
-    host: "%"
+    host: "{{ COURSE_DISCOVERY_MYSQL_MATCHER }}"
     password: "{{ item.password }}"
     priv: "{{ COURSE_DISCOVERY_DEFAULT_DB_NAME }}.*:ALL"
   with_items:
@@ -111,8 +111,8 @@
     - name: "{{ COMMON_MYSQL_MIGRATE_USER }}"
       password: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
   tags:
-    - devstack
-    - devstack:migrate
+    - migrate
+    - migrate:install
 
 - name: migrate database
   command: make migrate
@@ -124,8 +124,6 @@
   tags:
     - migrate
     - migrate:db
-    - devstack
-    - devstack:migrate
 
 - name: write out the supervisor wrapper
   template:

playbooks/roles/course_discovery/templates/edx/app/course_discovery/devstack.sh.j2

@@ -12,7 +12,7 @@ case $COMMAND in
 
         cd /edx/app/edx_ansible/edx_ansible/docker/plays
         sudo ansible-playbook course_discovery.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack' \
+            -t 'install:app-requirements,assets:gather,devstack,migrate' \
             --extra-vars="migrate_db=yes" \
             --extra-vars="@/ansible_overrides.yml"
 

playbooks/roles/credentials/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role credentials
+# 
+CREDENTIALS_GIT_IDENTITY: !!null
+
+# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
+# and a key being provided via NEWRELIC_LICENSE_KEY
+CREDENTIALS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ credentials_service_name }}"
+CREDENTIALS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+CREDENTIALS_NGINX_PORT: 18150
+CREDENTIALS_SSL_NGINX_PORT: 48150
+
+CREDENTIALS_DEFAULT_DB_NAME: 'credentials'
+CREDENTIALS_MYSQL: 'localhost'
+# MySQL usernames are limited to 16 characters
+CREDENTIALS_MYSQL_USER: 'credentials001'
+CREDENTIALS_MYSQL_PASSWORD: 'password'
+
+CREDENTIALS_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ CREDENTIALS_DEFAULT_DB_NAME }}'
+    USER: '{{ CREDENTIALS_MYSQL_USER }}'
+    PASSWORD: '{{ CREDENTIALS_MYSQL_PASSWORD }}'
+    HOST: '{{ CREDENTIALS_MYSQL }}'
+    PORT: '3306'
+    ATOMIC_REQUESTS: true
+    CONN_MAX_AGE: 60
+
+CREDENTIALS_DB_ADMIN_USER: 'root'
+CREDENTIALS_DB_ADMIN_PASSWORD: ''
+CREDENTIALS_MYSQL_MATCHER: 'localhost'
+
+CREDENTIALS_MEMCACHE: [ 'memcache' ]
+
+CREDENTIALS_CACHES:
+  default:
+    BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
+    KEY_PREFIX: 'default'
+    LOCATION: '{{ CREDENTIALS_MEMCACHE }}'
+
+CREDENTIALS_VERSION: "master"
+CREDENTIALS_DJANGO_SETTINGS_MODULE: "credentials.settings.production"
+CREDENTIALS_URL_ROOT: 'http://credentials:18150'
+CREDENTIALS_OAUTH_URL_ROOT: 'http://127.0.0.1:8000'
+
+CREDENTIALS_SECRET_KEY: 'Your secret key here'
+CREDENTIALS_TIME_ZONE: 'UTC'
+CREDENTIALS_LANGUAGE_CODE: 'en_US.UTF-8'
+
+# Used to automatically configure OAuth2 Client
+CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY : 'credentials-key'
+CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET : 'credentials-secret'
+CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+
+CREDENTIALS_PLATFORM_NAME: 'Your Platform Name Here'
+
+CREDENTIALS_SERVICE_CONFIG:
+  SECRET_KEY: '{{ CREDENTIALS_SECRET_KEY }}'
+  TIME_ZONE: '{{ CREDENTIALS_TIME_ZONE }}'
+  LANGUAGE_CODE: '{{ CREDENTIALS_LANGUAGE_CODE }}'
+
+  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ CREDENTIALS_OAUTH_URL_ROOT }}/oauth2'
+  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+
+  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ credentials_service_name }}/staticfiles"
+  # db config
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES: '{{ CREDENTIALS_DATABASES }}'
+  CACHES: '{{ CREDENTIALS_CACHES }}'
+
+  PLATFORM_NAME: '{{ CREDENTIALS_PLATFORM_NAME }}'
+
+  AWS_STORAGE_BUCKET_NAME: ''
+  AWS_ACCESS_KEY_ID: ''
+  AWS_SECRET_ACCESS_KEY: ''
+
+
+CREDENTIALS_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: credentials.git
+    VERSION: "{{ CREDENTIALS_VERSION }}"
+    DESTINATION: "{{ credentials_code_dir }}"
+    SSH_KEY: "{{ CREDENTIALS_GIT_IDENTITY }}"
+
+
+CREDENTIALS_GUNICORN_WORKERS: "2"
+CREDENTIALS_GUNICORN_EXTRA: ""
+CREDENTIALS_GUNICORN_EXTRA_CONF: ""
+CREDENTIALS_GUNICORN_WORKER_CLASS: "gevent"
+
+CREDENTIALS_HOSTNAME: '~^((stage|prod)-)?credentials.*'
+
+NGINX_CREDENTIALS_GUNICORN_HOSTS:
+  - 127.0.0.1
+
+#
+# vars are namespace with the module name.
+#
+credentials_role_name: credentials
+credentials_venv_dir: "{{ credentials_home }}/venvs/{{ credentials_service_name }}"
+
+credentials_migration_environment:
+  DJANGO_SETTINGS_MODULE: "{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}"
+  CREDENTIALS_CFG: "{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml"
+  PATH: "{{ credentials_venv_dir }}/bin:{{ ansible_env.PATH }}"
+  DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+  DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
+
+credentials_service_name: "{{ credentials_role_name }}"
+credentials_user: "{{ credentials_service_name }}"
+credentials_home: "{{ COMMON_APP_DIR }}/{{ credentials_service_name }}"
+credentials_code_dir: "{{ credentials_home }}/{{ credentials_service_name }}"
+
+credentials_nodeenv_dir: "{{ credentials_home }}/nodeenvs/{{ credentials_service_name }}"
+credentials_nodeenv_bin: "{{ credentials_nodeenv_dir }}/bin"
+credentials_node_modules_dir: "{{ credentials_code_dir }}/node_modules"
+credentials_node_bin: "{{ credentials_node_modules_dir }}/.bin"
+
+credentials_environment:
+  DJANGO_SETTINGS_MODULE: "{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}"
+  CREDENTIALS_CFG: "{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml"
+  PATH: "{{ credentials_venv_dir }}/bin:{{ ansible_env.PATH }}:{{ credentials_nodeenv_bin }}:{{ credentials_node_bin }}"
+
+credentials_gunicorn_host: "127.0.0.1"
+credentials_gunicorn_port: 8150
+credentials_gunicorn_timeout: 300
+
+credentials_log_dir: "{{ COMMON_LOG_DIR }}/{{ credentials_service_name }}"
+
+#
+# OS packages
+#
+
+credentials_debian_pkgs:
+  - libmysqlclient-dev
+  - libssl-dev
+  # Needed to manipulate images.
+  - libjpeg8-dev
+  - libpng12-dev
+
+credentials_redhat_pkgs: []

playbooks/roles/credentials/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role credentials
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - common
+  - supervisor
+  - role: edx_service
+    edx_service_name: "{{ credentials_service_name }}"
+    edx_service_config: "{{ CREDENTIALS_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ CREDENTIALS_REPOS }}"
+    edx_service_user: "{{ credentials_user }}"
+    edx_service_home: "{{ credentials_home }}"
+    edx_service_packages:
+      debian: "{{ credentials_debian_pkgs }}"
+      redhat: "{{ credentials_redhat_pkgs }}"

playbooks/roles/credentials/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role credentials
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: add gunicorn configuration file
+  template:
+    src: edx/app/credentials/credentials_gunicorn.py.j2
+    dest: "{{ credentials_home }}/credentials_gunicorn.py"
+  sudo_user: "{{ credentials_user }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: build virtualenv
+  command: "virtualenv {{ credentials_venv_dir }}"
+  args:
+    creates: "{{ credentials_venv_dir }}/bin/pip"
+  sudo_user: "{{ credentials_user }}"
+  environment: "{{ credentials_environment }}"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install application requirements
+  command: make prod-requirements
+  args:
+    chdir: "{{ credentials_code_dir }}"
+  sudo_user: "{{ credentials_user }}"
+  environment: "{{ credentials_environment }}"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: create nodeenv
+  shell: >
+    creates={{ credentials_nodeenv_dir }}
+    {{ credentials_venv_dir }}/bin/nodeenv {{ credentials_nodeenv_dir }} --prebuilt
+  sudo_user: "{{ credentials_user }}"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install development requirements
+  command: make requirements
+  args:
+    chdir: "{{ credentials_code_dir }}"
+  sudo_user: "{{ credentials_user }}"
+  environment: "{{ credentials_environment }}"
+  tags:
+    - devstack
+    - devstack:install
+
+- name: wait for database
+  wait_for:
+    host: "{{ CREDENTIALS_DATABASES.default.HOST }}"
+    port: "{{ CREDENTIALS_DATABASES.default.PORT }}"
+    delay: 2
+  tags:
+    - migrate
+    - migrate:install
+
+- name: create databases
+  mysql_db:
+    login_host: "{{ CREDENTIALS_DATABASES.default.HOST }}"
+    login_user: "{{ CREDENTIALS_DB_ADMIN_USER }}"
+    login_password: "{{ CREDENTIALS_DB_ADMIN_PASSWORD }}"
+    db: "{{ CREDENTIALS_DEFAULT_DB_NAME }}"
+    state: present
+    encoding: utf8
+  tags:
+    - migrate
+    - migrate:install
+
+- name: create database users
+  mysql_user:
+    login_host: "{{ CREDENTIALS_DATABASES.default.HOST }}"
+    login_user: "{{ CREDENTIALS_DB_ADMIN_USER }}"
+    login_password: "{{ CREDENTIALS_DB_ADMIN_PASSWORD }}"
+    name: "{{ item.name }}"
+    host: "{{ CREDENTIALS_MYSQL_MATCHER }}"
+    password: "{{ item.password }}"
+    priv: "{{ CREDENTIALS_DEFAULT_DB_NAME }}.*:ALL"
+  with_items:
+    - name: "{{ CREDENTIALS_DATABASES.default.USER }}"
+      password: "{{ CREDENTIALS_DATABASES.default.PASSWORD }}"
+    - name: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+      password: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
+  tags:
+    - migrate
+    - migrate:install
+
+- name: migrate database
+  command: make migrate
+  args:
+    chdir: "{{ credentials_code_dir }}"
+  sudo_user: "{{ credentials_user }}"
+  environment: "{{ credentials_migration_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+  tags:
+    - migrate
+    - migrate:db
+
+- name: write out the supervisor wrapper
+  template:
+    src: "edx/app/credentials/credentials.sh.j2"
+    dest: "{{ credentials_home }}/{{ credentials_service_name }}.sh"
+    mode: 0650
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: write supervisord config
+  template:
+    src: "edx/app/supervisor/conf.d.available/credentials.conf.j2"
+    dest: "{{ supervisor_available_dir }}/{{ credentials_service_name }}.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: write devstack script
+  template:
+    src: "edx/app/credentials/devstack.sh.j2"
+    dest: "{{ credentials_home }}/devstack.sh"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0744
+  tags:
+    - devstack
+    - devstack:install
+
+- name: setup the credentials env file
+  template:
+    src: "./{{ credentials_home }}/{{ credentials_service_name }}_env.j2"
+    dest: "{{ credentials_home }}/credentials_env"
+    owner: "{{ credentials_user }}"
+    group: "{{ credentials_user }}"
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: enable supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/{{ credentials_service_name }}.conf"
+    dest: "{{ supervisor_cfg_dir }}/{{ credentials_service_name }}.conf"
+    state: link
+    force: yes
+  when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
+
+- name: update supervisor configuration
+  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+  tags:
+    - manage
+    - manage:start
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: "{{ credentials_venv_dir }}/bin/{{ item }}"
+    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.credentials"
+    state: link
+  with_items:
+    - python
+    - pip
+    - django-admin.py
+  tags:
+    - install
+    - install:app-requirements
+
+- name: create symlinks from the repo dir
+  file:
+    src: "{{ credentials_code_dir }}/{{ item }}"
+    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.credentials"
+    state: link
+  with_items:
+    - manage.py
+  tags:
+    - install
+    - install:app-requirements
+
+- name: run collectstatic
+  command: make static
+  args:
+    chdir: "{{ credentials_code_dir }}"
+  sudo_user: "{{ credentials_user }}"
+  environment: "{{ credentials_environment }}"
+  tags:
+    - assets
+    - assets:gather
+
+- name: restart the application
+  supervisorctl: 
+    state: restarted
+    supervisorctl_path: "{{ supervisor_ctl }}"
+    config: "{{ supervisor_cfg }}"
+    name: "{{ credentials_service_name }}"
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"
+  tags:
+    - manage
+    - manage:start
+
+- name: Copying nginx configs for credentials
+  template: >
+    src=edx/app/nginx/sites-available/credentials.j2
+    dest={{ nginx_sites_available_dir }}/credentials
+    owner=root group={{ common_web_user }} mode=0640
+  notify: reload nginx
+  tags:
+    - install
+    - install:vhosts
+
+- name: Creating nginx config links for credentials
+  file: >
+    src={{ nginx_sites_available_dir }}/credentials
+    dest={{ nginx_sites_enabled_dir }}/credentials
+    state=link owner=root group=root
+  notify: reload nginx
+  tags:
+    - install
+    - install:vhosts

playbooks/roles/credentials/templates/edx/app/credentials/credentials.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% set credentials_venv_bin = credentials_home + "/venvs/" + credentials_service_name + "/bin" %}
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = credentials_venv_bin + '/newrelic-admin run-program ' + credentials_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = credentials_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ CREDENTIALS_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+source {{ credentials_home }}/credentials_env
+{{ executable }} -c {{ credentials_home }}/credentials_gunicorn.py {{ CREDENTIALS_GUNICORN_EXTRA }} credentials.wsgi:application

playbooks/roles/credentials/templates/edx/app/credentials/credentials_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in credentials_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/credentials/templates/edx/app/credentials/credentials_gunicorn.py.j2

+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+{{ ansible_managed }}
+"""
+
+timeout = {{ credentials_gunicorn_timeout }}
+bind = "{{ credentials_gunicorn_host }}:{{ credentials_gunicorn_port }}"
+pythonpath = "{{ credentials_code_dir }}"
+workers = {{ CREDENTIALS_GUNICORN_WORKERS }}
+worker_class = "{{ CREDENTIALS_GUNICORN_WORKER_CLASS }}"
+
+{{ CREDENTIALS_GUNICORN_EXTRA_CONF }}

playbooks/roles/credentials/templates/edx/app/credentials/devstack.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source {{ credentials_home }}/credentials_env
+COMMAND=$1
+
+case $COMMAND in
+    start)
+        {% set credentials_venv_bin = credentials_home + "/venvs/" + credentials_service_name + "/bin" %}
+        {% set executable = credentials_venv_bin + '/gunicorn' %}
+
+        cd /edx/app/edx_ansible/edx_ansible/docker/plays
+        sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook credentials.yml -c local -i '127.0.0.1,' \
+            -t 'install:app-requirements,assets:gather,devstack,migrate' \
+            --extra-vars="migrate_db=yes" \
+            --extra-vars="@/ansible_overrides.yml"
+
+        {{ executable }} -c {{ credentials_home }}/credentials_gunicorn.py \
+                                 {{ CREDENTIALS_GUNICORN_EXTRA }} \
+                                 --reload \
+                                 credentials.wsgi:application
+
+        ;;
+    open)
+        cd {{ credentials_code_dir }}/
+        . {{ credentials_venv_bin }}/activate
+        /bin/bash
+        ;;
+esac

playbooks/roles/credentials/templates/edx/app/nginx/sites-available/credentials.j2

+#
+# {{ ansible_managed }}
+#
+
+
+{% if nginx_default_sites is defined and "credentials" in nginx_default_sites %}
+  {% set default_site = "default" %}
+{% else %}
+  {% set default_site = "" %}
+{% endif %}
+
+upstream credentials_app_server {
+{% for host in NGINX_CREDENTIALS_GUNICORN_HOSTS %}
+    server {{ host }}:{{ credentials_gunicorn_port }} fail_timeout=0;
+{% endfor %}
+}
+
+server {
+  server_name {{ CREDENTIALS_HOSTNAME }};
+
+  {% if NGINX_ENABLE_SSL %}
+
+  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
+  listen {{ CREDENTIALS_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {% else %}
+  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
+  {% endif %}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ credentials_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% if NGINX_ROBOT_RULES|length > 0 %}
+  location /robots.txt {
+      root {{ nginx_app_dir }};
+      try_files $uri /robots.txt =404;
+  }
+  {% endif %}
+
+  location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://credentials_app_server;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+}

playbooks/roles/credentials/templates/edx/app/supervisor/conf.d.available/credentials.conf.j2

+#
+# {{ ansible_managed }}
+# 
+[program:{{ credentials_service_name }}]
+
+command={{ credentials_home }}/{{ credentials_service_name }}.sh
+user={{ common_web_user }}
+directory={{ credentials_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/launch_ec2/tasks/main.yml

@@ -78,7 +78,7 @@
     value: "{{ item[0].public_dns_name }}"
   with_nested:
     - ec2.instances
-    - ['studio', 'ecommerce', 'preview', 'programs', 'course-discovery']
+    - ['studio', 'ecommerce', 'preview', 'programs', 'course-discovery', 'credentials']
 
 
 - name: Add new instance to host group

playbooks/roles/local_dev/defaults/main.yml

@@ -62,6 +62,13 @@ localdev_accounts:
         repo: "programs"
       }
 
+    - {
+        user: "{{ credentials_user|default('None') }}",
+        home: "{{ credentials_home }}",
+        env: "credentials_env",
+        repo: "credentials"
+      }
+
 # Helpful system packages for local dev
 local_dev_pkgs:
     - vim

playbooks/roles/nginx/defaults/main.yml

@@ -73,6 +73,7 @@ NGINX_EDXAPP_ERROR_PAGES:
 CMS_HOSTNAME: '~^((stage|prod)-)?studio.*'
 ECOMMERCE_HOSTNAME: '~^((stage|prod)-)?ecommerce.*'
 PROGRAMS_HOSTNAME: '~^((stage|prod)-)?programs.*'
+CREDENTIALS_HOSTNAME: '~^((stage|prod)-)?credentials.*'
 
 nginx_template_dir: "edx/app/nginx/sites-available"
 
@@ -100,6 +101,8 @@ nginx_ecommerce_gunicorn_hosts:
   - 127.0.0.1
 nginx_programs_gunicorn_hosts:
   - 127.0.0.1
+nginx_credentails_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/tasks/main.yml

@@ -273,5 +273,3 @@
   tags:
     - manage
     - manage:start
-    - devstack
-    - devstack:manage

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/credentials.j2

+#
+# {{ ansible_managed }}
+#
+
+
+{% if "credentials" in nginx_default_sites %}
+  {% set default_site = "default" %}
+{% else %}
+  {% set default_site = "" %}
+{% endif %}
+
+upstream credentials_app_server {
+{% for host in NGINX_CREDENTIALS_GUNICORN_HOSTS %}
+    server {{ host }}:{{ credentials_gunicorn_port }} fail_timeout=0;
+{% endfor %}
+}
+
+server {
+  server_name {{ CREDENTIALS_HOSTNAME }};
+
+  {% if NGINX_ENABLE_SSL %}
+
+  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
+  listen {{ CREDENTIALS_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {% else %}
+  listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
+  {% endif %}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ credentials_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://credentials_app_server;
+  }
+{% if NGINX_SET_X_FORWARDED_HEADERS %}
+   if ($scheme != "https") {
+    rewrite ^ https://$host$uri permanent;
+   }
+ {% else %}
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+ {% endif %}
+}

playbooks/roles/oauth_client_setup/defaults/main.yml

@@ -35,6 +35,12 @@ oauth_client_setup_oauth2_clients:
         id: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY }}",
         secret: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}"
       }
+    - {
+        name: "{{ credentials_service_name | default('None') }}",
+        url_root: "{{ CREDENTIALS_URL_ROOT }}",
+        id: "{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY }}",
+        secret: "{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}"
+      }
 
 #
 # OS packages

util/jenkins/ansible-provision.sh

@@ -169,6 +169,10 @@ PROGRAMS_NGINX_PORT: 80
 PROGRAMS_SSL_NGINX_PORT: 443
 PROGRAMS_VERSION: $programs_version
 
+CREDENTIALS_NGINX_PORT: 80
+CREDENTIALS_SSL_NGINX_PORT: 443
+CREDENTIALS_VERSION: $credentials_version
+
 COURSE_DISCOVERY_NGINX_PORT: 80
 COURSE_DISCOVERY_SSL_NGINX_PORT: 443
 COURSE_DISCOVERY_VERSION: $course_discovery_version
@@ -254,6 +258,10 @@ PROGRAMS_LMS_URL_ROOT: "https://${deploy_host}"
 PROGRAMS_URL_ROOT: "https://programs-${deploy_host}"
 PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 
+CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
+CREDENTIALS_URL_ROOT: "https://credentials-${deploy_host}"
+CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
+
 COURSE_DISCOVERY_OAUTH_URL_ROOT: "https://${deploy_host}"
 COURSE_DISCOVERY_URL_ROOT: "https://course-discovery-${deploy_host}"
 COURSE_DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
@@ -302,7 +310,7 @@ EOF
 fi
 
 declare -A deploy
-roles="edxapp forum ecommerce programs course_discovery notifier xqueue xserver certs demo testcourses"
+roles="edxapp forum ecommerce programs credentials course_discovery notifier xqueue xserver certs demo testcourses"
 for role in $roles; do
     deploy[$role]=${!role}
 done