Added configuration for programs service

ECOM-2180

playbooks/roles/edxlocal/defaults/main.yml

@@ -14,6 +14,7 @@ edxlocal_databases:
   - "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}"
   - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}"
   - "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}"
+  - "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}"
 
 edxlocal_database_users:
   - {
@@ -41,3 +42,8 @@ edxlocal_database_users:
       user: "{{ EDXAPP_MYSQL_USER | default(None) }}",
       pass: "{{ EDXAPP_MYSQL_PASSWORD | default(None) }}"
     }
+  - {
+      db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ PROGRAMS_DATABASES.default.USER | default(None) }}",
+      pass: "{{ PROGRAMS_DATABASES.default.PASSWORD | default(None) }}"
+    }

playbooks/roles/local_dev/defaults/main.yml

@@ -55,6 +55,13 @@ localdev_accounts:
         repo: "edx_analytics_dashboard"
       }
 
+    - {
+        user: "{{ programs_user|default('None') }}",
+        home: "{{ programs_home }}",
+        env: "programs_env",
+        repo: "programs"
+      }
+
 # Helpful system packages for local dev
 local_dev_pkgs:
     - vim

playbooks/roles/nginx/defaults/main.yml

@@ -71,6 +71,7 @@ NGINX_EDXAPP_ERROR_PAGES:
 
 CMS_HOSTNAME: '~^((stage|prod)-)?studio.*'
 ECOMMERCE_HOSTNAME: '~^((stage|prod)-)?ecommerce.*'
+PROGRAMS_HOSTNAME: '~^((stage|prod)-)?programs.*'
 
 nginx_template_dir: "edx/app/nginx/sites-available"
 
@@ -96,6 +97,8 @@ nginx_edx_notes_api_gunicorn_hosts:
   - 127.0.0.1
 nginx_ecommerce_gunicorn_hosts:
   - 127.0.0.1
+nginx_programs_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/programs.j2

+#
+# {{ ansible_managed }}
+#
+
+
+{% if "programs" in nginx_default_sites %}
+  {% set default_site = "default" %}
+{% else %}
+  {% set default_site = "" %}
+{% endif %}
+
+upstream programs_app_server {
+{% for host in nginx_programs_gunicorn_hosts %}
+    server {{ host }}:{{ programs_gunicorn_port }} fail_timeout=0;
+{% endfor %}
+}
+
+server {
+  server_name {{ PROGRAMS_HOSTNAME }};
+
+  {% if NGINX_ENABLE_SSL %}
+
+  listen {{ PROGRAMS_NGINX_PORT }} {{ default_site }};
+  listen {{ PROGRAMS_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {% else %}
+  listen {{ PROGRAMS_NGINX_PORT }} {{ default_site }};
+  {% endif %}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ programs_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://programs_app_server;
+  }
+}
+

playbooks/roles/oauth_client_setup/defaults/main.yml

@@ -29,6 +29,12 @@ oauth_client_setup_oauth2_clients:
         id: "{{ INSIGHTS_OAUTH2_KEY }}",
         secret: "{{ INSIGHTS_OAUTH2_SECRET }}" 
       }
+    - {
+        name: "{{ PROGRAMS_OAUTH2_APP_CLIENT_NAME | default('None') }}",
+        url_root: "{{ PROGRAMS_URL_ROOT }}",
+        id: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY }}",
+        secret: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}"
+      }
 
 #
 # OS packages

playbooks/roles/programs/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role programs
+# 
+PROGRAMS_GIT_IDENTITY: !!null
+
+# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
+# and a key being provided via NEWRELIC_LICENSE_KEY
+PROGRAMS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ programs_service_name }}"
+PROGRAMS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+PROGRAMS_NGINX_PORT: 18140
+PROGRAMS_SSL_NGINX_PORT: 48140
+
+PROGRAMS_DEFAULT_DB_NAME: 'programs'
+
+PROGRAMS_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ PROGRAMS_DEFAULT_DB_NAME }}'
+    USER: 'programs001'
+    PASSWORD: 'password'
+    HOST: 'localhost'
+    PORT: '3306'
+    ATOMIC_REQUESTS: true
+    CONN_MAX_AGE: 60
+
+PROGRAMS_VERSION: "master"
+PROGRAMS_DJANGO_SETTINGS_MODULE: "programs.settings.production"
+PROGRAMS_URL_ROOT: 'http://localhost:8004'
+PROGRAMS_LMS_URL_ROOT: 'http://127.0.0.1:8000'
+
+PROGRAMS_SECRET_KEY: 'Your secret key here'
+PROGRAMS_TIME_ZONE: 'UTC'
+PROGRAMS_LANGUAGE_CODE: 'en-us'
+
+# Used to automatically configure OAuth2 Client
+PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY : 'some-secret'
+PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET : 'some-secret'
+PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+
+PROGRAMS_PLATFORM_NAME: 'Your Platform Name Here'
+
+PROGRAMS_SERVICE_CONFIG:
+  SECRET_KEY: '{{ PROGRAMS_SECRET_KEY }}'
+  TIME_ZONE: '{{ PROGRAMS_TIME_ZONE }}'
+  LANGUAGE_CODE: '{{ PROGRAMS_LANGUAGE_CODE }}'
+
+  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ PROGRAMS_LMS_URL_ROOT }}/oauth2'
+  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+
+  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ programs_service_name }}/staticfiles"
+  # db config
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES: '{{ PROGRAMS_DATABASES }}'
+
+  PLATFORM_NAME: '{{ PROGRAMS_PLATFORM_NAME }}'
+
+
+PROGRAMS_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: programs.git
+    VERSION: "{{ PROGRAMS_VERSION }}"
+    DESTINATION: "{{ programs_code_dir }}"
+    SSH_KEY: "{{ PROGRAMS_GIT_IDENTITY }}"
+
+
+PROGRAMS_GUNICORN_WORKERS: "2"
+PROGRAMS_GUNICORN_EXTRA: ""
+PROGRAMS_GUNICORN_EXTRA_CONF: ""
+PROGRAMS_GUNICORN_WORKER_CLASS: "gevent"
+
+#
+# vars are namespace with the module name.
+#
+programs_role_name: programs
+programs_venv_dir: "{{ programs_home }}/venvs/{{ programs_service_name }}"
+
+programs_environment:
+  DJANGO_SETTINGS_MODULE: "{{ PROGRAMS_DJANGO_SETTINGS_MODULE }}"
+  PROGRAMS_CFG: "{{ COMMON_CFG_DIR }}/{{ programs_service_name }}.yml"
+  PATH: "{{ programs_venv_dir }}/bin:${PATH}"
+
+programs_service_name: "programs"
+programs_user: "{{ programs_service_name }}"
+programs_home: "{{ COMMON_APP_DIR }}/{{ programs_service_name }}"
+programs_code_dir: "{{ programs_home }}/{{ programs_service_name }}"
+
+programs_gunicorn_host: "127.0.0.1"
+programs_gunicorn_port: 8140
+programs_gunicorn_timeout: 300
+
+programs_log_dir: "{{ COMMON_LOG_DIR }}/{{ programs_service_name }}"
+
+programs_requirements_base: "{{ programs_code_dir }}/requirements"
+programs_requirements:
+  - production.txt
+
+
+#
+# OS packages
+#
+
+programs_debian_pkgs:
+  - libmysqlclient-dev
+  - libssl-dev
+
+programs_redhat_pkgs: []

playbooks/roles/programs/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role programs
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - role: edx_service
+    edx_service_name: "{{ programs_service_name }}"
+    edx_service_config: "{{ PROGRAMS_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ PROGRAMS_REPOS }}"
+    edx_service_user: "{{ programs_user }}"
+    edx_service_home: "{{ programs_home }}"
+    edx_service_packages:
+      debian: "{{ programs_debian_pkgs }}"
+      redhat: "{{ programs_redhat_pkgs }}"
+  - supervisor

playbooks/roles/programs/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role programs
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: add gunicorn configuration file
+  template:
+    src: edx/app/programs/programs_gunicorn.py.j2
+    dest: "{{ programs_home }}/programs_gunicorn.py"
+  sudo_user: "{{ programs_user }}"
+
+- name: install application requirements
+  pip:
+    requirements: "{{ programs_requirements_base }}/{{ item }}"
+    virtualenv: "{{ programs_venv_dir }}"
+    state: present
+  sudo_user: "{{ programs_user }}"
+  with_items: "{{ programs_requirements }}"
+
+- name: migrate
+  shell: >
+    chdir={{ programs_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ programs_venv_dir }}/bin/python ./manage.py migrate --noinput
+  sudo_user: "{{ programs_user }}"
+  environment: "{{ programs_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: write out the supervisor wrapper
+  template:
+    src: "edx/app/programs/programs.sh.j2"
+    dest: "{{ programs_home }}/{{ programs_service_name }}.sh"
+    mode: 0650
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+
+- name: write supervisord config
+  template:
+    src: "edx/app/supervisor/conf.d.available/programs.conf.j2"
+    dest: "{{ supervisor_available_dir }}/{{ programs_service_name }}.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+
+- name: setup the programs env file
+  template:
+    src: "./{{ programs_home }}/{{ programs_service_name }}_env.j2"
+    dest: "{{ programs_home }}/programs_env"
+    owner: "{{ programs_user }}"
+    group: "{{ programs_user }}"
+    mode: 0644
+
+- name: enable supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/{{ programs_service_name }}.conf"
+    dest: "{{ supervisor_cfg_dir }}/{{ programs_service_name }}.conf"
+    state: link
+    force: yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: "{{ programs_venv_dir }}/bin/{{ item }}"
+    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.programs"
+    state: link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create symlinks from the repo dir
+  file:
+    src: "{{ programs_code_dir }}/{{ item }}"
+    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.programs"
+    state: link
+  with_items:
+  - manage.py
+
+- name: restart the application
+  supervisorctl: 
+    state: restarted
+    supervisorctl_path: "{{ supervisor_ctl }}"
+    config: "{{ supervisor_cfg }}"
+    name: "{{ programs_service_name }}"
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"

playbooks/roles/programs/templates/edx/app/programs/programs.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% set programs_venv_bin = programs_home + "/venvs/" + programs_service_name + "/bin" %}
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = programs_venv_bin + '/newrelic-admin run-program ' + programs_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = programs_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ PROGRAMS_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+source {{ programs_home }}/programs_env
+{{ executable }} -c {{ programs_home }}/programs_gunicorn.py {{ PROGRAMS_GUNICORN_EXTRA }} programs.wsgi:application

playbooks/roles/programs/templates/edx/app/programs/programs_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in programs_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/programs/templates/edx/app/programs/programs_gunicorn.py.j2

+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+{{ ansible_managed }}
+"""
+
+timeout = {{ programs_gunicorn_timeout }}
+bind = "{{ programs_gunicorn_host }}:{{ programs_gunicorn_port }}"
+pythonpath = "{{ programs_code_dir }}"
+workers = {{ PROGRAMS_GUNICORN_WORKERS }}
+worker_class = "{{ PROGRAMS_GUNICORN_WORKER_CLASS }}"
+
+{{ PROGRAMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/programs/templates/edx/app/supervisor/conf.d.available/programs.conf.j2

+#
+# {{ ansible_managed }}
+# 
+[program:{{ programs_service_name }}]
+
+command={{ programs_home }}/{{ programs_service_name }}.sh
+user={{ common_web_user }}
+directory={{ programs_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true