initial working version

playbooks/edx-east/edx_insights.yml

+- name: Deploy Analytics API
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: True
+  roles:
+    - role: nginx
+      nginx_sites:
+        - insights
+    - aws
+    - insights 
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/insights/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Defaults for role insights
+# 
+
+INSIGHTS_MEMCACHE: [ 'localhost:11211' ]
+
+INSIGHTS_CONFIG:
+  # assumes you are running https://github.com/edx/edx-analytics-data-api.git
+  # on port 8100
+  DATA_API_URL: 'http://127.0.0.1:8100/api/v0'
+  DATA_API_AUTH_TOKEN: 'edx'
+  SOCIAL_AUTH_EDX_OAUTH2_KEY: !!null
+  SOCIAL_AUTH_EDX_OAUTH2_SECRET: !!null
+  SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: !!null
+  SOCIAL_AUTH_EDX_OIDC_KEY: !!null
+  SOCIAL_AUTH_EDX_OIDC_SECRET: !!null
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: !!null
+  # This value should be the same as SOCIAL_AUTH_EDX_OIDC_SECRET
+  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: !!null
+  PLATFORM_NAME: 'edX'
+  APPLICATION_NAME: 'Insights'
+  CACHES:
+    default: &default_generic_cache
+      BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
+      KEY_PREFIX: 'default'
+      LOCATION: "{{ INSIGHTS_MEMCACHE }}"
+  TIME_ZONE: 'America/New_York'
+  LANGUAGE_CODE: 'en-us'
+  # email config
+  EMAIL_HOST: 'smtp.example.com'
+  EMAIL_HOST_PASSWORD: ""
+  EMAIL_HOST_USER: ""
+  EMAIL_PORT: 587
+  # static file config
+  STATICFILES_DIRS: ["{{ insights_code_dir }}/analytics_dashboard/static"]  
+  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ insights_service_name }}/staticfiles"
+  # db config
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES:
+    # rw user
+    default:
+      ENGINE: 'django.db.backends.mysql'
+      NAME: 'dashboard'
+      USER: 'rosencrantz'
+      PASSWORD: 'secret'
+      HOST: 'localhost'
+      PORT: '3306'
+      
+INSIGHTS_VERSION: "master"
+INSIGHTS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-analytics-api"
+INSIGHTS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+INSIGHTS_NGINX_PORT: "18110"
+INSIGHTS_GUNICORN_WORKERS: "2"
+INSIGHTS_GUNICORN_EXTRA: ""
+#
+# vars are namespace with the module name.
+#
+insights_environment:
+  DJANGO_SETTINGS_MODULE: "analytics_dashboard.settings.production"
+  ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml"
+
+insights_role_name: "insights"
+insights_service_name: "{{ insights_role_name }}"
+insights_user: "{{ insights_role_name }}"
+insights_app_dir: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
+insights_home: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
+insights_venv_base: "{{ insights_home }}/venvs"
+insights_venv_dir: "{{ insights_venv_base }}/{{ insights_service_name }}"
+insights_venv_bin: "{{ insights_venv_dir }}/bin"
+insights_code_dir: "{{ insights_app_dir }}/edx-analytics-dashboard"
+insights_python_path: "{{ insights_code_dir }}/analytics_dashboard"
+insights_conf_dir: "{{ insights_home }}"
+insights_log_dir: "{{ COMMON_LOG_DIR }}/{{ insights_service_name }}"
+
+insights_gunicorn_host: "127.0.0.1"
+insights_gunicorn_port: "8110"
+insights_gunicorn_timeout: "300"
+insights_wsgi: "analytics_dashboard.wsgi:application"
+
+insights_django_settings: "analytics_dashboard.settings.production"
+insights_source_repo: "git@{{ COMMON_GIT_MIRROR }}:/edx/edx-analytics-dashboard"
+insights_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+insights_git_ssh: "/tmp/insights_git_ssh"
+insights_git_identity_file: "{{ insights_home }}/git_identity"
+insights_manage: "{{ insights_code_dir }}/analytics_dashboard/manage.py"
+
+insights_requirements_base: "{{ insights_code_dir }}/requirements"
+insights_requirements:
+  - production.txt
+
+#
+# OS packages
+#
+insights_debian_pkgs:
+  - 'libmysqlclient-dev'
+
+insights_redhat_pkgs:
+  - 'community-mysql-devel'

playbooks/roles/insights/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role insights
+# 
+# Overview:
+# 
+#
+
+- name: "restart insights"
+  supervisorctl_local: >
+    name={{ insights_service_name }}
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  when: not disable_edx_services

playbooks/roles/insights/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role insights
+# 
+dependencies:
+  - role: edx_service
+    edx_role_name: "{{ insights_role_name }}"
+    edx_service_name: "{{ insights_service_name }}"
+  - supervisor

playbooks/roles/insights/tasks/deploy.yml

+---
+
+- name: create notifier/.ssh directory
+  file:
+    path="{{ insights_home }}.ssh" mode=2700 state=directory owner={{ insights_user }} group={{ insights_user }}
+
+- name: create ssh script for git (not authenticated)
+  template: >
+    src=tmp/git_ssh_noauth.sh.j2 dest={{ insights_git_ssh }}
+    owner={{ insights_user }} mode=750
+    
+- name: install read-only ssh key
+  copy: >
+    content="{{ COMMON_GIT_IDENTITY }}" dest={{ insights_git_identity_file }}
+    owner={{ insights_user }} group={{ insights_user }} mode=0600
+
+- name: setup the insights env file
+  template: >
+    src="edx/app/insights/insights_env.j2"
+    dest="{{ insights_app_dir }}/insights_env"
+    owner={{ insights_user }}
+    group={{ insights_user }}
+    mode=0644
+
+- name: checkout code
+  git: >
+    dest={{ insights_code_dir }} repo={{ insights_source_repo }} version={{ INSIGHTS_VERSION }}
+    accept_hostkey=yes
+  register: insights_code_checkout
+  notify: restart insights
+  sudo_user: "{{ insights_user }}"
+  environment:
+    GIT_SSH: "{{ insights_git_ssh }}"
+
+- name: write out app config file
+  template: >
+    src=edx/app/insights/insights.yaml.j2
+    dest={{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml
+    mode=0644 owner={{ insights_user }} group={{ insights_user }}
+  notify: restart insights
+
+- name: install application requirements
+  pip: >
+    requirements="{{ insights_requirements_base }}/{{ item }}"
+    virtualenv="{{ insights_venv_dir }}" state=present
+  sudo_user: "{{ insights_user }}"
+  notify: restart insights
+  with_items: insights_requirements
+  environment:
+    GIT_SSH: "{{ insights_git_ssh }}"
+
+- name: syncdb and migrate
+  shell: >
+    chdir={{ insights_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ insights_venv_bin }}/python {{ insights_manage }} syncdb --migrate --noinput
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run collectstatic
+  shell: >
+    chdir={{ insights_code_dir }}
+    {{ insights_venv_bin }}/python {{ insights_manage }} {{ item }}
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  with_items:
+    - "collectstatic --noinput"
+    - "compress"
+- name: write out the supervisior wrapper
+  template: >
+    src=edx/app/insights/insights.sh.j2
+    dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+  notify: restart insights
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/insights.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  notify: restart insights
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
+    state=link
+    force=yes
+  notify: restart insights
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ insights_venv_bin }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_role_name }}"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create manage.py symlink
+  file: >
+    src="{{ insights_manage }}"
+    dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_role_name }}"
+    state=link
+
+- name: remove read-only ssh key for the content repo
+  file: path={{ insights_git_identity_file }} state=absent

playbooks/roles/insights/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role insights
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- include: deploy.yml tags=deploy
\ No newline at end of file

playbooks/roles/insights/templates/edx/app/insights/git_ssh_noauth.sh.j2

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no "$@"

playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = insights_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ INSIGHTS_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+source {{ insights_app_dir }}/insights_env
+
+{{ executable }} --pythonpath={{ insights_python_path }} -b {{ insights_gunicorn_host }}:{{ insights_gunicorn_port }} -w {{ INSIGHTS_GUNICORN_WORKERS }} --timeout={{ insights_gunicorn_timeout }} {{ INSIGHTS_GUNICORN_EXTRA }} {{ insights_wsgi }}

playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2

+---
+# {{ ansible_managed }}
+
+{{ INSIGHTS_CONFIG | to_nice_yaml }}

playbooks/roles/insights/templates/edx/app/insights/insights_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in insights_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/insights/templates/edx/app/supervisor/conf.d.available/insights.conf.j2

+# {{ ansible_managed }}
+
+[program:{{ insights_service_name }}]
+
+command={{ insights_app_dir }}/insights.sh
+user={{ common_web_user }}
+directory={{ insights_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/insights/templates/tmp/git_ssh_noauth.sh.j2

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ insights_git_identity_file }} "$@"

playbooks/roles/nginx/defaults/main.yml

@@ -64,6 +64,8 @@ nginx_cms_gunicorn_hosts:
   - 127.0.0.1
 nginx_analytics_api_gunicorn_hosts:
   - 127.0.0.1
+nginx_insights_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2

+upstream insights_app_server {
+    {% for host in nginx_insights_gunicorn_hosts %}
+        server {{ host }}:{{ insights_gunicorn_port }} fail_timeout=0;
+    {% endfor %}
+}
+
+server {
+  listen {{ INSIGHTS_NGINX_PORT }} default_server;
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ insights_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  # No basic auth security on the heartbeat url, so that ELB can use it
+  location /status {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://insights_app_server;
+  }
+}