Merge branch 'e0d/edx-notes-api' into max/merge-refactor

Conflicts:
	CHANGELOG.md
	playbooks/roles/edxapp/tasks/deploy.yml
	playbooks/roles/insights/defaults/main.yml
	playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

CHANGELOG.md

 - Role: edxapp
   - Enabled combined login registration feature by default
 
+- Role: analytics_api
+  - Refactored name from `analytics-api` to `analytics_api`. This will require additional migrations when upgrading an existing server. While we recommend building from scratch, running the following command might work:
+  
+      ```
+      rm -rf /edx/app/analytics-api /edx/app/ /edx/app/nginx/sites-available/analytics-api.j2 /edx/app/supervisor/conf.d.available/analytics_api.conf
+      ```
+
 - Role: notifier
   - Refactored `NOTIFIER_HOME` and `NOTIFIER_USER` to `notifier_app_dir` and `notifier_user` to match other roles. This shouldn't change anything since users should've only been overriding COMMON_HOME.
 

playbooks/edx-east/analyticsapi.yml

@@ -9,9 +9,9 @@
   roles:
     - role: nginx
       nginx_sites:
-        - analytics-api
+        - analytics_api
     - aws
-    - analytics-api
+    - analytics_api
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/edx-east/edx_continuous_integration.yml

@@ -15,7 +15,7 @@
       - xqueue
       - xserver
       - certs
-      - analytics-api
+      - analytics_api
       nginx_default_sites:
       - lms
     - role: edxlocal
@@ -34,7 +34,7 @@
       when: XSERVER_GIT_IDENTITY|length > 0
     - certs
     - edx_ansible
-    - analytics-api
+    - analytics_api
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/edx-east/notes.yml

+- name: Deploy edX Notes API
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: True
+  roles:
+    - role: nginx
+      nginx_sites:
+        - edx-notes-api
+    - aws
+    - edx_notes_api 
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/edx_sandbox.yml

@@ -33,6 +33,11 @@
     - mongo
     - { role: 'edxapp', celery_worker: True }
     - edxapp
+    - notifier
+    - minos
+    - analytics_api
+    - insights
+    - edx_notes_api
     - demo
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - oraclejdk
@@ -40,6 +45,7 @@
     - forum
     - { role: notifier, NOTIFIER_DIGEST_TASK_INTERVAL: "5" }
     - { role: "xqueue", update_users: True }
+    - xqwatcher
     - role: ora
       when: ENABLE_LEGACY_ORA
     - certs

playbooks/roles/analytics-api/tasks/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-# Tasks for role analytics-api
-#
-# Overview:
-#
-# Install the Analytics Data API server, a python
-# django application that runs under gunicorn
-#
-# Dependencies:
-#
-# Example play:
-# - name: Deploy Analytics API
-#   hosts: all
-#   sudo: True
-#   gather_facts: True
-#   vars:
-#     ENABLE_DATADOG: False
-#     ENABLE_SPLUNKFORWARDER: False
-#     ENABLE_NEWRELIC: False
-#   roles:
-#     - aws
-#     - analytics-api
-#
-# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
-#
-
-- fail: msg="You must provide a private key for the analytics repo"
-  when: not ANALYTICS_API_GIT_IDENTITY
-
-- include: deploy.yml tags=deploy

playbooks/roles/analytics-api/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:analytics_api" : "{{ analytics_api_source_repo }} {{ analytics_api_code_checkout.after |truncate(7,True,'')}}"
-  when: analytics_api_code_checkout.after is defined

playbooks/roles/analytics-api/defaults/main.yml → playbooks/roles/analytics_api/defaults/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 ##
-# Defaults for role analytics-api
+# Defaults for role analytics_api
 #
 
 ANALYTICS_API_GIT_IDENTITY: !!null
@@ -51,7 +51,7 @@ ANALYTICS_API_EMAIL_HOST_PASSWORD: 'mail_password'
 ANALYTICS_API_EMAIL_PORT: 587
 ANALYTICS_API_AUTH_TOKEN: 'put-your-api-token-here'
 
-ANALYTICS_API_CONFIG:
+ANALYTICS_API_SERVICE_CONFIG:
   ANALYTICS_DATABASE: 'reports'
   SECRET_KEY: '{{ ANALYTICS_API_SECRET_KEY }}'
   TIME_ZONE: '{{ ANALYTICS_API_TIME_ZONE }}'
@@ -69,6 +69,15 @@ ANALYTICS_API_CONFIG:
     connect_timeout: 10
   DATABASES: '{{ ANALYTICS_API_DATABASES }}'
 
+ANALYTICS_API_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-analytics-data-api.git
+    VERSION: "{{ ANALYTICS_API_VERSION }}"
+    DESTINATION: "{{ analytics_api_code_dir }}"
+    SSH_KEY: "{{ ANALYTICS_API_GIT_IDENTITY }}"
+
 ANALYTICS_API_GUNICORN_WORKERS: "2"
 ANALYTICS_API_GUNICORN_EXTRA: ""
 ANALYTICS_API_GUNICORN_EXTRA_CONF: ""
@@ -78,27 +87,19 @@ ANALYTICS_API_GUNICORN_EXTRA_CONF: ""
 
 analytics_api_environment:
   DJANGO_SETTINGS_MODULE: "analyticsdataserver.settings.production"
-  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
-
-analytics_api_role_name: "analytics-api"
-analytics_api_service_name: "analytics-api"
-analytics_api_user: "analytics-api"
+  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yml"
 
-analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_service_name: "analytics_api"
+analytics_api_user: "{{ analytics_api_service_name }}"
 analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
-analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
-analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
-analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
-analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
+analytics_api_code_dir: "{{ analytics_api_home }}/{{ analytics_api_service_name }}"
 analytics_api_conf_dir: "{{ analytics_api_home }}"
+
 analytics_api_gunicorn_host: "127.0.0.1"
 analytics_api_gunicorn_port: "8100"
 analytics_api_gunicorn_timeout: "300"
 
 analytics_api_django_settings: "production"
-analytics_api_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/edx-analytics-data-api"
-analytics_api_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ analytics_api_git_identity_file }}"
-analytics_api_git_identity_file: "{{ analytics_api_app_dir }}/git-identity"
 
 analytics_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ analytics_api_service_name }}"
 

playbooks/roles/analytics-api/meta/main.yml → playbooks/roles/analytics_api/meta/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 #
-# Role includes for role analytics-api
+# Role includes for role analytics_api
 #
 # Example:
 #
@@ -21,6 +21,12 @@
 
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ analytics_api_role_name }}"
     edx_service_name: "{{ analytics_api_service_name }}"
+    edx_service_config: "{{ ANALYTICS_API_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ ANALYTICS_API_REPOS }}"
+    edx_service_user: "{{ analytics_api_user }}"
+    edx_service_home: "{{ analytics_api_home }}"
+    edx_service_packages:
+      debian: "{{ analytics_api_debian_pkgs }}"
+      redhat: "{{ analytics_api_redhat_pkgs }}"
   - supervisor

playbooks/roles/analytics-api/tasks/deploy.yml → playbooks/roles/analytics_api/tasks/main.yml

 ---
-
-- name: install read-only ssh key
-  copy: >
-    content="{{ ANALYTICS_API_GIT_IDENTITY }}" dest={{ analytics_api_git_identity_file }}
-    owner={{ analytics_api_user }} group={{ analytics_api_user }} mode=0600
-
-- name: setup the analytics-api env file
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for role analytics_api
+#
+# Overview:
+#
+# Install the Analytics Data API server, a python
+# django application that runs under gunicorn
+#
+# Dependencies:
+#
+# Example play:
+# - name: Deploy Analytics API
+#   hosts: all
+#   sudo: True
+#   gather_facts: True
+#   vars:
+#     ENABLE_DATADOG: False
+#     ENABLE_SPLUNKFORWARDER: False
+#     ENABLE_NEWRELIC: False
+#   roles:
+#     - aws
+#     - analytics_api
+#
+# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
+#
+
+- name: setup the analytics_api env file
   template: >
-    src="edx/app/analytics-api/analytics_api_env.j2"
-    dest="{{ analytics_api_app_dir }}/analytics_api_env"
+    src="edx/app/analytics_api/analytics_api_env.j2"
+    dest="{{ analytics_api_home }}/analytics_api_env"
     owner={{ analytics_api_user }}
     group={{ analytics_api_user }}
     mode=0644
 
-- name: checkout code
-  git: >
-    dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
-    accept_hostkey=yes
-    ssh_opts="{{ analytics_api_git_ssh_opts }}"
-  register: analytics_api_code_checkout
-  notify: "restart the analytics service"
-  sudo_user: "{{ analytics_api_user }}"
-
 - name: "add gunicorn configuration file"
   template: >
-    src=edx/app/analytics-api/analytics_api_gunicorn.py.j2
-    dest={{ analytics_api_app_dir }}/analytics_api_gunicorn.py
+    src=edx/app/analytics_api/analytics_api_gunicorn.py.j2
+    dest={{ analytics_api_home }}/analytics_api_gunicorn.py
   sudo_user: "{{ analytics_api_user }}"
-  notify: "restart the analytics service"
-
-- name: write out app config file
-  template: >
-    src=edx/app/analytics-api/analytics-api.yaml.j2
-    dest={{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml
-    mode=0644 owner={{ analytics_api_user }} group={{ analytics_api_user }}
-  notify: restart the analytics service
 
 - name: install application requirements
   pip: >
     requirements="{{ analytics_api_requirements_base }}/{{ item }}"
-    virtualenv="{{ analytics_api_venv_dir }}" state=present
+    virtualenv="{{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}"
+    state=present
   sudo_user: "{{ analytics_api_user }}"
-  notify: restart the analytics service
   with_items: analytics_api_requirements
 
 - name: migrate
@@ -49,7 +59,7 @@
     chdir={{ analytics_api_code_dir }}
     DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
     DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ analytics_api_venv_bin }}/python ./manage.py migrate --noinput
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python ./manage.py migrate --noinput
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
@@ -57,31 +67,29 @@
 - name: run collectstatic
   shell: >
     chdir={{ analytics_api_code_dir }}
-    {{ analytics_api_venv_bin }}/python manage.py collectstatic  --noinput
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py collectstatic  --noinput
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
 
 - name: create api users
   shell: >
     chdir={{ analytics_api_code_dir }}
-    {{ analytics_api_venv_bin }}/python manage.py set_api_key {{ item.key }} {{ item.value }}
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py set_api_key {{ item.key }} {{ item.value }}
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   with_dict: ANALYTICS_API_USERS
 
-- name: write out the supervisior wrapper
+- name: write out the supervisor wrapper
   template: >
-    src=edx/app/analytics-api/analytics-api.sh.j2
-    dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.sh
+    src=edx/app/analytics_api/analytics_api.sh.j2
+    dest={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
     mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
-  notify: restart the analytics service
 
 - name: write supervisord config
   template: >
-    src=edx/app/supervisor/conf.d.available/analytics-api.conf.j2
+    src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
     dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
-  notify: restart the analytics service
 
 - name: enable supervisor script
   file: >
@@ -89,7 +97,6 @@
     dest={{ supervisor_cfg_dir }}/{{ analytics_api_service_name }}.conf
     state=link
     force=yes
-  notify: restart the analytics service
   when: not disable_edx_services
 
 - name: update supervisor configuration
@@ -98,8 +105,8 @@
 
 - name: create symlinks from the venv bin dir
   file: >
-    src="{{ analytics_api_venv_bin }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    src="{{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics_api"
     state=link
   with_items:
   - python
@@ -109,13 +116,16 @@
 - name: create symlinks from the repo dir
   file: >
     src="{{ analytics_api_code_dir }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics_api"
     state=link
   with_items:
   - manage.py
 
-- name: remove read-only ssh key for the content repo
-  file: path={{ analytics_api_git_identity_file }} state=absent
-
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE
+- name: restart analytics_api
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ analytics_api_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2 → playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.sh.j2

@@ -2,6 +2,7 @@
 
 # {{ ansible_managed }}
 
+{% set analytics_api_venv_bin = analytics_api_home + "/venvs/" + analytics_api_service_name + "/bin/" %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
 {% set executable = analytics_api_venv_bin + '/newrelic-admin run-program ' + analytics_api_venv_bin + '/gunicorn' %}
 {% else %}
@@ -13,6 +14,6 @@ export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
 {% endif -%}
 
-source {{ analytics_api_app_dir }}/analytics_api_env
+source {{ analytics_api_home }}/analytics_api_env
 
-{{ executable }} -c {{ analytics_api_app_dir }}/analytics_api_gunicorn.py {{ ANALYTICS_API_GUNICORN_EXTRA }} analyticsdataserver.wsgi:application
+{{ executable }} -c {{ analytics_api_home }}/analytics_api_gunicorn.py {{ ANALYTICS_API_GUNICORN_EXTRA }} analyticsdataserver.wsgi:application

playbooks/roles/analytics-api/templates/edx/app/supervisor/conf.d.available/analytics-api.conf.j2 → playbooks/roles/analytics_api/templates/edx/app/supervisor/conf.d.available/analytics_api.conf.j2

@@ -2,7 +2,7 @@
 
 [program:{{ analytics_api_service_name }}]
 
-command={{ analytics_api_app_dir }}/analytics-api.sh
+command={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
 user={{ common_web_user }}
 directory={{ analytics_api_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log

playbooks/roles/common/defaults/main.yml

@@ -31,7 +31,9 @@ COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
 COMMON_NPM_MIRROR_URL: 'http://registry.npmjs.org'
 COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search="
 # do not include http/https
+COMMON_GIT_PROTOCOL: 'https'
 COMMON_GIT_MIRROR: 'github.com'
+COMMON_GIT_PATH: 'edx'
 # override this var to set a different hostname
 COMMON_HOSTNAME: ""
 

playbooks/roles/edx-shared-vars/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults shared across multiple roles.  They are prefixed with the
+# COMMON namespace and are mostly used by the common role.  They are
+# broken out into a separate role so the can be used with running common
+# 
+
+
+COMMON_MYSQL_READ_ONLY_USER: 'read_only'
+COMMON_MYSQL_READ_ONLY_PASS: 'password'
+COMMON_MYSQL_ADMIN_USER: 'admin'
+COMMON_MYSQL_ADMIN_PASS: 'password'
+COMMON_MYSQL_MIGRATE_USER: 'migrate'
+COMMON_MYSQL_MIGRATE_PASS: 'password'
+
+COMMON_MONGO_READ_ONLY_USER: 'read_only'
+COMMON_MONGO_READ_ONLY_PASS: !!null
+COMMON_ENABLE_DATADOG: False
+COMMON_ENABLE_NGINXTRA: False
+COMMON_ENABLE_SPLUNKFORWARDER: False
+COMMON_ENABLE_NEWRELIC: False
+# enables app reporting, you must enable newrelic
+# as well
+COMMON_ENABLE_NEWRELIC_APP: False
+COMMON_ENABLE_MINOS: False
+COMMON_TAG_EC2_INSTANCE: False
+common_debian_pkgs:
+  - ntp
+  - ack-grep
+  - lynx-cur
+  - logrotate
+  - mosh
+  - rsyslog
+  - screen
+  - tmux
+  - tree
+  - git
+  - unzip
+  - python2.7
+  - python-pip
+  - python2.7-dev
+  # Not installed by default on vagrant ubuntu
+  # boxes
+  - curl
+
+common_pip_pkgs:
+  - pip==1.5.6
+  - setuptools==3.6
+  - virtualenv==1.11.6
+  - virtualenvwrapper
+
+common_web_user: www-data
+common_web_group: www-data
+common_log_user: syslog
+
+common_git_ppa: "ppa:git-core/ppa"
+
+# Skip supervisor tasks
+# When set to true this flag will allow you to install everything but keep
+# supervisor from starting any of the services.
+# Service files will be placed in supervisor's conf.available.d but not linked
+# to supervisors 'conf.d' directory.
+disable_edx_services: False
+
+# Some apps run differently in dev mode(forums)
+# so different start scripts are generated in dev mode.
+devstack: False
+
+# Some cluster apps need special settings when in vagrant
+# due to eth0 always being the same IP address
+vagrant_cluster: False
+
+common_debian_variants:
+  - Ubuntu
+  - Debian
+
+common_redhat_variants:
+  - CentOS
+  - Red Hat Enterprise Linux
+  - Amazon
\ No newline at end of file

playbooks/roles/minos/handlers/main.yml → playbooks/roles/edx-shared-vars/handlers/main.yml

@@ -9,7 +9,7 @@
 #
 #
 #
-# Handlers for role minos
+# Handlers for role edx-shared-vars
 # 
 # Overview:
 # 

playbooks/roles/insights/handlers/main.yml → playbooks/roles/edx-shared-vars/meta/main.yml

@@ -7,18 +7,14 @@
 # code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
-#
-#
-# Handlers for role insights
-# 
-# Overview:
+##
+# Role includes for role edx-shared-vars
 # 
+# Example:
 #
-
-- name: "restart insights"
-  supervisorctl_local: >
-    name={{ insights_service_name }}
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: not disable_edx_services
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }

playbooks/roles/analytics-api/handlers/main.yml → playbooks/roles/edx-shared-vars/tasks/main.yml

@@ -9,16 +9,18 @@
 #
 #
 #
-# Handlers for role analytics-api
-#
+# Tasks for role edx-shared-vars
+# 
 # Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
 #
 #
 
-- name: "restart the analytics service"
-  supervisorctl_local: >
-    name={{ analytics_api_service_name }}
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: not disable_edx_services
\ No newline at end of file
+- name: stub ansible task
+  debug: msg="This is a stub task created by the ansible-role role"
+  notify: notify me

playbooks/roles/edx_notes_api/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx-notes-api
+# 
+
+EDX_NOTES_API_GUNICORN_WORKERS_EXTRA: ""
+EDX_NOTES_API_WORKERS_EXTRA_CONF: ""
+EDX_NOTES_API_LANG: en-us
+EDX_NOTES_API_WORKERS: !!null
+EDX_NOTES_API_DATASTORE_NAME: "{{ edx_notes_api_service_name }}"
+EDX_NOTES_API_MYSQL_DB_USER: notes001
+EDX_NOTES_API_MYSQL_DB_NAME: "{{ EDX_NOTES_API_DATASTORE_NAME }}"
+EDX_NOTES_API_MYSQL_DB_PASS: secret
+EDX_NOTES_API_MYSQL_HOST: localhost
+EDX_NOTES_API_ELASTICSEARCH_URL: http://127.0.0.1:9200
+EDX_NOTES_API_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-notes"
+# Change these values!!
+EDX_NOTES_API_SECRET_KEY: "i^,9%i=e=y/Nlpb=Mkx!j&,HD9d/17F][9P,FLdwM2+G6|]BEU"
+EDX_NOTES_API_CLIENT_ID: edx_notes_id
+EDX_NOTES_API_CLIENT_SECRET: edx_notes_secret
+EDX_NOTES_API_GIT_SSH_KEY: !!null
+
+EDX_NOTES_API_DATABASES:
+  # rw user
+  default:
+    ENGINE: django.db.backends.mysql
+    NAME: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}"
+    USER: "{{ EDX_NOTES_API_MYSQL_DB_USER }}"
+    PASSWORD: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}"
+    HOST: "{{ EDX_NOTES_API_MYSQL_HOST }}"
+    PORT: 3306
+EDX_NOTES_API_ALLOWED_HOSTS:
+  - localhost
+
+EDX_NOTES_API_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-notes-api.git
+    VERSION: master
+    DESTINATION: "{{ edx_notes_api_code_dir }}"
+    SSH_KEY: "{{ EDX_NOTES_API_GIT_SSH_KEY }}"
+
+#
+# This data structure will be written out to yaml configuration file
+# in /edx/etc
+#
+edx_notes_api_service_config:
+  ALLOWED_HOSTS: "{{ EDX_NOTES_API_ALLOWED_HOSTS }}"
+  # replace with your secret key
+  SECRET_KEY: '{{ EDX_NOTES_API_SECRET_KEY }}'
+  # replace with your oauth id and secret
+  CLIENT_ID: "{{ EDX_NOTES_API_CLIENT_ID }}"
+  CLIENT_SECRET: "{{ EDX_NOTES_API_CLIENT_SECRET }}"
+  ELASTICSEARCH_URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}"
+  ELASTICSEARCH_INDEX: "edx_notes"
+  # Number of rows to return by default in result.
+  RESULTS_DEFAULT_SIZE: 25
+  # Max number of rows to return in result.
+  RESULTS_MAX_SIZE: 250
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES: "{{ EDX_NOTES_API_DATABASES }}"
+  HAYSTACK_CONNECTIONS:
+    default:
+      ENGINE: 'notesserver.highlight.ElasticsearchSearchEngine'
+      URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}"
+      INDEX_NAME: '{{ EDX_NOTES_API_DATASTORE_NAME }}'
+  DISABLE_TOKEN_CHECK: True
+
+#
+# vars are namespace with the module name.
+#
+edx_notes_api_service_name: edx_notes_api
+edx_notes_api_user: "{{ edx_notes_api_service_name }}"
+edx_notes_api_home: "{{ COMMON_APP_DIR }}/{{ edx_notes_api_service_name }}"
+edx_notes_api_code_dir: "{{ edx_notes_api_home }}/{{ edx_notes_api_service_name }}"
+edx_notes_api_conf_dir: "{{ edx_notes_api_home }}"
+
+edx_notes_api_gunicorn_host: "127.0.0.1"
+edx_notes_api_gunicorn_port: "8120"
+edx_notes_api_gunicorn_timeout: "300"
+edx_notes_api_wsgi: notesserver.wsgi:application
+edx_notes_api_nginx_port: "18120"
+edx_notes_api_manage: "{{ edx_notes_api_code_dir }}/manage.py" 
+edx_notes_api_requirements_base: "{{ edx_notes_api_code_dir }}/requirements"
+# Application python requirements
+edx_notes_api_requirements:
+  - base.txt
+  - optional.txt
+
+#
+# OS packages
+#
+edx_notes_api_debian_pkgs:
+  - libmysqlclient-dev
+  - python-mysqldb
+
+edx_notes_api_redhat_pkgs: []

playbooks/roles/edx_notes_api/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role edx-notes-api
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+
+dependencies:
+  - role: edx_service
+    edx_service_name: "{{ edx_notes_api_service_name }}"
+    edx_service_config: "{{ edx_notes_api_service_config }}"
+    edx_service_repos: "{{ EDX_NOTES_API_REPOS }}"
+    edx_service_user: "{{ edx_notes_api_user }}"
+    edx_service_home: "{{ edx_notes_api_home }}"
+    edx_service_packages:
+      debian: "{{ edx_notes_api_debian_pkgs }}"
+      redhat: "{{ edx_notes_api_redhat_pkgs }}"
+  - supervisor

playbooks/roles/edx_notes_api/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Tasks for role edx-notes-api
+# 
+# Overview:
+#
+# Role for installing the edx-notes-api Django application, https://github.com/edx/edx-notes-api.
+#
+# Dependencies:
+#
+# For a complete picture of dependencies, see:
+#
+#  configuration/playbooks/role/edx-notes-api/meta/main.yml
+#  configuration/playbooks/edx-east/notes.yml
+#
+# Example play:
+#
+# - name: Deploy edX Notes API
+# hosts: all
+# sudo: True
+# gather_facts: True
+# vars:
+#   ENABLE_DATADOG: False
+#   ENABLE_SPLUNKFORWARDER: False
+#   ENABLE_NEWRELIC: True
+# roles:
+#   - role: nginx
+#     nginx_sites:
+#       - edx-notes-api
+#   - aws
+#   - edx-notes-api 
+#   - role: datadog
+#     when: COMMON_ENABLE_DATADOG
+#   - role: splunkforwarder
+#     when: COMMON_ENABLE_SPLUNKFORWARDER
+#   - role: newrelic
+#     when: COMMON_ENABLE_NEWRELIC
+
+- name: install application requirements
+  pip: >
+    requirements="{{ edx_notes_api_requirements_base }}/{{ item }}"
+    virtualenv="{{ edx_notes_api_home }}/venvs/{{ edx_notes_api_service_name }}"
+    state=present extra_args="--exists-action w"
+  sudo_user: "{{ edx_notes_api_user }}"
+  with_items: edx_notes_api_requirements
+
+- name: migrate
+  shell: >
+    chdir={{ edx_notes_api_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ edx_notes_api_home }}/venvs/{{ edx_notes_api_service_name }}/bin/python {{ edx_notes_api_manage }} migrate --noinput --settings="notesserver.settings.yaml_config"
+  sudo_user: "{{ edx_notes_api_user }}"
+  environment:
+    EDXNOTES_CONFIG_ROOT: "{{ COMMON_CFG_DIR }}"
+  #when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: write out gunicorn.py
+  template: >
+    src=edx/app/edx_notes_api/edx_notes_api_gunicorn.py.j2
+    dest={{ edx_notes_api_home }}/{{ edx_notes_api_service_name }}_gunicorn.py
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+  tags:
+    - deploy
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/edx_notes_api.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ edx_notes_api_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  tags:
+    - deploy
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ edx_notes_api_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ edx_notes_api_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+  tags:
+    - deploy
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+  tags:
+    - deploy
+
+- name: restart supervisor
+  supervisorctl_local: >
+    name={{ edx_notes_api_service_name }}
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  when: not disable_edx_services
+  tags:
+    - deploy
+
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ edx_notes_api_home }}/venvs/{{ edx_notes_api_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ edx_notes_api_service_name }}"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create manage.py symlink
+  file: >
+    src="{{ edx_notes_api_manage }}"
+    dest="{{ COMMON_BIN_DIR }}/manage.{{ edx_notes_api_service_name }}"
+    state=link
+
+- name: restart edx_notes_api
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ edx_notes_api_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"

playbooks/roles/edx_notes_api/templates/edx/app/edx_notes_api/edx_notes_api_gunicorn.py.j2

+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+
+{{ ansible_managed }}
+"""
+import multiprocessing
+
+preload_app = True
+timeout = {{ edx_notes_api_gunicorn_timeout }}
+bind = "{{ edx_notes_api_gunicorn_host }}:{{ edx_notes_api_gunicorn_port }}"
+pythonpath = "{{ edx_notes_api_code_dir }}"
+
+{% if EDX_NOTES_API_WORKERS %}
+workers = {{ EDX_NOTES_API_WORKERS }}
+{% else %}
+workers = (multiprocessing.cpu_count()-1) * 2 + 2
+{% endif %}
+
+{{ EDX_NOTES_API_WORKERS_EXTRA_CONF }}

playbooks/roles/edx_notes_api/templates/edx/app/supervisor/conf.d.available/edx_notes_api.conf.j2

+[program:{{ edx_notes_api_service_name }}]
+
+{% set edx_notes_api_venv_bin = edx_notes_api_home + '/venvs/' + edx_notes_api_service_name + '/bin' %}
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = edx_notes_api_venv_bin + '/newrelic-admin run-program ' + edx_notes_api_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = edx_notes_api_venv_bin + '/gunicorn' %}
+{% endif %}
+
+command={{ executable }} -c {{ edx_notes_api_home }}/edx_notes_api_gunicorn.py {{ EDX_NOTES_API_GUNICORN_WORKERS_EXTRA }} {{ edx_notes_api_wsgi }}
+
+user={{ common_web_user }}
+directory={{ edx_notes_api_code_dir }}
+
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDX_NOTES_API_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PID=/var/tmp/edx_notes_api.pid,PORT={{ edx_notes_api_gunicorn_port }},ADDRESS={{ edx_notes_api_gunicorn_host }},LANG={{ EDX_NOTES_API_LANG }},DJANGO_SETTINGS_MODULE=notesserver.settings.yaml_config,SERVICE_VARIANT="{{ edx_notes_api_service_name }}",EDXNOTES_CONFIG_ROOT="{{ COMMON_CFG_DIR }}"
+
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/edx_service/defaults/main.yml

@@ -14,12 +14,11 @@
 #
 # vars are namespace with the module name.
 #
-edx_service_role_name: edx_service
+edx_service_name: edx_service
 
 #
 # OS packages
 #
-
-edx_service_debian_pkgs: []
-
-edx_service_redhat_pkgs: []
+edx_service_packages:
+  debian: []
+  redhat: []

playbooks/roles/edx_service/tasks/main.yml

@@ -19,41 +19,64 @@
 # Example play:
 #
 # Rather than being included in the play, this role
-# is included as a dependency by other roles in the meta/mail.yml
+# is included as a dependency by other roles in the meta/main.yml
 # file.  The including role should add the following
 # depency definition.
 #
 # dependencies:
-#   - { role: edx_service, edx_service_name: "hotg" }
+#   - role: edx_service
+#     edx_service_name: "hotg"
+#     edx_service_config: "{{ structure_to_be_written_to_config_file_in_/edx/etc }}"
+#     edx_service_repos:
+#       - PROTOCOL: [https/ssh]
+#         DOMAIN: github.com
+#         PATH: edx
+#         REPO: hotg
+#         VERSION: master
+#         DESTINATION: "/edx/app/hotg/hotg"
+#         SSH_KEY: <required if PROTOCOL==ssh>
+#       - PROTOCOL
+#         ...
+#     edx_service_user: hotg_system_user
+#     edx_service_home: "/edx/app/hotg"
+#     edx_service_packages:
+#       debian: [ pkg1, pkg2, pkg3 ]      
+#       redhat: [ pkg4, pkg5 ]
 #
 
+# Generating an ssh key so service users can do a git
+# clone over ssh for public repositories without any
+# additional configuration
 - name: create application user
   user: >
     name="{{ edx_service_name }}"
-    home="{{ COMMON_APP_DIR }}/{{ edx_service_name }}"
-    createhome=no
+    home="{{ edx_service_home }}"
+    createhome=yes
     shell=/bin/false
+    generate_ssh_key=yes
 
-- name: create edx_service app and venv dir
+# Assumes that the home directory has been created above.
+- name: create edx_service app, venv, data, and staticfiles dirs
   file: >
-    path="{{ item }}"
+    path="{{ edx_service_home }}/{{ item }}"
     state=directory
     owner="{{ edx_service_name }}"
     group="{{ common_web_group }}"
   with_items:
-    - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}"
-    - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/venvs"
+    - "venvs"
+    - "data"
+    - "staticfiles"
 
-- name: create edx_service data and staticfiles dir
+- name: create /edx/var and /edx/etc dirs
   file: >
-    path="{{ item }}"
+    path="{{ item }}/{{ edx_service_name }}"
     state=directory
-    owner="{{ edx_service_name }}"
-    group="{{ common_web_group }}"
+    owner={{ edx_service_user }}
+    group={{ common_web_group }}
+    mode=0755
   with_items:
-    - "{{ COMMON_DATA_DIR }}/{{ edx_service_name }}/data"
-    - "{{ COMMON_DATA_DIR }}/{{ edx_service_name }}/staticfiles"
-
+    - /edx/var
+    - /edx/etc
 
 - name: create edx_service log dir
   file: >
@@ -64,14 +87,76 @@
   with_items:
     - "{{ COMMON_LOG_DIR }}/{{ edx_service_name }}"
 
-# Replace dashes with underscores to support roles that use
-# dashes (the role vars will contain underscores)
+- name: write out app config file
+  template: >
+    src=config.yml.j2
+    dest={{ COMMON_CFG_DIR  }}/{{ edx_service_name }}.yml
+    mode=0644
+  tags:
+    - deploy
+  when: edx_service_config is defined
+
 - name: install a bunch of system packages on which edx_service relies
   apt: pkg={{ item }} state=present
-  with_items: "{{ edx_service_name.replace('-', '_') }}_debian_pkgs"
+  with_items: edx_service_packages.debian
   when: ansible_distribution in common_debian_variants
 
 - name: install a bunch of system packages on which edx_service relies
   yum: pkg={{ item }} state=present
-  with_items: "{{ edx_service_name.replace('-', '_') }}_redhat_pkgs"
+  with_items: edx_service_name.redhat
   when: ansible_distribution in common_redhat_variants
+
+
+- name: set git fetch.prune to ignore deleted remote refs
+  shell: git config --global fetch.prune true
+  sudo_user: "{{ edx_service_user }}"
+  when: edx_service_repos is defined
+
+- name: validate git protocol
+  fail: msg='REPOS.PROTOCOL must be "https" or "ssh"'
+  when: (item.PROTOCOL != "https") and (item.PROTOCOL != "ssh") and edx_service_repos is defined
+  with_items: edx_service_repos
+
+- name: install read-only ssh key
+  copy: >
+    dest="{{ edx_service_home }}/.ssh/{{ item.REPO }}"
+    content="{{ item.SSH_KEY }}" owner={{ edx_service_user }}
+    group={{ edx_service_user }} mode=0600
+  when: item.PROTOCOL == "ssh" and edx_service_repos is defined
+  with_items: edx_service_repos
+
+- name: checkout code over ssh
+  git: >
+    repo=git@{{ item.DOMAIN }}:{{ item.PATH }}/{{ item.REPO }}
+    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
+    accept_hostkey=yes key_file={{ edx_service_home }}/.ssh/{{ item.REPO }}
+  sudo_user: "{{ edx_service_user }}"
+  register: code_checkout
+  when: item.PROTOCOL == "ssh" and edx_service_repos is defined
+  with_items: edx_service_repos
+
+- name: checkout code over https
+  git: >
+    repo=https://{{ item.DOMAIN }}/{{ item.PATH }}/{{ item.REPO }}
+    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
+  sudo_user: "{{ edx_service_user }}"
+  register: code_checkout
+  when: item.PROTOCOL == "https" and edx_service_repos is defined
+  with_items: edx_service_repos
+
+
+- name: get instance information
+  action: ec2_facts
+  #old syntax - should be fixed
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:{{edx_service_name}}" : "{{ item.0.DOMAIN }}/{{ item.0.PATH }}/{{ item.0.REPO }} {{ item.1.after |truncate(7,True,'') }}"
+  when: item.1.after is defined and ansible_ec2_instance_id is defined and edx_service_repos is defined
+  with_together: 
+    - edx_service_repos
+    - code_checkout.results
+
+#TODO: restart supervisor- depends on supervisor being refactored into this role

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.yaml.j2 → playbooks/roles/edx_service/templates/config.yml.j2

 ---
 # {{ ansible_managed }}
 
-{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
+{{ edx_service_config | to_nice_yaml }}

playbooks/roles/edxapp/tasks/deploy.yml

@@ -28,16 +28,9 @@
     force=yes owner={{ edxapp_user }} mode=0600
   when: EDXAPP_USE_GIT_IDENTITY
 
-- name: check if git repo exists before pruning
-  stat: path={{ edxapp_code_dir }}/.git
-  register: git_dir
-
-- name: git prune before checking out
-  shell: cd {{ edxapp_code_dir }} && git remote prune origin
-  environment:
-    GIT_SSH: "{{ edxapp_git_ssh }}"
+- name: set git fetch.prune to ignore deleted remote refs
+  shell: git config --global fetch.prune true
   sudo_user: "{{ edxapp_user }}"
-  when: git_dir.stat.exists and git_dir.stat.isdir
 
 # Do A Checkout
 - name: checkout edx-platform repo into {{ edxapp_code_dir }}

playbooks/roles/edxlocal/tasks/main.yml

@@ -90,25 +90,39 @@
     db={{ item }}
     state=present
     encoding=utf8
-  when: ANALYTICS_API_CONFIG is defined
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
   with_items:
-    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}"
-    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}"
+    - "{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['default']['NAME'] }}"
+    - "{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['reports']['NAME'] }}"
 
 
 - name: create api user for the analytics api
   mysql_user: >
     name=api001
     password=password
-    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT'
-  when: ANALYTICS_API_CONFIG is defined
+    priv='{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT'
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
 
 - name: create read-only reports user for the analytics-api
   mysql_user: >
     name=reports001
     password=password
-    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
-  when: ANALYTICS_API_CONFIG is defined
+    priv='{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
+
+- name: setup the edx-notes-api db user
+  mysql_user: >
+    name={{ EDX_NOTES_API_MYSQL_DB_USER }}
+    password={{ EDX_NOTES_API_MYSQL_DB_PASS }}
+    priv='{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE'
+  when: EDX_NOTES_API_MYSQL_DB_USER is defined
+
+- name: create a database for edx-notes-api
+  mysql_db: >
+    db={{ EDX_NOTES_API_MYSQL_DB_NAME }}
+    state=present
+    encoding=utf8
+  when: EDX_NOTES_API_MYSQL_DB_USER is defined
 
 - name: setup the migration db user
   mysql_user: >
@@ -121,18 +135,10 @@
     - "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
     - "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
     - "{{ ORA_MYSQL_DB_NAME|default('None') }}"
-
-- name: setup the migration db user for analytics
-  mysql_user: >
-    name={{ COMMON_MYSQL_MIGRATE_USER }}
-    password={{ COMMON_MYSQL_MIGRATE_PASS }}
-    priv='{{ item }}.*:ALL'
-    append_privs=yes
-  when: ANALYTICS_API_CONFIG is defined
-  with_items:
-    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}"
-    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}"
-
+    - "{{ EDX_NOTES_API_MYSQL_DB_NAME|default('None') }}"
+    - "{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['default']['NAME']|default('None') }}"
+    - "{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['reports']['NAME']|default('None') }}"
+    - "{{ INSIGHTS_DATABASES.default.NAME|default('None') }}"
 
 - name: setup the read-only db user
   mysql_user: >

playbooks/roles/insights/defaults/main.yml

@@ -11,8 +11,6 @@
 # Defaults for role insights
 # 
 
-INSIGHTS_GIT_IDENTITY: !!null
-
 INSIGHTS_MEMCACHE: [ 'localhost:11211' ]
 INSIGHTS_FEEDBACK_EMAIL: 'dashboard@example.com'
 INSIGHTS_MKTG_BASE: 'http://example.com'
@@ -107,31 +105,38 @@ INSIGHTS_CONFIG:
   LMS_COURSE_SHORTCUT_BASE_URL: "{{ INSIGHTS_LMS_COURSE_SHORTCUT_BASE_URL }}"
   COURSE_API_URL: "{{ INSIGHTS_COURSE_API_URL }}"
 
-INSIGHTS_VERSION: "master"
 INSIGHTS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-analytics-api"
 INSIGHTS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 INSIGHTS_NGINX_PORT: "18110"
 INSIGHTS_GUNICORN_WORKERS: "2"
 INSIGHTS_GUNICORN_EXTRA: ""
 INSIGHTS_COURSE_API_URL: "URL FOR COURSE API"
+
+INSIGHTS_VERSION: "master"
+INSIGHTS_GIT_IDENTITY: !!null
+
+INSIGHTS_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-analytics-dashboard.git
+    VERSION: "{{ INSIGHTS_VERSION }}"
+    DESTINATION: "{{ insights_code_dir }}"
+    SSH_KEY: "{{ INSIGHTS_GIT_IDENTITY }}"
+
 #
 # vars are namespace with the module name.
 #
 insights_environment:
   DJANGO_SETTINGS_MODULE: "analytics_dashboard.settings.production"
-  ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml"
+  ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yml"
 
-insights_role_name: "insights"
-insights_service_name: "{{ insights_role_name }}"
-insights_user: "{{ insights_role_name }}"
+insights_service_name: insights
+insights_user: "{{ insights_service_name }}"
 insights_app_dir: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
 insights_home: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
-insights_venv_base: "{{ insights_home }}/venvs"
-insights_venv_dir: "{{ insights_venv_base }}/{{ insights_service_name }}"
-insights_venv_bin: "{{ insights_venv_dir }}/bin"
-insights_code_dir: "{{ insights_app_dir }}/edx-analytics-dashboard"
-insights_python_path: "{{ insights_code_dir }}"
-insights_static_path: "{{ insights_code_dir }}/analytics_dashboard/static"
+insights_code_dir: "{{ insights_app_dir }}/edx_analytics_dashboard"
+insights_python_path: "{{ insights_code_dir }}/analytics_dashboard"
 insights_conf_dir: "{{ insights_home }}"
 insights_log_dir: "{{ COMMON_LOG_DIR }}/{{ insights_service_name }}"
 
@@ -146,10 +151,7 @@ insights_gunicorn_timeout: "300"
 insights_wsgi: "analytics_dashboard.wsgi:application"
 
 insights_django_settings: "analytics_dashboard.settings.production"
-insights_source_repo: "git@{{ COMMON_GIT_MIRROR }}:/edx/edx-analytics-dashboard"
-insights_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ insights_git_identity_file }}"
-insights_git_identity_file: "{{ insights_home }}/git-identity"
-insights_manage: "{{ insights_code_dir }}/manage.py"
+insights_manage: "{{ insights_code_dir }}/analytics_dashboard/manage.py"
 
 insights_requirements_base: "{{ insights_code_dir }}/requirements"
 insights_requirements:

playbooks/roles/insights/meta/main.yml

@@ -12,6 +12,12 @@
 # 
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ insights_role_name }}"
     edx_service_name: "{{ insights_service_name }}"
+    edx_service_config: "{{ INSIGHTS_CONFIG }}"
+    edx_service_repos: "{{ INSIGHTS_REPOS }}"
+    edx_service_user: "{{ insights_user }}"
+    edx_service_home: "{{ insights_home }}"
+    edx_service_packages:
+      debian: "{{ insights_debian_pkgs }}"
+      redhat: "{{ insights_redhat_pkgs }}"
   - supervisor

playbooks/roles/insights/tasks/deploy.yml

----
-
-- name: install read-only ssh key
-  copy: >
-    content="{{ INSIGHTS_GIT_IDENTITY }}" dest={{ insights_git_identity_file }}
-    owner={{ insights_user }} group={{ insights_user }} mode=0600
-
-- name: setup the insights env file
-  template: >
-    src="edx/app/insights/insights_env.j2"
-    dest="{{ insights_app_dir }}/insights_env"
-    owner={{ insights_user }}
-    group={{ insights_user }}
-    mode=0644
-
-- name: checkout code
-  git: >
-    dest={{ insights_code_dir }} repo={{ insights_source_repo }} version={{ INSIGHTS_VERSION }}
-    accept_hostkey=yes
-    ssh_opts="{{ insights_git_ssh_opts }}"
-  register: insights_code_checkout
-  notify: restart insights
-  sudo_user: "{{ insights_user }}"
-
-- name: write out app config file
-  template: >
-    src=edx/app/insights/insights.yaml.j2
-    dest={{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml
-    mode=0644 owner={{ insights_user }} group={{ insights_user }}
-  notify: restart insights
-
-- name: install application requirements
-  pip: >
-    requirements="{{ insights_requirements_base }}/{{ item }}"
-    virtualenv="{{ insights_venv_dir }}" state=present
-    extra_args="--exists-action w"
-  sudo_user: "{{ insights_user }}"
-  notify: restart insights
-  with_items: insights_requirements
-
-- name: create nodeenv
-  shell: >
-    creates={{ insights_nodeenv_dir }}
-    {{ insights_venv_bin }}/nodeenv {{ insights_nodeenv_dir }}
-  sudo_user: "{{ insights_user }}"
-
-- name: install node dependencies
-  npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
-  sudo_user: "{{ insights_user }}"
-
-- name: install bower dependencies
-  shell: >
-    chdir={{ insights_code_dir }}
-    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
-  sudo_user: "{{ insights_user }}"
-
-- name: migrate
-  shell: >
-    chdir={{ insights_code_dir }}
-    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
-    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ insights_venv_bin }}/python {{ insights_manage }} migrate --noinput
-  sudo_user: "{{ insights_user }}"
-  environment: "{{ insights_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-
-- name: run r.js optimizer
-  shell: >
-    chdir={{ insights_code_dir }}
-    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
-  sudo_user: "{{ insights_user }}"
-
-- name: run collectstatic
-  shell: >
-    chdir={{ insights_code_dir }}
-    {{ insights_venv_bin }}/python {{ insights_manage }} {{ item }}
-  sudo_user: "{{ insights_user }}"
-  environment: "{{ insights_environment }}"
-  with_items:
-    - "collectstatic --noinput"
-    - "compress"
-
-- name: compile translations
-  shell: >
-    chdir={{ insights_code_dir }}/analytics_dashboard
-    . {{ insights_venv_bin }}/activate && i18n_tool generate -v
-  sudo_user: "{{ insights_user }}"
-
-- name: write out the supervisior wrapper
-  template: >
-    src=edx/app/insights/insights.sh.j2
-    dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
-    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
-  notify: restart insights
-
-- name: write supervisord config
-  template: >
-    src=edx/app/supervisor/conf.d.available/insights.conf.j2
-    dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
-    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
-  notify: restart insights
-
-- name: enable supervisor script
-  file: >
-    src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
-    dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
-    state=link
-    force=yes
-  notify: restart insights
-  when: not disable_edx_services
-
-- name: update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-
-- name: create symlinks from the venv bin dir
-  file: >
-    src="{{ insights_venv_bin }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_role_name }}"
-    state=link
-  with_items:
-  - python
-  - pip
-  - django-admin.py
-
-- name: create manage.py symlink
-  file: >
-    src="{{ insights_manage }}"
-    dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_role_name }}"
-    state=link
-
-- name: remove read-only ssh key for the content repo
-  file: path={{ insights_git_identity_file }} state=absent
-
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/insights/tasks/main.yml

@@ -21,7 +21,115 @@
 #
 #
 
-- fail: msg="You must provide a private key for the Insights repo"
-  when: not INSIGHTS_GIT_IDENTITY
+- name: setup the insights env file
+  template: >
+    src="edx/app/insights/insights_env.j2"
+    dest="{{ insights_app_dir }}/insights_env"
+    owner={{ insights_user }}
+    group={{ insights_user }}
+    mode=0644
 
-- include: deploy.yml tags=deploy
+- name: install application requirements
+  pip: >
+    requirements="{{ insights_requirements_base }}/{{ item }}"
+    virtualenv="{{ insights_home }}/venvs/{{ insights_service_name }}" 
+    state=present extra_args="--exists-action w"
+  sudo_user: "{{ insights_user }}"
+  with_items: insights_requirements
+
+- name: create nodeenv
+  shell: >
+    creates={{ insights_nodeenv_dir }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/nodeenv {{ insights_nodeenv_dir }}
+  sudo_user: "{{ insights_user }}"
+
+- name: install node dependencies
+  npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
+  sudo_user: "{{ insights_user }}"
+
+- name: install bower dependencies
+  shell: >
+    chdir={{ insights_code_dir }}
+    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
+  sudo_user: "{{ insights_user }}"
+
+- name: migrate
+  shell: >
+    chdir={{ insights_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/python {{ insights_manage }} migrate --noinput
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run r.js optimizer
+  shell: >
+    chdir={{ insights_code_dir }}
+    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
+  sudo_user: "{{ insights_user }}"
+
+- name: run collectstatic
+  shell: >
+    chdir={{ insights_code_dir }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/python {{ insights_manage }} {{ item }}
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  with_items:
+    - "collectstatic --noinput"
+    - "compress"
+
+- name: compile translations
+  shell: >
+    chdir={{ insights_code_dir }}/analytics_dashboard
+    . {{ insights_home }}/venvs/{{ insights_service_name }}/bin/activate && i18n_tool generate -v
+  sudo_user: "{{ insights_user }}"
+
+- name: write out the supervisior wrapper
+  template: >
+    src=edx/app/insights/insights.sh.j2
+    dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/insights.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ insights_home }}/venvs/{{ insights_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_service_name }}"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create manage.py symlink
+  file: >
+    src="{{ insights_manage }}"
+    dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_service_name }}"
+    state=link
+
+- name: restart insights
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ insights_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"
\ No newline at end of file

playbooks/roles/insights/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:insights" : "{{ insights_source_repo }} {{ insights_code_checkout.after |truncate(7,True,'')}}"
-  when: insights_code_checkout.after is defined

playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2

@@ -2,6 +2,7 @@
 
 # {{ ansible_managed }}
 
+{% set insights_venv_bin = insights_home + '/venvs/' + insights_service_name + '/bin' %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
 {% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %}
 {% else %}

playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2

----
-# {{ ansible_managed }}
-
-{{ INSIGHTS_CONFIG | to_nice_yaml }}

playbooks/roles/mariadb/defaults/main.yml

@@ -78,6 +78,11 @@ MARIADB_USERS:
     priv: "*.*:CREATE USER"
     host: "{{ MARIADB_HOST_PRIV }}"
 
+  - name: "{{ EDX_NOTES_API_MYSQL_DB_USER|default('notes001') }}"
+    pass: "{{ EDX_NOTES_API_MYSQL_DB_PASS|default('secret') }}"
+    priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME|default('edx-notes-api') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
 MARIADB_ANALYTICS_USERS:
   - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['USER']|default('api001') }}"
     pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['PASSWORD']|default('password') }}"

playbooks/roles/minos/defaults/main.yml

@@ -12,17 +12,21 @@
 # 
 MINOS_GIT_IDENTITY: !!null
 
+MINOS_SERVICE_CONFIG:
+  aws_profile: !!null
+  s3_bucket: edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}
+  bucket_path: lifecycle/minos
+  voter_conf_d: "{{ minos_voter_cfg }}"
+
 #
 # vars are namespace with the module name.
 #
-minos_role_name: minos
-minos_service_name: "{{ minos_role_name }}"
-minos_data_dir: "{{ COMMON_DATA_DIR }}/minos"
-minos_app_dir: "{{ COMMON_APP_DIR }}/minos"
-minos_venv_dir: "{{ minos_app_dir }}/venvs/"
-minos_log_dir: "{{ COMMON_LOG_DIR }}/minos"
-minos_cfg_file: "{{ COMMON_CFG_DIR }}/minos/minos.yml"
-minos_voter_cfg: "{{ COMMON_CFG_DIR }}/minos/conf.d/"
+minos_service_name: minos
+minos_data_dir: "{{ COMMON_DATA_DIR }}/{{ minos_service_name }}"
+minos_app_dir: "{{ COMMON_APP_DIR }}/{{ minos_service_name }}"
+minos_log_dir: "{{ COMMON_LOG_DIR }}/{{ minos_service_name }}"
+minos_cfg_file: "{{ COMMON_CFG_DIR }}/{{ minos_service_name }}/minos.yml"
+minos_voter_cfg: "{{ COMMON_CFG_DIR }}/{{ minos_service_name }}/conf.d/"
 minos_git_ssh: "/tmp/git.sh"
 minos_git_identity: "{{ minos_app_dir }}/minos-git-identity"
 minos_edx_server_tools_repo: "git@github.com/edx-ops/edx-minos.git"

playbooks/roles/minos/meta/main.yml

@@ -20,5 +20,10 @@
 # }
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ minos_role_name }}"
-    edx_service_name: "{{ minos_service_name }}"
\ No newline at end of file
+    edx_service_name: "{{ minos_service_name }}"
+    edx_service_config: "{{ MINOS_SERVICE_CONFIG }}"
+    edx_service_user: root
+    edx_service_home: "{{ minos_app_dir }}"
+    edx_service_packages:
+      debian: "{{ minos_debian_pkgs }}"
+      redhat: "{{ minos_redhat_pkgs }}"
\ No newline at end of file

playbooks/roles/minos/tasks/main.yml

@@ -34,9 +34,6 @@
 #     - minos 
 #
 
-- name: gather ec2 facts
-  action: ec2_facts
-
 - name: create minos config directory
   file: >
     path={{ minos_voter_cfg }}
@@ -44,12 +41,6 @@
     owner=root
     group=root
     mode=0755
-
-- name: create minos config
-  template: >
-    dest={{ minos_cfg_file }}
-    src=edx/etc/minos/minos.yml.j2
-    mode=0755 owner=root group=root
   
 - name: create minos voters configs
   template: >
@@ -81,13 +72,13 @@
 
 - name: install read-only ssh key
   copy: >
-    content="{{ COMMON_GIT_IDENTITY }}" dest="{{ minos_git_identity }}"
+    content="{{ MINOS_GIT_IDENTITY }}" dest="{{ minos_git_identity }}"
     force=yes mode=0600
 
-- name : install python custom-requirements
+- name: install python custom-requirements
   pip: >
     name="{{ item }}"
-    virtualenv="{{ minos_venv_dir }}"
+    virtualenv="{{ minos_app_dir }}/venvs/"
     state=present
     extra_args="--exists-action w"
   environment:

playbooks/roles/minos/templates/edx/etc/minos/minos.yml.j2

----
-
-aws_profile: !!null
-s3_bucket: 'edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}'
-bucket_path: 'lifecycle/minos'
-voter_conf_d: '{{ minos_voter_cfg }}'

playbooks/roles/minos/templates/tmp/git-identity.sh.j2

 #!/bin/sh
-exec /usr/bin/ssh -o StrictHostKeyChecking=no {% if COMMON_GIT_IDENTITY %}-i {{ minos_git_identity }}{% endif %} "$@"
+exec /usr/bin/ssh -o StrictHostKeyChecking=no {% if MINOS_GIT_IDENTITY %}-i {{ minos_git_identity }}{% endif %} "$@"

playbooks/roles/nginx/defaults/main.yml

@@ -91,6 +91,8 @@ nginx_insights_gunicorn_hosts:
   - 127.0.0.1
 nginx_gitreload_gunicorn_hosts:
   - 127.0.0.1
+nginx_edx_notes_api_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2

+upstream {{ edx_notes_api_service_name }}_app_server {
+    {% for host in nginx_edx_notes_api_gunicorn_hosts %}
+        server {{ host }}:{{ edx_notes_api_gunicorn_port }} fail_timeout=0;
+    {% endfor %}
+}
+
+server {
+  listen {{ edx_notes_api_nginx_port }} default_server;
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://{{ edx_notes_api_service_name }}_app_server;
+  }
+}
+

playbooks/roles/xqwatcher/defaults/main.yml

@@ -51,10 +51,17 @@ XQWATCHER_COURSES:
           KWARGS:
             grader_root: "../data/exampleX-202x/graders/"
 
-XQWATCHER_GIT_IDENTITY: |
-  -----BEGIN RSA PRIVATE KEY-----
-  Your key if you need to access any private repositories
-  -----END RSA PRIVATE KEY-----
+XQWATCHER_GIT_IDENTITY: !!null
+XQWATCHER_VERSION: "master"
+
+XQWATCHER_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: xqueue-watcher.git
+    VERSION: "{{ XQWATCHER_VERSION }}"
+    DESTINATION: "{{ xqwatcher_code_dir }}"
+    SSH_KEY: "{{ XQWATCHER_GIT_IDENTITY }}"
 
 # depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
 # and a key being provided via NEWRELIC_LICENSE_KEY
@@ -64,25 +71,26 @@ XQWATCHER_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 #
 # vars are namespace with the module name.
 #
-xqwatcher_role_name: "xqwatcher"
 xqwatcher_service_name: "xqwatcher"
-xqwatcher_user: "xqwatcher"
-xqwatcher_module: "xqueue_watcher"
-
+xqwatcher_user: "{{ xqwatcher_service_name }}"
 xqwatcher_app_dir: "{{ COMMON_APP_DIR }}/{{ xqwatcher_service_name }}"
 xqwatcher_app_data: "{{ xqwatcher_app_dir }}/data"
-xqwatcher_venv_base: "{{ xqwatcher_app_dir }}/venvs"
-xqwatcher_venv_dir: "{{ xqwatcher_venv_base }}/{{ xqwatcher_service_name }}"
 xqwatcher_code_dir: "{{ xqwatcher_app_dir }}/src"
+
+#TODO: change this to /edx/etc after pulling xqwatcher.json out
 xqwatcher_conf_dir: "{{ xqwatcher_app_dir }}"
 
-xqwatcher_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/xqueue-watcher.git"
-xqwatcher_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ xqwatcher_git_identity }}"
-XQWATCHER_VERSION: "master"
-xqwatcher_git_identity: "{{ xqwatcher_app_dir }}/git-identity"
+#TODO: remove after refactoring out all the git stuff
+xqwatcher_course_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ XQWATCHER_GIT_IDENTITY }}"
 
 xqwatcher_requirements_file: "{{ xqwatcher_code_dir }}/requirements.txt"
 xqwatcher_log_dir: "{{ COMMON_LOG_DIR }}/{{ xqwatcher_service_name }}"
+xqwatcher_module: "xqueue_watcher"
+
+#Do not reference these outside of this file
+xqwatcher_venv_base: "{{ xqwatcher_app_dir }}/venvs"
+xqwatcher_venv_dir: "{{ xqwatcher_venv_base }}/{{ xqwatcher_service_name }}"
+
 
 #
 # supervisor related config

playbooks/roles/xqwatcher/handlers/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Handlers for role xqwatcher
-#
-# Overview:
-#
-#
-- name: restart xqwatcher
-  supervisorctl_local: >
-    state=restarted
-    supervisorctl_path={{ xqwatcher_supervisor_ctl }}
-    config={{ xqwatcher_supervisor_app_dir }}/supervisord.conf
-    name={{ xqwatcher_service_name }}
-  when: not disable_edx_services

playbooks/roles/xqwatcher/meta/main.yml

@@ -14,8 +14,13 @@
 # random corners of ansible/jinga/python variable expansion.
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ xqwatcher_role_name }}"
     edx_service_name: "{{ xqwatcher_service_name }}"
+    edx_service_repos: "{{ XQWATCHER_REPOS }}"
+    edx_service_user: "{{ xqwatcher_user }}"
+    edx_service_home: "{{ xqwatcher_app_dir }}"
+    edx_service_packages:
+        debian: "{{ xqwatcher_debian_pkgs }}"
+        redhat: "{{ xqwatcher_redhat_pkgs }}"
   - role: supervisor
     supervisor_app_dir: "{{ xqwatcher_supervisor_app_dir }}"
     supervisor_data_dir: "{{ xqwatcher_supervisor_data_dir }}"

playbooks/roles/xqwatcher/tasks/code_jail.yml

@@ -31,27 +31,27 @@
 
 - name: create jail virtualenv
   shell: >
-    /usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
+    /usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
   with_items: XQWATCHER_COURSES
 
 - name: write out requirements.txt
   template: >
     src=edx/app/xqwatcher/data/requirements.txt.j2
-    dest={{ xqwatcher_app_data }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}-requirements.txt
+    dest={{ xqwatcher_app_dir }}/data/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}-requirements.txt
     mode=0440 owner=root group=root
   with_items: XQWATCHER_COURSES
 
-- name : install course specific python requirements
+- name: install course specific python requirements
   pip: >
     requirements="{{ xqwatcher_app_data }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}-requirements.txt"
-    virtualenv="{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+    virtualenv="{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
     state=present
     extra_args="{{ XQWATCHER_PIP_EXTRA_ARGS }}"
   with_items: XQWATCHER_COURSES
 
 - name: give other read permissions to the virtualenv
   shell: >
-    chown -R {{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
+    chown -R {{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
   with_items: XQWATCHER_COURSES
 
 - name: start apparmor service

playbooks/roles/xqwatcher/tasks/deploy.yml

-- name: install read-only ssh key
-  copy: >
-    content="{{ XQWATCHER_GIT_IDENTITY }}" dest={{ xqwatcher_git_identity }}
-    owner={{ xqwatcher_user }} group={{ xqwatcher_user }} mode=0600
-
+#TODO: remove once xqwatcher.json can be pulled out into /edx/etc/
 - name: write out watcher config file
   template: >
     src=edx/app/xqwatcher/xqwatcher.json.j2
@@ -15,7 +11,4 @@
 
 - include: deploy_courses.yml
   tags:
-    - deploy-courses
-
-- name: remove read-only ssh key for the content repo
-  file: path={{ xqwatcher_git_identity }} state=absent
+    - deploy-courses
\ No newline at end of file

playbooks/roles/xqwatcher/tasks/deploy_courses.yml

@@ -4,7 +4,7 @@
 
 - name: checkout grader code
   git: >
-    dest={{ xqwatcher_app_data }}/{{ item.COURSE }} repo={{ item.GIT_REPO }}
+    dest={{ xqwatcher_app_dir }}/data/{{ item.COURSE }} repo={{ item.GIT_REPO }}
     version={{ item.GIT_REF }}
-    ssh_opts="{{ xqwatcher_git_ssh_opts }}"
+    ssh_opts="{{ xqwatcher_course_git_ssh_opts }}"
   with_items: XQWATCHER_COURSES

playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

@@ -2,18 +2,10 @@
 # The watcher can watch one or many queues and dispatch submissions
 # to the appropriate grader which lives in a separate SCM repository.
 
-- name: checkout watcher code
-  git: >
-    dest={{ xqwatcher_code_dir }} repo={{ xqwatcher_source_repo }} version={{ XQWATCHER_VERSION }}
-    accept_hostkey=yes
-    ssh_opts="{{ xqwatcher_git_ssh_opts }}"
-  sudo_user: "{{ xqwatcher_user }}"
-  register: xqwatcher_checkout
-
 - name: install application requirements
   pip: >
     requirements="{{ xqwatcher_requirements_file }}"
-    virtualenv="{{ xqwatcher_venv_dir }}" state=present
+    virtualenv="{{ xqwatcher_app_dir }}/venvs/{{ xqwatcher_service_name }}" state=present
   sudo: true
   sudo_user: "{{ xqwatcher_user }}"
 
@@ -41,7 +33,12 @@
 - name: update supervisor configuration
   shell: "{{ xqwatcher_supervisor_ctl }} -c {{ xqwatcher_supervisor_app_dir }}/supervisord.conf update"
   when: not disable_edx_services
-  notify: restart xqwatcher
 
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE
+- name: restart xqwatcher
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ xqwatcher_supervisor_ctl }}
+    config={{ xqwatcher_supervisor_app_dir }}/supervisord.conf
+    name={{ xqwatcher_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ xqwatcher_user }}"
\ No newline at end of file

playbooks/roles/xqwatcher/tasks/main.yml

@@ -100,12 +100,6 @@
     owner="{{ xqwatcher_user }}"
     group="{{ xqwatcher_user }}"
 
-- name: create app data dir
-  file: >
-    path="{{ xqwatcher_app_data }}"
-    state=directory
-    owner="{{ xqwatcher_user }}"
-    group="{{ xqwatcher_user }}"
 
 - include: code_jail.yml CODE_JAIL_COMPLAIN=false
 

playbooks/roles/xqwatcher/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:xqwatcher" : "{{ xqwatcher_source_repo }} {{ xqwatcher_checkout.after|truncate(7,True,'') }}"
-  when: xqwatcher_checkout.after is defined

playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2

 ;  {{ ansible_managed }}
 ;
 
+{% set xqwatcher_venv_dir = xqwatcher_app_dir + '/venvs/' + xqwatcher_service_name %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
 {% set executable = xqwatcher_venv_dir + '/bin/newrelic-admin run-program ' + xqwatcher_venv_dir + '/bin/python' %}
 {% else %}

playbooks/roles/xqwatcher/templates/etc/apparmor.d/code.jail.j2

 #include <tunables/global>
 
-{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python {
+{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python {
     #include <abstractions/base>
 
-    {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/** mr,
+    {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/** mr,
     #todo need a way of providing.
     # edxapp_code_dir /common/lib/sandbox-packages/** r,
     /tmp/codejail-*/ rix,

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-course-sandbox.j2

-{{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=({{ item.QUEUE.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_venv_base }}/{{ item.QUEUE.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=({{ item.QUEUE.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE.HANDLERS[0].CODEJAIL.name }}/bin/python
 {{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/bin/kill
 {{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-xqwatcher.j2

-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
 {{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill
 {{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill

playbooks/vagrant-cluster.yml

@@ -52,6 +52,6 @@
     MARIADB_CREATE_DBS: yes
   vars_files:
     - "group_vars/all"
-    - "roles/analytics-api/defaults/main.yml"
+    - "roles/analytics_api/defaults/main.yml"
   roles:
     - mariadb

playbooks/vagrant-devstack.yml

@@ -30,5 +30,5 @@
     - browsermob-proxy
     - local_dev
     - demo
-    - role: analytics-api
+    - role: analytics_api
       when: ANALYTICS_API_GIT_IDENTITY

playbooks/vagrant-fullstack.yml

@@ -40,6 +40,6 @@
     - forum
     - { role: "xqueue", update_users: True }
     - certs
-    - role: analytics-api
+    - role: analytics_api
       when: ANALYTICS_API_GIT_IDENTITY
     - edx_ansible