Merge pull request #1232 from edx/jarv/settings-from-yaml

Jarv/settings from yaml

playbooks/edx-east/analyticsapi.yml

@@ -8,7 +8,7 @@
     ENABLE_NEWRELIC: False
   roles:
     - aws
-    - analytics_api 
+    - analytics-api
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/roles/analytics_api/defaults/main.yml → playbooks/roles/analytics-api/defaults/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 ##
-# Defaults for role analytics_api
+# Defaults for role analytics-api
 #
 
 ANALYTICS_API_GIT_IDENTITY: !!null
@@ -21,16 +21,16 @@ ANALYTICS_API_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 #
 # vars are namespace with the module name.
 #
-analytics_api_role_name: "analytics_api"
-analytics_api_service_name: "analytics_api"
-analytics_api_user: "analytics_api"
+analytics_api_role_name: "analytics-api"
+analytics_api_service_name: "analytics-api"
+analytics_api_user: "analytics-api"
 
 analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
 analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
 analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
 analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
 analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
-analytics_api_code_dir: "{{ analytics_api_app_dir }}/src/edx-analytics-data-api"
+analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
 analytics_api_conf_dir: "{{ analytics_api_home }}"
 analytics_api_gunicorn_host: "0.0.0.0"
 analytics_api_gunicorn_port: "8080"
@@ -57,20 +57,28 @@ ANALYTICS_API_CONFIG:
   LANGUAGE_CODE: 'en-us'
   # email config
   EMAIL_HOST: 'smtp.example.com'
-  EMAIL_HOST_PASSWORD: ''
-  EMAIL_HOST_USER: ''
+  EMAIL_HOST_PASSWORD: ""
+  EMAIL_HOST_USER: ""
   EMAIL_PORT: 587
   API_AUTH_TOKEN: 'put-your-api-token-here'
   # db config
   DATABASES:
+    # rw user
     default:
       ENGINE: 'django.db.backends.mysql'
-      NAME: 'name'
-      USER: 'user'
+      NAME: 'analytics-api'
+      USER: 'api001'
+      PASSWORD: 'password'
+      HOST: 'localhost'
+      PORT: '3306'
+    # read-only user
+    reports:
+      ENGINE: 'django.db.backends.mysql'
+      NAME: 'reports'
+      USER: 'reports001'
       PASSWORD: 'password'
       HOST: 'localhost'
       PORT: '3306'
-
 #
 # OS packages
 #

playbooks/roles/analytics_api/meta/main.yml → playbooks/roles/analytics-api/meta/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 #
-# Role includes for role analytics_api
+# Role includes for role analytics-api
 #
 # Example:
 #
@@ -23,4 +23,4 @@ dependencies:
   - role: edx_service
     edx_role_name: "{{ analytics_api_role_name }}"
     edx_service_name: "{{ analytics_api_service_name }}"
-  - supervisor
+  - supervisor
\ No newline at end of file

playbooks/roles/analytics_api/tasks/deploy.yml → playbooks/roles/analytics-api/tasks/deploy.yml

@@ -10,6 +10,7 @@
     dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
     accept_hostkey=yes
     ssh_opts="{{ analytics_api_git_ssh_opts }}"
+  sudo_user: "{{ analytics_api_user }}"
 
 - name: install application requirements
   pip: >
@@ -20,19 +21,19 @@
 
 - name: write out app config file
   template: >
-    src=edx/app/analytics_api/analytics_api.yaml.j2
-    dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.yaml
+    src=edx/app/analytics-api/analytics-api.yaml.j2
+    dest={{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml
     mode=0644 owner={{ analytics_api_user }} group={{ analytics_api_user }}
 
 - name: write out the supervisior wrapper
   template: >
-    src=edx/app/analytics_api/analytics_api.sh.j2
+    src=edx/app/analytics-api/analytics-api.sh.j2
     dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.sh
     mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
 
 - name: write supervisord config
   template: >
-    src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
+    src=edx/app/supervisor/conf.d.available/analytics-api.conf.j2
     dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
 

playbooks/roles/analytics_api/tasks/main.yml → playbooks/roles/analytics-api/tasks/main.yml

@@ -7,7 +7,7 @@
 # code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
-# Tasks for role analytics_api
+# Tasks for role analytics-api
 #
 # Overview:
 #
@@ -27,9 +27,9 @@
 #     ENABLE_NEWRELIC: False
 #   roles:
 #     - aws
-#     - analytics_api
+#     - analytics-api
 #
-# ansible-playbook -i 'api.example.com,' ./analytics_api.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
+# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
 #
 
 - fail: msg="You must provide an private key for the analytics repo"

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% if COMMON_ENABLE_NEWRELIC %}
+{% set executable = analytics_api_venv_bin + '/newrelic-admin run-program ' + analytics_api_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = analytics_api_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC %}
+export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.production"
+export ANALYTICS_API_CFG="{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+
+{{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.yaml → playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.yaml.j2

 ---
 # {{ ansible_managed }}
 
-{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
+{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
\ No newline at end of file

playbooks/roles/analytics_api/templates/edx/app/supervisor/conf.d.available/analytics_api.conf.j2 → playbooks/roles/analytics-api/templates/edx/app/supervisor/conf.d.available/analytics-api.conf.j2

@@ -2,7 +2,7 @@
 
 [program:{{ analytics_api_service_name }}]
 
-command={{ analytics_api_app_dir }}/analytics_api.sh
+command={{ analytics_api_app_dir }}/analytics-api.sh
 user={{ common_web_user }}
 directory={{ analytics_api_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% if COMMON_ENABLE_NEWRELIC %}
-{% set executable = analytics_api_venv_bin + '/newrelic-admin run-program ' + analytics_api_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = analytics_api_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC %}
-export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-#
-# These blocks output environment vars that will
-# eventually be replaced by loading a yaml file in
-# the django settings.  That file is currently also
-# being created, but the var reading doesn't exist
-{% for name,value in ANALYTICS_ENV_VARS.items() %}
-{% if value %}
-export {{ name }}="{{ value }}"
-{% endif -%}
-{% endfor %}
-
-{% for name,value in ANALYTICS_DATABASES.default.items() %}
-{% if value %}
-export DATABASE_{{ name }}="{{ value }}"
-{% endif -%}
-{% endfor %}
-
-{{ executable }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.yaml.j2

----
-# {{ ansible_managed }}
-
-{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
\ No newline at end of file

playbooks/roles/edx_service/tasks/main.yml

@@ -43,7 +43,6 @@
   with_items:
     - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}"
     - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/venvs"
-    - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/data"
 
 - name: create edx_service log dir
   file: >
@@ -54,13 +53,14 @@
   with_items:
     - "{{ COMMON_LOG_DIR }}/{{ edx_service_name }}"
 
-
+# Replace dashes with underscores to support roles that use
+# dashes (the role vars will contain underscores)
 - name: install a bunch of system packages on which edx_service relies
   apt: pkg={{ item }} state=present
-  with_items: "{{ edx_service_name }}_debian_pkgs"
+  with_items: "{{ edx_service_name.replace('-', '_') }}_debian_pkgs"
   when: ansible_distribution in common_debian_variants
 
 - name: install a bunch of system packages on which edx_service relies
   yum: pkg={{ item }} state=present
-  with_items: "{{ edx_service_name }}_redhat_pkgs"
+  with_items: "{{ edx_service_name.replace('-', '_') }}_redhat_pkgs"
   when: ansible_distribution in common_redhat_variants

playbooks/roles/edxlocal/tasks/main.yml

@@ -18,12 +18,14 @@
     name={{ EDXAPP_MYSQL_USER_MIGRATE }}
     password={{ EDXAPP_MYSQL_PASSWORD_MIGRATE}}
     priv='{{EDXAPP_MYSQL_DB_NAME}}.*:ALL'
+  when: EDXAPP_MYSQL_USER_MIGRATE is defined
 
 - name: setup the edxapp db user
   mysql_user: >
     name={{ EDXAPP_MYSQL_USER }}
     password={{ EDXAPP_MYSQL_PASSWORD }}
     priv='{{EDXAPP_MYSQL_DB_NAME}}.*:ALL'
+  when: EDXAPP_MYSQL_USER is defined
 
 - name: create a database for edxapp
   mysql_db: >
@@ -31,7 +33,7 @@
     state=present
     encoding=utf8
   when: EDXAPP_MYSQL_USER is defined
-    
+
 - name: setup the xqueue db user
   mysql_user: >
     name={{ XQUEUE_MYSQL_USER }}
@@ -51,6 +53,7 @@
     name={{ ORA_MYSQL_USER }}
     password={{ ORA_MYSQL_PASSWORD }}
     priv='{{ORA_MYSQL_DB_NAME}}.*:ALL'
+  when: ORA_MYSQL_USER is defined
 
 - name: create a database for ora
   mysql_db: >
@@ -74,6 +77,29 @@
     encoding=utf8
   when: DISCERN_MYSQL_USER is defined and not disable_edx_services
 
+- name: create databases for analytics api
+  mysql_db: >
+    db={{ item }}
+    state=present
+    encoding=utf8
+  with_items:
+    - 'reports'
+    - 'analytics-api'
+  when: analytics_api_role_name is defined
+
+- name: create api user for the analytics api
+  mysql_user: >
+    name=api001
+    password=password
+    priv='analytics-api.*:ALL/reports.*:SELECT'
+  when: analytics_api_role_name is defined
+
+- name: create read-only reports user for the analytics-api
+  mysql_user: >
+    name=reports001
+    password=password
+    priv='reports.*:SELECT'
+  when: analytics_api_role_name is defined
 
 - name: install memcached
   apt: pkg=memcached state=present