Merge pull request #1822 from edx/max/git-refactor

Refactor analytics_api to use edx_service

CHANGELOG.md

+- Role: analytics_api
+  - Refactored name from `analytics-api` to `analytics_api`. This will require additional migrations when upgrading an existing server. While we recommend building from scratch, you can also run the following command:
+  
+      ```
+      rm -rf /edx/app/analytics-api /edx/app/ /edx/app/nginx/sites-available/analytics-api.j2 /edx/app/supervisor/conf.d.available/analytics_api.conf
+      ```
+
 - Role: notifier
   - Refactored `NOTIFIER_HOME` and `NOTIFIER_USER` to `notifier_app_dir` and `notifier_user` to match other roles. This shouldn't change anything since users should've only been overriding COMMON_HOME.
 

playbooks/edx-east/analyticsapi.yml

@@ -9,9 +9,9 @@
   roles:
     - role: nginx
       nginx_sites:
-        - analytics-api
+        - analytics_api
     - aws
-    - analytics-api
+    - analytics_api
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/edx-east/edx_continuous_integration.yml

@@ -15,7 +15,7 @@
       - xqueue
       - xserver
       - certs
-      - analytics-api
+      - analytics_api
       nginx_default_sites:
       - lms
     - role: edxlocal
@@ -34,7 +34,7 @@
       when: XSERVER_GIT_IDENTITY|length > 0
     - certs
     - edx_ansible
-    - analytics-api
+    - analytics_api
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/roles/analytics-api/handlers/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Handlers for role analytics-api
-#
-# Overview:
-#
-#
-
-- name: "restart the analytics service"
-  supervisorctl_local: >
-    name={{ analytics_api_service_name }}
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: not disable_edx_services
\ No newline at end of file

playbooks/roles/analytics-api/tasks/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-# Tasks for role analytics-api
-#
-# Overview:
-#
-# Install the Analytics Data API server, a python
-# django application that runs under gunicorn
-#
-# Dependencies:
-#
-# Example play:
-# - name: Deploy Analytics API
-#   hosts: all
-#   sudo: True
-#   gather_facts: True
-#   vars:
-#     ENABLE_DATADOG: False
-#     ENABLE_SPLUNKFORWARDER: False
-#     ENABLE_NEWRELIC: False
-#   roles:
-#     - aws
-#     - analytics-api
-#
-# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
-#
-
-- fail: msg="You must provide a private key for the analytics repo"
-  when: not ANALYTICS_API_GIT_IDENTITY
-
-- include: deploy.yml tags=deploy

playbooks/roles/analytics-api/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:analytics_api" : "{{ analytics_api_source_repo }} {{ analytics_api_code_checkout.after |truncate(7,True,'')}}"
-  when: analytics_api_code_checkout.after is defined

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.yaml.j2

----
-# {{ ansible_managed }}
-
-{{ ANALYTICS_API_CONFIG | to_nice_yaml }}

playbooks/roles/analytics-api/defaults/main.yml → playbooks/roles/analytics_api/defaults/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 ##
-# Defaults for role analytics-api
+# Defaults for role analytics_api
 #
 
 ANALYTICS_API_GIT_IDENTITY: !!null
@@ -51,7 +51,7 @@ ANALYTICS_API_EMAIL_HOST_PASSWORD: 'mail_password'
 ANALYTICS_API_EMAIL_PORT: 587
 ANALYTICS_API_AUTH_TOKEN: 'put-your-api-token-here'
 
-ANALYTICS_API_CONFIG:
+ANALYTICS_API_SERVICE_CONFIG:
   ANALYTICS_DATABASE: 'reports'
   SECRET_KEY: '{{ ANALYTICS_API_SECRET_KEY }}'
   TIME_ZONE: '{{ ANALYTICS_API_TIME_ZONE }}'
@@ -69,6 +69,15 @@ ANALYTICS_API_CONFIG:
     connect_timeout: 10
   DATABASES: '{{ ANALYTICS_API_DATABASES }}'
 
+ANALYTICS_API_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-analytics-data-api.git
+    VERSION: "{{ ANALYTICS_API_VERSION }}"
+    DESTINATION: "{{ analytics_api_code_dir }}"
+    SSH_KEY: "{{ ANALYTICS_API_GIT_IDENTITY }}"
+
 ANALYTICS_API_GUNICORN_WORKERS: "2"
 ANALYTICS_API_GUNICORN_EXTRA: ""
 ANALYTICS_API_GUNICORN_EXTRA_CONF: ""
@@ -78,27 +87,19 @@ ANALYTICS_API_GUNICORN_EXTRA_CONF: ""
 
 analytics_api_environment:
   DJANGO_SETTINGS_MODULE: "analyticsdataserver.settings.production"
-  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
-
-analytics_api_role_name: "analytics-api"
-analytics_api_service_name: "analytics-api"
-analytics_api_user: "analytics-api"
+  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yml"
 
-analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_service_name: "analytics_api"
+analytics_api_user: "{{ analytics_api_service_name }}"
 analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
-analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
-analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
-analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
-analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
+analytics_api_code_dir: "{{ analytics_api_home }}/{{ analytics_api_service_name }}"
 analytics_api_conf_dir: "{{ analytics_api_home }}"
+
 analytics_api_gunicorn_host: "127.0.0.1"
 analytics_api_gunicorn_port: "8100"
 analytics_api_gunicorn_timeout: "300"
 
 analytics_api_django_settings: "production"
-analytics_api_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/edx-analytics-data-api"
-analytics_api_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ analytics_api_git_identity_file }}"
-analytics_api_git_identity_file: "{{ analytics_api_app_dir }}/git-identity"
 
 analytics_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ analytics_api_service_name }}"
 

playbooks/roles/analytics-api/meta/main.yml → playbooks/roles/analytics_api/meta/main.yml

@@ -8,7 +8,7 @@
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
 #
-# Role includes for role analytics-api
+# Role includes for role analytics_api
 #
 # Example:
 #
@@ -21,6 +21,9 @@
 
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ analytics_api_role_name }}"
     edx_service_name: "{{ analytics_api_service_name }}"
+    edx_service_config: "{{ ANALYTICS_API_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ ANALYTICS_API_REPOS }}"
+    edx_service_user: "{{ analytics_api_user }}"
+    edx_service_home: "{{ analytics_api_home }}"
   - supervisor

playbooks/roles/analytics-api/tasks/deploy.yml → playbooks/roles/analytics_api/tasks/main.yml

 ---
-
-- name: install read-only ssh key
-  copy: >
-    content="{{ ANALYTICS_API_GIT_IDENTITY }}" dest={{ analytics_api_git_identity_file }}
-    owner={{ analytics_api_user }} group={{ analytics_api_user }} mode=0600
-
-- name: setup the analytics-api env file
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for role analytics_api
+#
+# Overview:
+#
+# Install the Analytics Data API server, a python
+# django application that runs under gunicorn
+#
+# Dependencies:
+#
+# Example play:
+# - name: Deploy Analytics API
+#   hosts: all
+#   sudo: True
+#   gather_facts: True
+#   vars:
+#     ENABLE_DATADOG: False
+#     ENABLE_SPLUNKFORWARDER: False
+#     ENABLE_NEWRELIC: False
+#   roles:
+#     - aws
+#     - analytics_api
+#
+# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
+#
+
+- name: setup the analytics_api env file
   template: >
-    src="edx/app/analytics-api/analytics_api_env.j2"
-    dest="{{ analytics_api_app_dir }}/analytics_api_env"
+    src="edx/app/analytics_api/analytics_api_env.j2"
+    dest="{{ analytics_api_home }}/analytics_api_env"
     owner={{ analytics_api_user }}
     group={{ analytics_api_user }}
     mode=0644
 
-- name: checkout code
-  git: >
-    dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
-    accept_hostkey=yes
-    ssh_opts="{{ analytics_api_git_ssh_opts }}"
-  register: analytics_api_code_checkout
-  notify: "restart the analytics service"
-  sudo_user: "{{ analytics_api_user }}"
-
 - name: "add gunicorn configuration file"
   template: >
-    src=edx/app/analytics-api/analytics_api_gunicorn.py.j2
-    dest={{ analytics_api_app_dir }}/analytics_api_gunicorn.py
+    src=edx/app/analytics_api/analytics_api_gunicorn.py.j2
+    dest={{ analytics_api_home }}/analytics_api_gunicorn.py
   sudo_user: "{{ analytics_api_user }}"
-  notify: "restart the analytics service"
-
-- name: write out app config file
-  template: >
-    src=edx/app/analytics-api/analytics-api.yaml.j2
-    dest={{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml
-    mode=0644 owner={{ analytics_api_user }} group={{ analytics_api_user }}
-  notify: restart the analytics service
 
 - name: install application requirements
   pip: >
     requirements="{{ analytics_api_requirements_base }}/{{ item }}"
-    virtualenv="{{ analytics_api_venv_dir }}" state=present
+    virtualenv="{{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}"
+    state=present
   sudo_user: "{{ analytics_api_user }}"
-  notify: restart the analytics service
   with_items: analytics_api_requirements
 
 - name: migrate
@@ -49,7 +59,7 @@
     chdir={{ analytics_api_code_dir }}
     DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
     DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ analytics_api_venv_bin }}/python ./manage.py migrate --noinput
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python ./manage.py migrate --noinput
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
@@ -57,31 +67,29 @@
 - name: run collectstatic
   shell: >
     chdir={{ analytics_api_code_dir }}
-    {{ analytics_api_venv_bin }}/python manage.py collectstatic  --noinput
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py collectstatic  --noinput
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
 
 - name: create api users
   shell: >
     chdir={{ analytics_api_code_dir }}
-    {{ analytics_api_venv_bin }}/python manage.py set_api_key {{ item.key }} {{ item.value }}
+    {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py set_api_key {{ item.key }} {{ item.value }}
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   with_dict: ANALYTICS_API_USERS
 
-- name: write out the supervisior wrapper
+- name: write out the supervisor wrapper
   template: >
-    src=edx/app/analytics-api/analytics-api.sh.j2
-    dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.sh
+    src=edx/app/analytics_api/analytics_api.sh.j2
+    dest={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
     mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
-  notify: restart the analytics service
 
 - name: write supervisord config
   template: >
-    src=edx/app/supervisor/conf.d.available/analytics-api.conf.j2
+    src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
     dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
-  notify: restart the analytics service
 
 - name: enable supervisor script
   file: >
@@ -89,7 +97,6 @@
     dest={{ supervisor_cfg_dir }}/{{ analytics_api_service_name }}.conf
     state=link
     force=yes
-  notify: restart the analytics service
   when: not disable_edx_services
 
 - name: update supervisor configuration
@@ -98,8 +105,8 @@
 
 - name: create symlinks from the venv bin dir
   file: >
-    src="{{ analytics_api_venv_bin }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    src="{{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics_api"
     state=link
   with_items:
   - python
@@ -109,13 +116,16 @@
 - name: create symlinks from the repo dir
   file: >
     src="{{ analytics_api_code_dir }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics_api"
     state=link
   with_items:
   - manage.py
 
-- name: remove read-only ssh key for the content repo
-  file: path={{ analytics_api_git_identity_file }} state=absent
-
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE
+- name: restart analytics_api
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ analytics_api_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2 → playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.sh.j2

@@ -2,6 +2,7 @@
 
 # {{ ansible_managed }}
 
+{% set analytics_api_venv_bin = analytics_api_home + "/venvs/" + analytics_api_service_name + "/bin/" %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
 {% set executable = analytics_api_venv_bin + '/newrelic-admin run-program ' + analytics_api_venv_bin + '/gunicorn' %}
 {% else %}
@@ -13,6 +14,6 @@ export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
 {% endif -%}
 
-source {{ analytics_api_app_dir }}/analytics_api_env
+source {{ analytics_api_home }}/analytics_api_env
 
-{{ executable }} -c {{ analytics_api_app_dir }}/analytics_api_gunicorn.py {{ ANALYTICS_API_GUNICORN_EXTRA }} analyticsdataserver.wsgi:application
+{{ executable }} -c {{ analytics_api_home }}/analytics_api_gunicorn.py {{ ANALYTICS_API_GUNICORN_EXTRA }} analyticsdataserver.wsgi:application

playbooks/roles/analytics-api/templates/edx/app/supervisor/conf.d.available/analytics-api.conf.j2 → playbooks/roles/analytics_api/templates/edx/app/supervisor/conf.d.available/analytics_api.conf.j2

@@ -2,7 +2,7 @@
 
 [program:{{ analytics_api_service_name }}]
 
-command={{ analytics_api_app_dir }}/analytics-api.sh
+command={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
 user={{ common_web_user }}
 directory={{ analytics_api_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log

playbooks/roles/edx_service/tasks/main.yml

@@ -64,6 +64,17 @@
     - "data"
     - "staticfiles"
 
+- name: create /edx/var and /edx/etc dirs
+  file: >
+    path="{{ item }}/{{ edx_service_name }}"
+    state=directory
+    owner={{ edx_service_user }}
+    group={{ common_web_group }}
+    mode=0755
+  with_items:
+    - /edx/var
+    - /edx/etc
+
 - name: create edx_service log dir
   file: >
     path="{{ item }}"

playbooks/vagrant-cluster.yml

@@ -52,6 +52,6 @@
     MARIADB_CREATE_DBS: yes
   vars_files:
     - "group_vars/all"
-    - "roles/analytics-api/defaults/main.yml"
+    - "roles/analytics_api/defaults/main.yml"
   roles:
     - mariadb

playbooks/vagrant-devstack.yml

@@ -30,5 +30,5 @@
     - browsermob-proxy
     - local_dev
     - demo
-    - role: analytics-api
+    - role: analytics_api
       when: ANALYTICS_API_GIT_IDENTITY

playbooks/vagrant-fullstack.yml

@@ -41,6 +41,6 @@
     - forum
     - { role: "xqueue", update_users: True }
     - certs
-    - role: analytics-api
+    - role: analytics_api
       when: ANALYTICS_API_GIT_IDENTITY
     - edx_ansible