Merge pull request #4209 from edx/ammar/veda-deliver-worker-role

veda delivery worker role

CHANGELOG.md

+- Role: veda_delivery_worker
+  - New role added to run [video delivery worker](https://github.com/edx/edx-video-pipeline/blob/master/bin/deliver)
+
 - Role: veda_web_frontend
   - New role added for [edx-video-pipeline](https://github.com/edx/edx-video-pipeline)
 

playbooks/edx-east/edx_continuous_integration.yml

@@ -38,6 +38,7 @@
     - ecommerce
     - credentials
     - veda_web_frontend
+    - veda_delivery_worker
     - oauth_client_setup
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx-east/veda_delivery_worker.yml

@@ -2,4 +2,7 @@
   hosts: all
   gather_facts: True
   roles:
+    - aws
     - veda_delivery_worker
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/roles/edx_ansible/templates/update.j2

@@ -14,7 +14,7 @@ IFS=","
 
     <repo> - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, xserver, configuration,
     read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, course_discovery,
-    notifier, video_web_frontend
+    notifier, video_web_frontend, video_delivery_worker
     <version> - can be a commit or tag
 
 EO
@@ -62,6 +62,7 @@ repos_to_cmd["ecommerce"]="$edx_ansible_cmd ecommerce.yml -e 'ECOMMERCE_VERSION=
 repos_to_cmd["discovery"]="$edx_ansible_cmd discovery.yml -e 'DISCOVERY_VERSION=$2'"
 repos_to_cmd["notifier"]="$edx_ansible_cmd notifier.yml -e 'NOTIFIER_VERSION=$2'"
 repos_to_cmd["video_web_frontend"]="$edx_ansible_cmd veda_web_frontend.yml -e 'VEDA_WEB_FRONTEND_VERSION=$2'"
+repos_to_cmd["video_delivery_worker"]="$edx_ansible_cmd veda_delivery_worker.yml -e 'VEDA_DELIVERY_WORKER_VERSION=$2'"
 
 
 if [[ -z $1 || -z $2 ]]; then

playbooks/roles/veda_delivery_worker/defaults/main.yml

 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for veda_delivery_worker role
+#
+
+veda_delivery_worker_service_name: 'veda_delivery_worker'
+veda_delivery_worker_user: '{{ veda_delivery_worker_service_name }}'
+veda_delivery_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_delivery_worker_service_name }}'
+veda_delivery_worker_code_dir: '{{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}'
+veda_delivery_worker_venv_bin: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}/bin'
+veda_delivery_worker_service_config: !!null
+
+veda_delivery_worker_requirements_base: '{{ veda_delivery_worker_code_dir }}'
+veda_delivery_worker_requirements:
+  - requirements.txt
+
+veda_delivery_worker_environment:
+  PATH: '{{ veda_delivery_worker_venv_bin }}:{{ ansible_env.PATH }}'
+  PYTHONPATH: "{{ veda_delivery_worker_code_dir }}"
+  VEDA_DELIVERY_WORKER_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_delivery_worker_service_name }}.yml'
+
+# Requires that New Relic be enabled via COMMON_ENABLE_NEWRELIC, and that
+# a key be provided via NEWRELIC_LICENSE_KEY.
+VEDA_DELIVERY_WORKER_NEWRELIC_APPNAME: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ veda_delivery_worker_service_name }}'
+
+VEDA_DELIVERY_WORKER_GIT_IDENTITY: !!null
+VEDA_DELIVERY_WORKER_VERSION: 'master'
+
+VEDA_DELIVERY_WORKER_REPOS:
+  - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}'
+    DOMAIN: '{{ COMMON_GIT_MIRROR }}'
+    PATH: '{{ COMMON_GIT_PATH }}'
+    REPO: edx-video-pipeline.git
+    VERSION: '{{ VEDA_DELIVERY_WORKER_VERSION }}'
+    DESTINATION: '{{ veda_delivery_worker_code_dir }}'
+    SSH_KEY: '{{ VEDA_DELIVERY_WORKER_GIT_IDENTITY }}'

playbooks/roles/veda_delivery_worker/meta/main.yml

 ---
-
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for veda_delivery_worker role
+#
 dependencies:
-  - video_pipeline_base
+  - common
+  - supervisor
+  - role: video_pipeline_base
+    video_pipeline_base_service_name: '{{ veda_delivery_worker_service_name }}'
+  - role: edx_service
+    edx_service_name: "{{ veda_delivery_worker_service_name }}"
+    edx_service_config: "{{ veda_delivery_worker_service_config }}"
+    edx_service_repos: "{{ VEDA_DELIVERY_WORKER_REPOS }}"
+    edx_service_user: "{{ veda_delivery_worker_user }}"
+    edx_service_home: "{{ veda_delivery_worker_home }}"

playbooks/roles/veda_delivery_worker/tasks/main.yml

 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for veda_delivery_worker role
+#
+
+- name: install application requirements
+  pip:
+    requirements: '{{ veda_delivery_worker_requirements_base }}/{{ item }}'
+    virtualenv: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}'
+    state: present
+  become_user: '{{ veda_delivery_worker_user }}'
+  with_items: "{{ veda_delivery_worker_requirements }}"
+  tags:
+    - install
+    - install:app-requirements
+
+# TODO! Update VEDA_WORKING directory path with a better path
+# https://openedx.atlassian.net/browse/EDUCATOR-1851
+- name: create worker VEDA_WORKING directory
+  file:
+    path: "{{ veda_delivery_worker_home }}/{{ item }}"
+    state: directory
+    owner: "{{ veda_delivery_worker_service_name }}"
+    group: "{{ common_web_group }}"
+  with_items:
+    - "VEDA_WORKING"
+  tags:
+    - install
+    - install:base
+
+- name: write out the supervisor wrapper
+  template:
+    src: edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2
+    dest: '{{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}.sh'
+    mode: 0650
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+  tags:
+    - install
+    - install:configuration
+
+- name: write supervisord config
+  template:
+    src: edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2
+    dest: '{{ supervisor_available_dir }}/{{ veda_delivery_worker_service_name }}.conf'
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: setup the veda_delivery_worker env file
+  template:
+    src: edx/app/veda_delivery_worker/veda_delivery_worker_env.j2
+    dest: '{{ veda_delivery_worker_home }}/veda_delivery_worker_env'
+    owner: '{{ veda_delivery_worker_user }}'
+    group: '{{ veda_delivery_worker_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: enable supervisor script
+  file:
+    src: '{{ supervisor_available_dir }}/{{ veda_delivery_worker_service_name }}.conf'
+    dest: '{{ supervisor_cfg_dir }}/{{ veda_delivery_worker_service_name }}.conf'
+    state: link
+    force: yes
+  when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
+
+- name: update supervisor configuration
+  shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update'
+  when: not disable_edx_services
+  tags:
+    - manage
+    - manage:start
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}/bin/{{ item }}'
+    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_delivery_worker'
+    state: link
+  with_items:
+  - python
+  - pip
+  tags:
+    - install
+    - install:configuration
+
+- name: restart the applicaton
+  supervisorctl:
+    state: restarted
+    supervisorctl_path: '{{ supervisor_ctl }}'
+    config: '{{ supervisor_cfg }}'
+    name: '{{ veda_delivery_worker_service_name }}'
+  when: not disable_edx_services
+  become_user: '{{ supervisor_service_user }}'
+  tags:
+    - manage
+    - manage:start

playbooks/roles/veda_delivery_worker/templates/edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2

+#
+# {{ ansible_managed }}
+#
+[program:{{ veda_delivery_worker_service_name }}]
+
+command={{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}.sh
+user={{ common_web_user }}
+directory={{ veda_delivery_worker_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% set veda_delivery_worker_venv_bin = veda_delivery_worker_home + '/venvs/' + veda_delivery_worker_service_name + '/bin' %}
+{% set executable = veda_delivery_worker_venv_bin + '/python' %}
+
+source {{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}_env
+{{ executable }} {{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}/bin/deliver

playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in veda_delivery_worker_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/video_pipeline_base/meta/main.yml

+---
+
+# Allow this role to be duplicated in dependencies.
+allow_duplicates: yes