Merge pull request #4240 from edx/ammar/veda-pipeline-worker

veda pipeline worker

.gitignore

@@ -30,3 +30,6 @@ playbooks/edx-east/travis-test.yml
 
 ## Ansible Artifacts
 *.retry
+
+### VisualStudioCode ###
+.vscode/*

CHANGELOG.md

+- Role: veda_pipeline_worker
+  - New role to run all (`deliver, ingest, youtubecallback`) [video pipeline workers](https://github.com/edx/edx-video-pipeline/blob/master/bin/)
+
 - Role: veda_ffmpeg
   - New role added to compile ffmpeg for video pipeline. It will be used as a dependency for video pipeline roles.
 

playbooks/edx-east/edx_continuous_integration.yml

@@ -38,7 +38,6 @@
     - ecommerce
     - credentials
     - veda_web_frontend
-    - veda_delivery_worker
     - oauth_client_setup
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx-east/veda_pipeline_worker.yml

+- name: Deploy edX VEDA pipeline Worker
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - veda_pipeline_worker
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/roles/edx_ansible/templates/update.j2

@@ -14,7 +14,7 @@ IFS=","
 
     <repo> - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, xserver, configuration,
     read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, course_discovery,
-    notifier, video_web_frontend, video_delivery_worker
+    notifier, video_web_frontend, video_delivery_worker, veda_pipeline_worker
     <version> - can be a commit or tag
 
 EO
@@ -63,6 +63,7 @@ repos_to_cmd["discovery"]="$edx_ansible_cmd discovery.yml -e 'DISCOVERY_VERSION=
 repos_to_cmd["notifier"]="$edx_ansible_cmd notifier.yml -e 'NOTIFIER_VERSION=$2'"
 repos_to_cmd["video_web_frontend"]="$edx_ansible_cmd veda_web_frontend.yml -e 'VEDA_WEB_FRONTEND_VERSION=$2'"
 repos_to_cmd["video_delivery_worker"]="$edx_ansible_cmd veda_delivery_worker.yml -e 'VEDA_DELIVERY_WORKER_VERSION=$2'"
+repos_to_cmd["veda_pipeline_worker"]="$edx_ansible_cmd veda_pipeline_worker.yml -e 'VEDA_PIPELINE_WORKER_VERSION=$2'"
 
 
 if [[ -z $1 || -z $2 ]]; then

playbooks/roles/veda_pipeline_worker/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for veda_pipeline_worker role
+#
+
+veda_pipeline_worker_service_name: 'veda_pipeline_worker'
+veda_pipeline_worker_user: '{{ veda_pipeline_worker_service_name }}'
+veda_pipeline_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_pipeline_worker_service_name }}'
+veda_pipeline_worker_code_dir: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}'
+veda_pipeline_worker_venv_bin: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin'
+veda_pipeline_worker_service_config: !!null
+veda_pipeline_worker_variants:
+  - deliver
+  - ingest
+  - youtubecallback
+
+veda_pipeline_worker_requirements_base: '{{ veda_pipeline_worker_code_dir }}'
+veda_pipeline_worker_requirements:
+  - requirements.txt
+
+veda_pipeline_worker_deliver_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/DELIVER_WORK_DIR'
+veda_pipeline_worker_ingest_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/INGEST_WORK_DIR'
+
+veda_pipeline_worker_directories:
+  DELIVER_WORK_DIR: '{{ veda_pipeline_worker_deliver_work_dir }}'
+  INGEST_WORK_DIR: '{{ veda_pipeline_worker_ingest_work_dir }}'
+
+veda_pipeline_worker_config_other: {}
+veda_pipeline_worker_config_extra: '{{ veda_pipeline_worker_config_other | combine(veda_pipeline_worker_directories) }}'
+
+veda_pipeline_worker_environment:
+  PATH: '{{ veda_pipeline_worker_venv_bin }}:{{ ansible_env.PATH }}'
+  PYTHONPATH: "{{ veda_pipeline_worker_code_dir }}"
+  VIDEO_PIPELINE_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_pipeline_worker_service_name }}.yml'
+  DJANGO_SETTINGS_MODULE: "VEDA.settings.production"
+
+VEDA_PIPELINE_WORKER_GIT_IDENTITY: !!null
+VEDA_PIPELINE_WORKER_VERSION: 'master'
+
+VEDA_PIPELINE_WORKER_REPOS:
+  - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}'
+    DOMAIN: '{{ COMMON_GIT_MIRROR }}'
+    PATH: '{{ COMMON_GIT_PATH }}'
+    REPO: edx-video-pipeline.git
+    VERSION: '{{ VEDA_PIPELINE_WORKER_VERSION }}'
+    DESTINATION: '{{ veda_pipeline_worker_code_dir }}'
+    SSH_KEY: '{{ VEDA_PIPELINE_WORKER_GIT_IDENTITY }}'

playbooks/roles/veda_pipeline_worker/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for veda_pipeline_worker role
+#
+dependencies:
+  - common
+  - supervisor
+  - role: video_pipeline_base
+    video_pipeline_base_service_name: '{{ veda_pipeline_worker_service_name }}'
+    video_pipeline_base_config_extra: '{{ veda_pipeline_worker_config_extra }}'
+  - role: edx_service
+    edx_service_name: "{{ veda_pipeline_worker_service_name }}"
+    edx_service_config: "{{ veda_pipeline_worker_service_config }}"
+    edx_service_repos: "{{ VEDA_PIPELINE_WORKER_REPOS }}"
+    edx_service_user: "{{ veda_pipeline_worker_user }}"
+    edx_service_home: "{{ veda_pipeline_worker_home }}"

playbooks/roles/veda_pipeline_worker/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for veda_pipeline_worker role
+#
+
+- name: install application requirements
+  pip:
+    requirements: '{{ veda_pipeline_worker_requirements_base }}/{{ item }}'
+    virtualenv: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}'
+    state: present
+  become_user: '{{ veda_pipeline_worker_user }}'
+  with_items: "{{ veda_pipeline_worker_requirements }}"
+  tags:
+    - install
+    - install:app-requirements
+
+# TODO! Update VEDA_WORKING directory path with a better path
+# https://openedx.atlassian.net/browse/EDUCATOR-1851
+- name: create worker VEDA_WORKING directory
+  file:
+    path: "{{ veda_pipeline_worker_home }}/{{ item }}"
+    state: directory
+    owner: "{{ veda_pipeline_worker_service_name }}"
+    group: "{{ common_web_group }}"
+  with_items:
+    - "VEDA_WORKING"
+  tags:
+    - install
+    - install:base
+
+- name: create worker work directories
+  file:
+    path: "{{ item.value }}"
+    state: directory
+    owner: "{{ veda_pipeline_worker_service_name }}"
+    group: "{{ common_web_group }}"
+  with_dict: "{{ veda_pipeline_worker_directories }}"
+  tags:
+    - install
+    - install:base
+
+- name: write out the supervisor wrapper
+  template:
+    src: edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2
+    dest: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.sh'
+    mode: 0650
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+  with_items: "{{ veda_pipeline_worker_variants }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: write supervisord config
+  template:
+    src: edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2
+    dest: '{{ supervisor_available_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf'
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+    mode: 0644
+  with_items: "{{ veda_pipeline_worker_variants }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: setup the veda_pipeline_worker env file
+  template:
+    src: edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2
+    dest: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_env'
+    owner: '{{ veda_pipeline_worker_user }}'
+    group: '{{ veda_pipeline_worker_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: enable supervisor script
+  file:
+    src: '{{ supervisor_available_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf'
+    dest: '{{ supervisor_cfg_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf'
+    state: link
+    force: yes
+  with_items: "{{ veda_pipeline_worker_variants }}"
+  when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
+
+- name: update supervisor configuration
+  shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update'
+  when: not disable_edx_services
+  tags:
+    - manage
+    - manage:start
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin/{{ item }}'
+    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_pipeline_worker'
+    state: link
+  with_items:
+  - python
+  - pip
+  tags:
+    - install
+    - install:configuration
+
+- name: restart the applicaton
+  supervisorctl:
+    state: restarted
+    supervisorctl_path: '{{ supervisor_ctl }}'
+    config: '{{ supervisor_cfg }}'
+    name: '{{ veda_pipeline_worker_service_name }}_{{ item }}'
+  with_items: "{{ veda_pipeline_worker_variants }}"
+  when: not disable_edx_services
+  become_user: '{{ supervisor_service_user }}'
+  tags:
+    - manage
+    - manage:start

playbooks/roles/veda_pipeline_worker/templates/edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2

+#
+# {{ ansible_managed }}
+#
+[program:{{ veda_pipeline_worker_service_name }}_{{ item }}]
+
+command={{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.sh
+user={{ common_web_user }}
+directory={{ veda_pipeline_worker_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source {{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_env
+{{ COMMON_BIN_DIR }}/python.{{ veda_pipeline_worker_service_name }} {{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}/bin/{{ item }}

playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in veda_pipeline_worker_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/video_pipeline_base/defaults/main.yml

@@ -36,8 +36,8 @@ VIDEO_PIPELINE_BASE_MYSQL_USER: "veda001"
 VIDEO_PIPELINE_BASE_MYSQL_PASSWORD: "password"
 
 VIDEO_PIPELINE_BASE_RABBITMQ_BROKER: "localhost"
-VIDEO_PIPELINE_BASE_RABBITMQ_USER: "pipeline"
-VIDEO_PIPELINE_BASE_RABBITMQ_PASS: "pipeline"
+VIDEO_PIPELINE_BASE_RABBITMQ_USER: "celery"
+VIDEO_PIPELINE_BASE_RABBITMQ_PASS: "celery"
 
 # video pipeline config overrides
 
@@ -84,7 +84,7 @@ VIDEO_PIPELINE_BASE_HOST_ENVIRONMENT: "sandbox"
 
 # video pipeline config
 
-VIDEO_PIPELINE_BASE_CONFIG:
+video_pipeline_base_config_default:
   DATABASES:
       default:
           ENGINE: 'django.db.backends.mysql'
@@ -99,6 +99,8 @@ VIDEO_PIPELINE_BASE_CONFIG:
   # django staticfile settings
   STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ video_pipeline_base_service_name }}/staticfiles"
 
+  SERVICE_VARIANT_NAME: "{{ video_pipeline_base_service_name }}"
+
   # ---
   # AWS Buckets, Prefixes
   # ---
@@ -182,3 +184,6 @@ VIDEO_PIPELINE_BASE_CONFIG:
   sg_server_path: "{{ VIDEO_PIPELINE_BASE_SG_SERVER_PATH }}"
   sg_script_name: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_NAME }}"
   sg_script_key: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_KEY }}"
+
+video_pipeline_base_config_extra: {}
+VIDEO_PIPELINE_BASE_CONFIG: '{{ video_pipeline_base_config_default | combine(video_pipeline_base_config_extra) }}'

util/jenkins/ansible-provision.sh

@@ -359,8 +359,9 @@ EOF
 fi
 
 veda_web_frontend="true"
+veda_pipeline_worker="false"
 declare -A deploy
-roles="edxapp forum ecommerce credentials discovery veda_web_frontend notifier xqueue xserver certs demo testcourses"
+roles="edxapp forum ecommerce credentials discovery veda_web_frontend veda_pipeline_worker notifier xqueue xserver certs demo testcourses"
 
 for role in $roles; do
     deploy[$role]=${!role}