Merge pull request #4247 from edx/ammar/veda-encode-worker

veda encode worker

playbooks/edx-east/veda_encode_worker.yml

 - name: Deploy edX VEDA Encode Worker
   hosts: all
+  become: True
   gather_facts: True
   roles:
     - veda_encode_worker

playbooks/roles/edx_ansible/templates/update.j2

@@ -14,7 +14,7 @@ IFS=","
 
     <repo> - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, xserver, configuration,
     read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, course_discovery,
-    notifier, video_web_frontend, video_delivery_worker, veda_pipeline_worker
+    notifier, video_web_frontend, video_delivery_worker, veda_pipeline_worker, video_encode_worker
     <version> - can be a commit or tag
 
 EO
@@ -64,6 +64,7 @@ repos_to_cmd["notifier"]="$edx_ansible_cmd notifier.yml -e 'NOTIFIER_VERSION=$2'
 repos_to_cmd["video_web_frontend"]="$edx_ansible_cmd veda_web_frontend.yml -e 'VEDA_WEB_FRONTEND_VERSION=$2'"
 repos_to_cmd["video_delivery_worker"]="$edx_ansible_cmd veda_delivery_worker.yml -e 'VEDA_DELIVERY_WORKER_VERSION=$2'"
 repos_to_cmd["veda_pipeline_worker"]="$edx_ansible_cmd veda_pipeline_worker.yml -e 'VEDA_PIPELINE_WORKER_VERSION=$2'"
+repos_to_cmd["video_encode_worker"]="$edx_ansible_cmd veda_encode_worker.yml -e 'VEDA_ENCODE_WORKER_VERSION=$2'"
 
 
 if [[ -z $1 || -z $2 ]]; then

playbooks/roles/veda_encode_worker/defaults/main.yml

 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for veda_encode_worker role
+#
+
+veda_encode_worker_service_name: 'veda_encode_worker'
+veda_encode_worker_user: '{{ veda_encode_worker_service_name }}'
+veda_encode_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_encode_worker_service_name }}'
+veda_encode_worker_code_dir: '{{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}'
+veda_encode_worker_venv_bin: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}/bin'
+veda_encode_worker_data_dir: '{{ COMMON_DATA_DIR }}/{{ veda_encode_worker_service_name }}/ENCODE_WORKDIR'
+veda_encode_worker_service_config: !!null
+
+veda_encode_worker_directories:
+  ENCODE_WORK_DIR: '{{ veda_encode_worker_data_dir }}'
+
+veda_encode_worker_config_other: {}
+veda_encode_worker_config_extra: '{{ veda_encode_worker_config_other | combine(veda_encode_worker_directories) }}'
+
+veda_encode_worker_requirements_base: '{{ veda_encode_worker_code_dir }}'
+veda_encode_worker_requirements:
+  - requirements.txt
+
+veda_encode_worker_environment:
+  PATH: '{{ veda_encode_worker_venv_bin }}:{{ ansible_env.PATH }}'
+  PYTHONPATH: "{{ veda_encode_worker_code_dir }}"
+  VEDA_ENCODE_WORKER_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_encode_worker_service_name }}.yml'
+
+VEDA_ENCODE_WORKER_GIT_IDENTITY: !!null
+VEDA_ENCODE_WORKER_VERSION: 'master'
+
+VEDA_ENCODE_WORKER_REPOS:
+  - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}'
+    DOMAIN: '{{ COMMON_GIT_MIRROR }}'
+    PATH: '{{ COMMON_GIT_PATH }}'
+    REPO: edx-video-worker.git
+    VERSION: '{{ VEDA_ENCODE_WORKER_VERSION }}'
+    DESTINATION: '{{ veda_encode_worker_code_dir }}'
+    SSH_KEY: '{{ VEDA_ENCODE_WORKER_GIT_IDENTITY }}'

playbooks/roles/veda_encode_worker/meta/main.yml

 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for veda_encode_worker role
+#
+dependencies:
+  - common
+  - supervisor
+  - veda_ffmpeg
+  - role: video_pipeline_base
+    video_pipeline_base_service_name: '{{ veda_encode_worker_service_name }}'
+    video_pipeline_base_config_extra: '{{ veda_encode_worker_config_extra }}'
+  - role: edx_service
+    edx_service_name: "{{ veda_encode_worker_service_name }}"
+    edx_service_config: "{{ veda_encode_worker_service_config }}"
+    edx_service_repos: "{{ VEDA_ENCODE_WORKER_REPOS }}"
+    edx_service_user: "{{ veda_encode_worker_user }}"
+    edx_service_home: "{{ veda_encode_worker_home }}"

playbooks/roles/veda_encode_worker/tasks/main.yml

 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for veda_encode_worker role
+#
+
+- name: install application requirements
+  pip:
+    requirements: '{{ veda_encode_worker_requirements_base }}/{{ item }}'
+    virtualenv: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}'
+    state: present
+  become_user: '{{ veda_encode_worker_user }}'
+  with_items: "{{ veda_encode_worker_requirements }}"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: create worker work directories
+  file:
+    path: "{{ item.value }}"
+    state: directory
+    owner: "{{ veda_encode_worker_service_name }}"
+    group: "{{ common_web_group }}"
+  with_dict: "{{ veda_encode_worker_directories }}"
+  tags:
+    - install
+    - install:base
+
+- name: write out the supervisor wrapper
+  template:
+    src: edx/app/veda_encode_worker/veda_encode_worker.sh.j2
+    dest: '{{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}.sh'
+    mode: 0650
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+  tags:
+    - install
+    - install:configuration
+
+- name: write supervisord config
+  template:
+    src: edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2
+    dest: '{{ supervisor_available_dir }}/{{ veda_encode_worker_service_name }}.conf'
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: setup the veda_encode_worker env file
+  template:
+    src: edx/app/veda_encode_worker/veda_encode_worker_env.j2
+    dest: '{{ veda_encode_worker_home }}/veda_encode_worker_env'
+    owner: '{{ veda_encode_worker_user }}'
+    group: '{{ veda_encode_worker_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: enable supervisor script
+  file:
+    src: '{{ supervisor_available_dir }}/{{ veda_encode_worker_service_name }}.conf'
+    dest: '{{ supervisor_cfg_dir }}/{{ veda_encode_worker_service_name }}.conf'
+    state: link
+    force: yes
+  when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
+
+- name: update supervisor configuration
+  shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update'
+  when: not disable_edx_services
+  tags:
+    - manage
+    - manage:start
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}/bin/{{ item }}'
+    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_encode_worker'
+    state: link
+  with_items:
+  - python
+  - pip
+  tags:
+    - install
+    - install:configuration
+
+- name: restart the applicaton
+  supervisorctl:
+    state: restarted
+    supervisorctl_path: '{{ supervisor_ctl }}'
+    config: '{{ supervisor_cfg }}'
+    name: '{{ veda_encode_worker_service_name }}'
+  when: not disable_edx_services
+  become_user: '{{ supervisor_service_user }}'
+  tags:
+    - manage
+    - manage:start

playbooks/roles/veda_encode_worker/templates/edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2

+#
+# {{ ansible_managed }}
+#
+[program:{{ veda_encode_worker_service_name }}]
+
+command={{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}.sh
+user={{ common_web_user }}
+directory={{ veda_encode_worker_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source {{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}_env
+source {{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}/worker.sh

playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in veda_encode_worker_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/video_pipeline_base/defaults/main.yml

@@ -46,7 +46,6 @@ VIDEO_PIPELINE_BASE_EDX_CLOUDFRONT_PREFIX: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_EDX_S3_INGEST_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_EDX_S3_ENDPOINT_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET: "SET-ME-PLEASE"
-VIDEO_PIPELINE_BASE_VEDA_UPLOAD_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_S3_HOTSTORE_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_DELIVERABLE_BUCKET: "SET-ME-PLEASE"
 
@@ -119,9 +118,9 @@ video_pipeline_base_config_default:
 
   # VEDA Internal
   veda_s3_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET }}"
+  veda_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET }}"
   veda_s3_hotstore_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_HOTSTORE_BUCKET }}"
   veda_deliverable_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_DELIVERABLE_BUCKET }}"
-  veda_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_UPLOAD_BUCKET }}"
   veda_base_url: "{{ VIDEO_PIPELINE_BASE_VEDA_BASE_URL }}"
   s3_base_url: https://s3.amazonaws.com
 

util/jenkins/ansible-provision.sh

@@ -360,8 +360,9 @@ fi
 
 veda_web_frontend="true"
 veda_pipeline_worker="false"
+veda_encode_worker="false"
 declare -A deploy
-roles="edxapp forum ecommerce credentials discovery veda_web_frontend veda_pipeline_worker notifier xqueue xserver certs demo testcourses"
+roles="edxapp forum ecommerce credentials discovery veda_web_frontend veda_pipeline_worker veda_encode_worker notifier xqueue xserver certs demo testcourses"
 
 for role in $roles; do
     deploy[$role]=${!role}