Merge pull request #4273 from edx/ammar/veda-sandbox

veda sandbox

playbooks/edx-east/edx_continuous_integration.yml

@@ -38,6 +38,9 @@
     - ecommerce
     - credentials
     - veda_web_frontend
+    - veda_pipeline_worker
+    - veda_encode_worker
+    - video_pipeline_integration
     - oauth_client_setup
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx-east/veda_encode_worker.yml

@@ -3,4 +3,7 @@
   become: True
   gather_facts: True
   roles:
+    - aws
     - veda_encode_worker
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/edx-east/video_pipeline_integration.yml

+- name: Create edxapp video pipeline integration
+  hosts: all
+  become: True
+  gather_facts: True
+  vars_files:
+    - "roles/common_vars/defaults/main.yml"
+    - "roles/edxapp/defaults/main.yml"
+  roles:
+    - video_pipeline_integration

playbooks/roles/edx_ansible/templates/update.j2

@@ -14,7 +14,7 @@ IFS=","
 
     <repo> - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, xserver, configuration,
     read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, course_discovery,
-    notifier, video_web_frontend, video_delivery_worker, veda_pipeline_worker, video_encode_worker
+    notifier, video_web_frontend, video_delivery_worker, veda_pipeline_worker, video_encode_worker, veda_ffmpeg
     <version> - can be a commit or tag
 
 EO
@@ -65,6 +65,7 @@ repos_to_cmd["video_web_frontend"]="$edx_ansible_cmd veda_web_frontend.yml -e 'V
 repos_to_cmd["video_delivery_worker"]="$edx_ansible_cmd veda_delivery_worker.yml -e 'VEDA_DELIVERY_WORKER_VERSION=$2'"
 repos_to_cmd["veda_pipeline_worker"]="$edx_ansible_cmd veda_pipeline_worker.yml -e 'VEDA_PIPELINE_WORKER_VERSION=$2'"
 repos_to_cmd["video_encode_worker"]="$edx_ansible_cmd veda_encode_worker.yml -e 'VEDA_ENCODE_WORKER_VERSION=$2'"
+repos_to_cmd["veda_ffmpeg"]="$edx_ansible_cmd veda_ffmpeg.yml -e 'VEDA_FFMPEG_VERSION=$2'"
 
 
 if [[ -z $1 || -z $2 ]]; then

playbooks/roles/edxapp/defaults/main.yml

@@ -461,7 +461,7 @@ EDXAPP_EXTRA_REQUIREMENTS: []
 # Example:
 # EDXAPP_PRIVATE_REQUIREMENTS:
 #   - name: git+https://git.myproject.org/MyProject#egg=MyProject
-EDXAPP_PRIVATE_REQUIREMENTS: 
+EDXAPP_PRIVATE_REQUIREMENTS:
     # For Harvard courses:
     - name: git+https://github.com/open-craft/problem-builder.git@v2.7.7#egg=xblock-problem-builder==2.7.7
     # Oppia XBlock
@@ -797,6 +797,11 @@ EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/enterprise/api/v1
 EDXAPP_ENTERPRISE_SERVICE_WORKER_EMAIL: "enterprise_worker@example.com"
 EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME: "enterprise_worker"
 
+EDXAPP_VEDA_SERVICE_CLIENT_NAME: "veda"
+EDXAPP_VEDA_SERVICE_API_URL: "{{ EDXAPP_LMS_BASE_SCHEME | default('https') }}://veda-{{ EDXAPP_LMS_BASE }}/api/"
+EDXAPP_VEDA_SERVICE_USER_EMAIL: "veda_service_user@example.com"
+EDXAPP_VEDA_SERVICE_USER_NAME: "veda_service_user"
+
 EDXAPP_ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES:
   - audit
   - honor
@@ -1475,3 +1480,7 @@ SERVICE_WORKER_USERS:
     username: "{{ EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME }}"
     is_staff: true
     is_superuser: false
+  - email: "{{ EDXAPP_VEDA_SERVICE_USER_EMAIL }}"
+    username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME }}"
+    is_staff: true
+    is_superuser: false

playbooks/roles/edxapp/tasks/deploy.yml

@@ -247,7 +247,7 @@
     - install
     - install:app-requirements
 
-#install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json 
+#install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json
 - name: install node dependencies
   shell: "{{ edxapp_nodeenv_bin }}/npm install"
   args:

playbooks/roles/veda_encode_worker/tasks/main.yml

@@ -27,6 +27,7 @@
     state: directory
     owner: "{{ veda_encode_worker_service_name }}"
     group: "{{ common_web_group }}"
+    mode: 0775
   with_dict: "{{ veda_encode_worker_directories }}"
   tags:
     - install

playbooks/roles/veda_ffmpeg/defaults/main.yml

@@ -13,6 +13,6 @@
 
 VEDA_FFMPEG_GIT_URL: 'https://github.com/yro/v_videocompile'
 # used /opt path to clone the repo and then install ffmpeg using the cloned repo
-veda_ffmpeg_root: '/opt/veda_ffmpeg'
+veda_ffmpeg_root: '/var/tmp/veda_ffmpeg'
 VEDA_FFMPEG_VERSION: 'master'
 

playbooks/roles/veda_ffmpeg/tasks/main.yml

@@ -15,6 +15,7 @@
     repo: "{{ VEDA_FFMPEG_GIT_URL }}"
     dest: "{{ veda_ffmpeg_root }}"
     version: "{{ VEDA_FFMPEG_VERSION }}"
+  become: False
   tags:
     - install
     - install:code
@@ -29,6 +30,7 @@
 
 - name: compile v_videocompile
   command: "v_videocompile"
+  become: False
   tags:
     - install
     - install:system-requirements

playbooks/roles/veda_pipeline_worker/defaults/main.yml

@@ -28,10 +28,12 @@ veda_pipeline_worker_requirements:
 
 veda_pipeline_worker_deliver_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/DELIVER_WORK_DIR'
 veda_pipeline_worker_ingest_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/INGEST_WORK_DIR'
+veda_pipeline_worker_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/VEDA_WORKING'
 
 veda_pipeline_worker_directories:
   DELIVER_WORK_DIR: '{{ veda_pipeline_worker_deliver_work_dir }}'
   INGEST_WORK_DIR: '{{ veda_pipeline_worker_ingest_work_dir }}'
+  VEDA_WORKING: '{{ veda_pipeline_worker_work_dir }}'
 
 veda_pipeline_worker_config_other: {}
 veda_pipeline_worker_config_extra: '{{ veda_pipeline_worker_config_other | combine(veda_pipeline_worker_directories) }}'

playbooks/roles/veda_pipeline_worker/meta/main.yml

@@ -13,6 +13,7 @@
 dependencies:
   - common
   - supervisor
+  - veda_ffmpeg
   - role: video_pipeline_base
     video_pipeline_base_service_name: '{{ veda_pipeline_worker_service_name }}'
     video_pipeline_base_config_extra: '{{ veda_pipeline_worker_config_extra }}'

playbooks/roles/veda_pipeline_worker/tasks/main.yml

@@ -21,30 +21,29 @@
     - install
     - install:app-requirements
 
-# TODO! Update VEDA_WORKING directory path with a better path
-# https://openedx.atlassian.net/browse/EDUCATOR-1851
-- name: create worker VEDA_WORKING directory
+- name: create worker work directories
   file:
-    path: "{{ veda_pipeline_worker_home }}/{{ item }}"
+    path: "{{ item.value }}"
     state: directory
     owner: "{{ veda_pipeline_worker_service_name }}"
     group: "{{ common_web_group }}"
-  with_items:
-    - "VEDA_WORKING"
+    mode: 0775
+  with_dict: "{{ veda_pipeline_worker_directories }}"
   tags:
     - install
     - install:base
 
-- name: create worker work directories
+- name: create symlinks from the venv bin dir
   file:
-    path: "{{ item.value }}"
-    state: directory
-    owner: "{{ veda_pipeline_worker_service_name }}"
-    group: "{{ common_web_group }}"
-  with_dict: "{{ veda_pipeline_worker_directories }}"
+    src: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin/{{ item }}'
+    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_pipeline_worker'
+    state: link
+  with_items:
+  - python
+  - pip
   tags:
     - install
-    - install:base
+    - install:configuration
 
 - name: write out the supervisor wrapper
   template:
@@ -100,18 +99,6 @@
     - manage
     - manage:start
 
-- name: create symlinks from the venv bin dir
-  file:
-    src: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin/{{ item }}'
-    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_pipeline_worker'
-    state: link
-  with_items:
-  - python
-  - pip
-  tags:
-    - install
-    - install:configuration
-
 - name: restart the applicaton
   supervisorctl:
     state: restarted

playbooks/roles/veda_web_frontend/defaults/main.yml

@@ -20,6 +20,18 @@ veda_web_frontend_home: "{{ COMMON_APP_DIR }}/{{ veda_web_frontend_service_name
 veda_web_frontend_service_home: "{{ COMMON_APP_DIR }}/{{ veda_web_frontend_service_name }}"
 veda_web_frontend_code_dir: "{{ veda_web_frontend_service_home }}/{{ veda_web_frontend_service_name }}"
 veda_web_frontend_venv_dir: "{{ veda_web_frontend_service_home }}/venvs/{{ veda_web_frontend_service_name }}"
+veda_web_frontend_work_dir: "{{ COMMON_DATA_DIR }}/{{ veda_web_frontend_service_name }}/VEDA_WORKING"
+
+veda_web_frontend_directories:
+  VEDA_WORKING: '{{ veda_web_frontend_work_dir }}'
+
+veda_web_frontend_config_other: {}
+veda_web_frontend_config_extra: "{{ veda_web_frontend_config_other | combine(veda_web_frontend_directories) }}"
+
+VEDA_WEB_FRONTEND_LOAD_DATA: false
+veda_web_frontend_post_migrate_commands:
+  - command: 'python manage.py loaddata encodes'
+    when: '{{ VEDA_WEB_FRONTEND_LOAD_DATA }}'
 
 #
 # OS packages

playbooks/roles/veda_web_frontend/meta/main.yml

@@ -13,6 +13,7 @@
 dependencies:
   - role: video_pipeline_base
     video_pipeline_base_service_name: '{{ veda_web_frontend_service_name }}'
+    video_pipeline_base_config_extra: '{{ veda_web_frontend_config_extra }}'
   - role: edx_django_service
     edx_django_service_repo: '{{ VEDA_WEB_FRONTEND_REPO }}'
     edx_django_service_version: '{{ VEDA_WEB_FRONTEND_VERSION }}'
@@ -35,3 +36,4 @@ dependencies:
     edx_django_service_db_password: '{{ VEDA_WEB_FRONTEND_MYSQL_PASSWORD }}'
     edx_django_service_use_python3: false
     edx_django_service_config: '{{ VEDA_WEB_FRONTEND_SERVICE_CONFIG }}'
+    edx_django_service_post_migrate_commands: '{{ veda_web_frontend_post_migrate_commands }}'

playbooks/roles/veda_web_frontend/tasks/main.yml

@@ -10,6 +10,16 @@
 ##
 # Tasks for role veda_web_frontend
 #
+- name: create work directory
+  file:
+    path: "{{ veda_web_frontend_work_dir }}"
+    state: directory
+    owner: "{{ veda_web_frontend_service_name }}"
+    group: "{{ common_web_group }}"
+    mode: 0775
+  tags:
+    - install
+    - install:base
 
 # This is creating a client in VEDA application not LMS.
 - name: create OAuth application clients

playbooks/roles/video_pipeline_base/defaults/main.yml

@@ -41,9 +41,19 @@ VIDEO_PIPELINE_BASE_RABBITMQ_PASS: "celery"
 
 # video pipeline config overrides
 
-VIDEO_PIPELINE_BASE_EDX_S3_INGEST_PREFIX: "ingest/"
+VIDEO_PIPELINE_BASE_EDX_S3_INGEST:
+  BUCKET: "SET-ME-PLEASE"
+  ROOT_PATH: "ingest/"
+
+VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES:
+  BUCKET: "SET-ME-PLEASE"
+  ROOT_PATH: "video-images/"
+
+VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS:
+  BUCKET: "SET-ME-PLEASE"
+  ROOT_PATH: "video-transcripts/"
+
 VIDEO_PIPELINE_BASE_EDX_CLOUDFRONT_PREFIX: "SET-ME-PLEASE"
-VIDEO_PIPELINE_BASE_EDX_S3_INGEST_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_EDX_S3_ENDPOINT_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_S3_HOTSTORE_BUCKET: "SET-ME-PLEASE"
@@ -53,9 +63,6 @@ VIDEO_PIPELINE_BASE_VEDA_BASE_URL: "{{ VIDEO_PIPELINE_BASE_URL_ROOT }}"
 VIDEO_PIPELINE_BASE_VEDA_ACCESS_KEY_ID: "SET-ME-PLEASE"
 VIDEO_PIPELINE_BASE_VEDA_SECRET_ACCESS_KEY: "SET-ME-PLEASE"
 
-VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES_BUCKET: "SET-ME-PLEASE"
-VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS_BUCKET: "SET-ME-PLEASE"
-
 VIDEO_PIPELINE_BASE_ADMIN_EMAIL: "veda@example.com"
 VIDEO_PIPELINE_BASE_VEDA_NOREPLY_EMAIL: "veda-noreply@example.com"
 
@@ -104,17 +111,20 @@ video_pipeline_base_config_default:
   # AWS Buckets, Prefixes
   # ---
   # Studio/Platform
-  edx_s3_ingest_prefix: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST_PREFIX }}"
-  edx_s3_ingest_bucket: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST_BUCKET }}"
+  edx_s3_ingest_prefix: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST.ROOT_PATH }}"
+  edx_s3_ingest_bucket: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST.BUCKET }}"
   edx_s3_endpoint_bucket: "{{ VIDEO_PIPELINE_BASE_EDX_S3_ENDPOINT_BUCKET }}"
 
   # CF
   edx_cloudfront_prefix: "{{ VIDEO_PIPELINE_BASE_EDX_CLOUDFRONT_PREFIX }}"
 
   # Images
-  aws_video_images_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES_BUCKET }}"
-  aws_video_images_prefix: "video-images/"
+  aws_video_images_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES.BUCKET }}"
+  aws_video_images_prefix: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES.ROOT_PATH }}"
 
+  # Transcripts
+  aws_video_transcripts_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS.BUCKET }}"
+  aws_video_transcripts_prefix: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS.ROOT_PATH }}"
 
   # VEDA Internal
   veda_s3_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET }}"
@@ -124,11 +134,6 @@ video_pipeline_base_config_default:
   veda_base_url: "{{ VIDEO_PIPELINE_BASE_VEDA_BASE_URL }}"
   s3_base_url: https://s3.amazonaws.com
 
-
-  # Transcripts
-  aws_video_transcripts_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS_BUCKET }}"
-  aws_video_transcripts_prefix: video-transcripts/
-
   # cielo24 api urls
   cielo24_api_base_url: 'https://{{ VIDEO_PIPELINE_BASE_CIELO24_API_ENVIRONMENT }}.cielo24.com/api'
 
@@ -184,5 +189,8 @@ video_pipeline_base_config_default:
   sg_script_name: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_NAME }}"
   sg_script_key: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_KEY }}"
 
+  lms_base_url: "{{ video_pipeline_base_val_base_url }}"
+  instance_prefix: "{{ ansible_ec2_public_ipv4 }}"
+
 video_pipeline_base_config_extra: {}
 VIDEO_PIPELINE_BASE_CONFIG: '{{ video_pipeline_base_config_default | combine(video_pipeline_base_config_extra) }}'

playbooks/roles/video_pipeline_integration/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role video_pipeline_integration
+#
+
+#
+# vars are namespaced with the module name.
+#
+
+EDXAPP_VIDEO_PIPELINE_INTEGRATION_CONFIG:
+  - client_name: "{{ EDXAPP_VEDA_SERVICE_CLIENT_NAME }}"
+    api_url: "{{ EDXAPP_VEDA_SERVICE_API_URL }}"
+    service_username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME }}"

playbooks/roles/video_pipeline_integration/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role video_pipeline_integration
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: create edxapp video pipeline integration
+  shell: >
+    {{ COMMON_BIN_DIR }}/python.edxapp {{ COMMON_BIN_DIR }}/manage.edxapp lms --settings={{ COMMON_EDXAPP_SETTINGS }}
+    create_video_pipeline_integration {{ item.client_name}} {{ item.api_url }} {{ item.service_username }} --enabled
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ edxapp_user }}"
+  with_items: "{{ EDXAPP_VIDEO_PIPELINE_INTEGRATION_CONFIG }}"
+  when: CREATE_SERVICE_WORKER_USERS and CREATE_EDXAPP_VIDEO_PIPELINE_INTEGRATION is defined

util/jenkins/ansible-provision.sh

@@ -369,9 +369,10 @@ fi
 veda_web_frontend=${video_pipeline:-false}
 veda_pipeline_worker=${video_pipeline:-false}
 veda_encode_worker=${video_encode_worker:-false}
+video_pipeline_integration=${video_pipeline:-false}
 
 declare -A deploy
-roles="edxapp forum ecommerce credentials discovery veda_web_frontend veda_pipeline_worker veda_encode_worker notifier xqueue xserver certs demo testcourses"
+roles="edxapp forum ecommerce credentials discovery veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration notifier xqueue xserver certs demo testcourses"
 
 for role in $roles; do
     deploy[$role]=${!role}