video pipeline role

ansible_dev.cfg

+[defaults]
+jinja2_extensions = jinja2.ext.do
+private_key_file = /Users/mammar/.vagrant.d/boxes/ubuntu_xenial/0/virtualbox/vagrant_private_key
+remote_port = 2222
+remote_user = vagrant

pipeline-vagrantfile

+Vagrant.require_version ">= 1.8.7"
+unless Vagrant.has_plugin?("vagrant-vbguest")
+  raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
+end
+
+VAGRANTFILE_API_VERSION = "2"
+
+MEMORY = 1024
+CPU_COUNT = 2
+
+vm_guest_ip = "192.168.33.10"
+
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  # Creates a devstack from a base Ubuntu 16.04 image for virtualbox
+  config.vm.box = "ubuntu_xenial"
+
+  config.vm.network :private_network, ip: vm_guest_ip, nic_type: "virtio"
+
+  config.vm.network "private_network", ip: "192.168.33.10"
+  config.vm.network :forwarded_port, guest: 18381, host: 18381    # Course discovery
+
+  config.ssh.insert_key = true
+
+ # Enable X11 forwarding so we can interact with GUI applications
+  config.ssh.forward_x11 = true
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
+    vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
+    vb.customize ['modifyvm', :id, '--nictype1', 'virtio']
+  end
+
+  # Make LC_ALL default to en_US.UTF-8 instead of en_US.
+  # See: https://github.com/mitchellh/vagrant/issues/1188
+  config.vm.provision "shell", inline: 'echo \'LC_ALL="en_US.UTF-8"\' > /etc/default/locale'
+
+#   # Get ready for ansible on this box.
+#   config.vm.provision "shell", path: '../../../util/install/ansible-bootstrap.sh'
+
+  # Use vagrant-vbguest plugin to make sure Guest Additions are in sync
+  config.vbguest.auto_reboot = true
+  config.vbguest.auto_update = true
+
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "/Users/mammar/Documents/edx/devstack/src/configuration/playbooks/edx-east/video_pipeline.yml"
+    # ansible.verbose = "vvv"
+    ansible.extra_vars = {
+        migrate_db: 'yes',
+        disable_edx_services: 'false',
+        edx_django_service_has_static_assets: 'yes',
+        edx_django_service_version: 'ammar/changes-for-config',
+        edx_django_service_use_python3: 'false'
+    }
+  end
+end

playbooks/edx-east/video_pipeline.yml

+- name: Deploy edX Video Pipeline Service
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_NEWRELIC: False
+  roles:
+    - role: nginx
+      nginx_default_sites:
+        - video_pipeline
+    - video_pipeline

playbooks/roles/video_pipeline/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx-video-pipeline
+#
+
+
+#
+# vars are namespace with the module name.
+#
+video_pipeline_service_name: "video_pipeline"
+video_pipeline_gunicorn_port: 8555
+
+video_pipeline_user: "{{ video_pipeline_service_name }}"
+video_pipeline_home: "{{ COMMON_APP_DIR }}/{{ video_pipeline_service_name }}"
+
+# edx-video-pipeline uses Python2.7
+edx_django_service_use_python3: false
+
+#
+# OS packages
+#
+
+video_pipeline_debian_pkgs:
+  - libffi6
+  - libffi-dev
+
+VIDEO_PIPELINE_NGINX_PORT: "1{{ video_pipeline_gunicorn_port }}"
+VIDEO_PIPELINE_SSL_NGINX_PORT: "4{{ video_pipeline_gunicorn_port }}"
+
+VIDEO_PIPELINE_DEFAULT_DB_NAME: 'pipeline'
+VIDEO_PIPELINE_MYSQL: 'localhost'
+# MySQL usernames are limited to 16 characters
+VIDEO_PIPELINE_MYSQL_USER: 'pipeline001'
+VIDEO_PIPELINE_MYSQL_PASSWORD: 'password'
+VIDEO_PIPELINE_MYSQL_PORT: 'password'
+
+VIDEO_PIPELINE_MEMCACHE: [ 'memcache' ]
+
+VIDEO_PIPELINE_VERSION: "master"
+VIDEO_PIPELINE_DJANGO_SETTINGS_MODULE: "VEDA.settings"
+VIDEO_PIPELINE_URL_ROOT: 'http://pipeline:{{ VIDEO_PIPELINE_NGINX_PORT }}'
+VIDEO_PIPELINE_LOGOUT_URL: '{{ VIDEO_PIPELINE_URL_ROOT }}/logout/'
+
+VIDEO_PIPELINE_SECRET_KEY: 'Your secret key here'
+VIDEO_PIPELINE_LANGUAGE_CODE: 'en-us'
+
+# Used to automatically configure OAuth2 Client
+VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_KEY : 'pipeline-key'
+VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_SECRET : 'pipeline-secret'
+VIDEO_PIPELINE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+
+VIDEO_PIPELINE_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ video_pipeline_service_name }}'
+VIDEO_PIPELINE_MEDIA_ROOT: '{{ VIDEO_PIPELINE_DATA_DIR }}/media'
+VIDEO_PIPELINE_MEDIA_URL: '/media/'
+
+VIDEO_PIPELINE_MEDIA_STORAGE_BACKEND:
+  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
+  MEDIA_ROOT: '{{ VIDEO_PIPELINE_MEDIA_ROOT }}'
+  MEDIA_URL: '{{ VIDEO_PIPELINE_MEDIA_URL }}'
+
+VIDEO_PIPELINE_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage'
+
+VIDEO_PIPELINE_GUNICORN_EXTRA: ''
+
+VIDEO_PIPELINE_EXTRA_APPS: []
+
+# See edx_django_service_automated_users for an example of what this should be
+VIDEO_PIPELINE_AUTOMATED_USERS: {}
+
+# instance_config.yaml variable overrides
+
+VIDEO_PIPELINE_EDX_S3_INGEST_PREFIX: 'video_pipeline_edx_s3_ingest_prefix'
+VIDEO_PIPELINE_EDX_S3_INGEST_BUCKET: 'video_pipeline_edx_s3_ingest_bucket'
+VIDEO_PIPELINE_EDX_S3_ENDPOINT_BUCKET: 'video_pipeline_edx_s3_endpoint_bucket'
+VIDEO_PIPELINE_EDX_CLOUDFRONT_PREFIX: 'video_pipeline_edx_cloudfront_prefix'
+VIDEO_PIPELINE_AWS_VIDEO_IMAGES_BUCKET: 'video_pipeline_aws_video_images_bucket'
+VIDEO_PIPELINE_VEDA_S3_UPLOAD_BUCKET: 'video_pipeline_veda_s3_upload_bucket'
+VIDEO_PIPELINE_VEDA_S3_HOTSTORE_BUCKET: 'video_pipeline_veda_s3_hotstore_bucket'
+VIDEO_PIPELINE_VEDA_DELIVERABLE_BUCKET: 'video_pipeline_veda_deliverable_bucket'
+VIDEO_PIPELINE_VEDA_BASE_URL: 'video_pipeline_veda_base_url'
+VIDEO_PIPELINE_AWS_VIDEO_TRANSCRIPTS_BUCKET: 'video_pipeline_aws_video_transcripts_bucket'
+VIDEO_PIPELINE_CIELO24_API_ENVIRONMENT: 'video_pipeline_cielo24_api_environment'
+VIDEO_PIPELINE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN: 'video_pipeline_transcript_provider_request_token'
+VIDEO_PIPELINE_VEDA_SECRET_ACCESS_KEY: 'video_pipeline_veda_secret_access_key'
+VIDEO_PIPELINE_VEDA_ACCESS_KEY_ID: 'video_pipeline_veda_access_key_id'
+VIDEO_PIPELINE_VEDA_NOREPLY_EMAIL: 'video_pipeline_veda_noreply_email'
+VIDEO_PIPELINE_ADMIN_EMAIL: 'video_pipeline_admin_email'
+VIDEO_PIPELINE_VEDA_API_URL: 'video_pipeline_veda_api_url'
+VIDEO_PIPELINE_VEDA_AUTH_URL: 'video_pipeline_veda_auth_url'
+VIDEO_PIPELINE_VEDA_CLIENT_ID: 'video_pipeline_veda_client_id'
+VIDEO_PIPELINE_VEDA_SECRET_KEY: 'video_pipeline_veda_secret_key'
+VIDEO_PIPELINE_VEDA_TOKEN_URL: 'video_pipeline_veda_token_url'
+VIDEO_PIPELINE_VAL_API_URL: 'video_pipeline_val_api_url'
+VIDEO_PIPELINE_VAL_TOKEN_URL: 'video_pipeline_val_token_url'
+VIDEO_PIPELINE_VAL_VIDEO_IMAGES_URL: 'video_pipeline_val_video_images_url'
+VIDEO_PIPELINE_VAL_CLIENT_ID: 'video_pipeline_val_client_id'
+VIDEO_PIPELINE_VAL_SECRET_KEY: 'video_pipeline_val_secret_key'
+VIDEO_PIPELINE_VAL_PASSWORD: 'video_pipeline_val_password'
+VIDEO_PIPELINE_VAL_USERNAME: 'video_pipeline_val_username'
+VIDEO_PIPELINE_VAL_TRANSCRIPT_CREATE_URL: 'video_pipeline_val_transcript_create_url'
+VIDEO_PIPELINE_VAL_VIDEO_TRANSCRIPT_STATUS_URL: 'video_pipeline_val_video_transcript_status_url'
+VIDEO_PIPELINE_RABBITMQ_BROKER: 'video_pipeline_rabbitmq_broker'
+VIDEO_PIPELINE_RABBITMQ_PASS: 'video_pipeline_rabbitmq_pass'
+VIDEO_PIPELINE_RABBITMQ_USER: 'video_pipeline_rabbitmq_user'
+VIDEO_PIPELINE_SG_SERVER_PATH: 'video_pipeline_sg_server_path'
+VIDEO_PIPELINE_SG_SCRIPT_NAME: 'video_pipeline_sg_script_name'
+VIDEO_PIPELINE_SG_SCRIPT_KEY: 'video_pipeline_sg_script_key'
+VIDEO_PIPELINE_THREEPLAY_FTPHOST: 'video_pipeline_threeplay_ftphost'
+VIDEO_PIPELINE_XUETANG_API_URL: 'video_pipeline_xuetang_api_url'
+VIDEO_PIPELINE_XUETANG_API_SHARED_SECRET: 'video_pipeline_xuetang_api_shared_secret'

playbooks/roles/video_pipeline/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role edx-video-pipeline
+#
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role
+#   my_role_var0: 'foo'
+#   my_role_var1: 'bar'
+# }
+dependencies:
+  - role: edx_django_service
+    edx_django_service_repo: 'edx-video-pipeline'
+    edx_django_service_version: '{{ VIDEO_PIPELINE_VERSION }}'
+    edx_django_service_name: '{{ video_pipeline_service_name }}'
+    edx_django_service_user: '{{ video_pipeline_user }}'
+    edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ video_pipeline_service_name }}'
+    edx_django_service_debian_pkgs_extra: '{{ video_pipeline_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ video_pipeline_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ VIDEO_PIPELINE_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ video_pipeline_environment }}'
+    edx_django_service_gunicorn_extra: '{{ VIDEO_PIPELINE_GUNICORN_EXTRA }}'
+    edx_django_service_wsgi_name: 'edx-video-pipeline.VEDA'
+    edx_django_service_nginx_port: '{{ VIDEO_PIPELINE_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ VIDEO_PIPELINE_SSL_NGINX_PORT }}'
+    edx_django_service_language_code: '{{ VIDEO_PIPELINE_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ VIDEO_PIPELINE_SECRET_KEY }}'
+    edx_django_service_staticfiles_storage: '{{ VIDEO_PIPELINE_STATICFILES_STORAGE }}'
+    edx_django_service_media_storage_backend: '{{ VIDEO_PIPELINE_MEDIA_STORAGE_BACKEND }}'
+    edx_django_service_memcache: '{{ VIDEO_PIPELINE_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ VIDEO_PIPELINE_MYSQL }}'
+    edx_django_service_default_db_name: '{{ VIDEO_PIPELINE_DEFAULT_DB_NAME }}'
+    edx_django_service_default_db_atomic_requests: false
+    edx_django_service_db_user: '{{ VIDEO_PIPELINE_MYSQL_USER }}'
+    edx_django_service_db_password: '{{ VIDEO_PIPELINE_MYSQL_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ VIDEO_PIPELINE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_extra_apps: '{{ VIDEO_PIPELINE_EXTRA_APPS }}'
+    edx_django_service_automated_users: '{{ VIDEO_PIPELINE_AUTOMATED_USERS }}'

playbooks/roles/video_pipeline/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role edx-video-pipeline
+#
+# Overview: This role's tasks come from edx_django_service.
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: Create video pipeline configuration
+  template:
+    src: "templates/instance_config.yaml.j2"
+    dest: "{{ edx_django_service_code_dir }}/instance_config.yaml"

playbooks/roles/video_pipeline/templates/instance_config.yaml.j2

+---
+# ---
+# Database information
+# ---
+
+DATABASES:
+    default:
+        ENGINE: 'django.db.backends.mysql'
+        NAME: '{{ VIDEO_PIPELINE_DEFAULT_DB_NAME }}'
+        USER: '{{ VIDEO_PIPELINE_MYSQL_USER }}'
+        PASSWORD: '{{ VIDEO_PIPELINE_MYSQL_PASSWORD }}'
+        HOST: '{{ VIDEO_PIPELINE_MYSQL }}'
+        PORT: '3306'
+
+django_secret_key: '{{ VIDEO_PIPELINE_SECRET_KEY }}'
+
+# Django DEBUG global
+# (set to false in prod)
+debug: false
+
+# ---
+# AWS Buckets, Prefixes
+# ---
+# Studio/Platform
+edx_s3_ingest_prefix: '{{ VIDEO_PIPELINE_EDX_S3_INGEST_PREFIX }}'
+edx_s3_ingest_bucket: '{{ VIDEO_PIPELINE_EDX_S3_INGEST_BUCKET }}'
+edx_s3_endpoint_bucket: '{{ VIDEO_PIPELINE_EDX_S3_ENDPOINT_BUCKET }}'
+
+# CF
+edx_cloudfront_prefix: '{{ VIDEO_PIPELINE_EDX_CLOUDFRONT_PREFIX }}'
+
+# Images
+aws_video_images_bucket: '{{ VIDEO_PIPELINE_AWS_VIDEO_IMAGES_BUCKET }}'
+aws_video_images_prefix: "video-images/"
+
+
+# VEDA Internal
+veda_s3_upload_bucket: '{{ VIDEO_PIPELINE_VEDA_S3_UPLOAD_BUCKET }}'
+veda_s3_hotstore_bucket: '{{ VIDEO_PIPELINE_VEDA_S3_HOTSTORE_BUCKET }}'
+veda_deliverable_bucket: '{{ VIDEO_PIPELINE_VEDA_DELIVERABLE_BUCKET }}'
+# Settings
+multi_upload_barrier: 2000000000
+veda_base_url: '{{ VIDEO_PIPELINE_VEDA_BASE_URL }}'
+s3_base_url: https://s3.amazonaws.com
+
+
+# Transcripts
+aws_video_transcripts_bucket: '{{ VIDEO_PIPELINE_AWS_VIDEO_TRANSCRIPTS_BUCKET }}'
+aws_video_transcripts_prefix: video-transcripts/
+
+# cielo24 api urls
+cielo24_api_base_url: 'https://{{ VIDEO_PIPELINE_CIELO24_API_ENVIRONMENT }}.cielo24.com/api'
+
+# 3playmedia api urls
+three_play_api_base_url: https://api.3playmedia.com/
+three_play_api_transcript_url: https://static.3playmedia.com/
+
+# a token identifying a valid request from transcript provider
+transcript_provider_request_token: '{{ VIDEO_PIPELINE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN }}'
+
+# Ingest Secret
+# TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.
+veda_secret_access_key: '{{ VIDEO_PIPELINE_VEDA_SECRET_ACCESS_KEY }}'
+veda_access_key_id: '{{ VIDEO_PIPELINE_VEDA_ACCESS_KEY_ID }}'
+
+# ---
+# email vars
+# ---
+veda_noreply_email: '{{ VIDEO_PIPELINE_VEDA_NOREPLY_EMAIL }}'
+admin_email: '{{ VIDEO_PIPELINE_ADMIN_EMAIL }}'
+
+# ---
+# VEDA API
+# ---
+## VEDA API Auth
+veda_api_url: '{{ VIDEO_PIPELINE_VEDA_API_URL }}'
+veda_auth_url: '{{ VIDEO_PIPELINE_VEDA_AUTH_URL }}'
+veda_client_id: '{{ VIDEO_PIPELINE_VEDA_CLIENT_ID }}'
+veda_secret_key: '{{ VIDEO_PIPELINE_VEDA_SECRET_KEY }}'
+veda_token_url: '{{ VIDEO_PIPELINE_VEDA_TOKEN_URL }}'
+
+# ---
+# VAL
+# ---
+val_api_url: '{{ VIDEO_PIPELINE_VAL_API_URL }}'
+val_token_url: '{{ VIDEO_PIPELINE_VAL_TOKEN_URL }}'
+val_video_images_url: '{{ VIDEO_PIPELINE_VAL_VIDEO_IMAGES_URL }}'
+# Credentials
+val_client_id: '{{ VIDEO_PIPELINE_VAL_CLIENT_ID }}'
+val_secret_key: '{{ VIDEO_PIPELINE_VAL_SECRET_KEY }}'
+val_password: '{{ VIDEO_PIPELINE_VAL_PASSWORD }}'
+val_username: '{{ VIDEO_PIPELINE_VAL_USERNAME }}'
+val_transcript_create_url: '{{ VIDEO_PIPELINE_VAL_TRANSCRIPT_CREATE_URL }}'
+val_video_transcript_status_url: '{{ VIDEO_PIPELINE_VAL_VIDEO_TRANSCRIPT_STATUS_URL }}'
+
+# ---
+# Celery Info
+# ---
+celery_app_name: veda_production
+# can do multiple queues like so: foo,bar,baz
+main_celery_queue: encode_worker
+celery_receiver_queue: encode_worker
+largefile_celery_queue: large_encode_worker
+celery_stat_queue: transcode_stat
+largefile_queue_barrier: 1000000000
+celery_threads: 1
+
+rabbitmq_broker: '{{ VIDEO_PIPELINE_RABBITMQ_BROKER }}'
+rabbitmq_pass: '{{ VIDEO_PIPELINE_RABBITMQ_PASS }}'
+rabbitmq_user: '{{ VIDEO_PIPELINE_RABBITMQ_USER }}'
+
+# ---
+# Shotgun Variables (internal mediateam)
+# ---
+sg_server_path: '{{ VIDEO_PIPELINE_SG_SERVER_PATH }}'
+sg_script_name: '{{ VIDEO_PIPELINE_SG_SCRIPT_NAME }}'
+sg_script_key: '{{ VIDEO_PIPELINE_SG_SCRIPT_KEY }}'
+
+# ---
+# Endpoints
+# ---
+threeplay_ftphost: '{{ VIDEO_PIPELINE_THREEPLAY_FTPHOST }}'
+xuetang_api_url: '{{ VIDEO_PIPELINE_XUETANG_API_URL }}'
+xuetang_api_shared_secret: '{{ VIDEO_PIPELINE_XUETANG_API_SHARED_SECRET }}'
+
+## Encoding Config
+ffmpeg_compiled: "ffmpeg"
+ffprobe_compiled: "ffprobe"
+target_aspect_ratio: 1.7777778
+
+# ----------
+##---
+# This is a list of encodes and their respective course
+# boolean matches
+encode_dict:
+    review_proc:
+        - review
+    mobile_override:
+        - override
+    s3_proc:
+        - mobile_high
+        - mobile_low
+        - audio_mp3
+        - desktop_webm
+        - desktop_mp4
+        - hls
+
+    yt_proc:
+        - youtube
+
+##---
+# This is a list of encode profiles and their val profile matches
+# boolean matches
+val_profile_dict:
+    mobile_low:
+        - mobile_low
+
+    desktop_mp4:
+        - desktop_mp4
+
+    override:
+        - desktop_mp4
+        - mobile_low
+        - mobile_high
+
+    mobile_high:
+        - mobile_high
+
+    audio_mp3:
+        - audio_mp3
+
+    desktop_webm:
+        - desktop_webm
+
+    youtube:
+        - youtube
+    review:
+    hls:
+        - hls
+
+#--
+# Heal settings
+heal_start: 1
+heal_end: 144
+
+...