Commit 0dcec6ee by muhammad-ammar

refactoring

parent a6ac5443
......@@ -8,6 +8,6 @@
roles:
- role: nginx
nginx_default_sites:
- pipeline
- video_pipeline
- aws
- role: video_pipeline
......@@ -11,167 +11,175 @@
# Defaults for common variables for video_pipeline and video_worker roles
#
video_common_veda_url_root: "http://127.0.0.1:8555"
video_common_veda_static_root: !!null
video_common_veda_django_secret_key: !!null
video_common_veda_mysql_db_name: !!null
video_common_veda_mysql_host: !!null
video_common_veda_mysql_user: !!null
video_common_veda_mysql_password: !!null
video_common_service_name: !!null
# Used to automatically configure OAuth2 Client
SOCIAL_AUTH_EDX_OIDC_KEY: "pipeline-key"
SOCIAL_AUTH_EDX_OIDC_SECRET: "pipeline-secret"
VIDEO_COMMON_SOCIAL_AUTH_EDX_OIDC_KEY: "pipeline-key"
VIDEO_COMMON_SOCIAL_AUTH_EDX_OIDC_SECRET: "pipeline-secret"
# video pipeline config overrides
EDX_CLOUDFRONT_PREFIX: "SET-ME-PLEASE"
EDX_S3_INGEST_PREFIX: "ingest/"
EDX_S3_INGEST_BUCKET: "SET-ME-PLEASE"
EDX_S3_ENDPOINT_BUCKET: "SET-ME-PLEASE"
VEDA_S3_UPLOAD_BUCKET: "SET-ME-PLEASE"
VEDA_UPLOAD_BUCKET: "SET-ME-PLEASE"
VEDA_S3_HOTSTORE_BUCKET: "SET-ME-PLEASE"
VEDA_DELIVERABLE_BUCKET: "SET-ME-PLEASE"
VEDA_URL_ROOT: "SET-ME-PLEASE"
VEDA_BASE_URL: "{{ VEDA_URL_ROOT }}"
VEDA_ACCESS_KEY_ID: "SET-ME-PLEASE"
VEDA_SECRET_ACCESS_KEY: "SET-ME-PLEASE"
AWS_VIDEO_IMAGES_BUCKET: "SET-ME-PLEASE"
AWS_VIDEO_TRANSCRIPTS_BUCKET: "SET-ME-PLEASE"
ADMIN_EMAIL: "veda@example.com"
VEDA_NOREPLY_EMAIL: "veda-noreply@example.com"
CIELO24_API_ENVIRONMENT: "sandbox"
TRANSCRIPT_PROVIDER_REQUEST_TOKEN: "SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING"
VEDA_API_URL: "{{ VEDA_BASE_URL }}/api"
VEDA_OAUTH_URL: "{{ VEDA_BASE_URL }}/veda_auth"
VEDA_OAUTH_TOKEN_URL: "{{ VEDA_BASE_URL }}/o/token"
VEDA_OAUTH_CLIENT_ID: "veda-client-id"
VEDA_OAUTH_CLIENT_SECRET: "veda-client-secret"
VEDA_OAUTH_CLIENT_NAME: "video-pipeline"
VAL_BASE_URL: "http://127.0.0.1:8010"
VAL_API_URL: "{{ VAL_BASE_URL }}/api/val/v0/videos"
VAL_TOKEN_URL: "{{ VAL_BASE_URL }}/oauth2/access_token"
VAL_VIDEO_IMAGES_URL: "{{ VAL_BASE_URL }}/api/val/v0/videos/video-images/update/"
VAL_CLIENT_ID: "{{ SOCIAL_AUTH_EDX_OIDC_KEY }}"
VAL_SECRET_KEY: "{{ SOCIAL_AUTH_EDX_OIDC_SECRET }}"
VAL_USERNAME: "staff"
VAL_PASSWORD: "edx"
VAL_TRANSCRIPT_CREATE_URL: "{{ VAL_BASE_URL }}/api/val/v0/videos/video-transcripts/create/"
VAL_VIDEO_TRANSCRIPT_STATUS_URL: "{{ VAL_BASE_URL }}/api/val/v0/videos/status/"
RABBITMQ_BROKER: "localhost"
RABBITMQ_PASS: "pipeline"
RABBITMQ_USER: "pipeline"
SG_SERVER_PATH: "SET-ME-PLEASE"
SG_SCRIPT_NAME: "SET-ME-PLEASE"
SG_SCRIPT_KEY: "SET-ME-PLEASE"
THREEPLAY_FTPHOST: "SET-ME-PLEASE"
XUETANG_API_URL: "SET-ME-PLEASE"
XUETANG_API_SHARED_SECRET: "SET-ME-PLEASE"
HOST_ENVIRONMENT: "sandbox"
VIDEO_COMMON_EDX_CLOUDFRONT_PREFIX: "SET-ME-PLEASE"
VIDEO_COMMON_EDX_S3_INGEST_PREFIX: "ingest/"
VIDEO_COMMON_EDX_S3_INGEST_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_EDX_S3_ENDPOINT_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_S3_UPLOAD_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_UPLOAD_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_S3_HOTSTORE_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_DELIVERABLE_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_BASE_URL: "{{ video_common_veda_url_root }}"
VIDEO_COMMON_VEDA_ACCESS_KEY_ID: "SET-ME-PLEASE"
VIDEO_COMMON_VEDA_SECRET_ACCESS_KEY: "SET-ME-PLEASE"
VIDEO_COMMON_AWS_VIDEO_IMAGES_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_AWS_VIDEO_TRANSCRIPTS_BUCKET: "SET-ME-PLEASE"
VIDEO_COMMON_ADMIN_EMAIL: "veda@example.com"
VIDEO_COMMON_VEDA_NOREPLY_EMAIL: "veda-noreply@example.com"
VIDEO_COMMON_CIELO24_API_ENVIRONMENT: "sandbox"
VIDEO_COMMON_TRANSCRIPT_PROVIDER_REQUEST_TOKEN: "SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING"
VIDEO_COMMON_VEDA_API_URL: "{{ VIDEO_COMMON_VEDA_BASE_URL }}/api"
VIDEO_COMMON_VEDA_OAUTH_URL: "{{ VIDEO_COMMON_VEDA_BASE_URL }}/veda_auth"
VIDEO_COMMON_VEDA_OAUTH_TOKEN_URL: "{{ VIDEO_COMMON_VEDA_BASE_URL }}/o/token"
VIDEO_COMMON_VEDA_OAUTH_CLIENT_ID: "veda-client-id"
VIDEO_COMMON_VEDA_OAUTH_CLIENT_SECRET: "veda-client-secret"
VIDEO_COMMON_VEDA_OAUTH_CLIENT_NAME: "video-pipeline"
VIDEO_COMMON_VAL_BASE_URL: "http://127.0.0.1:8010"
VIDEO_COMMON_VAL_API_URL: "{{ VIDEO_COMMON_VAL_BASE_URL }}/api/val/v0/videos"
VIDEO_COMMON_VAL_TOKEN_URL: "{{ VIDEO_COMMON_VAL_BASE_URL }}/oauth2/access_token"
VIDEO_COMMON_VAL_VIDEO_IMAGES_URL: "{{ VIDEO_COMMON_VAL_BASE_URL }}/api/val/v0/videos/video-images/update/"
VIDEO_COMMON_VAL_CLIENT_ID: "{{ VIDEO_COMMON_SOCIAL_AUTH_EDX_OIDC_KEY }}"
VIDEO_COMMON_VAL_SECRET_KEY: "{{ VIDEO_COMMON_SOCIAL_AUTH_EDX_OIDC_SECRET }}"
VIDEO_COMMON_VAL_USERNAME: "staff"
VIDEO_COMMON_VAL_PASSWORD: "edx"
VIDEO_COMMON_VAL_TRANSCRIPT_CREATE_URL: "{{ VIDEO_COMMON_VAL_BASE_URL }}/api/val/v0/videos/video-transcripts/create/"
VIDEO_COMMON_VAL_VIDEO_TRANSCRIPT_STATUS_URL: "{{ VIDEO_COMMON_VAL_BASE_URL }}/api/val/v0/videos/status/"
VIDEO_COMMON_RABBITMQ_BROKER: "localhost"
VIDEO_COMMON_RABBITMQ_PASS: "pipeline"
VIDEO_COMMON_RABBITMQ_USER: "pipeline"
VIDEO_COMMON_SG_SERVER_PATH: "SET-ME-PLEASE"
VIDEO_COMMON_SG_SCRIPT_NAME: "SET-ME-PLEASE"
VIDEO_COMMON_SG_SCRIPT_KEY: "SET-ME-PLEASE"
VIDEO_COMMON_THREEPLAY_FTPHOST: "SET-ME-PLEASE"
VIDEO_COMMON_XUETANG_API_URL: "SET-ME-PLEASE"
VIDEO_COMMON_XUETANG_API_SHARED_SECRET: "SET-ME-PLEASE"
VIDEO_COMMON_HOST_ENVIRONMENT: "sandbox"
# video pipeline config
VEDA_COMMON_CONFIG:
VIDEO_COMMON_CONFIG:
DATABASES:
default:
ENGINE: 'django.db.backends.mysql'
NAME: '{{ VEDA_MYSQL_DB_NAME }}'
USER: '{{ VEDA_MYSQL_USER }}'
PASSWORD: '{{ VEDA_MYSQL_PASSWORD }}'
HOST: '{{ VEDA_MYSQL_HOST }}'
NAME: "{{ video_common_veda_mysql_db_name }}"
USER: "{{ video_common_veda_mysql_user }}"
PASSWORD: "{{ video_common_veda_mysql_password }}"
HOST: "{{ video_common_veda_mysql_host }}"
PORT: 3306
django_secret_key: '{{ VEDA_DJANGO_SECRET_KEY }}'
django_secret_key: "{{ video_common_veda_django_secret_key }}"
# Django DEBUG global
debug: false
# django staticfile settings
STATIC_ROOT_PATH: '{{ VEDA_STATIC_ROOT }}'
STATIC_ROOT_PATH: "{{ video_common_veda_static_root }}"
# ---
# AWS Buckets, Prefixes
# ---
# Studio/Platform
edx_s3_ingest_prefix: '{{ EDX_S3_INGEST_PREFIX }}'
edx_s3_ingest_bucket: '{{ EDX_S3_INGEST_BUCKET }}'
edx_s3_endpoint_bucket: '{{ EDX_S3_ENDPOINT_BUCKET }}'
edx_s3_ingest_prefix: "{{ VIDEO_COMMON_EDX_S3_INGEST_PREFIX }}"
edx_s3_ingest_bucket: "{{ VIDEO_COMMON_EDX_S3_INGEST_BUCKET }}"
edx_s3_endpoint_bucket: "{{ VIDEO_COMMON_EDX_S3_ENDPOINT_BUCKET }}"
# CF
edx_cloudfront_prefix: '{{ EDX_CLOUDFRONT_PREFIX }}'
edx_cloudfront_prefix: "{{ VIDEO_COMMON_EDX_CLOUDFRONT_PREFIX }}"
# Images
aws_video_images_bucket: '{{ AWS_VIDEO_IMAGES_BUCKET }}'
aws_video_images_bucket: "{{ VIDEO_COMMON_AWS_VIDEO_IMAGES_BUCKET }}"
aws_video_images_prefix: "video-images/"
# VEDA Internal
veda_s3_upload_bucket: '{{ VEDA_S3_UPLOAD_BUCKET }}'
veda_s3_hotstore_bucket: '{{ VEDA_S3_HOTSTORE_BUCKET }}'
veda_deliverable_bucket: '{{ VEDA_DELIVERABLE_BUCKET }}'
veda_upload_bucket: '{{ VEDA_UPLOAD_BUCKET }}'
veda_s3_upload_bucket: "{{ VIDEO_COMMON_VEDA_S3_UPLOAD_BUCKET }}"
veda_s3_hotstore_bucket: "{{ VIDEO_COMMON_VEDA_S3_HOTSTORE_BUCKET }}"
veda_deliverable_bucket: "{{ VIDEO_COMMON_VEDA_DELIVERABLE_BUCKET }}"
veda_upload_bucket: "{{ VIDEO_COMMON_VEDA_UPLOAD_BUCKET }}"
# Settings
multi_upload_barrier: 2000000000
veda_base_url: '{{ VEDA_BASE_URL }}'
veda_base_url: "{{ VIDEO_COMMON_VEDA_BASE_URL }}"
s3_base_url: https://s3.amazonaws.com
# Transcripts
aws_video_transcripts_bucket: '{{ AWS_VIDEO_TRANSCRIPTS_BUCKET }}'
aws_video_transcripts_bucket: "{{ VIDEO_COMMON_AWS_VIDEO_TRANSCRIPTS_BUCKET }}"
aws_video_transcripts_prefix: video-transcripts/
# cielo24 api urls
cielo24_api_base_url: 'https://{{ CIELO24_API_ENVIRONMENT }}.cielo24.com/api'
cielo24_api_base_url: 'https://{{ VIDEO_COMMON_CIELO24_API_ENVIRONMENT }}.cielo24.com/api'
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
# a token identifying a valid request from transcript provider
transcript_provider_request_token: '{{ TRANSCRIPT_PROVIDER_REQUEST_TOKEN }}'
transcript_provider_request_token: "{{ VIDEO_COMMON_TRANSCRIPT_PROVIDER_REQUEST_TOKEN }}"
# Ingest Secret
veda_secret_access_key: '{{ VEDA_SECRET_ACCESS_KEY }}'
veda_access_key_id: '{{ VEDA_ACCESS_KEY_ID }}'
veda_secret_access_key: "{{ VIDEO_COMMON_VEDA_SECRET_ACCESS_KEY }}"
veda_access_key_id: "{{ VIDEO_COMMON_VEDA_ACCESS_KEY_ID }}"
# ---
# email vars
# ---
veda_noreply_email: '{{ VEDA_NOREPLY_EMAIL }}'
admin_email: '{{ ADMIN_EMAIL }}'
veda_noreply_email: "{{ VIDEO_COMMON_VEDA_NOREPLY_EMAIL }}"
admin_email: "{{ VIDEO_COMMON_ADMIN_EMAIL }}"
# ---
# VEDA API
# ---
## VEDA API Auth
veda_api_url: '{{ VEDA_API_URL }}'
veda_auth_url: '{{ VEDA_OAUTH_URL }}'
veda_client_id: '{{ VEDA_OAUTH_CLIENT_ID }}'
veda_secret_key: '{{ VEDA_OAUTH_CLIENT_SECRET }}'
veda_token_url: '{{ VEDA_OAUTH_TOKEN_URL }}'
veda_api_url: "{{ VIDEO_COMMON_VEDA_API_URL }}"
veda_auth_url: "{{ VIDEO_COMMON_VEDA_OAUTH_URL }}"
veda_client_id: "{{ VIDEO_COMMON_VEDA_OAUTH_CLIENT_ID }}"
veda_secret_key: "{{ VIDEO_COMMON_VEDA_OAUTH_CLIENT_SECRET }}"
veda_token_url: "{{ VIDEO_COMMON_VEDA_OAUTH_TOKEN_URL }}"
# ---
# VAL
# ---
val_api_url: '{{ VAL_API_URL }}'
val_token_url: '{{ VAL_TOKEN_URL }}'
val_video_images_url: '{{ VAL_VIDEO_IMAGES_URL }}'
val_api_url: "{{ VIDEO_COMMON_VAL_API_URL }}"
val_token_url: "{{ VIDEO_COMMON_VAL_TOKEN_URL }}"
val_video_images_url: "{{ VIDEO_COMMON_VAL_VIDEO_IMAGES_URL }}"
# Credentials
val_client_id: '{{ VAL_CLIENT_ID }}'
val_secret_key: '{{ VAL_SECRET_KEY }}'
val_password: '{{ VAL_PASSWORD }}'
val_username: '{{ VAL_USERNAME }}'
val_transcript_create_url: '{{ VAL_TRANSCRIPT_CREATE_URL }}'
val_video_transcript_status_url: '{{ VAL_VIDEO_TRANSCRIPT_STATUS_URL }}'
val_client_id: "{{ VIDEO_COMMON_VAL_CLIENT_ID }}"
val_secret_key: "{{ VIDEO_COMMON_VAL_SECRET_KEY }}"
val_password: "{{ VIDEO_COMMON_VAL_PASSWORD }}"
val_username: "{{ VIDEO_COMMON_VAL_USERNAME }}"
val_transcript_create_url: "{{ VIDEO_COMMON_VAL_TRANSCRIPT_CREATE_URL }}"
val_video_transcript_status_url: "{{ VIDEO_COMMON_VAL_VIDEO_TRANSCRIPT_STATUS_URL }}"
# ---
# Celery Info
# ---
celery_app_name: 'veda_{{ HOST_ENVIRONMENT }}'
celery_app_name: "veda_{{ VIDEO_COMMON_HOST_ENVIRONMENT }}"
# can do multiple queues like so: foo,bar,baz
main_celery_queue: encode_worker
celery_receiver_queue: encode_worker
......@@ -180,23 +188,23 @@ VEDA_COMMON_CONFIG:
largefile_queue_barrier: 1000000000
celery_threads: 1
rabbitmq_broker: '{{ RABBITMQ_BROKER }}'
rabbitmq_pass: '{{ RABBITMQ_PASS }}'
rabbitmq_user: '{{ RABBITMQ_USER }}'
rabbitmq_broker: "{{ VIDEO_COMMON_RABBITMQ_BROKER }}"
rabbitmq_pass: "{{ VIDEO_COMMON_RABBITMQ_PASS }}"
rabbitmq_user: "{{ VIDEO_COMMON_RABBITMQ_USER }}"
# ---
# Shotgun Variables (internal mediateam)
# ---
sg_server_path: '{{ SG_SERVER_PATH }}'
sg_script_name: '{{ SG_SCRIPT_NAME }}'
sg_script_key: '{{ SG_SCRIPT_KEY }}'
sg_server_path: "{{ VIDEO_COMMON_SG_SERVER_PATH }}"
sg_script_name: "{{ VIDEO_COMMON_SG_SCRIPT_NAME }}"
sg_script_key: "{{ VIDEO_COMMON_SG_SCRIPT_KEY }}"
# ---
# Endpoints
# ---
threeplay_ftphost: '{{ THREEPLAY_FTPHOST }}'
xuetang_api_url: '{{ XUETANG_API_URL }}'
xuetang_api_shared_secret: '{{ XUETANG_API_SHARED_SECRET }}'
threeplay_ftphost: "{{ VIDEO_COMMON_THREEPLAY_FTPHOST }}"
xuetang_api_url: "{{ VIDEO_COMMON_XUETANG_API_URL }}"
xuetang_api_shared_secret: "{{ VIDEO_COMMON_XUETANG_API_SHARED_SECRET }}"
## Encoding Config
ffmpeg_compiled: "ffmpeg"
......
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS
# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Tasks for role video_common
#
#
- name: Write out app config file
template:
src: "config.yml.j2"
dest: "{{ COMMON_CFG_DIR }}/{{ video_common_service_name }}.yml"
mode: "0644"
---
# {{ ansible_managed }}
{{ VIDEO_COMMON_CONFIG | to_nice_yaml }}
......@@ -36,6 +36,7 @@ video_pipeline_deliver_worker: "{{ video_pipeline_service_name }}_deliver"
video_pipeline_debian_pkgs:
- libffi6
- libffi-dev
- ffmpeg
VIDEO_PIPELINE_HOST_ENVIRONMENT: 'sandbox'
......@@ -80,9 +81,6 @@ VIDEO_PIPELINE_STATICFILES_STORAGE: "django.contrib.staticfiles.storage.StaticFi
VIDEO_PIPELINE_HOSTNAME: '~^((stage|prod)-)?pipeline.*'
NGINX_VIDEO_PIPELINE_GUNICORN_HOSTS:
- 127.0.0.1
VIDEO_PIPELINE_GUNICORN_EXTRA: ""
VIDEO_PIPELINE_EXTRA_APPS: []
......
......@@ -19,14 +19,6 @@
# my_role_var1: 'bar'
# }
dependencies:
- role: video_common_vars
VEDA_STATIC_ROOT: '{{ VIDEO_PIPELINE_STATIC_ROOT }}'
VEDA_DJANGO_SECRET_KEY: '{{ VIDEO_PIPELINE_SECRET_KEY }}'
VEDA_MYSQL_DB_NAME: '{{ VIDEO_PIPELINE_DEFAULT_DB_NAME }}'
VEDA_MYSQL_HOST: '{{ VIDEO_PIPELINE_MYSQL_HOST }}'
VEDA_MYSQL_USER: '{{ VIDEO_PIPELINE_MYSQL_USER }}'
VEDA_MYSQL_PASSWORD: '{{ VIDEO_PIPELINE_MYSQL_PASSWORD }}'
VEDA_URL_ROOT: '{{ VIDEO_PIPELINE_URL_ROOT }}'
- role: edx_django_service
edx_django_service_repo: 'edx-video-pipeline'
edx_django_service_version: '{{ VIDEO_PIPELINE_VERSION }}'
......@@ -57,5 +49,13 @@ dependencies:
edx_django_service_extra_apps: '{{ VIDEO_PIPELINE_EXTRA_APPS }}'
edx_django_service_automated_users: '{{ VIDEO_PIPELINE_AUTOMATED_USERS }}'
edx_django_service_use_python3: false
nginx_edx_django_service_gunicorn_hosts: '{{ NGINX_VIDEO_PIPELINE_GUNICORN_HOSTS }}'
edx_django_service_config: '{{ VEDA_COMMON_CONFIG }}'
edx_django_service_config: {}
- role: video_common_vars
video_common_service_name: '{{ video_pipeline_service_name }}'
video_common_veda_url_root: '{{ VIDEO_PIPELINE_URL_ROOT }}'
video_common_veda_static_root: '{{ VIDEO_PIPELINE_STATIC_ROOT }}'
video_common_veda_django_secret_key: '{{ VIDEO_PIPELINE_SECRET_KEY }}'
video_common_veda_mysql_db_name: '{{ VIDEO_PIPELINE_DEFAULT_DB_NAME }}'
video_common_veda_mysql_host: '{{ VIDEO_PIPELINE_MYSQL_HOST }}'
video_common_veda_mysql_user: '{{ VIDEO_PIPELINE_MYSQL_USER }}'
video_common_veda_mysql_password: '{{ VIDEO_PIPELINE_MYSQL_PASSWORD }}'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment