root config

.gitignore

+
+.idea/
+
+*.pyc
+
+static/admin/

.pep8

+[pep8]
+max-line-length = 120
+ignore = E402
+exclude = dependencies, watchdog.py, veda_deliver_xuetang.py, scripts

.travis.yml

+language: python
+python:
+  - "2.7"
+sudo: required
+
+install: pip install -r requirements.txt
+install: pip install pep8
+
+# build tests
+script:
+ - python VEDA/tests/test_build.py
+ - pep8
+ - cd control/tests nosetests
+

README.rst

+===========
+edx-video-pipeline
+===========
+
+Video encode automation django app/control node for edx-platform
+----------------------------------------------------------------
+
+**The video pipeline performs the following tasks**
+- Ingest (Discovery, Cataloging, Sending tasks to worker cluster)
+- Delivery
+- Storage
+- Maintenance
+
+The video pipeline seeks modularity between parts, and for each part to operate as cleanly and independently as possible.
+Each course's workflow operates independently, and workflows can be configured to serve a variety of endpoints.
+
+INGEST:
+Currently we ingest remote video from edx-platform via the Studio video upload tool. The videos are discovered by the video pipeline and ingested upon succcessful upload, renamed to an internal ID schema, and routed to the appropriate transcode task cluster.
+
+TRANSCODE:
+code for this is housed at https://github.com/edx/edx-video-worker
+
+DELIVERY:
+Uploads product videos to specific third-party destinations (YT, AWS, 3Play, cielo24), retrieves URLs/Statuses/products.
+
+STORAGE:
+A specified AWS S3 bucket=
+
+MAINTENANCE:
+Logging, Data dumping, Celery node status and queue information
+
+
+.. image:: https://travis-ci.org/edx/edx-video-pipeline.svg?branch=master
+    :target: https://travis-ci.org/edx/edx-video-pipeline

instance_config.yaml

+---
+# ---
+# Database information
+# ---
+# SANDBOX
+DATABASES:
+    default:
+        ENGINE: django.db.backends.sqlite3
+        NAME: sandbox.db
+
+
+## PRODUCTION
+#DATABASES:
+#    default:
+#        ENGINE: 'django.db.backends.mysql'
+#        NAME:
+#        USER:
+#        PASSWORD:
+#        HOST:
+#        PORT: '3306'
+
+django_secret_key:
+
+# ---
+# Authentication keys, VEDA AWS Account
+# ---
+veda_s3_upload_bucket:
+veda_s3_hotstore_bucket:
+veda_deliverable_bucket:
+veda_access_key_id:
+veda_secret_access_key:
+multi_upload_barrier: 2000000000
+
+# ---
+# edX AWS Account
+# ---
+edx_aws_user:
+edx_access_key_id:
+edx_secret_access_key:
+edx_s3_ingest_bucket:
+edx_s3_endpoint_bucket:
+edx_cloudfront_prefix:
+
+# ---
+# email vars
+# ---
+veda_noreply_email:
+admin_email:
+
+# ---
+# VAL user creds
+# ---
+val_api_url:
+val_client_id:
+val_password:
+val_secret_key:
+val_token_url:
+val_username:
+
+# ---
+# AWS Storage config for video images
+# ---
+aws_video_images_access_key:
+aws_video_images_secret_key:
+aws_video_images_bucket:
+aws_video_images_prefix:
+
+# ---
+## Celery Info
+# ---
+celery_app_name:
+# can do multiple queues like so: foo,bar,baz
+main_celery_queue: encode_worker
+largefile_queue_barrier: 1000000000
+largefile_celery_queue: large_encode_worker
+celery_stat_queue: transcode_stat
+celery_threads: 1
+
+rabbitmq_broker:
+rabbitmq_pass:
+rabbitmq_user:
+
+# ---
+# Shotgun Variables (internal mediateam)
+# ---
+sg_server_path:
+sg_script_name:
+sg_script_key:
+
+# ---
+# Endpoints
+# ---
+threeplay_ftphost:
+xuetang_api_url:
+xuetang_api_shared_secret:
+
+# ----------
+##---
+# This is a list of encodes and their respective course
+# boolean matches
+encode_dict:
+    review_proc:
+        - review
+    mobile_override:
+        - override
+    s3_proc:
+        - mobile_high
+        - mobile_low
+        - audio_mp3
+        - desktop_webm
+        - desktop_mp4
+        - hls
+
+    yt_proc:
+        - youtube
+
+##---
+# This is a list of encode profiles and their val profile matches
+# boolean matches
+val_profile_dict:
+    mobile_low:
+        - mobile_low
+
+    desktop_mp4:
+        - desktop_mp4
+
+    override:
+        - desktop_mp4
+        - mobile_low
+        - mobile_high
+
+    mobile_high:
+        - mobile_high
+
+    audio_mp3:
+        - audio_mp3
+
+    desktop_webm:
+        - desktop_webm
+
+    youtube:
+        - youtube
+    review:
+    hls:
+        - hls
+
+#--
+# Heal settings
+heal_start: 1
+heal_end: 144
+
+...

manage.py

+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "VEDA.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

requirements.txt

+## NOTE: This is not a working req file -- merely a collection for notes.
+django<1.10
+djangorestframework
+oauth2_provider
+django-cors-headers
+django-oauth-toolkit==0.11.0
+django-filter
+newrelic
+uwsgi
+pysftp
+boto
+pyyaml

uwsgi_params

+
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;