Revert "Merge pull request #3 from edx/debug"

This reverts commit b8dc1339, reversing
changes made to f530eb7d.

README.rst

 ===========
-veda-django
+edx-video-pipeline
 ===========
 
-cookie-cutter django project for veda deployments
--------------------------------------------------
+Video encode automation django app/control node for edx-platform
+----------------------------------------------------------------
 
-[] / 2017.01
-~~~~~~~~~~~~~~~~
+**The video pipeline performs the following tasks**
+- Ingest (Discovery, Cataloging, Sending tasks to worker cluster)
+- Delivery
+- Storage
+- Maintenance
 
-.. image:: https://travis-ci.org/yro/veda-django.svg?branch=master
-    :target: https://travis-ci.org/yro/veda-django
+The video pipeline seeks modularity between parts, and for each part to operate as cleanly and independently as possible.
+Each course's workflow operates independently, and workflows can be configured to serve a variety of endpoints.
 
+INGEST:
+Currently we ingest remote video from edx-platform via the Studio video upload tool. The videos are discovered by the video pipeline and ingested upon succcessful upload, renamed to an internal ID schema, and routed to the appropriate transcode task cluster.
+
+TRANSCODE:
+code for this is housed at https://github.com/edx/edx-video-worker
+
+DELIVERY:
+Uploads product videos to specific third-party destinations (YT, AWS, 3Play, cielo24), retrieves URLs/Statuses/products.
+
+STORAGE:
+A specified AWS S3 bucket=
+
+MAINTENANCE:
+Logging, Data dumping, Celery node status and queue information
+
+
+.. image:: https://travis-ci.org/edx/edx-video-pipeline.svg?branch=master
+    :target: https://travis-ci.org/edx/edx-video-pipeline

VEDA/django_secrets.py

-"""
-VEDA's Django Secrets
-NEVER SHARE ANYTHING IN HERE, like, EVER
---assume unchanged in git--
-"""
-
-import os
-
-DJANGO_SECRET_KEY = ""
-
-DJANGO_DB_USER = ""
-DJANGO_DB_PASS = ""
-
-DJANGO_ADMIN = ('', '')
-
-SANDBOX_TOKEN = None
-
-if SANDBOX_TOKEN is not None and SANDBOX_TOKEN in os.path.dirname(__file__):
-    DEBUG = True
-    DBHOST = ''
-    veda_dbname = ""
-else:
-    DEBUG = False
-    DBHOST = ''
-    veda_dbname = ""

VEDA_OS01/api.py

 """
-VEDA_API METHODS
+Pipeline API METHODS
 
 1. cheap-o token authorizer
 This is a super hacky way to finish the Oauth2 Flow, but I need to move on
@@ -15,7 +15,7 @@ from oauth2_provider import models
 from rest_framework.authtoken.models import Token
 from django.contrib.auth.models import User
 
-from veda_env import *
+from pipeline_env import *
 
 primary_directory = os.path.dirname(__file__)
 sys.path.append(primary_directory)

VEDA_OS01/veda_env.py → VEDA_OS01/pipeline_env.py

@@ -16,7 +16,7 @@ project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
 
-os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'video_common.settings'
 
 django.setup()
 

VEDA_OS01/views.py

@@ -118,4 +118,4 @@ def user_login(request):
 
 
 if __name__ == "__main__":
-    course_view()
+    pass

bin/deliver

@@ -9,7 +9,7 @@ if project_path not in sys.path:
     sys.path.append(project_path)
 
 
-class DeliverCli():
+class DeliverCli:
     """
     Deliver
 
@@ -43,12 +43,12 @@ class DeliverCli():
             )
 
         self.args = parser.parse_args()
-        self._PARSE_ARGS()
+        self._parse_args()
 
-    def _PARSE_ARGS(self):
+    def _parse_args(self):
         self.list = self.args.list
 
-    def _RUN(self):
+    def run(self):
         """
         Launch Celery Delivery Worker
 
@@ -75,9 +75,9 @@ class DeliverCli():
 
 
 def main():
-    DC = DeliverCli()
-    DC.get_args()
-    DC._RUN()
+    deliverinstance = DeliverCli()
+    deliverinstance.get_args()
+    deliverinstance.run()
 
 
 if __name__ == '__main__':

bin/heal

@@ -21,7 +21,7 @@ Command Line Interface
 """
 
 
-class HealCli():
+class HealCli:
 
     def __init__(self, **kwargs):
         self.logging = kwargs.get('logging', True)

bin/ingest

@@ -24,7 +24,6 @@ class IngestCli():
 
         self.course_list = []
 
-
     def get_args(self):
         parser = argparse.ArgumentParser()
         parser.usage = '''
@@ -45,16 +44,13 @@ class IngestCli():
             )
 
         self.args = parser.parse_args()
+        self._parse_args()
 
-        self._PARSE_ARGS()
-
-
-    def _PARSE_ARGS(self):
+    def _parse_args(self):
         self.course_id = self.args.courseid
         self.list = self.args.list
 
-
-    def _RUN(self):
+    def run(self):
         """
         Loop, constant video retreival from Remotes
 
@@ -75,7 +71,7 @@ class IngestCli():
 def main():
     IC = IngestCli()
     IC.get_args()
-    IC._RUN()
+    IC.run()
 
 
 if __name__ == '__main__':

bin/loop.py

@@ -20,7 +20,7 @@ from youtube_callback.daemon import generate_course_list
 from youtube_callback.sftp_id_retrieve import callfunction
 
 
-class DaemonCli():
+class DaemonCli:
 
     def __init__(self):
         self.args = None

bin/youtubecallback

@@ -16,7 +16,7 @@ Youtube Callback
 
 Command Line Interface
 """
-# TODO: Add email daemon alert
+
 
 class YoutubeCallbackCli():
 
@@ -50,13 +50,13 @@ class YoutubeCallbackCli():
 
         self.args = parser.parse_args()
 
-        self._PARSE_ARGS()
+        self._parse_args()
 
-    def _PARSE_ARGS(self):
+    def _parse_args(self):
         self.course_id = self.args.courseid
         self.list = self.args.list
 
-    def _RUN(self):
+    def run(self):
 
         if self.list is True:
             self.listcourses()
@@ -93,11 +93,10 @@ class YoutubeCallbackCli():
             print course.edx_classid
 
 
-
 def main():
     YTCC = YoutubeCallbackCli()
     YTCC.get_args()
-    YTCC._RUN()
+    YTCC.run()
 
 if __name__ == '__main__':
     sys.exit(main())

control/celeryapp.py

@@ -10,9 +10,9 @@ Start Celery Worker
 
 """
 try:
-    from control.veda_env import *
+    from control.control_env import *
 except:
-    from veda_env import *
+    from control_env import *
 
 try:
     from control.veda_deliver import VedaDelivery
@@ -20,8 +20,8 @@ except:
     from veda_deliver import VedaDelivery
 
 auth_yaml = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)),
-    'veda_auth.yaml'
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    'instance_config.yaml'
 )
 with open(auth_yaml, 'r') as stream:
     try:

control/veda_env.py → control/control_env.py

 #!/usr/bin/env python
+"""
+VEDA Environment variables
+
+"""
 
 import os
 import sys
@@ -6,19 +10,11 @@ import django
 from django.utils.timezone import utc
 from django.db import reset_queries
 
-"""
-VEDA Environment variables
-
-"""
-
-"""
-Import Django Shit
-"""
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
 
-os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'video_common.settings'
 
 django.setup()
 

control/veda_deliver.py

@@ -34,7 +34,7 @@ and upload to the appropriate endpoint via the approp. methods
 """
 homedir = expanduser("~")
 
-from veda_env import *
+from control_env import *
 from veda_utils import ErrorObject, Output, Metadata, VideoProto
 from veda_video_validation import Validation
 from veda_val import VALAPICall

control/veda_deliver_cielo.py

@@ -22,7 +22,7 @@ priority =
 turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently
 
 """
-from veda_env import *
+from control_env import *
 from veda_utils import ErrorObject, Output
 
 requests.packages.urllib3.disable_warnings()

control/veda_deliver_youtube.py

@@ -11,7 +11,7 @@ Youtube Dynamic Upload
 Note: This represents early VEDA work, but is functional
 
 """
-from veda_env import *
+from control_env import *
 
 
 def printTotals(transferred, toBeTransferred):

control/veda_encode.py

@@ -13,7 +13,7 @@ Get a list of needed encodes from VEDA
 * Protected against extant URLs *
 
 """
-from veda_env import *
+from control_env import *
 from dependencies.shotgun_api3 import Shotgun
 
 newrelic.agent.initialize(

control/veda_file_discovery.py

@@ -27,7 +27,7 @@ Currently:
     Local (watchfolder w/o edit priv.)
 
 """
-from veda_env import *
+from control_env import *
 from veda_utils import ErrorObject
 from veda_file_ingest import VideoProto, VedaIngest
 from veda_val import VALAPICall

control/veda_file_ingest.py

@@ -31,7 +31,7 @@ This just takes discovered
     - Studio Uploads
     - FTP Uploads
 """
-from veda_env import *
+from control_env import *
 from veda_hotstore import Hotstore
 from veda_video_validation import Validation
 from veda_utils import ErrorObject, Output, Report

control/veda_heal.py

@@ -31,7 +31,7 @@ Roll through videos, check for completion
 
 
 """
-from veda_env import *
+from control_env import *
 
 from veda_encode import VedaEncode
 from veda_val import VALAPICall

control/veda_utils.py

@@ -19,7 +19,7 @@ newrelic.agent.initialize(
 Let's do some quick and dirty error handling & logging
 
 """
-from control.veda_env import *
+from control.control_env import *
 from control.veda_encode import VedaEncode
 
 

control/veda_val.py

@@ -36,7 +36,7 @@ Send data to VAL, either Video ID data or endpoint URLs
 "imported": _IMPORTED,
 
 '''
-from veda_env import *
+from control_env import *
 from control.veda_utils import ErrorObject, Output
 
 

control/veda_video_validation.py

@@ -23,7 +23,7 @@ image files (which read as 0:00 duration or N/A)
 Mismatched Durations (within 5 sec)
 
 """
-from veda_env import *
+from control_env import *
 
 
 class Validation():

frontend/abvid_reporting.py

@@ -9,7 +9,7 @@ import yaml
 ABVID REPORTING - email / etc.
 
 '''
-from veda_env import *
+from frontend_env import *
 
 '''
 v1 = Video.objects.filter(edx_id = upload_info['edx_id'])

frontend/abvid_validate.py

@@ -7,7 +7,7 @@ import sys
 import datetime
 
 
-from veda_env import *
+from frontend_env import *
 
 
 def create_record(upload_data):

frontend/course_validate.py

@@ -9,7 +9,7 @@ import datetime
 
 from django.utils.timezone import utc
 
-from veda_env import *
+from frontend_env import *
 
 """
 Import Django Shit

frontend/veda_env.py → frontend/frontend_env.py

 #!/usr/bin/env python
+"""
+VEDA Environment variables
+
+"""
 
 import os
 import sys
@@ -6,14 +10,6 @@ import django
 from django.utils.timezone import utc
 
 
-"""
-VEDA Environment variables
-
-"""
-
-"""
-Import Django Shit
-"""
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
@@ -28,12 +24,6 @@ from VEDA_OS01.models import Video
 from VEDA_OS01.models import URL
 from VEDA_OS01.models import VedaUpload
 
-
-"""
-Generalized display and such
-
-"""
-
 """
 TERM COLORS
 """

frontend/views.py

@@ -13,7 +13,7 @@ from django.http import HttpResponse
 from django.template import RequestContext, loader
 from django.http import HttpResponseRedirect
 
-from veda_env import *
+from frontend_env import *
 from course_validate import VEDACat
 from abvid_validate import validate_incoming, create_record, send_to_pipeline
 

manage.py

@@ -3,7 +3,7 @@ import os
 import sys
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "VEDA.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "video_common.settings")
 
     from django.core.management import execute_from_command_line
 

requirements.txt

 ## NOTE: This is not a working req file -- merely a collection for notes.
-### TODO: Make this a working file
-
 django<1.10
 djangorestframework
 oauth2_provider
@@ -12,11 +10,3 @@ uwsgi
 pysftp
 boto
 pyyaml
-
-# below is a yum install
-# python-devel mysql-devel
-# gcc
-# mysql-python
-# git
-# pcre-devel
-# nginx

scripts/resolution_fix.py

@@ -14,7 +14,7 @@ sys.path.append(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 # import abvid_reporting
-from veda_env import *
+from pipeline_env import *
 from veda_heal import VedaHeal
 
 sick_list = [

templates/course_form.html

@@ -2,7 +2,7 @@
 <html lang="en" xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <meta http-equiv="content-type" content="text/html; charset=UTF-8">
-<title>edX/VEDA New Course Input</title>
+<title>edX Video Pipeline New Course Input</title>
 <script src='//ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js'></script>
 
 <link rel="icon" type="image/ico" href="https://s3.amazonaws.com/mediateam.edx.org/favicon.ico"/>
@@ -46,7 +46,7 @@ var institution_list = {{ institution_list|safe }}
     <div id="data"></div>
     </div>
     <div id="initial_title">
-    <span>VEDA Course Addition Tool</span>
+    <span>edX Video Pipeline Course Addition Tool</span>
     </div>
     <div id="forminput" class="course_input">
 
@@ -83,7 +83,7 @@ var institution_list = {{ institution_list|safe }}
     </div>
 
 <div class="footer">
-veda_os i/o version 0.90 (B)
+edx-video-pipeline i/o version 1.0.0
 </p>
 <a href="../admin/logout/">logout</a>
 

templates/index.html

@@ -2,7 +2,7 @@
 <html lang="en" xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <meta http-equiv="content-type" content="text/html; charset=UTF-8">
-<title>edX/VEDA</title>
+<title>edX video pipeline</title>
 <link rel="shortcut icon" type="image/png" href="/static/img/MPT-Logo.png"/>
 
  {% load staticfiles %}

templates/js/course_form.js

@@ -204,7 +204,7 @@ function create_post(inst) {
                 $("#institutional").empty();
 
                 var $inst_name = $("<div>", {id: "institution_title", class: "data"});
-                $inst_name.append('<span class="tiny_titles">VEDA Course Addition Tool : </span><br>')
+                $inst_name.append('<span class="tiny_titles">edX Video Pipeline Course Addition Tool : </span><br>')
                 if (json.length == 0) {
                     $inst_name.append(('NEW INSTITUTION'));
                 }
@@ -614,7 +614,7 @@ function submit_data () {
             $('#return').append('<h3>Success!</h3>');
             $('#return').append('<span class=\"advisory\" style=\"margin-left: 49px;\">Paste into edX Studio Advanced Settings > <br>Video Upload Credentials</span>')
             $('#return').append('<span class=\"final_data\"> &nbsp;\"course_video_upload_token\": \"'+json['studio_hex']+'\"&nbsp;<span><br>')
-            $('#return').append('<span class=\"advisory\" style=\"margin-left: 49px;\">VEDA Code : ' + json['course_code'] + '</span><br>')
+            $('#return').append('<span class=\"advisory\" style=\"margin-left: 49px;\">Pipeline Code : ' + json['course_code'] + '</span><br>')
 
             // Reset Button
             $('#rstb').attr('value', 'New')

templates/upload_video.html

@@ -2,7 +2,7 @@
 <html lang="en" xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <meta http-equiv="content-type" content="text/html; charset=UTF-8">
-<title>edX/VEDA Video Upload</title>
+<title>edX Video Pipeline Video Upload</title>
 <link rel="shortcut icon" type="image/png" href="/static/img/MPT-Logo.png"/>
 
     {% load staticfiles %}
@@ -71,7 +71,7 @@ $('#inst_lookup').hide()
 </script>
 <div class="footer">
 <span class="help"><a target="_blank" href="http://edx.readthedocs.org/projects/edx-partner-course-staff/en/latest/set_up_course/setting_up_student_view.html#add-a-course-about-video-to-edx-org">Help?</a></span><br>
-veda_os i/o <br>
+edx-video-pipeline i/o <br>
 version 1.1
 </div>
 </body>

video_common/django_secrets.py

+"""
+Video Pipeline Django Secrets Shim
+
+This acts as a django-secret shimmer until we can finish pushing all changes to terraform/prod
+
+"""
+import os
+import yaml
+
+read_yaml = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    'instance_config.yaml'
+)
+
+with open(read_yaml, 'r') as stream:
+    try:
+        return_dict = yaml.load(stream)
+    except yaml.YAMLError as exc:
+        return_dict = None
+
+
+DJANGO_SECRET_KEY = return_dict['django_secret_key']
+DJANGO_ADMIN = ('', '')
+DEBUG = True
+DATABASES = return_dict['DATABASES']

VEDA/settings.py → video_common/settings.py

@@ -22,7 +22,7 @@ if DATABASES is None:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.mysql',
-            'NAME': veda_dbname,
+            'NAME': pipeline_dbname,
             'USER': DJANGO_DB_USER,
             'PASSWORD': DJANGO_DB_PASS,
             'HOST': DBHOST,
@@ -102,10 +102,10 @@ X_FRAME_OPTIONS = 'DENY'
 SECURE_SSL_REDIRECT = False
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_BROWSER_XSS_FILTER = True
-ROOT_URLCONF = 'VEDA.urls'
+ROOT_URLCONF = 'video_common.urls'
 
 # Python dotted path to the WSGI application used by Django's runserver.
-WSGI_APPLICATION = 'VEDA.wsgi.application'
+WSGI_APPLICATION = 'video_common.wsgi.application'
 
 
 REST_FRAMEWORK = {

VEDA/urls.py → video_common/urls.py

@@ -5,13 +5,14 @@ sys.path.append(os.path.abspath(__file__))
 os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
 
 from django.conf import settings
-from rest_framework.routers import DefaultRouter
+from rest_framework import routers
+# from rest_framework.routers import DefaultRouter
 from django.conf.urls import patterns, include, url
 from django.contrib import admin
 
 from VEDA_OS01 import views
 
-router = DefaultRouter()
+router = routers.DefaultRouter()
 admin.autodiscover()
 
 router.register(r'courses', views.CourseViewSet)

VEDA/wsgi.py → video_common/wsgi.py

 """
-...
 """
 import os
 from django.core.wsgi import get_wsgi_application
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.environ.setdefault("PYTHON_EGG_CACHE", BASE_DIR + "/egg_cache")
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "VEDA.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "video_common.settings")
 application = get_wsgi_application()

youtube_callback/daemon.py

@@ -20,7 +20,7 @@ project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
 
-os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'video_common.settings'
 
 django.setup()
 

youtube_callback/sftp_id_retrieve.py

@@ -21,7 +21,7 @@ from django.utils.timezone import utc
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
-os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'video_common.settings'
 django.setup()
 
 from VEDA_OS01.models import Video, Encode, URL