Skip to content

E
edx-video-pipeline
Unverified Commit c51c966d by Gregory Martin Committed by GitHub
Options

Merge pull request #95 from edx/yro/update_logging 

Yro/update logging
parents 249d340e bac906f7
Showing with 450 additions and 569 deletions
VEDA/urls.py
...@@ -4,9 +4,9 @@ import os ...@@ -4,9 +4,9 @@ import os
sys.path.append(os.path.abspath(__file__)) sys.path.append(os.path.abspath(__file__))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
# pep8: disable=E402
from django.conf import settings from django.conf import settings
from rest_framework import routers from rest_framework import routers
# from rest_framework.routers import DefaultRouter
from django.conf.urls import patterns, include, url from django.conf.urls import patterns, include, url
from django.contrib import admin from django.contrib import admin
......
VEDA_OS01/fixtures/destinations.json
...@@ -56,15 +56,15 @@ ...@@ -56,15 +56,15 @@
"fields": { "fields": {
"course_name": "Veda Sandbox Test Course", "course_name": "Veda Sandbox Test Course",
"course_hold": true, "course_hold": true,
"institution": "XXX", "institution": "EDX",
"edx_classid": "XXXXX", "edx_classid": "DEMOX",
"semesterid": "2017", "semesterid": "2017",
"yt_proc": false, "yt_proc": false,
"tp_proc": false, "tp_proc": false,
"c24_proc": false, "c24_proc": false,
"s3_proc": true, "s3_proc": true,
"local_storedir": "course-v1:VEDA+VEDA201+2015_T1", "local_storedir": "course-v1:VEDA+VEDA201+2015_T1",
"studio_hex": "xxxx" "studio_hex": "shared_course_token"
} }
} }
] ]
VEDA_OS01/fixtures/encodes.json
...@@ -160,5 +160,18 @@ ...@@ -160,5 +160,18 @@
"encode_resolution": "720", "encode_resolution": "720",
"product_spec": "desktop_webm" "product_spec": "desktop_webm"
} }
},
{
"model": "VEDA_OS01.course",
"pk": 1,
"fields": {
"course_name": "Demo Course",
"institution": "EDX",
"edx_classid": "DEMOX",
"semesterid": 2018,
"yt_proc": false,
"local_storedir": "this/is/an/exemplar",
"studio_hex": "shared_course_token"
}
} }
] ]
bin/deliver
...@@ -9,7 +9,7 @@ if project_path not in sys.path: ...@@ -9,7 +9,7 @@ if project_path not in sys.path:
sys.path.append(project_path) sys.path.append(project_path)
class DeliverCli: class DeliverCli(object):
""" """
Deliver Deliver
......
bin/heal
...@@ -4,11 +4,12 @@ Deliver ...@@ -4,11 +4,12 @@ Deliver
Command Line Interface Command Line Interface
""" """
import os
import sys
import argparse import argparse
import datetime import datetime
from datetime import timedelta from datetime import timedelta
import logging
import os
import sys
import pytz import pytz
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
...@@ -21,8 +22,12 @@ from VEDA_OS01.models import Course, Video ...@@ -21,8 +22,12 @@ from VEDA_OS01.models import Course, Video
from VEDA_OS01.transcripts import retrieve_three_play_translations from VEDA_OS01.transcripts import retrieve_three_play_translations
from VEDA.utils import get_config from VEDA.utils import get_config
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class HealCli: class HealCli(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.logging = kwargs.get('logging', True) self.logging = kwargs.get('logging', True)
...@@ -93,8 +98,8 @@ def main(): ...@@ -93,8 +98,8 @@ def main():
retrieve_three_play_translations() retrieve_three_play_translations()
return return
print '%s - %s: %s' % ('Healing', 'VEDA ID', veda_id) LOGGER.info('%s - %s: %s' % ('Healing', 'VEDA ID', veda_id))
print '%s - %s: %s' % ('Healing', 'Course', course_id) LOGGER.info('%s - %s: %s' % ('Healing', 'Course', course_id))
if veda_id is None and course_id is None and schedule is False: if veda_id is None and course_id is None and schedule is False:
VH = VedaHeal() VH = VedaHeal()
......
bin/ingest
#!/usr/bin/env python #!/usr/bin/env python
"""
Ingest
Command Line Interface
"""
import os import os
import sys import sys
import argparse import argparse
...@@ -9,13 +15,8 @@ if project_path not in sys.path: ...@@ -9,13 +15,8 @@ if project_path not in sys.path:
from control.veda_utils import EmailAlert from control.veda_utils import EmailAlert
"""
Ingest
Command Line Interface
"""
class IngestCli(): class IngestCli(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.args = None self.args = None
......
bin/loop.py
#!/usr/bin/env python #!/usr/bin/env python
"""
This is a cheapo way to get a pager (using SES)
"""
import os import os
import sys import sys
import argparse import argparse
import logging
from django.db import reset_queries from django.db import reset_queries
import resource import resource
import time import time
...@@ -13,11 +18,6 @@ project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ...@@ -13,11 +18,6 @@ project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path: if project_path not in sys.path:
sys.path.append(project_path) sys.path.append(project_path)
"""
This is a cheapo way to get a pager (using SES)
"""
import django import django
django.setup() django.setup()
...@@ -25,8 +25,12 @@ from control.veda_file_discovery import FileDiscovery ...@@ -25,8 +25,12 @@ from control.veda_file_discovery import FileDiscovery
from youtube_callback.daemon import generate_course_list from youtube_callback.daemon import generate_course_list
from youtube_callback.sftp_id_retrieve import callfunction from youtube_callback.sftp_id_retrieve import callfunction
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class DaemonCli: class DaemonCli(object):
def __init__(self): def __init__(self):
self.args = None self.args = None
...@@ -80,7 +84,7 @@ class DaemonCli: ...@@ -80,7 +84,7 @@ class DaemonCli:
reset_queries() reset_queries()
x += 1 x += 1
if x >= 100: if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
x = 0 x = 0
def youtube_daemon(self): def youtube_daemon(self):
...@@ -88,12 +92,12 @@ class DaemonCli: ...@@ -88,12 +92,12 @@ class DaemonCli:
while True: while True:
self.course_list = generate_course_list() self.course_list = generate_course_list()
for course in self.course_list: for course in self.course_list:
print "%s%s: Callback" % (course.institution, course.edx_classid) LOGGER.info('%s%s: Callback' % (course.institution, course.edx_classid))
callfunction(course) callfunction(course)
x += 1 x += 1
if x >= 100: if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
x = 0 x = 0
reset_queries() reset_queries()
......
bin/youtubecallback
...@@ -18,7 +18,7 @@ Command Line Interface ...@@ -18,7 +18,7 @@ Command Line Interface
""" """
class YoutubeCallbackCli(): class YoutubeCallbackCli(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.args = None self.args = None
...@@ -54,14 +54,10 @@ class YoutubeCallbackCli(): ...@@ -54,14 +54,10 @@ class YoutubeCallbackCli():
def _parse_args(self): def _parse_args(self):
self.course_id = self.args.courseid self.course_id = self.args.courseid
self.list = self.args.list
def run(self): def run(self):
if self.list is True: self.loop()
self.listcourses()
else:
self.loop()
def loop(self): def loop(self):
""" """
...@@ -82,16 +78,6 @@ class YoutubeCallbackCli(): ...@@ -82,16 +78,6 @@ class YoutubeCallbackCli():
E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon') E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
E1.email() E1.email()
def listcourses(self):
"""
list and exit
:return:
"""
self.course_list = generate_course_list()
for course in self.course_list:
print course.institution
print course.edx_classid
def main(): def main():
YTCC = YoutubeCallbackCli() YTCC = YoutubeCallbackCli()
......
control/celeryapp.py
...@@ -4,15 +4,20 @@ Start Celery Worker ...@@ -4,15 +4,20 @@ Start Celery Worker
from __future__ import absolute_import from __future__ import absolute_import
import os
from celery import Celery from celery import Celery
from VEDA.utils import get_config import logging
import os
import sys
from VEDA.utils import get_config
try: try:
from control.veda_deliver import VedaDelivery from control.veda_deliver import VedaDelivery
except ImportError: except ImportError:
from veda_deliver import VedaDelivery from veda_deliver import VedaDelivery
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
auth_dict = get_config() auth_dict = get_config()
...@@ -42,7 +47,7 @@ app.conf.update( ...@@ -42,7 +47,7 @@ app.conf.update(
@app.task(name='worker_encode') @app.task(name='worker_encode')
def worker_task_fire(veda_id, encode_profile, jobid): def worker_task_fire(veda_id, encode_profile, jobid):
print '[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile) LOGGER.info('[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile))
return 1 return 1
......
control/old_veda_deliver_cielo.py
import requests
import ast
import urllib
""" """
Cielo24 API Job Start and Download Cielo24 API Job Start and Download
Options (reflected in Course.models): Options (reflected in Course.models):
...@@ -14,10 +10,17 @@ priority = ...@@ -14,10 +10,17 @@ priority =
priority (48h) priority (48h)
turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently
""" """
import logging
import requests
import ast
import urllib
from control_env import * from control_env import *
from veda_utils import ErrorObject
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Cielo24TranscriptOld(object): class Cielo24TranscriptOld(object):
...@@ -77,11 +80,9 @@ class Cielo24TranscriptOld(object): ...@@ -77,11 +80,9 @@ class Cielo24TranscriptOld(object):
videoID=video_query, videoID=video_query,
encode_url__icontains='_DTH.mp4', encode_url__icontains='_DTH.mp4',
).latest() ).latest()
if video_query.inst_class.c24_username is None: if video_query.inst_class.c24_username is None:
ErrorObject.print_error( LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API : Course record incomplete'.format(id=self.veda_id))
message='Cielo24 Record Incomplete',
)
return None return None
c24_defaults = { c24_defaults = {
...@@ -102,10 +103,8 @@ class Cielo24TranscriptOld(object): ...@@ -102,10 +103,8 @@ class Cielo24TranscriptOld(object):
# Generate Token # Generate Token
r1 = requests.get(token_url) r1 = requests.get(token_url)
if r1.status_code > 299: if r1.status_code > 299:
ErrorObject.print_error( LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API access'.format(id=self.veda_id))
message='Cielo24 API Access Error', return
)
return None
api_token = ast.literal_eval(r1.text)["ApiToken"] api_token = ast.literal_eval(r1.text)["ApiToken"]
return api_token return api_token
...@@ -161,7 +160,6 @@ class Cielo24TranscriptOld(object): ...@@ -161,7 +160,6 @@ class Cielo24TranscriptOld(object):
urllib.quote_plus(self.c24_defaults['url']) urllib.quote_plus(self.c24_defaults['url'])
)) ))
) )
print str(r4.status_code) + ' : Cielo24 Status Code'
return ast.literal_eval(r4.text)['TaskId'] return ast.literal_eval(r4.text)['TaskId']
......
control/tests/test_encode.py
...@@ -81,30 +81,3 @@ class TestEncode(TestCase): ...@@ -81,30 +81,3 @@ class TestEncode(TestCase):
).delete() ).delete()
encode_list = self.E.determine_encodes() encode_list = self.E.determine_encodes()
self.assertTrue(len(encode_list) == baseline) self.assertTrue(len(encode_list) == baseline)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
'''
Save for poss future test
# import celeryapp
# co = Course.objects.get(institution='XXX', edx_classid='C93BC')
# vid = 'XXXC93BC2016-V003500'
# v = VedaEncode(course_object=co, veda_id=vid)
# encode_list = v.determine_encodes()
# for e in encode_list:
# veda_id = vid
# encode_profile = e
# jobid = uuid.uuid1().hex[0:10]
# # celeryapp.worker_task_fire.apply_async(
# # (veda_id, encode_profile, jobid),
# # queue='encode_worker'
# # )
'''
control/tests/test_file_discovery.py
...@@ -170,22 +170,6 @@ class TestFileDiscovery(TestCase): ...@@ -170,22 +170,6 @@ class TestFileDiscovery(TestCase):
self.assertTrue(mock_validate_and_feed_to_ingest.called) self.assertTrue(mock_validate_and_feed_to_ingest.called)
@ddt.data( @ddt.data(
('veda/working', '[File Ingest] S3 Ingest Connection Failure'),
(None, '[File Ingest] No Working Node directory')
)
@ddt.unpack
@patch('control.veda_file_discovery.ErrorObject.print_error')
@patch('boto.s3.connection.S3Connection')
def test_discover_studio_ingested_video_exceptions(self, work_dir, error_message, mocked_s3_conn, mock_error):
"""
Tests 'FileDiscovery.discover_studio_ingested_videos' exception cases.
"""
mocked_s3_conn.side_effect = S3ResponseError('Error', 'Timeout')
file_discovery_instance = FileDiscovery(node_work_directory=work_dir)
file_discovery_instance.discover_studio_ingested_videos()
mock_error.assert_called_with(message=error_message)
@ddt.data(
(None, 'invalid_course_key'), (None, 'invalid_course_key'),
('non-existent-hex', None) ('non-existent-hex', None)
) )
......
control/veda_deliver.py
...@@ -27,7 +27,7 @@ from VEDA_OS01 import utils ...@@ -27,7 +27,7 @@ from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider, from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus) TranscriptStatus)
from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
from veda_utils import ErrorObject, Metadata, Output, VideoProto from veda_utils import Metadata, Output, VideoProto
from veda_val import VALAPICall from veda_val import VALAPICall
from veda_video_validation import Validation from veda_video_validation import Validation
...@@ -49,7 +49,7 @@ boto.config.set('Boto', 'http_socket_timeout', '100') ...@@ -49,7 +49,7 @@ boto.config.set('Boto', 'http_socket_timeout', '100')
homedir = expanduser("~") homedir = expanduser("~")
class VedaDelivery: class VedaDelivery(object):
def __init__(self, veda_id, encode_profile, **kwargs): def __init__(self, veda_id, encode_profile, **kwargs):
self.veda_id = veda_id self.veda_id = veda_id
...@@ -71,7 +71,7 @@ class VedaDelivery: ...@@ -71,7 +71,7 @@ class VedaDelivery:
Check the destination, route via available methods, Check the destination, route via available methods,
throw error if method is not extant throw error if method is not extant
""" """
LOGGER.info('[VIDEO_DELIVER] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile)) LOGGER.info('[DELIVERY] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile))
if self.encode_profile == 'hls': if self.encode_profile == 'hls':
# HLS encodes are a pass through # HLS encodes are a pass through
self.hls_run() self.hls_run()
...@@ -137,7 +137,7 @@ class VedaDelivery: ...@@ -137,7 +137,7 @@ class VedaDelivery:
self.status = self._DETERMINE_STATUS() self.status = self._DETERMINE_STATUS()
self._UPDATE_DATA() self._UPDATE_DATA()
self._CLEANUP() self._CLEANUP()
LOGGER.info('[DELIVERY] {video_id} : complete'.format(video_id=self.veda_id))
# We only want to generate transcripts when all the encodings(except for YT and Review) are done. # We only want to generate transcripts when all the encodings(except for YT and Review) are done.
if utils.is_video_ready(self.video_query.edx_id, ignore_encodes=['review', 'youtube']): if utils.is_video_ready(self.video_query.edx_id, ignore_encodes=['review', 'youtube']):
self.start_transcription() self.start_transcription()
...@@ -217,14 +217,14 @@ class VedaDelivery: ...@@ -217,14 +217,14 @@ class VedaDelivery:
conn = S3Connection() conn = S3Connection()
bucket = conn.get_bucket(self.auth_dict['veda_deliverable_bucket']) bucket = conn.get_bucket(self.auth_dict['veda_deliverable_bucket'])
except NoAuthHandlerFound: except NoAuthHandlerFound:
LOGGER.error('[VIDEO_DELIVER] BOTO/S3 Communication error') LOGGER.error('[DELIVERY] {url} : BOTO/S3 Communication error'.format(url=self.hotstore_url))
return return
except S3ResponseError: except S3ResponseError:
LOGGER.error('[VIDEO_DELIVER] Invalid Storage Bucket') LOGGER.error('[DELIVERY] {url} : Invalid Storage Bucket'.format(url=self.hotstore_url))
return return
source_key = bucket.get_key(self.encoded_file) source_key = bucket.get_key(self.encoded_file)
if source_key is None: if source_key is None:
LOGGER.error('[VIDEO_DELIVER] S3 Intake Object NOT FOUND') LOGGER.error('[DELIVERY] {url} : S3 Intake Object not found'.format(url=self.hotstore_url))
return return
source_key.get_contents_to_filename( source_key.get_contents_to_filename(
...@@ -245,7 +245,7 @@ class VedaDelivery: ...@@ -245,7 +245,7 @@ class VedaDelivery:
VM._METADATA() VM._METADATA()
if not isinstance(self.video_proto.duration, int) and ':' not in self.video_proto.duration: if not isinstance(self.video_proto.duration, int) and ':' not in self.video_proto.duration:
print 'Duration Failure' LOGGER.error('[DELIVERY] {id} : Duration Failure'.format(id=self.video_proto.veda_id))
return return
self.video_proto.duration = Output._seconds_from_string( self.video_proto.duration = Output._seconds_from_string(
...@@ -358,8 +358,8 @@ class VedaDelivery: ...@@ -358,8 +358,8 @@ class VedaDelivery:
self.encoded_file self.encoded_file
) )
): ):
print 'WARNING -- NO FILE' LOGGER.error('[DELIVERY] {file} : No file for routing'.format(file=self.encoded_file))
return None return
''' '''
Destination Nicks: Destination Nicks:
S31 S31
...@@ -382,10 +382,8 @@ class VedaDelivery: ...@@ -382,10 +382,8 @@ class VedaDelivery:
""" """
Throw error Throw error
""" """
ErrorObject.print_error( LOGGER.error('[DELIVERY] No method')
message='Deliverable - No Method', return
)
return None
def AWS_UPLOAD(self): def AWS_UPLOAD(self):
""" """
...@@ -426,10 +424,7 @@ class VedaDelivery: ...@@ -426,10 +424,7 @@ class VedaDelivery:
try: try:
conn = boto.connect_s3() conn = boto.connect_s3()
except S3ResponseError: except S3ResponseError:
ErrorObject.print_error( LOGGER.error('[DELIVERY] s3 Connection Error')
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
return False return False
delv_bucket = conn.get_bucket( delv_bucket = conn.get_bucket(
self.auth_dict['edx_s3_endpoint_bucket'] self.auth_dict['edx_s3_endpoint_bucket']
...@@ -481,17 +476,11 @@ class VedaDelivery: ...@@ -481,17 +476,11 @@ class VedaDelivery:
try: try:
c = boto.connect_s3() c = boto.connect_s3()
except S3ResponseError: except S3ResponseError:
ErrorObject.print_error( LOGGER.error('[DELIVERY] s3 Connection Error')
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
return False return False
b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket']) b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket'])
if b is None: if b is None:
ErrorObject.print_error( LOGGER.error('[DELIVERY] s3 Connection Error')
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
return False return False
""" """
...@@ -540,7 +529,7 @@ class VedaDelivery: ...@@ -540,7 +529,7 @@ class VedaDelivery:
try: try:
api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key
except TranscriptCredentials.DoesNotExist: except TranscriptCredentials.DoesNotExist:
LOGGER.warn('[cielo24] Unable to find api_key for org=%s', org) LOGGER.warn('[DELIVERY] Unable to find cielo24 api_key for org=%s', org)
return None return None
s3_video_url = build_url( s3_video_url = build_url(
...@@ -630,7 +619,7 @@ class VedaDelivery: ...@@ -630,7 +619,7 @@ class VedaDelivery:
except TranscriptCredentials.DoesNotExist: except TranscriptCredentials.DoesNotExist:
LOGGER.warning( LOGGER.warning(
'Transcript preference is not found for provider=%s, video=%s', '[DELIVERY] : Transcript preference is not found for provider=%s, video=%s',
self.video_query.provider, self.video_query.provider,
self.video_query.studio_id, self.video_query.studio_id,
) )
...@@ -647,13 +636,11 @@ class VedaDelivery: ...@@ -647,13 +636,11 @@ class VedaDelivery:
if self.encode_profile != 'desktop_mp4': if self.encode_profile != 'desktop_mp4':
return None return None
C24 = Cielo24TranscriptOld( cielojob = Cielo24TranscriptOld(
veda_id=self.video_query.edx_id veda_id=self.video_query.edx_id
) )
output = C24.perform_transcription() cielojob.perform_transcription()
print '[ %s ] : %s' % ( LOGGER.info('[DELIVERY] {id} : Cielo24 job sent '.format(id=self.video_query.edx_id))
'Cielo24 JOB', self.video_query.edx_id
)
def _THREEPLAY_UPLOAD(self): def _THREEPLAY_UPLOAD(self):
""" """
...@@ -675,9 +662,7 @@ class VedaDelivery: ...@@ -675,9 +662,7 @@ class VedaDelivery:
try: try:
ftp1.login(user, passwd) ftp1.login(user, passwd)
except: except:
ErrorObject.print_error( LOGGER.error('[DELIVERY] {file} : 3Play Authentication Failure'.format(file=self.encoded_file))
message='3Play Authentication Failure'
)
try: try:
ftp1.cwd( ftp1.cwd(
self.video_query.inst_class.tp_speed self.video_query.inst_class.tp_speed
...@@ -704,8 +689,10 @@ class VedaDelivery: ...@@ -704,8 +689,10 @@ class VedaDelivery:
def YOUTUBE_SFTP(self, review=False): def YOUTUBE_SFTP(self, review=False):
if self.video_query.inst_class.yt_proc is False: if self.video_query.inst_class.yt_proc is False:
if self.video_query.inst_class.review_proc is False: if self.video_query.inst_class.review_proc is False:
print 'NO YOUTUBE' LOGGER.error(
return None '[DELIVERY] {id} : Youtube called, youtube processing off'.format(id=self.video_proto.veda_id)
)
return
DY = DeliverYoutube( DY = DeliverYoutube(
veda_id=self.video_query.edx_id, veda_id=self.video_query.edx_id,
......
control/veda_deliver_youtube.py
import os
import os.path
import sys
import time
import pysftp
""" """
Youtube Dynamic Upload Youtube Dynamic Upload
Note: This represents early VEDA work, but is functional Note: This is early VEDA work, but is functional. Ideally deprecated in favor of a no-youtube workflow, this code
is only maintained, and not prioritized for refactoring
""" """
import logging
import os.path
import pysftp
import paramiko
import time
from control_env import * from control_env import *
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def printTotals(transferred, toBeTransferred): def printTotals(transferred, toBeTransferred):
""" """
Optional upload logging method
"""
'''
try: try:
sys.stdout.write('\r') sys.stdout.write('\r')
sys.stdout.write("Transferred: {0}\tOut of: {1}\r".format(transferred, toBeTransferred)) sys.stdout.write("Transferred: {0}\tOut of: {1}\r".format(transferred, toBeTransferred))
sys.stdout.flush() sys.stdout.flush()
except: except:
print 'Callback Failing' LOGGER.error('Callback Failing')
""" '''
return None return
class DeliverYoutube(): class DeliverYoutube(object):
def __init__(self, veda_id, encode_profile): def __init__(self, veda_id, encode_profile):
self.veda_id = veda_id self.veda_id = veda_id
...@@ -120,7 +129,7 @@ class DeliverYoutube(): ...@@ -120,7 +129,7 @@ class DeliverYoutube():
'playlist_id', 'playlist_id',
'require_paid_subscription' 'require_paid_subscription'
] ]
print "%s : %s" % ("Generate CSV", str(self.video.edx_id)) LOGGER.info('[YOUTUBE] {id} : Generating sidecar metadata CSV'.format(id=str(self.video.edx_id)))
''' '''
# TODO: Refactor this into centrally located util for escaping bad chars # TODO: Refactor this into centrally located util for escaping bad chars
if self.video.client_title is not None: if self.video.client_title is not None:
...@@ -142,7 +151,6 @@ class DeliverYoutube(): ...@@ -142,7 +151,6 @@ class DeliverYoutube():
This is where we can add or subtract file attributes as needed This is where we can add or subtract file attributes as needed
""" """
print self.file
metadata_dict = { metadata_dict = {
'filename': self.file, 'filename': self.file,
'channel': self.course.yt_channel, 'channel': self.course.yt_channel,
...@@ -154,7 +162,7 @@ class DeliverYoutube(): ...@@ -154,7 +162,7 @@ class DeliverYoutube():
# Header Row # Header Row
output = ','.join(([c for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES])) + '\n' output = ','.join(([c for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES])) + '\n'
# Data Row # Data Row
output += ','.join(([metadata_dict.get(c, '') for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES])) # + '\n' <--NO output += ','.join(([metadata_dict.get(c, '') for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES]))
with open(os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'), 'w') as c1: with open(os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'), 'w') as c1:
c1.write('%s %s' % (output, '\n')) c1.write('%s %s' % (output, '\n'))
...@@ -190,50 +198,42 @@ class DeliverYoutube(): ...@@ -190,50 +198,42 @@ class DeliverYoutube():
d1.write('') d1.write('')
cnopts = pysftp.CnOpts() cnopts = pysftp.CnOpts()
cnopts.hostkeys = None cnopts.hostkeys = None
try:
with pysftp.Connection( with pysftp.Connection(
'partnerupload.google.com', 'partnerupload.google.com',
username=self.course.yt_logon, username=self.course.yt_logon,
private_key=self.youtubekey, private_key=self.youtubekey,
port=19321, port=19321,
cnopts=cnopts cnopts=cnopts
) as s1: ) as s1:
print "Go for YT : " + str(self.video.edx_id) LOGGER.info('[YOUTUBE] {id} : Ready for youtube SFTP upload'.format(id=str(self.video.edx_id)))
# Upload file, sidecar metadata,
s1.mkdir(remote_directory, mode=660) # and (google required) empty delivery.complete file
s1.cwd(remote_directory) s1.mkdir(remote_directory, mode=660)
s1.put( s1.cwd(remote_directory)
os.path.join(WORK_DIRECTORY, self.file), s1.put(
callback=printTotals os.path.join(WORK_DIRECTORY, self.file),
) callback=printTotals
print )
s1.put( s1.put(
os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'), os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'),
callback=printTotals callback=printTotals
) )
print s1.put(
s1.put( os.path.join(
os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'youtube_callback',
'youtube_callback', 'static_files',
'static_files', 'delivery.complete'
'delivery.complete' ),
), callback=printTotals,
callback=printTotals, confirm=False,
confirm=False, preserve_mtime=False
preserve_mtime=False )
) except paramiko.ssh_exception.AuthenticationException:
print LOGGER.info('[YOUTUBE] {file} : Paramiko Authentication Exception'.format(file=str(self.file)))
os.remove(os.path.join( os.remove(os.path.join(
WORK_DIRECTORY, WORK_DIRECTORY,
self.video.edx_id + '_100.csv' self.video.edx_id + '_100.csv'
)) ))
def main():
pass
if __name__ == "__main__":
sys.exit(main())
control/veda_encode.py
"""
Get a list of needed encodes from VEDA
import os * Protected against extant URLs *
import sys
import uuid
import django """
from control_env import * from control_env import *
from dependencies.shotgun_api3 import Shotgun from dependencies.shotgun_api3 import Shotgun
from dependencies.shotgun_api3.lib.xmlrpclib import ProtocolError from dependencies.shotgun_api3.lib.xmlrpclib import ProtocolError
from VEDA.utils import get_config from VEDA.utils import get_config
"""
Get a list of needed encodes from VEDA
* Protected against extant URLs *
"""
class VedaEncode(object): class VedaEncode(object):
......
control/veda_file_discovery.py
...@@ -22,7 +22,6 @@ from control_env import * ...@@ -22,7 +22,6 @@ from control_env import *
from VEDA.utils import extract_course_org, get_config from VEDA.utils import extract_course_org, get_config
from veda_file_ingest import VedaIngest, VideoProto from veda_file_ingest import VedaIngest, VideoProto
from VEDA_OS01.models import TranscriptCredentials from VEDA_OS01.models import TranscriptCredentials
from veda_utils import ErrorObject
from veda_val import VALAPICall from veda_val import VALAPICall
try: try:
...@@ -31,6 +30,8 @@ except: ...@@ -31,6 +30,8 @@ except:
pass pass
boto.config.set('Boto', 'http_socket_timeout', '100') boto.config.set('Boto', 'http_socket_timeout', '100')
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
...@@ -47,12 +48,12 @@ class FileDiscovery(object): ...@@ -47,12 +48,12 @@ class FileDiscovery(object):
Crawl VEDA Upload bucket Crawl VEDA Upload bucket
""" """
if self.node_work_directory is None: if self.node_work_directory is None:
print '[Discovery Error] No Workdir' LOGGER.error('[DISCOVERY] No Workdir')
return return
try: try:
conn = boto.connect_s3() conn = boto.connect_s3()
except NoAuthHandlerFound: except NoAuthHandlerFound:
print '[Discovery Error] BOTO Auth Handler' LOGGER.error('[DISCOVERY] BOTO Auth Handler')
return return
try: try:
self.bucket = conn.get_bucket(self.auth_dict['veda_s3_upload_bucket']) self.bucket = conn.get_bucket(self.auth_dict['veda_s3_upload_bucket'])
...@@ -211,7 +212,7 @@ class FileDiscovery(object): ...@@ -211,7 +212,7 @@ class FileDiscovery(object):
key.get_contents_to_filename(os.path.join(self.node_work_directory, file_name)) key.get_contents_to_filename(os.path.join(self.node_work_directory, file_name))
file_ingested = True file_ingested = True
except S3DataError: except S3DataError:
LOGGER.exception('[File Ingest] Error downloading the file into node working directory.') LOGGER.error('[DISCOVERY] Error downloading the file into node working directory.')
return file_ingested return file_ingested
def parse_transcript_preferences(self, course_id, transcript_preferences): def parse_transcript_preferences(self, course_id, transcript_preferences):
...@@ -233,7 +234,7 @@ class FileDiscovery(object): ...@@ -233,7 +234,7 @@ class FileDiscovery(object):
# have associated 3rd party transcription provider API keys. # have associated 3rd party transcription provider API keys.
transcript_preferences = None transcript_preferences = None
except ValueError: except ValueError:
LOGGER.exception('[File Discovery] Invalid transcripts preferences=%s', transcript_preferences) LOGGER.error('[DISCOVERY] Invalid transcripts preferences=%s', transcript_preferences)
transcript_preferences = None transcript_preferences = None
return transcript_preferences return transcript_preferences
...@@ -250,11 +251,12 @@ class FileDiscovery(object): ...@@ -250,11 +251,12 @@ class FileDiscovery(object):
if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']: if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name)) self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
except S3ResponseError: except S3ResponseError:
ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure') LOGGER.error('[DISCOVERY] S3 Ingest Connection Failure')
except NoAuthHandlerFound: except NoAuthHandlerFound:
ErrorObject.print_error(message='[Discovery Error] BOTO Auth Handler') LOGGER.error('[DISCOVERY] BOTO Auth Handler')
else: else:
ErrorObject.print_error(message='[File Ingest] No Working Node directory') LOGGER.error('[DISCOVERY] No Working Node directory')
def validate_metadata_and_feed_to_ingest(self, video_s3_key): def validate_metadata_and_feed_to_ingest(self, video_s3_key):
""" """
......
control/veda_file_ingest.py
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
Discovered file ingest/insert/job triggering Discovered file ingest/insert/job triggering
""" """
import datetime import datetime
import logging import logging
import subprocess import subprocess
...@@ -82,7 +81,10 @@ class VedaIngest(object): ...@@ -82,7 +81,10 @@ class VedaIngest(object):
self.val_insert() self.val_insert()
self.rename() self.rename()
self.archived = self.store() self.archived = self.store()
LOGGER.info('[INGEST] {studio_id} | {video_id} : Video in hot store'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
if self.video_proto.valid is False: if self.video_proto.valid is False:
self.abvid_report() self.abvid_report()
self.complete = True self.complete = True
...@@ -90,7 +92,8 @@ class VedaIngest(object): ...@@ -90,7 +92,8 @@ class VedaIngest(object):
os.remove(self.full_filename) os.remove(self.full_filename)
return None return None
LOGGER.info('[VIDEO_INGEST : Ingested] {video_id} : {datetime}'.format( LOGGER.info('[INGEST] {studio_id} | {video_id} : Ingested {datetime}'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id, video_id=self.video_proto.veda_id,
datetime=str(datetime.datetime.utcnow())) datetime=str(datetime.datetime.utcnow()))
) )
...@@ -179,7 +182,12 @@ class VedaIngest(object): ...@@ -179,7 +182,12 @@ class VedaIngest(object):
)) ))
if not os.path.exists(self.full_filename): if not os.path.exists(self.full_filename):
LOGGER.exception('[VIDEO_INGEST] File Not Found %s', self.video_proto.veda_id) LOGGER.exception(
'[INGEST] {studio_id} | {video_id} : Local file not found'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
)
)
return return
""" """
...@@ -253,7 +261,12 @@ class VedaIngest(object): ...@@ -253,7 +261,12 @@ class VedaIngest(object):
v1.client_title = final_string v1.client_title = final_string
v1.save() v1.save()
self.complete = True self.complete = True
return None LOGGER.info('[INGEST] {studio_id} | {video_id} : Database record complete'.format(
studio_id=self.video_proto.studio_id,
video_id=self.video_proto.veda_id
)
)
return
# Update transcription preferences for the Video # Update transcription preferences for the Video
if self.video_proto.process_transcription: if self.video_proto.process_transcription:
...@@ -301,11 +314,17 @@ class VedaIngest(object): ...@@ -301,11 +314,17 @@ class VedaIngest(object):
s1 += 1 s1 += 1
v1.client_title = final_string v1.client_title = final_string
v1.save() v1.save()
except Exception: except Exception:
# Log the exception and raise. # Log the exception and raise.
LOGGER.exception('[VIDEO_INGEST] - Cataloging of video=%s failed.', self.video_proto.veda_id) LOGGER.exception('[INGEST] {studio_id} | {video_id} : Video catalog failed.'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
raise raise
LOGGER.info('[INGEST] {studio_id} | {video_id} : Video record cataloged'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
def val_insert(self): def val_insert(self):
if self.video_proto.abvid_serial: if self.video_proto.abvid_serial:
...@@ -316,12 +335,12 @@ class VedaIngest(object): ...@@ -316,12 +335,12 @@ class VedaIngest(object):
else: else:
val_status = 'ingest' val_status = 'ingest'
VAC = VALAPICall( val_call = VALAPICall(
video_proto=self.video_proto, video_proto=self.video_proto,
val_status=val_status, val_status=val_status,
platform_course_url="" # Empty record for initial status update platform_course_url="" # Empty record for initial status update
) )
VAC.call() val_call.call()
def abvid_report(self): def abvid_report(self):
if self.video_proto.abvid_serial is None: if self.video_proto.abvid_serial is None:
...@@ -333,6 +352,9 @@ class VedaIngest(object): ...@@ -333,6 +352,9 @@ class VedaIngest(object):
youtube_id='' youtube_id=''
) )
email_report.upload_status() email_report.upload_status()
LOGGER.info('[INGEST] {video_id} : About video reported'.format(
video_id=self.video_proto.veda_id
))
self.complete = True self.complete = True
def rename(self): def rename(self):
......
control/veda_heal.py
...@@ -91,11 +91,17 @@ class VedaHeal(object): ...@@ -91,11 +91,17 @@ class VedaHeal(object):
) )
# Misqueued Task # Misqueued Task
if task_result == 1: if task_result == 1:
LOGGER.error('[ENQUEUE ERROR] : {id}'.format(id=v.edx_id)) LOGGER.error('[ENQUEUE] {studio_id} | {video_id} : queueing call'.format(
studio_id=v.studio_id,
video_id=v.edx_id
))
continue continue
# Update Status # Update Status
LOGGER.info('[ENQUEUE] : {id}'.format(id=v.edx_id)) LOGGER.info('[ENQUEUE] {studio_id} | {video_id}: file enqueued for encoding'.format(
studio_id=v.studio_id,
video_id=v.edx_id
))
Video.objects.filter(edx_id=v.edx_id).update( Video.objects.filter(edx_id=v.edx_id).update(
video_trans_status='Queue' video_trans_status='Queue'
) )
...@@ -104,7 +110,7 @@ class VedaHeal(object): ...@@ -104,7 +110,7 @@ class VedaHeal(object):
""" """
Determine expected and completed encodes Determine expected and completed encodes
""" """
LOGGER.info('[ENQUEUE] : {id}'.format(id=video_object.edx_id)) LOGGER.info('[ENQUEUE] : {id}'.format(id=video_object.studio_id))
if self.freezing_bug is True: if self.freezing_bug is True:
if video_object.video_trans_status == 'Corrupt File': if video_object.video_trans_status == 'Corrupt File':
self.val_status = 'file_corrupt' self.val_status = 'file_corrupt'
...@@ -135,9 +141,9 @@ class VedaHeal(object): ...@@ -135,9 +141,9 @@ class VedaHeal(object):
pass pass
requeued_encodes = self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object) requeued_encodes = self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object)
LOGGER.info('[ENQUEUE] : {id} : {status} : {encodes}'.format( LOGGER.info('[ENQUEUE] {studio_id} | {video_id}: encoding {encodes}'.format(
id=video_object.edx_id, studio_id=video_object.studio_id,
status=self.val_status, video_id=video_object.edx_id,
encodes=requeued_encodes encodes=requeued_encodes
)) ))
...@@ -197,7 +203,7 @@ class VedaHeal(object): ...@@ -197,7 +203,7 @@ class VedaHeal(object):
mark file corrupt -- just run the query again with mark file corrupt -- just run the query again with
no veda_id no veda_id
""" """
# TODO: Adapt to alert for >24h dead videos
try: try:
expected_encodes.remove('hls') expected_encodes.remove('hls')
except ValueError: except ValueError:
......
control/veda_hotstore.py
import boto import boto
import logging
import os import os
import shutil import shutil
import sys import sys
...@@ -10,7 +11,6 @@ from boto.s3.key import Key ...@@ -10,7 +11,6 @@ from boto.s3.key import Key
from boto.exception import S3ResponseError from boto.exception import S3ResponseError
from os.path import expanduser from os.path import expanduser
from veda_utils import ErrorObject
from VEDA.utils import get_config from VEDA.utils import get_config
try: try:
...@@ -21,6 +21,10 @@ boto.config.set('Boto', 'http_socket_timeout', '100') ...@@ -21,6 +21,10 @@ boto.config.set('Boto', 'http_socket_timeout', '100')
homedir = expanduser("~") homedir = expanduser("~")
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Hotstore(object): class Hotstore(object):
""" """
...@@ -44,9 +48,7 @@ class Hotstore(object): ...@@ -44,9 +48,7 @@ class Hotstore(object):
return False return False
if not os.path.exists(self.upload_filepath): if not os.path.exists(self.upload_filepath):
ErrorObject().print_error( LOGGER.error('[HOTSTORE] Local file not found')
message='Hotstore: File Not Found'
)
return False return False
self.upload_filesize = os.stat(self.upload_filepath).st_size self.upload_filesize = os.stat(self.upload_filepath).st_size
...@@ -68,9 +70,7 @@ class Hotstore(object): ...@@ -68,9 +70,7 @@ class Hotstore(object):
self.auth_dict['veda_s3_hotstore_bucket'] self.auth_dict['veda_s3_hotstore_bucket']
) )
except S3ResponseError: except S3ResponseError:
ErrorObject().print_error( LOGGER.error('[HOTSTORE] No hotstore bucket connection')
message='Hotstore: Bucket Connectivity'
)
return False return False
else: else:
try: try:
...@@ -79,9 +79,7 @@ class Hotstore(object): ...@@ -79,9 +79,7 @@ class Hotstore(object):
self.auth_dict['edx_s3_endpoint_bucket'] self.auth_dict['edx_s3_endpoint_bucket']
) )
except S3ResponseError: except S3ResponseError:
ErrorObject().print_error( LOGGER.error('[HOTSTORE] No endpoint bucket connection')
message='Endpoint: Bucket Connectivity'
)
return False return False
upload_key = Key(delv_bucket) upload_key = Key(delv_bucket)
...@@ -128,24 +126,18 @@ class Hotstore(object): ...@@ -128,24 +126,18 @@ class Hotstore(object):
c = boto.connect_s3() c = boto.connect_s3()
b = c.lookup(self.auth_dict['veda_s3_hotstore_bucket']) b = c.lookup(self.auth_dict['veda_s3_hotstore_bucket'])
except S3ResponseError: except S3ResponseError:
ErrorObject().print_error( LOGGER.error('[HOTSTORE] : No hotstore bucket connection')
message='Hotstore: Bucket Connectivity'
)
return False return False
else: else:
try: try:
c = boto.connect_s3() c = boto.connect_s3()
b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket']) b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket'])
except S3ResponseError: except S3ResponseError:
ErrorObject().print_error( LOGGER.error('[HOTSTORE] : No endpoint bucket connection')
message='Endpoint: Bucket Connectivity'
)
return False return False
if b is None: if b is None:
ErrorObject().print_error( LOGGER.error('[HOTSTORE] : s3 Bucket Error - no object')
message='Deliverable Fail: s3 Bucket Error'
)
return False return False
""" """
...@@ -180,11 +172,3 @@ class Hotstore(object): ...@@ -180,11 +172,3 @@ class Hotstore(object):
os.chdir(homedir) os.chdir(homedir)
shutil.rmtree(os.path.join(path_to_multipart, filename.split('.')[0])) shutil.rmtree(os.path.join(path_to_multipart, filename.split('.')[0]))
return True return True
def main():
pass
if __name__ == '__main__':
sys.exit(main())
control/veda_utils.py
"""
Quick and dirty output handling
import os """
import sys
import datetime
import boto.ses import boto.ses
import hashlib import datetime
import subprocess import subprocess
"""
Let's do some quick and dirty error handling & logging
"""
from control.control_env import * from control.control_env import *
from control.veda_encode import VedaEncode from control.veda_encode import VedaEncode
from VEDA.utils import get_config from VEDA.utils import get_config
class EmailAlert(): class EmailAlert(object):
"""
Send alert emails VIA AWS SES for Course About Video Statuses
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.auth_dict = get_config() self.auth_dict = get_config()
self.message = kwargs.get('message', None) self.message = kwargs.get('message', None)
...@@ -42,24 +40,9 @@ class EmailAlert(): ...@@ -42,24 +40,9 @@ class EmailAlert():
) )
class ErrorObject(object):
"""
Unspecified errors with a message
"""
@staticmethod
def print_error(message):
decorator = "***************E*R*R*O*R*******************"
outgoing = '\n%s \n\n%s \n\n%s\n' % (
NODE_COLORS_BLUE + decorator + NODE_COLORS_END,
message,
NODE_COLORS_BLUE + decorator + NODE_COLORS_END,
)
print outgoing
class Output(object): class Output(object):
""" """
Various reporting methods Display/Reporting methods
""" """
@staticmethod @staticmethod
def _seconds_from_string(duration): def _seconds_from_string(duration):
...@@ -109,7 +92,7 @@ class Output(object): ...@@ -109,7 +92,7 @@ class Output(object):
sys.stdout.flush() sys.stdout.flush()
class Report(): class Report(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.auth_dict = get_config() self.auth_dict = get_config()
...@@ -197,8 +180,12 @@ class Report(): ...@@ -197,8 +180,12 @@ class Report():
) )
class VideoProto(): class VideoProto(object):
"""
Video object abstraction,
intended as a record before object is recorded in DB
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.s3_filename = kwargs.get('s3_filename', None) self.s3_filename = kwargs.get('s3_filename', None)
self.client_title = kwargs.get('client_title', None) self.client_title = kwargs.get('client_title', None)
...@@ -217,8 +204,11 @@ class VideoProto(): ...@@ -217,8 +204,11 @@ class VideoProto():
self.val_id = kwargs.get('val_id', None) self.val_id = kwargs.get('val_id', None)
class Metadata(): class Metadata(object):
"""
Centralized video metadata probe
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.video_proto = kwargs.get('video_proto', None) self.video_proto = kwargs.get('video_proto', None)
self.video_object = kwargs.get( self.video_object = kwargs.get(
...@@ -270,11 +260,12 @@ class Metadata(): ...@@ -270,11 +260,12 @@ class Metadata():
self.video_proto.resolution = vid_breakout[3].strip() self.video_proto.resolution = vid_breakout[3].strip()
def _FAULT(self, video_object): def _FAULT(self, video_object):
if self.video_object is None:
return []
""" """
Is there anything to do with this? Find missing encodes
""" """
if self.video_object is None:
return []
# Check for object viability against prior findings
if video_object.video_trans_status == 'Corrupt File': if video_object.video_trans_status == 'Corrupt File':
return [] return []
...@@ -288,15 +279,13 @@ class Metadata(): ...@@ -288,15 +279,13 @@ class Metadata():
return [] return []
""" # Determine encodes
Finally, determine encodes encode = VedaEncode(
"""
E = VedaEncode(
course_object=video_object.inst_class, course_object=video_object.inst_class,
veda_id=video_object.edx_id veda_id=video_object.edx_id
) )
encode_list = E.determine_encodes() encode_list = encode.determine_encodes()
if encode_list is not None: if encode_list is not None:
if 'mobile_high' in encode_list: if 'mobile_high' in encode_list:
...@@ -321,26 +310,20 @@ class Metadata(): ...@@ -321,26 +310,20 @@ class Metadata():
) )
return [] return []
""" # get baseline // if there are == encodes and baseline,
get baseline // if there are == encodes and baseline, # mark file corrupt -- just run the query again with no veda_id.
mark file corrupt -- just run the query again with
no veda_id # kwarg override:
"""
"""
This overrides
"""
if self.freezing_bug is False and self.val_status != 'file_complete': if self.freezing_bug is False and self.val_status != 'file_complete':
self.val_status = 'transcode_queue' self.val_status = 'transcode_queue'
return encode_list return encode_list
E2 = VedaEncode( encode_two = VedaEncode(
course_object=video_object.inst_class, course_object=video_object.inst_class,
) )
E2.determine_encodes() encode_two.determine_encodes()
if len(E2.encode_list) == len(encode_list) and len(encode_list) > 1: if len(encode_two.encode_list) == len(encode_list) and len(encode_list) > 1:
""" # Mark File Corrupt, accounting for migrated legacy URLs
Mark File Corrupt, accounting for migrated URLs
"""
url_test = URL.objects.filter( url_test = URL.objects.filter(
videoID=Video.objects.filter( videoID=Video.objects.filter(
edx_id=video_object.edx_id edx_id=video_object.edx_id
......
control/veda_val.py
"""
Send data to VAL, either Video ID data or endpoint URLs
"""
import logging import logging
import os
import sys
import requests import requests
import ast import ast
import json import json
import datetime
import yaml
from VEDA.utils import get_config
requests.packages.urllib3.disable_warnings() from control_env import *
from control.veda_utils import Output, VideoProto
"""
Send data to VAL, either Video ID data or endpoint URLs
""" LOGGER = logging.getLogger(__name__)
requests.packages.urllib3.disable_warnings()
''' '''
"upload": _UPLOADING, "upload": _UPLOADING,
...@@ -31,13 +28,9 @@ Send data to VAL, either Video ID data or endpoint URLs ...@@ -31,13 +28,9 @@ Send data to VAL, either Video ID data or endpoint URLs
"imported": _IMPORTED, "imported": _IMPORTED,
''' '''
from control_env import *
from control.veda_utils import ErrorObject, Output
LOGGER = logging.getLogger(__name__)
class VALAPICall(): class VALAPICall(object):
def __init__(self, video_proto, val_status, **kwargs): def __init__(self, video_proto, val_status, **kwargs):
"""VAL Data""" """VAL Data"""
...@@ -63,16 +56,17 @@ class VALAPICall(): ...@@ -63,16 +56,17 @@ class VALAPICall():
self.auth_dict = kwargs.get('CONFIG_DATA', self._AUTH()) self.auth_dict = kwargs.get('CONFIG_DATA', self._AUTH())
def call(self): def call(self):
if self.auth_dict is None: if not self.auth_dict:
return None return None
""" """
Errors covered in other methods Errors covered in other methods
""" """
if self.val_token is None: if not self.val_token:
self.val_tokengen() self.val_tokengen()
if self.video_object is not None: if self.video_object:
self.send_object_data() self.send_object_data()
return
if self.video_proto is not None: if self.video_proto is not None:
self.send_val_data() self.send_val_data()
...@@ -93,10 +87,8 @@ class VALAPICall(): ...@@ -93,10 +87,8 @@ class VALAPICall():
r = requests.post(self.auth_dict['val_token_url'], data=payload, timeout=self.auth_dict['global_timeout']) r = requests.post(self.auth_dict['val_token_url'], data=payload, timeout=self.auth_dict['global_timeout'])
if r.status_code != 200: if r.status_code != 200:
ErrorObject.print_error( LOGGER.error('[API] : VAL Token generation')
message='Token Gen Fail: VAL\nCheck VAL Config' return
)
return None
self.val_token = ast.literal_eval(r.text)['access_token'] self.val_token = ast.literal_eval(r.text)['access_token']
self.headers = { self.headers = {
...@@ -109,10 +101,8 @@ class VALAPICall(): ...@@ -109,10 +101,8 @@ class VALAPICall():
Rather than rewrite the protocol to fit the veda models, Rather than rewrite the protocol to fit the veda models,
we'll shoehorn the model into the VideoProto model we'll shoehorn the model into the VideoProto model
""" """
class VideoProto():
platform_course_url = []
self.video_proto = VideoProto() self.video_proto = VideoProto()
self.video_proto.s3_filename = self.video_object.studio_id self.video_proto.s3_filename = self.video_object.studio_id
self.video_proto.veda_id = self.video_object.edx_id self.video_proto.veda_id = self.video_object.edx_id
self.video_proto.client_title = self.video_object.client_title self.video_proto.client_title = self.video_object.client_title
...@@ -144,7 +134,7 @@ class VALAPICall(): ...@@ -144,7 +134,7 @@ class VALAPICall():
## "PUT" for extant objects to video/id -- ## "PUT" for extant objects to video/id --
cannot send duplicate course records cannot send duplicate course records
''' '''
if self.val_token is None: if not self.val_token:
return False return False
if self.video_proto.s3_filename is None or \ if self.video_proto.s3_filename is None or \
...@@ -188,13 +178,12 @@ class VALAPICall(): ...@@ -188,13 +178,12 @@ class VALAPICall():
val_courses = [] val_courses = []
if self.val_status != 'invalid_token': if self.val_status != 'invalid_token':
for f in self.video_object.inst_class.local_storedir.split(','): for f in self.video_object.inst_class.local_storedir.split(','):
if f.strip() not in val_courses: if f.strip() not in val_courses and len(f.strip()) > 0:
val_courses.append({f.strip(): None}) val_courses.append({f.strip(): None})
if len(val_courses) == 0: for g in self.video_proto.platform_course_url:
for g in self.video_proto.platform_course_url: if g.strip() not in val_courses:
if g.strip() not in val_courses: val_courses.append({g.strip(): None})
val_courses.append({g.strip(): None})
self.val_data = { self.val_data = {
'client_video_id': self.video_proto.client_title, 'client_video_id': self.video_proto.client_title,
...@@ -213,10 +202,8 @@ class VALAPICall(): ...@@ -213,10 +202,8 @@ class VALAPICall():
) )
if r1.status_code != 200 and r1.status_code != 404: if r1.status_code != 200 and r1.status_code != 404:
ErrorObject.print_error( LOGGER.error('[API] : VAL Communication')
message='R1 : VAL Communication Fail: VAL\nCheck VAL Config' return
)
return None
if r1.status_code == 404: if r1.status_code == 404:
self.send_404() self.send_404()
...@@ -246,7 +233,7 @@ class VALAPICall(): ...@@ -246,7 +233,7 @@ class VALAPICall():
self.auth_dict['val_profile_dict'][self.encode_profile] self.auth_dict['val_profile_dict'][self.encode_profile]
except KeyError: except KeyError:
return return
if self.endpoint_url is not None: if self.endpoint_url:
for p in self.auth_dict['val_profile_dict'][self.encode_profile]: for p in self.auth_dict['val_profile_dict'][self.encode_profile]:
self.encode_data.append(dict( self.encode_data.append(dict(
...@@ -257,7 +244,7 @@ class VALAPICall(): ...@@ -257,7 +244,7 @@ class VALAPICall():
)) ))
test_list = [] test_list = []
if self.video_proto.veda_id is not None: if self.video_proto.veda_id:
url_query = URL.objects.filter( url_query = URL.objects.filter(
videoID=Video.objects.filter( videoID=Video.objects.filter(
edx_id=self.video_proto.veda_id edx_id=self.video_proto.veda_id
...@@ -327,13 +314,7 @@ class VALAPICall(): ...@@ -327,13 +314,7 @@ class VALAPICall():
) )
if r2.status_code > 299: if r2.status_code > 299:
ErrorObject.print_error( LOGGER.error('[API] : VAL POST/PUT {code}'.format(code=r2.status_code))
message='%s\n %s\n %s\n' % (
'R2 : VAL POST/PUT Fail: VAL',
'Check VAL Config',
r2.status_code
)
)
def send_200(self, val_api_return): def send_200(self, val_api_return):
""" """
...@@ -372,13 +353,13 @@ class VALAPICall(): ...@@ -372,13 +353,13 @@ class VALAPICall():
headers=self.headers, headers=self.headers,
timeout=self.auth_dict['global_timeout'] timeout=self.auth_dict['global_timeout']
) )
LOGGER.info('[VAL] : {id} : {status} : {code}'.format( LOGGER.info('[API] {id} : {status} sent to VAL {code}'.format(
id=self.video_proto.val_id, id=self.video_proto.val_id,
status=self.val_status, status=self.val_status,
code=r4.status_code) code=r4.status_code)
) )
if r4.status_code > 299: if r4.status_code > 299:
LOGGER.error('[VAL] : POST/PUT Fail : Check Config : {status}'.format(status=r4.status_code)) LOGGER.error('[API] : VAL POST/PUT : {status}'.format(status=r4.status_code))
def update_val_transcript(self, video_id, lang_code, name, transcript_format, provider): def update_val_transcript(self, video_id, lang_code, name, transcript_format, provider):
""" """
...@@ -404,7 +385,7 @@ class VALAPICall(): ...@@ -404,7 +385,7 @@ class VALAPICall():
if not response.ok: if not response.ok:
LOGGER.error( LOGGER.error(
'update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s', '[API] : VAL update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s',
video_id, video_id,
provider, provider,
response.status_code, response.status_code,
...@@ -432,16 +413,8 @@ class VALAPICall(): ...@@ -432,16 +413,8 @@ class VALAPICall():
if not response.ok: if not response.ok:
LOGGER.error( LOGGER.error(
'update_video_status failed -- video_id=%s -- status=%s -- text=%s', '[API] : VAL Update_video_status failed -- video_id=%s -- status=%s -- text=%s',
video_id, video_id,
response.status_code, response.status_code,
response.text response.text
) )
def main():
pass
if __name__ == '__main__':
sys.exit(main())
control/veda_video_validation.py
import os
import sys
import subprocess
import fnmatch
import django
from control.control_env import FFPROBE
from VEDA_OS01.models import Video
""" """
VEDA Intake/Product Final Testing Suite VEDA Intake/Product Final Testing Suite
...@@ -19,6 +9,18 @@ Mismatched Durations (within 5 sec) ...@@ -19,6 +9,18 @@ Mismatched Durations (within 5 sec)
""" """
import logging
import os
import subprocess
import sys
from control.control_env import FFPROBE
from VEDA_OS01.models import Video
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Validation(object): class Validation(object):
""" """
...@@ -39,60 +41,70 @@ class Validation(object): ...@@ -39,60 +41,70 @@ class Validation(object):
def validate(self): def validate(self):
""" """
Test #1 - assumes file is in 'work' directory of node Video validation probe
""" """
# Test #1
# Assumes file is in 'work' directory of node.
# Probe for metadata, ditch on common/found errors
ff_command = ' '.join(( ff_command = ' '.join((
FFPROBE, FFPROBE,
"\"" + self.videofile + "\"" "\"" + self.videofile + "\""
)) ))
"""
Test if size is zero
"""
if int(os.path.getsize(self.videofile)) == 0:
print 'Corrupt: Invalid' video_duration = None
if int(os.path.getsize(self.videofile)) == 0:
LOGGER.info('[VALIDATION] {id} : CORRUPT/File size is zero'.format(id=self.videofile))
return False return False
p = subprocess.Popen(ff_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) p = subprocess.Popen(ff_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
for line in iter(p.stdout.readline, b''): for line in iter(p.stdout.readline, b''):
if "Invalid data found when processing input" in line: if "Invalid data found when processing input" in line:
print 'Corrupt: Invalid' LOGGER.info('[VALIDATION] {id} : CORRUPT/Invalid data on input'.format(id=self.videofile))
return False return False
if "multiple edit list entries, a/v desync might occur, patch welcome" in line: if "multiple edit list entries, a/v desync might occur, patch welcome" in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Desync error'.format(id=self.videofile))
return False return False
if "command not found" in line: if "command not found" in line:
print line LOGGER.info('[VALIDATION] {id} : CORRUPT/Atypical file error'.format(id=self.videofile))
return False return False
if "Duration: " in line: if "Duration: " in line:
if "Duration: 00:00:00.0" in line: if "Duration: 00:00:00.0" in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration is zero'.format(id=self.videofile))
return False return False
elif "Duration: N/A, " in line: elif "Duration: N/A, " in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration N/A'.format(id=self.videofile))
return False return False
video_duration = line.split(',')[0][::-1].split(' ')[0][::-1] video_duration = line.split(',')[0][::-1].split(' ')[0][::-1]
try: if not video_duration:
str(video_duration) LOGGER.info('[VALIDATION] {id} : CORRUPT/No Duration'.format(id=self.videofile))
except: p.kill()
return False return False
p.kill() p.kill()
""" # Test #2
Compare Product to DB averages - pass within 5 sec # Compare Product to DB averages
# pass is durations within 5 sec or each other
"""
if self.mezzanine is True: if self.mezzanine is True:
# Return if original/source rawfile
LOGGER.info('[VALIDATION] {id} : VALID/Mezzanine file'.format(id=self.videofile))
return True return True
if self.veda_id is None: if self.veda_id is None:
print 'Error: Validation, encoded file no comparison ID' LOGGER.info('[VALIDATION] {id} : CORRUPT/Validation, No VEDA ID'.format(id=self.videofile))
return False return False
try: try:
video_query = Video.objects.filter(edx_id=self.veda_id).latest() video_query = Video.objects.filter(edx_id=self.veda_id).latest()
except: except:
LOGGER.info(
'[VALIDATION] {id} : CORRUPT/Validation, No recorded ID for comparison'.format(id=self.videofile)
)
return False return False
product_duration = float( product_duration = float(
...@@ -105,21 +117,10 @@ class Validation(object): ...@@ -105,21 +117,10 @@ class Validation(object):
duration=video_query.video_orig_duration duration=video_query.video_orig_duration
) )
) )
"""
Final Test
"""
if (data_duration - 5) <= product_duration <= (data_duration + 5): if (data_duration - 5) <= product_duration <= (data_duration + 5):
LOGGER.info('[VALIDATION] {id} : VALID'.format(id=self.videofile))
return True return True
else: else:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration mismatch'.format(id=self.videofile))
return False return False
def main():
pass
# V = Validation(videofile='/Users/ernst/VEDA_WORKING/fecf210f-0e94-4627-8ac3-46c2338e5897.mp4')
# print V.validate()
# # def __init__(self, videofile, **kwargs):
if __name__ == '__main__':
sys.exit(main())
frontend/abvid_validate.py
''' """
About Video Input and Validation About Video Input and Validation
'''
import os """
import sys
import datetime
import datetime
import logging
from frontend_env import * from frontend_env import *
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def create_record(upload_data): def create_record(upload_data):
""" """
...@@ -22,7 +25,6 @@ def create_record(upload_data): ...@@ -22,7 +25,6 @@ def create_record(upload_data):
file_valid=False, file_valid=False,
file_complete=False, file_complete=False,
) )
try: try:
ul1.save() ul1.save()
return True return True
...@@ -32,7 +34,7 @@ def create_record(upload_data): ...@@ -32,7 +34,7 @@ def create_record(upload_data):
def validate_incoming(upload_data): def validate_incoming(upload_data):
ul2 = VedaUpload.objects.filter( VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial'] video_serial=upload_data['abvid_serial']
).update( ).update(
upload_date=datetime.datetime.utcnow().replace(tzinfo=utc), upload_date=datetime.datetime.utcnow().replace(tzinfo=utc),
...@@ -42,26 +44,24 @@ def validate_incoming(upload_data): ...@@ -42,26 +44,24 @@ def validate_incoming(upload_data):
def send_to_pipeline(upload_data): def send_to_pipeline(upload_data):
VedaUpload.objects.filter(
ul3 = VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial'] video_serial=upload_data['abvid_serial']
).update( ).update(
file_valid=upload_data['success'], file_valid=upload_data['success'],
) )
if upload_data['success'] == 'true': if upload_data['success'] == 'true':
print 'Sending File to Pipeline' LOGGER.info('[ABOUT_VIDEO] {ul_id} : Sending File to Pipeline'.format(
ul_id=upload_data['abvid_serial']
else: ))
ul3 = VedaUpload.objects.filter( return True
video_serial=upload_data['abvid_serial']
).update(
comment='Failed Upload',
)
return False
if __name__ == '__main__': # Failed upload
upload_data = {} VedaUpload.objects.filter(
upload_data['abvid_serial'] = '19e1e1c78e' video_serial=upload_data['abvid_serial']
upload_data['success'] = 'true' ).update(
send_to_pipeline(upload_data) comment='Failed Upload',
)
LOGGER.info('[ABOUT_VIDEO] {ul_id} : Failed upload'.format(
ul_id=upload_data['abvid_serial']
))
return False
frontend/course_validate.py
''' """
Validate Course / Predict Inputs for advanced fields Validate Course / Predict Inputs for advanced fields
''' """
import os
import uuid import uuid
import json import json
import yaml import yaml
import datetime import datetime
from django.utils.timezone import utc
from frontend_env import * from frontend_env import *
"""
Import Django Shit
"""
class VEDACat(): class VEDACat(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
...@@ -290,12 +283,10 @@ class VEDACat(): ...@@ -290,12 +283,10 @@ class VEDACat():
return return_dict return return_dict
def simple_majority(self, attribute_list): def simple_majority(self, attribute_list):
''' """
Simple Majority Finder Simple Majority Finder
Dumbly just figures out the field attribute for 'most' of them Field attribute for 'most' of them (>50.0%) per attrib
"""
'''
comparitor = {'None': 0} comparitor = {'None': 0}
for a in attribute_list: for a in attribute_list:
in_it = False in_it = False
...@@ -321,13 +312,3 @@ class VEDACat(): ...@@ -321,13 +312,3 @@ class VEDACat():
data['majority'] = False data['majority'] = False
return data return data
###############
def main():
V = VEDACat()
print V.veda_model
if __name__ == '__main__':
sys.exit(main())
instance_config.yaml
...@@ -2,88 +2,74 @@ ...@@ -2,88 +2,74 @@
# --- # ---
# Database information # Database information
# --- # ---
# SANDBOX
#DATABASES:
# default:
# ENGINE: django.db.backends.sqlite3
# NAME: sandbox.db
## PRODUCTION
DATABASES: DATABASES:
default: default:
ENGINE: 'django.db.backends.mysql' ENGINE: django.db.backends.sqlite3
NAME: 'pipeline' NAME: sandbox.db
USER: 'pipeline001'
PASSWORD: 'password' SECRET_KEY: "a_random_string"
HOST: 'localhost'
PORT: 3306 FERNET_KEYS: ["a_random_string"]
SECRET_KEY: "" debug: True
# Fernet keys # Frontend S3 Auth
FERNET_KEYS: [] veda_secret_access_key: A_RANDOM_STRING
veda_access_key_id: A_RANDOM_STRING
# JWT AUTH settings # JWT AUTH settings
JWT_AUTH: JWT_AUTH:
JWT_SECRET_KEY: JWT_SECRET_KEY: A_RANDOM_STRING
JWT_ISSUER: JWT_ISSUER:
JWT_AUDIENCE: JWT_AUDIENCE: A_RANDOM_STRING
JWT_VERIFY_AUDIENCE: JWT_VERIFY_AUDIENCE: true
# --- # ---
# AWS Buckets, Prefixes # AWS
# --- # ---
# Studio/Platform #veda_s3_upload_bucket:
edx_s3_ingest_prefix: #veda_s3_hotstore_bucket:
edx_s3_ingest_bucket: #veda_deliverable_bucket:
edx_s3_endpoint_bucket: #veda_upload_bucket:
#edx_s3_ingest_prefix:
# CF #edx_s3_ingest_bucket:
edx_cloudfront_prefix: #edx_s3_endpoint_bucket:
#edx_cloudfront_prefix:
# Images #aws_video_images_bucket:
aws_video_images_bucket: #aws_video_images_prefix:
aws_video_images_prefix: "video-images/"
s3_base_url:
# VEDA Internal
veda_s3_upload_bucket:
veda_s3_hotstore_bucket:
veda_deliverable_bucket:
# Settings
veda_base_url: veda_base_url:
s3_base_url: https://s3.amazonaws.com
# Transcripts
aws_video_transcripts_bucket: aws_video_transcripts_bucket:
aws_video_transcripts_prefix: video-transcripts/ aws_video_transcripts_prefix: video-transcripts/
# cielo24 api urls
cielo24_api_base_url: https://sandbox.cielo24.com/api
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
# a token identifying a valid request from transcript provider # a token identifying a valid request from transcript provider
transcript_provider_request_token: testtoken transcript_provider_request_token:
# Ingest Secret
# TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.
veda_secret_access_key:
veda_access_key_id:
# --- # ---
# email vars # email vars
# --- # ---
veda_noreply_email: veda_noreply_email: admin@example.com
admin_email: admin_email: admin@example.com
lms_base_url:
instance_prefix: ''
# --- # ---
# VEDA API # VAL user creds
# --- # ---
val_token_url:
val_api_url:
val_video_images_url:
val_transcript_create_url:
val_video_transcript_status_url:
val_client_id:
val_secret_key:
val_username: admin@example.com
val_password:
## VEDA API Auth ## VEDA API Auth
veda_api_url: veda_api_url:
veda_auth_url: veda_auth_url:
...@@ -92,24 +78,21 @@ veda_secret_key: ...@@ -92,24 +78,21 @@ veda_secret_key:
veda_token_url: veda_token_url:
# --- # ---
# VAL # Celery Info
# --- # ---
val_api_url: celery_app_name:
val_token_url: # can do multiple queues like so: foo,bar,baz
val_video_images_url: celery_worker_queue:
# Credentials celery_deliver_queue:
val_client_id: 'clientkey' celery_heal_queue:
val_secret_key: 'secretkey' celery_threads: 1
val_password:
val_username:
val_transcript_create_url:
val_video_transcript_status_url:
# Celery Worker Config Information
rabbitmq_broker: rabbitmq_broker:
rabbitmq_pass: rabbitmq_pass:
rabbitmq_user: rabbitmq_user:
onsite_worker: False
# --- # ---
# Shotgun Variables (internal mediateam) # Shotgun Variables (internal mediateam)
# --- # ---
...@@ -117,10 +100,4 @@ sg_server_path: ...@@ -117,10 +100,4 @@ sg_server_path:
sg_script_name: sg_script_name:
sg_script_key: sg_script_key:
# --- ...
# Endpoints
# ---
threeplay_ftphost:
lms_base_url: ''
instance_prefix: ''
scripts/celery_cancel_task.py
...@@ -15,4 +15,4 @@ tasks_to_cancel = [ ...@@ -15,4 +15,4 @@ tasks_to_cancel = [
] ]
for t in tasks_to_cancel: for t in tasks_to_cancel:
revoke(t, terminate=True) revoke(t, terminate=True)
\ No newline at end of file
scripts/reencode_crawler.py
...@@ -11,9 +11,7 @@ if project_path not in sys.path: ...@@ -11,9 +11,7 @@ if project_path not in sys.path:
from control.veda_heal import VedaHeal from control.veda_heal import VedaHeal
from VEDA_OS01.models import URL, Encode, Video from VEDA_OS01.models import URL, Encode, Video
""" # Set globals
Set your globals here
"""
CRAWL_SUFFIX = 'HLS' CRAWL_SUFFIX = 'HLS'
CRAWL_START = datetime.datetime.strptime('Feb 1 2017 00:01', '%b %d %Y %H:%M') CRAWL_START = datetime.datetime.strptime('Feb 1 2017 00:01', '%b %d %Y %H:%M')
CRAWL_EPOCH = datetime.datetime.strptime('Apr 12 2017 00:01', '%b %d %Y %H:%M') CRAWL_EPOCH = datetime.datetime.strptime('Apr 12 2017 00:01', '%b %d %Y %H:%M')
...@@ -22,9 +20,7 @@ BATCH_SIZE = 10 ...@@ -22,9 +20,7 @@ BATCH_SIZE = 10
SLEEP_TIME = 1200 SLEEP_TIME = 1200
# NOTE: MSXSBVCP2017-V002600_100 class ReEncodeCrawler(object):
class ReEncodeCrawler:
def __init__(self): def __init__(self):
self.crawl_start = CRAWL_START self.crawl_start = CRAWL_START
...@@ -78,13 +74,3 @@ def main(): ...@@ -78,13 +74,3 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())
'''
Manual HEAL IDs
GEOB3DAE2017-V006300
'''
static_config.yaml
...@@ -24,6 +24,17 @@ ffmpeg_compiled: "ffmpeg" ...@@ -24,6 +24,17 @@ ffmpeg_compiled: "ffmpeg"
ffprobe_compiled: "ffprobe" ffprobe_compiled: "ffprobe"
target_aspect_ratio: 1.7777778 target_aspect_ratio: 1.7777778
# ---
# Endpoints
# ---
# cielo24 api urls
cielo24_api_base_url: 'https://sandbox.cielo24.com/api'
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
threeplay_ftphost: ftp.3playmedia.com
# This is a list of encodes and their respective course # This is a list of encodes and their respective course
# boolean matches # boolean matches
encode_dict: encode_dict:
......
test_requirements.txt
...@@ -14,4 +14,4 @@ pytest-django-ordering==1.0.1 ...@@ -14,4 +14,4 @@ pytest-django-ordering==1.0.1
pylint==1.7.1 pylint==1.7.1
pylint-celery==0.3 pylint-celery==0.3
pylint-django==0.7.2 pylint-django==0.7.2
responses==0.6.1 responses==0.6.1
\ No newline at end of file
youtube_callback/sftp_id_retrieve.py
...@@ -17,11 +17,10 @@ import django ...@@ -17,11 +17,10 @@ import django
import pysftp import pysftp
from django.utils.timezone import utc from django.utils.timezone import utc
from control.veda_utils import ErrorObject, Metadata, VideoProto from control.veda_utils import Metadata, VideoProto
from control.veda_val import VALAPICall from control.veda_val import VALAPICall
from frontend.abvid_reporting import report_status from frontend.abvid_reporting import report_status
from VEDA_OS01.models import URL, Encode, Video from VEDA_OS01.models import URL, Encode, Video
from youtube_callback.daemon import get_course
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path: if project_path not in sys.path:
...@@ -30,23 +29,20 @@ if project_path not in sys.path: ...@@ -30,23 +29,20 @@ if project_path not in sys.path:
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
django.setup() django.setup()
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
"""
Defaults:
"""
homedir = expanduser("~") homedir = expanduser("~")
workdir = os.path.join(homedir, 'download_data_holding') workdir = os.path.join(homedir, 'download_data_holding')
YOUTUBE_LOOKBACK_DAYS = 4 YOUTUBE_LOOKBACK_DAYS = 4
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def callfunction(course): def callfunction(course):
""" """
:param course: :param course:
:return:
""" """
if os.path.exists(workdir): if os.path.exists(workdir):
shutil.rmtree(workdir) shutil.rmtree(workdir)
...@@ -55,11 +51,14 @@ def callfunction(course): ...@@ -55,11 +51,14 @@ def callfunction(course):
xml_downloader(course) xml_downloader(course)
for file in os.listdir(workdir): for file in os.listdir(workdir):
print file
upload_data = domxml_parser(file) upload_data = domxml_parser(file)
if upload_data is not None: if upload_data is not None:
print upload_data LOGGER.info('[YOUTUBE_CALLBACK] : {inst}{clss} {upload_data}'.format(
inst=course.institution,
clss=course.edx_classid,
upload_data=upload_data
))
urlpatch(upload_data) urlpatch(upload_data)
...@@ -90,11 +89,20 @@ def xml_downloader(course): ...@@ -90,11 +89,20 @@ def xml_downloader(course):
for d in s1.listdir_attr(): for d in s1.listdir_attr():
crawl_sftp(d=d, s1=s1) crawl_sftp(d=d, s1=s1)
except AuthenticationException: except AuthenticationException:
LOGGER.info("{inst}{clss} : Authentication Failed".format(inst=course.institution, clss=course.edx_classid)) LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : Authentication Failed".format(
inst=course.institution,
clss=course.edx_classid
))
except SSHException: except SSHException:
LOGGER.info("{inst}{clss} : Authentication Failed".format(inst=course.institution, clss=course.edx_classid)) LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : Authentication Failed".format(
inst=course.institution,
clss=course.edx_classid
))
except IOError: except IOError:
LOGGER.info("{inst}{clss} : List Dir Failed".format(inst=course.institution, clss=course.edx_classid)) LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : List Dir Failed".format(
inst=course.institution,
clss=course.edx_classid
))
def crawl_sftp(d, s1): def crawl_sftp(d, s1):
...@@ -139,7 +147,6 @@ def crawl_sftp(d, s1): ...@@ -139,7 +147,6 @@ def crawl_sftp(d, s1):
x += 1 x += 1
else: else:
break break
print "%s : %s" % (f.filename, file_to_find)
s1.get( s1.get(
f.filename, f.filename,
os.path.join(workdir, file_to_find) os.path.join(workdir, file_to_find)
...@@ -283,7 +290,6 @@ def urlpatch(upload_data): ...@@ -283,7 +290,6 @@ def urlpatch(upload_data):
bitrate='0', bitrate='0',
s3_filename=test_id.studio_id s3_filename=test_id.studio_id
) )
print test_id.video_orig_duration
VF = Metadata( VF = Metadata(
video_object=test_id video_object=test_id
) )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment