Skip to content

E
edx-video-pipeline
Commit bac906f7 by Gregory Martin
Options

Update Logging, standardize formatting.

parent 249d340e
Showing with 450 additions and 569 deletions
VEDA/urls.py
......@@ -4,9 +4,9 @@ import os
sys.path.append(os.path.abspath(__file__))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
# pep8: disable=E402
from django.conf import settings
from rest_framework import routers
# from rest_framework.routers import DefaultRouter
from django.conf.urls import patterns, include, url
from django.contrib import admin
......
VEDA_OS01/fixtures/destinations.json
......@@ -56,15 +56,15 @@
"fields": {
"course_name": "Veda Sandbox Test Course",
"course_hold": true,
"institution": "XXX",
"edx_classid": "XXXXX",
"institution": "EDX",
"edx_classid": "DEMOX",
"semesterid": "2017",
"yt_proc": false,
"tp_proc": false,
"c24_proc": false,
"s3_proc": true,
"local_storedir": "course-v1:VEDA+VEDA201+2015_T1",
"studio_hex": "xxxx"
"studio_hex": "shared_course_token"
}
}
]
VEDA_OS01/fixtures/encodes.json
......@@ -160,5 +160,18 @@
"encode_resolution": "720",
"product_spec": "desktop_webm"
}
},
{
"model": "VEDA_OS01.course",
"pk": 1,
"fields": {
"course_name": "Demo Course",
"institution": "EDX",
"edx_classid": "DEMOX",
"semesterid": 2018,
"yt_proc": false,
"local_storedir": "this/is/an/exemplar",
"studio_hex": "shared_course_token"
}
}
]
bin/deliver
......@@ -9,7 +9,7 @@ if project_path not in sys.path:
sys.path.append(project_path)
class DeliverCli:
class DeliverCli(object):
"""
Deliver
......
bin/heal
......@@ -4,11 +4,12 @@ Deliver
Command Line Interface
"""
import os
import sys
import argparse
import datetime
from datetime import timedelta
import logging
import os
import sys
import pytz
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
......@@ -21,8 +22,12 @@ from VEDA_OS01.models import Course, Video
from VEDA_OS01.transcripts import retrieve_three_play_translations
from VEDA.utils import get_config
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class HealCli:
class HealCli(object):
def __init__(self, **kwargs):
self.logging = kwargs.get('logging', True)
......@@ -93,8 +98,8 @@ def main():
retrieve_three_play_translations()
return
print '%s - %s: %s' % ('Healing', 'VEDA ID', veda_id)
print '%s - %s: %s' % ('Healing', 'Course', course_id)
LOGGER.info('%s - %s: %s' % ('Healing', 'VEDA ID', veda_id))
LOGGER.info('%s - %s: %s' % ('Healing', 'Course', course_id))
if veda_id is None and course_id is None and schedule is False:
VH = VedaHeal()
......
bin/ingest
#!/usr/bin/env python
"""
Ingest
Command Line Interface
"""
import os
import sys
import argparse
......@@ -9,13 +15,8 @@ if project_path not in sys.path:
from control.veda_utils import EmailAlert
"""
Ingest
Command Line Interface
"""
class IngestCli():
class IngestCli(object):
def __init__(self, **kwargs):
self.args = None
......
bin/loop.py
#!/usr/bin/env python
"""
This is a cheapo way to get a pager (using SES)
"""
import os
import sys
import argparse
import logging
from django.db import reset_queries
import resource
import time
......@@ -13,11 +18,6 @@ project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
sys.path.append(project_path)
"""
This is a cheapo way to get a pager (using SES)
"""
import django
django.setup()
......@@ -25,8 +25,12 @@ from control.veda_file_discovery import FileDiscovery
from youtube_callback.daemon import generate_course_list
from youtube_callback.sftp_id_retrieve import callfunction
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class DaemonCli:
class DaemonCli(object):
def __init__(self):
self.args = None
......@@ -80,7 +84,7 @@ class DaemonCli:
reset_queries()
x += 1
if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
x = 0
def youtube_daemon(self):
......@@ -88,12 +92,12 @@ class DaemonCli:
while True:
self.course_list = generate_course_list()
for course in self.course_list:
print "%s%s: Callback" % (course.institution, course.edx_classid)
LOGGER.info('%s%s: Callback' % (course.institution, course.edx_classid))
callfunction(course)
x += 1
if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
x = 0
reset_queries()
......
bin/youtubecallback
......@@ -18,7 +18,7 @@ Command Line Interface
"""
class YoutubeCallbackCli():
class YoutubeCallbackCli(object):
def __init__(self, **kwargs):
self.args = None
......@@ -54,14 +54,10 @@ class YoutubeCallbackCli():
def _parse_args(self):
self.course_id = self.args.courseid
self.list = self.args.list
def run(self):
if self.list is True:
self.listcourses()
else:
self.loop()
self.loop()
def loop(self):
"""
......@@ -82,16 +78,6 @@ class YoutubeCallbackCli():
E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
E1.email()
def listcourses(self):
"""
list and exit
:return:
"""
self.course_list = generate_course_list()
for course in self.course_list:
print course.institution
print course.edx_classid
def main():
YTCC = YoutubeCallbackCli()
......
control/celeryapp.py
......@@ -4,15 +4,20 @@ Start Celery Worker
from __future__ import absolute_import
import os
from celery import Celery
from VEDA.utils import get_config
import logging
import os
import sys
from VEDA.utils import get_config
try:
from control.veda_deliver import VedaDelivery
except ImportError:
from veda_deliver import VedaDelivery
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
auth_dict = get_config()
......@@ -42,7 +47,7 @@ app.conf.update(
@app.task(name='worker_encode')
def worker_task_fire(veda_id, encode_profile, jobid):
print '[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile)
LOGGER.info('[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile))
return 1
......
control/old_veda_deliver_cielo.py
import requests
import ast
import urllib
"""
Cielo24 API Job Start and Download
Options (reflected in Course.models):
......@@ -14,10 +10,17 @@ priority =
priority (48h)
turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently
"""
import logging
import requests
import ast
import urllib
from control_env import *
from veda_utils import ErrorObject
requests.packages.urllib3.disable_warnings()
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Cielo24TranscriptOld(object):
......@@ -77,11 +80,9 @@ class Cielo24TranscriptOld(object):
videoID=video_query,
encode_url__icontains='_DTH.mp4',
).latest()
if video_query.inst_class.c24_username is None:
ErrorObject.print_error(
message='Cielo24 Record Incomplete',
)
LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API : Course record incomplete'.format(id=self.veda_id))
return None
c24_defaults = {
......@@ -102,10 +103,8 @@ class Cielo24TranscriptOld(object):
# Generate Token
r1 = requests.get(token_url)
if r1.status_code > 299:
ErrorObject.print_error(
message='Cielo24 API Access Error',
)
return None
LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API access'.format(id=self.veda_id))
return
api_token = ast.literal_eval(r1.text)["ApiToken"]
return api_token
......@@ -161,7 +160,6 @@ class Cielo24TranscriptOld(object):
urllib.quote_plus(self.c24_defaults['url'])
))
)
print str(r4.status_code) + ' : Cielo24 Status Code'
return ast.literal_eval(r4.text)['TaskId']
......
control/tests/test_encode.py
......@@ -81,30 +81,3 @@ class TestEncode(TestCase):
).delete()
encode_list = self.E.determine_encodes()
self.assertTrue(len(encode_list) == baseline)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
'''
Save for poss future test
# import celeryapp
# co = Course.objects.get(institution='XXX', edx_classid='C93BC')
# vid = 'XXXC93BC2016-V003500'
# v = VedaEncode(course_object=co, veda_id=vid)
# encode_list = v.determine_encodes()
# for e in encode_list:
# veda_id = vid
# encode_profile = e
# jobid = uuid.uuid1().hex[0:10]
# # celeryapp.worker_task_fire.apply_async(
# # (veda_id, encode_profile, jobid),
# # queue='encode_worker'
# # )
'''
control/tests/test_file_discovery.py
......@@ -170,22 +170,6 @@ class TestFileDiscovery(TestCase):
self.assertTrue(mock_validate_and_feed_to_ingest.called)
@ddt.data(
('veda/working', '[File Ingest] S3 Ingest Connection Failure'),
(None, '[File Ingest] No Working Node directory')
)
@ddt.unpack
@patch('control.veda_file_discovery.ErrorObject.print_error')
@patch('boto.s3.connection.S3Connection')
def test_discover_studio_ingested_video_exceptions(self, work_dir, error_message, mocked_s3_conn, mock_error):
"""
Tests 'FileDiscovery.discover_studio_ingested_videos' exception cases.
"""
mocked_s3_conn.side_effect = S3ResponseError('Error', 'Timeout')
file_discovery_instance = FileDiscovery(node_work_directory=work_dir)
file_discovery_instance.discover_studio_ingested_videos()
mock_error.assert_called_with(message=error_message)
@ddt.data(
(None, 'invalid_course_key'),
('non-existent-hex', None)
)
......
control/veda_deliver.py
......@@ -27,7 +27,7 @@ from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus)
from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
from veda_utils import ErrorObject, Metadata, Output, VideoProto
from veda_utils import Metadata, Output, VideoProto
from veda_val import VALAPICall
from veda_video_validation import Validation
......@@ -49,7 +49,7 @@ boto.config.set('Boto', 'http_socket_timeout', '100')
homedir = expanduser("~")
class VedaDelivery:
class VedaDelivery(object):
def __init__(self, veda_id, encode_profile, **kwargs):
self.veda_id = veda_id
......@@ -71,7 +71,7 @@ class VedaDelivery:
Check the destination, route via available methods,
throw error if method is not extant
"""
LOGGER.info('[VIDEO_DELIVER] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile))
LOGGER.info('[DELIVERY] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile))
if self.encode_profile == 'hls':
# HLS encodes are a pass through
self.hls_run()
......@@ -137,7 +137,7 @@ class VedaDelivery:
self.status = self._DETERMINE_STATUS()
self._UPDATE_DATA()
self._CLEANUP()
LOGGER.info('[DELIVERY] {video_id} : complete'.format(video_id=self.veda_id))
# We only want to generate transcripts when all the encodings(except for YT and Review) are done.
if utils.is_video_ready(self.video_query.edx_id, ignore_encodes=['review', 'youtube']):
self.start_transcription()
......@@ -217,14 +217,14 @@ class VedaDelivery:
conn = S3Connection()
bucket = conn.get_bucket(self.auth_dict['veda_deliverable_bucket'])
except NoAuthHandlerFound:
LOGGER.error('[VIDEO_DELIVER] BOTO/S3 Communication error')
LOGGER.error('[DELIVERY] {url} : BOTO/S3 Communication error'.format(url=self.hotstore_url))
return
except S3ResponseError:
LOGGER.error('[VIDEO_DELIVER] Invalid Storage Bucket')
LOGGER.error('[DELIVERY] {url} : Invalid Storage Bucket'.format(url=self.hotstore_url))
return
source_key = bucket.get_key(self.encoded_file)
if source_key is None:
LOGGER.error('[VIDEO_DELIVER] S3 Intake Object NOT FOUND')
LOGGER.error('[DELIVERY] {url} : S3 Intake Object not found'.format(url=self.hotstore_url))
return
source_key.get_contents_to_filename(
......@@ -245,7 +245,7 @@ class VedaDelivery:
VM._METADATA()
if not isinstance(self.video_proto.duration, int) and ':' not in self.video_proto.duration:
print 'Duration Failure'
LOGGER.error('[DELIVERY] {id} : Duration Failure'.format(id=self.video_proto.veda_id))
return
self.video_proto.duration = Output._seconds_from_string(
......@@ -358,8 +358,8 @@ class VedaDelivery:
self.encoded_file
)
):
print 'WARNING -- NO FILE'
return None
LOGGER.error('[DELIVERY] {file} : No file for routing'.format(file=self.encoded_file))
return
'''
Destination Nicks:
S31
......@@ -382,10 +382,8 @@ class VedaDelivery:
"""
Throw error
"""
ErrorObject.print_error(
message='Deliverable - No Method',
)
return None
LOGGER.error('[DELIVERY] No method')
return
def AWS_UPLOAD(self):
"""
......@@ -426,10 +424,7 @@ class VedaDelivery:
try:
conn = boto.connect_s3()
except S3ResponseError:
ErrorObject.print_error(
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
LOGGER.error('[DELIVERY] s3 Connection Error')
return False
delv_bucket = conn.get_bucket(
self.auth_dict['edx_s3_endpoint_bucket']
......@@ -481,17 +476,11 @@ class VedaDelivery:
try:
c = boto.connect_s3()
except S3ResponseError:
ErrorObject.print_error(
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
LOGGER.error('[DELIVERY] s3 Connection Error')
return False
b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket'])
if b is None:
ErrorObject.print_error(
message='Deliverable Fail: s3 Connection Error\n \
Check node_config DELIVERY_ENDPOINT'
)
LOGGER.error('[DELIVERY] s3 Connection Error')
return False
"""
......@@ -540,7 +529,7 @@ class VedaDelivery:
try:
api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key
except TranscriptCredentials.DoesNotExist:
LOGGER.warn('[cielo24] Unable to find api_key for org=%s', org)
LOGGER.warn('[DELIVERY] Unable to find cielo24 api_key for org=%s', org)
return None
s3_video_url = build_url(
......@@ -630,7 +619,7 @@ class VedaDelivery:
except TranscriptCredentials.DoesNotExist:
LOGGER.warning(
'Transcript preference is not found for provider=%s, video=%s',
'[DELIVERY] : Transcript preference is not found for provider=%s, video=%s',
self.video_query.provider,
self.video_query.studio_id,
)
......@@ -647,13 +636,11 @@ class VedaDelivery:
if self.encode_profile != 'desktop_mp4':
return None
C24 = Cielo24TranscriptOld(
cielojob = Cielo24TranscriptOld(
veda_id=self.video_query.edx_id
)
output = C24.perform_transcription()
print '[ %s ] : %s' % (
'Cielo24 JOB', self.video_query.edx_id
)
cielojob.perform_transcription()
LOGGER.info('[DELIVERY] {id} : Cielo24 job sent '.format(id=self.video_query.edx_id))
def _THREEPLAY_UPLOAD(self):
"""
......@@ -675,9 +662,7 @@ class VedaDelivery:
try:
ftp1.login(user, passwd)
except:
ErrorObject.print_error(
message='3Play Authentication Failure'
)
LOGGER.error('[DELIVERY] {file} : 3Play Authentication Failure'.format(file=self.encoded_file))
try:
ftp1.cwd(
self.video_query.inst_class.tp_speed
......@@ -704,8 +689,10 @@ class VedaDelivery:
def YOUTUBE_SFTP(self, review=False):
if self.video_query.inst_class.yt_proc is False:
if self.video_query.inst_class.review_proc is False:
print 'NO YOUTUBE'
return None
LOGGER.error(
'[DELIVERY] {id} : Youtube called, youtube processing off'.format(id=self.video_proto.veda_id)
)
return
DY = DeliverYoutube(
veda_id=self.video_query.edx_id,
......
control/veda_deliver_youtube.py
import os
import os.path
import sys
import time
import pysftp
"""
Youtube Dynamic Upload
Note: This represents early VEDA work, but is functional
Note: This is early VEDA work, but is functional. Ideally deprecated in favor of a no-youtube workflow, this code
is only maintained, and not prioritized for refactoring
"""
import logging
import os.path
import pysftp
import paramiko
import time
from control_env import *
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def printTotals(transferred, toBeTransferred):
"""
Optional upload logging method
"""
'''
try:
sys.stdout.write('\r')
sys.stdout.write("Transferred: {0}\tOut of: {1}\r".format(transferred, toBeTransferred))
sys.stdout.flush()
except:
print 'Callback Failing'
"""
return None
LOGGER.error('Callback Failing')
'''
return
class DeliverYoutube():
class DeliverYoutube(object):
def __init__(self, veda_id, encode_profile):
self.veda_id = veda_id
......@@ -120,7 +129,7 @@ class DeliverYoutube():
'playlist_id',
'require_paid_subscription'
]
print "%s : %s" % ("Generate CSV", str(self.video.edx_id))
LOGGER.info('[YOUTUBE] {id} : Generating sidecar metadata CSV'.format(id=str(self.video.edx_id)))
'''
# TODO: Refactor this into centrally located util for escaping bad chars
if self.video.client_title is not None:
......@@ -142,7 +151,6 @@ class DeliverYoutube():
This is where we can add or subtract file attributes as needed
"""
print self.file
metadata_dict = {
'filename': self.file,
'channel': self.course.yt_channel,
......@@ -154,7 +162,7 @@ class DeliverYoutube():
# Header Row
output = ','.join(([c for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES])) + '\n'
# Data Row
output += ','.join(([metadata_dict.get(c, '') for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES])) # + '\n' <--NO
output += ','.join(([metadata_dict.get(c, '') for c in YOUTUBE_DEFAULT_CSV_COLUMNNAMES]))
with open(os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'), 'w') as c1:
c1.write('%s %s' % (output, '\n'))
......@@ -190,50 +198,42 @@ class DeliverYoutube():
d1.write('')
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
with pysftp.Connection(
'partnerupload.google.com',
username=self.course.yt_logon,
private_key=self.youtubekey,
port=19321,
cnopts=cnopts
) as s1:
print "Go for YT : " + str(self.video.edx_id)
s1.mkdir(remote_directory, mode=660)
s1.cwd(remote_directory)
s1.put(
os.path.join(WORK_DIRECTORY, self.file),
callback=printTotals
)
print
s1.put(
os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'),
callback=printTotals
)
print
s1.put(
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'youtube_callback',
'static_files',
'delivery.complete'
),
callback=printTotals,
confirm=False,
preserve_mtime=False
)
print
try:
with pysftp.Connection(
'partnerupload.google.com',
username=self.course.yt_logon,
private_key=self.youtubekey,
port=19321,
cnopts=cnopts
) as s1:
LOGGER.info('[YOUTUBE] {id} : Ready for youtube SFTP upload'.format(id=str(self.video.edx_id)))
# Upload file, sidecar metadata,
# and (google required) empty delivery.complete file
s1.mkdir(remote_directory, mode=660)
s1.cwd(remote_directory)
s1.put(
os.path.join(WORK_DIRECTORY, self.file),
callback=printTotals
)
s1.put(
os.path.join(WORK_DIRECTORY, self.video.edx_id + '_100.csv'),
callback=printTotals
)
s1.put(
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'youtube_callback',
'static_files',
'delivery.complete'
),
callback=printTotals,
confirm=False,
preserve_mtime=False
)
except paramiko.ssh_exception.AuthenticationException:
LOGGER.info('[YOUTUBE] {file} : Paramiko Authentication Exception'.format(file=str(self.file)))
os.remove(os.path.join(
WORK_DIRECTORY,
self.video.edx_id + '_100.csv'
))
def main():
pass
if __name__ == "__main__":
sys.exit(main())
control/veda_encode.py
"""
Get a list of needed encodes from VEDA
import os
import sys
import uuid
* Protected against extant URLs *
import django
"""
from control_env import *
from dependencies.shotgun_api3 import Shotgun
from dependencies.shotgun_api3.lib.xmlrpclib import ProtocolError
from VEDA.utils import get_config
"""
Get a list of needed encodes from VEDA
* Protected against extant URLs *
"""
class VedaEncode(object):
......
control/veda_file_discovery.py
......@@ -22,7 +22,6 @@ from control_env import *
from VEDA.utils import extract_course_org, get_config
from veda_file_ingest import VedaIngest, VideoProto
from VEDA_OS01.models import TranscriptCredentials
from veda_utils import ErrorObject
from veda_val import VALAPICall
try:
......@@ -31,6 +30,8 @@ except:
pass
boto.config.set('Boto', 'http_socket_timeout', '100')
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
LOGGER = logging.getLogger(__name__)
......@@ -47,12 +48,12 @@ class FileDiscovery(object):
Crawl VEDA Upload bucket
"""
if self.node_work_directory is None:
print '[Discovery Error] No Workdir'
LOGGER.error('[DISCOVERY] No Workdir')
return
try:
conn = boto.connect_s3()
except NoAuthHandlerFound:
print '[Discovery Error] BOTO Auth Handler'
LOGGER.error('[DISCOVERY] BOTO Auth Handler')
return
try:
self.bucket = conn.get_bucket(self.auth_dict['veda_s3_upload_bucket'])
......@@ -211,7 +212,7 @@ class FileDiscovery(object):
key.get_contents_to_filename(os.path.join(self.node_work_directory, file_name))
file_ingested = True
except S3DataError:
LOGGER.exception('[File Ingest] Error downloading the file into node working directory.')
LOGGER.error('[DISCOVERY] Error downloading the file into node working directory.')
return file_ingested
def parse_transcript_preferences(self, course_id, transcript_preferences):
......@@ -233,7 +234,7 @@ class FileDiscovery(object):
# have associated 3rd party transcription provider API keys.
transcript_preferences = None
except ValueError:
LOGGER.exception('[File Discovery] Invalid transcripts preferences=%s', transcript_preferences)
LOGGER.error('[DISCOVERY] Invalid transcripts preferences=%s', transcript_preferences)
transcript_preferences = None
return transcript_preferences
......@@ -250,11 +251,12 @@ class FileDiscovery(object):
if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
except S3ResponseError:
ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
LOGGER.error('[DISCOVERY] S3 Ingest Connection Failure')
except NoAuthHandlerFound:
ErrorObject.print_error(message='[Discovery Error] BOTO Auth Handler')
LOGGER.error('[DISCOVERY] BOTO Auth Handler')
else:
ErrorObject.print_error(message='[File Ingest] No Working Node directory')
LOGGER.error('[DISCOVERY] No Working Node directory')
def validate_metadata_and_feed_to_ingest(self, video_s3_key):
"""
......
control/veda_file_ingest.py
......@@ -2,7 +2,6 @@
Discovered file ingest/insert/job triggering
"""
import datetime
import logging
import subprocess
......@@ -82,7 +81,10 @@ class VedaIngest(object):
self.val_insert()
self.rename()
self.archived = self.store()
LOGGER.info('[INGEST] {studio_id} | {video_id} : Video in hot store'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
if self.video_proto.valid is False:
self.abvid_report()
self.complete = True
......@@ -90,7 +92,8 @@ class VedaIngest(object):
os.remove(self.full_filename)
return None
LOGGER.info('[VIDEO_INGEST : Ingested] {video_id} : {datetime}'.format(
LOGGER.info('[INGEST] {studio_id} | {video_id} : Ingested {datetime}'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id,
datetime=str(datetime.datetime.utcnow()))
)
......@@ -179,7 +182,12 @@ class VedaIngest(object):
))
if not os.path.exists(self.full_filename):
LOGGER.exception('[VIDEO_INGEST] File Not Found %s', self.video_proto.veda_id)
LOGGER.exception(
'[INGEST] {studio_id} | {video_id} : Local file not found'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
)
)
return
"""
......@@ -253,7 +261,12 @@ class VedaIngest(object):
v1.client_title = final_string
v1.save()
self.complete = True
return None
LOGGER.info('[INGEST] {studio_id} | {video_id} : Database record complete'.format(
studio_id=self.video_proto.studio_id,
video_id=self.video_proto.veda_id
)
)
return
# Update transcription preferences for the Video
if self.video_proto.process_transcription:
......@@ -301,11 +314,17 @@ class VedaIngest(object):
s1 += 1
v1.client_title = final_string
v1.save()
except Exception:
# Log the exception and raise.
LOGGER.exception('[VIDEO_INGEST] - Cataloging of video=%s failed.', self.video_proto.veda_id)
LOGGER.exception('[INGEST] {studio_id} | {video_id} : Video catalog failed.'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
raise
LOGGER.info('[INGEST] {studio_id} | {video_id} : Video record cataloged'.format(
studio_id=self.video_proto.s3_filename,
video_id=self.video_proto.veda_id
))
def val_insert(self):
if self.video_proto.abvid_serial:
......@@ -316,12 +335,12 @@ class VedaIngest(object):
else:
val_status = 'ingest'
VAC = VALAPICall(
val_call = VALAPICall(
video_proto=self.video_proto,
val_status=val_status,
platform_course_url="" # Empty record for initial status update
)
VAC.call()
val_call.call()
def abvid_report(self):
if self.video_proto.abvid_serial is None:
......@@ -333,6 +352,9 @@ class VedaIngest(object):
youtube_id=''
)
email_report.upload_status()
LOGGER.info('[INGEST] {video_id} : About video reported'.format(
video_id=self.video_proto.veda_id
))
self.complete = True
def rename(self):
......
control/veda_heal.py
......@@ -91,11 +91,17 @@ class VedaHeal(object):
)
# Misqueued Task
if task_result == 1:
LOGGER.error('[ENQUEUE ERROR] : {id}'.format(id=v.edx_id))
LOGGER.error('[ENQUEUE] {studio_id} | {video_id} : queueing call'.format(
studio_id=v.studio_id,
video_id=v.edx_id
))
continue
# Update Status
LOGGER.info('[ENQUEUE] : {id}'.format(id=v.edx_id))
LOGGER.info('[ENQUEUE] {studio_id} | {video_id}: file enqueued for encoding'.format(
studio_id=v.studio_id,
video_id=v.edx_id
))
Video.objects.filter(edx_id=v.edx_id).update(
video_trans_status='Queue'
)
......@@ -104,7 +110,7 @@ class VedaHeal(object):
"""
Determine expected and completed encodes
"""
LOGGER.info('[ENQUEUE] : {id}'.format(id=video_object.edx_id))
LOGGER.info('[ENQUEUE] : {id}'.format(id=video_object.studio_id))
if self.freezing_bug is True:
if video_object.video_trans_status == 'Corrupt File':
self.val_status = 'file_corrupt'
......@@ -135,9 +141,9 @@ class VedaHeal(object):
pass
requeued_encodes = self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object)
LOGGER.info('[ENQUEUE] : {id} : {status} : {encodes}'.format(
id=video_object.edx_id,
status=self.val_status,
LOGGER.info('[ENQUEUE] {studio_id} | {video_id}: encoding {encodes}'.format(
studio_id=video_object.studio_id,
video_id=video_object.edx_id,
encodes=requeued_encodes
))
......@@ -197,7 +203,7 @@ class VedaHeal(object):
mark file corrupt -- just run the query again with
no veda_id
"""
# TODO: Adapt to alert for >24h dead videos
try:
expected_encodes.remove('hls')
except ValueError:
......
control/veda_hotstore.py
import boto
import logging
import os
import shutil
import sys
......@@ -10,7 +11,6 @@ from boto.s3.key import Key
from boto.exception import S3ResponseError
from os.path import expanduser
from veda_utils import ErrorObject
from VEDA.utils import get_config
try:
......@@ -21,6 +21,10 @@ boto.config.set('Boto', 'http_socket_timeout', '100')
homedir = expanduser("~")
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Hotstore(object):
"""
......@@ -44,9 +48,7 @@ class Hotstore(object):
return False
if not os.path.exists(self.upload_filepath):
ErrorObject().print_error(
message='Hotstore: File Not Found'
)
LOGGER.error('[HOTSTORE] Local file not found')
return False
self.upload_filesize = os.stat(self.upload_filepath).st_size
......@@ -68,9 +70,7 @@ class Hotstore(object):
self.auth_dict['veda_s3_hotstore_bucket']
)
except S3ResponseError:
ErrorObject().print_error(
message='Hotstore: Bucket Connectivity'
)
LOGGER.error('[HOTSTORE] No hotstore bucket connection')
return False
else:
try:
......@@ -79,9 +79,7 @@ class Hotstore(object):
self.auth_dict['edx_s3_endpoint_bucket']
)
except S3ResponseError:
ErrorObject().print_error(
message='Endpoint: Bucket Connectivity'
)
LOGGER.error('[HOTSTORE] No endpoint bucket connection')
return False
upload_key = Key(delv_bucket)
......@@ -128,24 +126,18 @@ class Hotstore(object):
c = boto.connect_s3()
b = c.lookup(self.auth_dict['veda_s3_hotstore_bucket'])
except S3ResponseError:
ErrorObject().print_error(
message='Hotstore: Bucket Connectivity'
)
LOGGER.error('[HOTSTORE] : No hotstore bucket connection')
return False
else:
try:
c = boto.connect_s3()
b = c.lookup(self.auth_dict['edx_s3_endpoint_bucket'])
except S3ResponseError:
ErrorObject().print_error(
message='Endpoint: Bucket Connectivity'
)
LOGGER.error('[HOTSTORE] : No endpoint bucket connection')
return False
if b is None:
ErrorObject().print_error(
message='Deliverable Fail: s3 Bucket Error'
)
LOGGER.error('[HOTSTORE] : s3 Bucket Error - no object')
return False
"""
......@@ -180,11 +172,3 @@ class Hotstore(object):
os.chdir(homedir)
shutil.rmtree(os.path.join(path_to_multipart, filename.split('.')[0]))
return True
def main():
pass
if __name__ == '__main__':
sys.exit(main())
control/veda_utils.py
"""
Quick and dirty output handling
import os
import sys
import datetime
"""
import boto.ses
import hashlib
import datetime
import subprocess
"""
Let's do some quick and dirty error handling & logging
"""
from control.control_env import *
from control.veda_encode import VedaEncode
from VEDA.utils import get_config
class EmailAlert():
class EmailAlert(object):
"""
Send alert emails VIA AWS SES for Course About Video Statuses
"""
def __init__(self, **kwargs):
self.auth_dict = get_config()
self.message = kwargs.get('message', None)
......@@ -42,24 +40,9 @@ class EmailAlert():
)
class ErrorObject(object):
"""
Unspecified errors with a message
"""
@staticmethod
def print_error(message):
decorator = "***************E*R*R*O*R*******************"
outgoing = '\n%s \n\n%s \n\n%s\n' % (
NODE_COLORS_BLUE + decorator + NODE_COLORS_END,
message,
NODE_COLORS_BLUE + decorator + NODE_COLORS_END,
)
print outgoing
class Output(object):
"""
Various reporting methods
Display/Reporting methods
"""
@staticmethod
def _seconds_from_string(duration):
......@@ -109,7 +92,7 @@ class Output(object):
sys.stdout.flush()
class Report():
class Report(object):
def __init__(self, **kwargs):
self.auth_dict = get_config()
......@@ -197,8 +180,12 @@ class Report():
)
class VideoProto():
class VideoProto(object):
"""
Video object abstraction,
intended as a record before object is recorded in DB
"""
def __init__(self, **kwargs):
self.s3_filename = kwargs.get('s3_filename', None)
self.client_title = kwargs.get('client_title', None)
......@@ -217,8 +204,11 @@ class VideoProto():
self.val_id = kwargs.get('val_id', None)
class Metadata():
class Metadata(object):
"""
Centralized video metadata probe
"""
def __init__(self, **kwargs):
self.video_proto = kwargs.get('video_proto', None)
self.video_object = kwargs.get(
......@@ -270,11 +260,12 @@ class Metadata():
self.video_proto.resolution = vid_breakout[3].strip()
def _FAULT(self, video_object):
if self.video_object is None:
return []
"""
Is there anything to do with this?
Find missing encodes
"""
if self.video_object is None:
return []
# Check for object viability against prior findings
if video_object.video_trans_status == 'Corrupt File':
return []
......@@ -288,15 +279,13 @@ class Metadata():
return []
"""
Finally, determine encodes
"""
E = VedaEncode(
# Determine encodes
encode = VedaEncode(
course_object=video_object.inst_class,
veda_id=video_object.edx_id
)
encode_list = E.determine_encodes()
encode_list = encode.determine_encodes()
if encode_list is not None:
if 'mobile_high' in encode_list:
......@@ -321,26 +310,20 @@ class Metadata():
)
return []
"""
get baseline // if there are == encodes and baseline,
mark file corrupt -- just run the query again with
no veda_id
"""
"""
This overrides
"""
# get baseline // if there are == encodes and baseline,
# mark file corrupt -- just run the query again with no veda_id.
# kwarg override:
if self.freezing_bug is False and self.val_status != 'file_complete':
self.val_status = 'transcode_queue'
return encode_list
E2 = VedaEncode(
encode_two = VedaEncode(
course_object=video_object.inst_class,
)
E2.determine_encodes()
if len(E2.encode_list) == len(encode_list) and len(encode_list) > 1:
"""
Mark File Corrupt, accounting for migrated URLs
"""
encode_two.determine_encodes()
if len(encode_two.encode_list) == len(encode_list) and len(encode_list) > 1:
# Mark File Corrupt, accounting for migrated legacy URLs
url_test = URL.objects.filter(
videoID=Video.objects.filter(
edx_id=video_object.edx_id
......
control/veda_val.py
"""
Send data to VAL, either Video ID data or endpoint URLs
"""
import logging
import os
import sys
import requests
import ast
import json
import datetime
import yaml
from VEDA.utils import get_config
requests.packages.urllib3.disable_warnings()
"""
Send data to VAL, either Video ID data or endpoint URLs
from control_env import *
from control.veda_utils import Output, VideoProto
"""
LOGGER = logging.getLogger(__name__)
requests.packages.urllib3.disable_warnings()
'''
"upload": _UPLOADING,
......@@ -31,13 +28,9 @@ Send data to VAL, either Video ID data or endpoint URLs
"imported": _IMPORTED,
'''
from control_env import *
from control.veda_utils import ErrorObject, Output
LOGGER = logging.getLogger(__name__)
class VALAPICall():
class VALAPICall(object):
def __init__(self, video_proto, val_status, **kwargs):
"""VAL Data"""
......@@ -63,16 +56,17 @@ class VALAPICall():
self.auth_dict = kwargs.get('CONFIG_DATA', self._AUTH())
def call(self):
if self.auth_dict is None:
if not self.auth_dict:
return None
"""
Errors covered in other methods
"""
if self.val_token is None:
if not self.val_token:
self.val_tokengen()
if self.video_object is not None:
if self.video_object:
self.send_object_data()
return
if self.video_proto is not None:
self.send_val_data()
......@@ -93,10 +87,8 @@ class VALAPICall():
r = requests.post(self.auth_dict['val_token_url'], data=payload, timeout=self.auth_dict['global_timeout'])
if r.status_code != 200:
ErrorObject.print_error(
message='Token Gen Fail: VAL\nCheck VAL Config'
)
return None
LOGGER.error('[API] : VAL Token generation')
return
self.val_token = ast.literal_eval(r.text)['access_token']
self.headers = {
......@@ -109,10 +101,8 @@ class VALAPICall():
Rather than rewrite the protocol to fit the veda models,
we'll shoehorn the model into the VideoProto model
"""
class VideoProto():
platform_course_url = []
self.video_proto = VideoProto()
self.video_proto.s3_filename = self.video_object.studio_id
self.video_proto.veda_id = self.video_object.edx_id
self.video_proto.client_title = self.video_object.client_title
......@@ -144,7 +134,7 @@ class VALAPICall():
## "PUT" for extant objects to video/id --
cannot send duplicate course records
'''
if self.val_token is None:
if not self.val_token:
return False
if self.video_proto.s3_filename is None or \
......@@ -188,13 +178,12 @@ class VALAPICall():
val_courses = []
if self.val_status != 'invalid_token':
for f in self.video_object.inst_class.local_storedir.split(','):
if f.strip() not in val_courses:
if f.strip() not in val_courses and len(f.strip()) > 0:
val_courses.append({f.strip(): None})
if len(val_courses) == 0:
for g in self.video_proto.platform_course_url:
if g.strip() not in val_courses:
val_courses.append({g.strip(): None})
for g in self.video_proto.platform_course_url:
if g.strip() not in val_courses:
val_courses.append({g.strip(): None})
self.val_data = {
'client_video_id': self.video_proto.client_title,
......@@ -213,10 +202,8 @@ class VALAPICall():
)
if r1.status_code != 200 and r1.status_code != 404:
ErrorObject.print_error(
message='R1 : VAL Communication Fail: VAL\nCheck VAL Config'
)
return None
LOGGER.error('[API] : VAL Communication')
return
if r1.status_code == 404:
self.send_404()
......@@ -246,7 +233,7 @@ class VALAPICall():
self.auth_dict['val_profile_dict'][self.encode_profile]
except KeyError:
return
if self.endpoint_url is not None:
if self.endpoint_url:
for p in self.auth_dict['val_profile_dict'][self.encode_profile]:
self.encode_data.append(dict(
......@@ -257,7 +244,7 @@ class VALAPICall():
))
test_list = []
if self.video_proto.veda_id is not None:
if self.video_proto.veda_id:
url_query = URL.objects.filter(
videoID=Video.objects.filter(
edx_id=self.video_proto.veda_id
......@@ -327,13 +314,7 @@ class VALAPICall():
)
if r2.status_code > 299:
ErrorObject.print_error(
message='%s\n %s\n %s\n' % (
'R2 : VAL POST/PUT Fail: VAL',
'Check VAL Config',
r2.status_code
)
)
LOGGER.error('[API] : VAL POST/PUT {code}'.format(code=r2.status_code))
def send_200(self, val_api_return):
"""
......@@ -372,13 +353,13 @@ class VALAPICall():
headers=self.headers,
timeout=self.auth_dict['global_timeout']
)
LOGGER.info('[VAL] : {id} : {status} : {code}'.format(
LOGGER.info('[API] {id} : {status} sent to VAL {code}'.format(
id=self.video_proto.val_id,
status=self.val_status,
code=r4.status_code)
)
if r4.status_code > 299:
LOGGER.error('[VAL] : POST/PUT Fail : Check Config : {status}'.format(status=r4.status_code))
LOGGER.error('[API] : VAL POST/PUT : {status}'.format(status=r4.status_code))
def update_val_transcript(self, video_id, lang_code, name, transcript_format, provider):
"""
......@@ -404,7 +385,7 @@ class VALAPICall():
if not response.ok:
LOGGER.error(
'update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s',
'[API] : VAL update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s',
video_id,
provider,
response.status_code,
......@@ -432,16 +413,8 @@ class VALAPICall():
if not response.ok:
LOGGER.error(
'update_video_status failed -- video_id=%s -- status=%s -- text=%s',
'[API] : VAL Update_video_status failed -- video_id=%s -- status=%s -- text=%s',
video_id,
response.status_code,
response.text
)
def main():
pass
if __name__ == '__main__':
sys.exit(main())
control/veda_video_validation.py
import os
import sys
import subprocess
import fnmatch
import django
from control.control_env import FFPROBE
from VEDA_OS01.models import Video
"""
VEDA Intake/Product Final Testing Suite
......@@ -19,6 +9,18 @@ Mismatched Durations (within 5 sec)
"""
import logging
import os
import subprocess
import sys
from control.control_env import FFPROBE
from VEDA_OS01.models import Video
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Validation(object):
"""
......@@ -39,60 +41,70 @@ class Validation(object):
def validate(self):
"""
Test #1 - assumes file is in 'work' directory of node
Video validation probe
"""
# Test #1
# Assumes file is in 'work' directory of node.
# Probe for metadata, ditch on common/found errors
ff_command = ' '.join((
FFPROBE,
"\"" + self.videofile + "\""
))
"""
Test if size is zero
"""
if int(os.path.getsize(self.videofile)) == 0:
print 'Corrupt: Invalid'
video_duration = None
if int(os.path.getsize(self.videofile)) == 0:
LOGGER.info('[VALIDATION] {id} : CORRUPT/File size is zero'.format(id=self.videofile))
return False
p = subprocess.Popen(ff_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
for line in iter(p.stdout.readline, b''):
if "Invalid data found when processing input" in line:
print 'Corrupt: Invalid'
LOGGER.info('[VALIDATION] {id} : CORRUPT/Invalid data on input'.format(id=self.videofile))
return False
if "multiple edit list entries, a/v desync might occur, patch welcome" in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Desync error'.format(id=self.videofile))
return False
if "command not found" in line:
print line
LOGGER.info('[VALIDATION] {id} : CORRUPT/Atypical file error'.format(id=self.videofile))
return False
if "Duration: " in line:
if "Duration: 00:00:00.0" in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration is zero'.format(id=self.videofile))
return False
elif "Duration: N/A, " in line:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration N/A'.format(id=self.videofile))
return False
video_duration = line.split(',')[0][::-1].split(' ')[0][::-1]
try:
str(video_duration)
except:
if not video_duration:
LOGGER.info('[VALIDATION] {id} : CORRUPT/No Duration'.format(id=self.videofile))
p.kill()
return False
p.kill()
"""
Compare Product to DB averages - pass within 5 sec
"""
# Test #2
# Compare Product to DB averages
# pass is durations within 5 sec or each other
if self.mezzanine is True:
# Return if original/source rawfile
LOGGER.info('[VALIDATION] {id} : VALID/Mezzanine file'.format(id=self.videofile))
return True
if self.veda_id is None:
print 'Error: Validation, encoded file no comparison ID'
LOGGER.info('[VALIDATION] {id} : CORRUPT/Validation, No VEDA ID'.format(id=self.videofile))
return False
try:
video_query = Video.objects.filter(edx_id=self.veda_id).latest()
except:
LOGGER.info(
'[VALIDATION] {id} : CORRUPT/Validation, No recorded ID for comparison'.format(id=self.videofile)
)
return False
product_duration = float(
......@@ -105,21 +117,10 @@ class Validation(object):
duration=video_query.video_orig_duration
)
)
"""
Final Test
"""
if (data_duration - 5) <= product_duration <= (data_duration + 5):
LOGGER.info('[VALIDATION] {id} : VALID'.format(id=self.videofile))
return True
else:
LOGGER.info('[VALIDATION] {id} : CORRUPT/Duration mismatch'.format(id=self.videofile))
return False
def main():
pass
# V = Validation(videofile='/Users/ernst/VEDA_WORKING/fecf210f-0e94-4627-8ac3-46c2338e5897.mp4')
# print V.validate()
# # def __init__(self, videofile, **kwargs):
if __name__ == '__main__':
sys.exit(main())
frontend/abvid_validate.py
'''
"""
About Video Input and Validation
'''
import os
import sys
import datetime
"""
import datetime
import logging
from frontend_env import *
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def create_record(upload_data):
"""
......@@ -22,7 +25,6 @@ def create_record(upload_data):
file_valid=False,
file_complete=False,
)
try:
ul1.save()
return True
......@@ -32,7 +34,7 @@ def create_record(upload_data):
def validate_incoming(upload_data):
ul2 = VedaUpload.objects.filter(
VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial']
).update(
upload_date=datetime.datetime.utcnow().replace(tzinfo=utc),
......@@ -42,26 +44,24 @@ def validate_incoming(upload_data):
def send_to_pipeline(upload_data):
ul3 = VedaUpload.objects.filter(
VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial']
).update(
file_valid=upload_data['success'],
)
if upload_data['success'] == 'true':
print 'Sending File to Pipeline'
else:
ul3 = VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial']
).update(
comment='Failed Upload',
)
return False
LOGGER.info('[ABOUT_VIDEO] {ul_id} : Sending File to Pipeline'.format(
ul_id=upload_data['abvid_serial']
))
return True
if __name__ == '__main__':
upload_data = {}
upload_data['abvid_serial'] = '19e1e1c78e'
upload_data['success'] = 'true'
send_to_pipeline(upload_data)
# Failed upload
VedaUpload.objects.filter(
video_serial=upload_data['abvid_serial']
).update(
comment='Failed Upload',
)
LOGGER.info('[ABOUT_VIDEO] {ul_id} : Failed upload'.format(
ul_id=upload_data['abvid_serial']
))
return False
frontend/course_validate.py
'''
"""
Validate Course / Predict Inputs for advanced fields
'''
import os
"""
import uuid
import json
import yaml
import datetime
from django.utils.timezone import utc
from frontend_env import *
"""
Import Django Shit
"""
class VEDACat():
class VEDACat(object):
def __init__(self, **kwargs):
......@@ -290,12 +283,10 @@ class VEDACat():
return return_dict
def simple_majority(self, attribute_list):
'''
"""
Simple Majority Finder
Dumbly just figures out the field attribute for 'most' of them
'''
Field attribute for 'most' of them (>50.0%) per attrib
"""
comparitor = {'None': 0}
for a in attribute_list:
in_it = False
......@@ -321,13 +312,3 @@ class VEDACat():
data['majority'] = False
return data
###############
def main():
V = VEDACat()
print V.veda_model
if __name__ == '__main__':
sys.exit(main())
instance_config.yaml
......@@ -2,88 +2,74 @@
# ---
# Database information
# ---
# SANDBOX
#DATABASES:
# default:
# ENGINE: django.db.backends.sqlite3
# NAME: sandbox.db
## PRODUCTION
DATABASES:
default:
ENGINE: 'django.db.backends.mysql'
NAME: 'pipeline'
USER: 'pipeline001'
PASSWORD: 'password'
HOST: 'localhost'
PORT: 3306
ENGINE: django.db.backends.sqlite3
NAME: sandbox.db
SECRET_KEY: "a_random_string"
FERNET_KEYS: ["a_random_string"]
SECRET_KEY: ""
debug: True
# Fernet keys
FERNET_KEYS: []
# Frontend S3 Auth
veda_secret_access_key: A_RANDOM_STRING
veda_access_key_id: A_RANDOM_STRING
# JWT AUTH settings
JWT_AUTH:
JWT_SECRET_KEY:
JWT_ISSUER:
JWT_AUDIENCE:
JWT_VERIFY_AUDIENCE:
JWT_SECRET_KEY: A_RANDOM_STRING
JWT_ISSUER:
JWT_AUDIENCE: A_RANDOM_STRING
JWT_VERIFY_AUDIENCE: true
# ---
# AWS Buckets, Prefixes
# AWS
# ---
# Studio/Platform
edx_s3_ingest_prefix:
edx_s3_ingest_bucket:
edx_s3_endpoint_bucket:
# CF
edx_cloudfront_prefix:
# Images
aws_video_images_bucket:
aws_video_images_prefix: "video-images/"
# VEDA Internal
veda_s3_upload_bucket:
veda_s3_hotstore_bucket:
veda_deliverable_bucket:
# Settings
#veda_s3_upload_bucket:
#veda_s3_hotstore_bucket:
#veda_deliverable_bucket:
#veda_upload_bucket:
#edx_s3_ingest_prefix:
#edx_s3_ingest_bucket:
#edx_s3_endpoint_bucket:
#edx_cloudfront_prefix:
#aws_video_images_bucket:
#aws_video_images_prefix:
s3_base_url:
veda_base_url:
s3_base_url: https://s3.amazonaws.com
# Transcripts
aws_video_transcripts_bucket:
aws_video_transcripts_prefix: video-transcripts/
# cielo24 api urls
cielo24_api_base_url: https://sandbox.cielo24.com/api
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
# a token identifying a valid request from transcript provider
transcript_provider_request_token: testtoken
# Ingest Secret
# TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.
veda_secret_access_key:
veda_access_key_id:
transcript_provider_request_token:
# ---
# email vars
# ---
veda_noreply_email:
admin_email:
veda_noreply_email: admin@example.com
admin_email: admin@example.com
lms_base_url:
instance_prefix: ''
# ---
# VEDA API
# VAL user creds
# ---
val_token_url:
val_api_url:
val_video_images_url:
val_transcript_create_url:
val_video_transcript_status_url:
val_client_id:
val_secret_key:
val_username: admin@example.com
val_password:
## VEDA API Auth
veda_api_url:
veda_auth_url:
......@@ -92,24 +78,21 @@ veda_secret_key:
veda_token_url:
# ---
# VAL
# Celery Info
# ---
val_api_url:
val_token_url:
val_video_images_url:
# Credentials
val_client_id: 'clientkey'
val_secret_key: 'secretkey'
val_password:
val_username:
val_transcript_create_url:
val_video_transcript_status_url:
celery_app_name:
# can do multiple queues like so: foo,bar,baz
celery_worker_queue:
celery_deliver_queue:
celery_heal_queue:
celery_threads: 1
# Celery Worker Config Information
rabbitmq_broker:
rabbitmq_pass:
rabbitmq_user:
onsite_worker: False
# ---
# Shotgun Variables (internal mediateam)
# ---
......@@ -117,10 +100,4 @@ sg_server_path:
sg_script_name:
sg_script_key:
# ---
# Endpoints
# ---
threeplay_ftphost:
lms_base_url: ''
instance_prefix: ''
...
scripts/celery_cancel_task.py
......@@ -15,4 +15,4 @@ tasks_to_cancel = [
]
for t in tasks_to_cancel:
revoke(t, terminate=True)
\ No newline at end of file
revoke(t, terminate=True)
scripts/reencode_crawler.py
......@@ -11,9 +11,7 @@ if project_path not in sys.path:
from control.veda_heal import VedaHeal
from VEDA_OS01.models import URL, Encode, Video
"""
Set your globals here
"""
# Set globals
CRAWL_SUFFIX = 'HLS'
CRAWL_START = datetime.datetime.strptime('Feb 1 2017 00:01', '%b %d %Y %H:%M')
CRAWL_EPOCH = datetime.datetime.strptime('Apr 12 2017 00:01', '%b %d %Y %H:%M')
......@@ -22,9 +20,7 @@ BATCH_SIZE = 10
SLEEP_TIME = 1200
# NOTE: MSXSBVCP2017-V002600_100
class ReEncodeCrawler:
class ReEncodeCrawler(object):
def __init__(self):
self.crawl_start = CRAWL_START
......@@ -78,13 +74,3 @@ def main():
if __name__ == '__main__':
sys.exit(main())
'''
Manual HEAL IDs
GEOB3DAE2017-V006300
'''
static_config.yaml
......@@ -24,6 +24,17 @@ ffmpeg_compiled: "ffmpeg"
ffprobe_compiled: "ffprobe"
target_aspect_ratio: 1.7777778
# ---
# Endpoints
# ---
# cielo24 api urls
cielo24_api_base_url: 'https://sandbox.cielo24.com/api'
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
threeplay_ftphost: ftp.3playmedia.com
# This is a list of encodes and their respective course
# boolean matches
encode_dict:
......
test_requirements.txt
......@@ -14,4 +14,4 @@ pytest-django-ordering==1.0.1
pylint==1.7.1
pylint-celery==0.3
pylint-django==0.7.2
responses==0.6.1
responses==0.6.1
\ No newline at end of file
youtube_callback/sftp_id_retrieve.py
......@@ -17,11 +17,10 @@ import django
import pysftp
from django.utils.timezone import utc
from control.veda_utils import ErrorObject, Metadata, VideoProto
from control.veda_utils import Metadata, VideoProto
from control.veda_val import VALAPICall
from frontend.abvid_reporting import report_status
from VEDA_OS01.models import URL, Encode, Video
from youtube_callback.daemon import get_course
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
......@@ -30,23 +29,20 @@ if project_path not in sys.path:
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
django.setup()
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
"""
Defaults:
"""
homedir = expanduser("~")
workdir = os.path.join(homedir, 'download_data_holding')
YOUTUBE_LOOKBACK_DAYS = 4
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
def callfunction(course):
"""
:param course:
:return:
"""
if os.path.exists(workdir):
shutil.rmtree(workdir)
......@@ -55,11 +51,14 @@ def callfunction(course):
xml_downloader(course)
for file in os.listdir(workdir):
print file
upload_data = domxml_parser(file)
if upload_data is not None:
print upload_data
LOGGER.info('[YOUTUBE_CALLBACK] : {inst}{clss} {upload_data}'.format(
inst=course.institution,
clss=course.edx_classid,
upload_data=upload_data
))
urlpatch(upload_data)
......@@ -90,11 +89,20 @@ def xml_downloader(course):
for d in s1.listdir_attr():
crawl_sftp(d=d, s1=s1)
except AuthenticationException:
LOGGER.info("{inst}{clss} : Authentication Failed".format(inst=course.institution, clss=course.edx_classid))
LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : Authentication Failed".format(
inst=course.institution,
clss=course.edx_classid
))
except SSHException:
LOGGER.info("{inst}{clss} : Authentication Failed".format(inst=course.institution, clss=course.edx_classid))
LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : Authentication Failed".format(
inst=course.institution,
clss=course.edx_classid
))
except IOError:
LOGGER.info("{inst}{clss} : List Dir Failed".format(inst=course.institution, clss=course.edx_classid))
LOGGER.error("[YOUTUBE_CALLBACK] : {inst}{clss} : List Dir Failed".format(
inst=course.institution,
clss=course.edx_classid
))
def crawl_sftp(d, s1):
......@@ -139,7 +147,6 @@ def crawl_sftp(d, s1):
x += 1
else:
break
print "%s : %s" % (f.filename, file_to_find)
s1.get(
f.filename,
os.path.join(workdir, file_to_find)
......@@ -283,7 +290,6 @@ def urlpatch(upload_data):
bitrate='0',
s3_filename=test_id.studio_id
)
print test_id.video_orig_duration
VF = Metadata(
video_object=test_id
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment