Skip to content

E
edx-video-pipeline
Commit 56b28ce3 by Gregory Martin
Options

Debug

parent e0e2d4c9
Showing with 71 additions and 88 deletions
bin/heal
#!/usr/bin/env python
"""
Deliver
Command Line Interface
"""
import os
import sys
import argparse
......@@ -14,12 +19,7 @@ from control.celeryapp import maintainer_healer
from control.veda_heal import VedaHeal
from VEDA_OS01.models import Course, Video
from VEDA_OS01.transcripts import retrieve_three_play_translations
"""
Deliver
Command Line Interface
"""
from VEDA.utils import get_config
class HealCli:
......@@ -30,6 +30,10 @@ class HealCli:
self.binscript = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'heal')
def schedule(self):
"""
Reschedule Heal process via Celery ETA
"""
auth_dict = get_config()
go_time = datetime.datetime.now(pytz.timezone("America/New_York")) \
.replace(hour=0, minute=0, second=0, microsecond=0) \
.astimezone(pytz.utc) + timedelta(days=1)
......
control/tests/test_heal.py
......@@ -125,7 +125,7 @@ class HealTests(TestCase):
'video_active': True,
},
{
'edx_id': '1',
'edx_id': '2',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': False,
......
control/veda_file_discovery.py
"""
multi-point videofile discovery
Currently:
Amazon S3 (studio-ingest as well as about/marketing
video ingest
)
Local (watchfolder w/o edit priv.)
"""
import json
import logging
import os.path
import boto
import boto.s3
from boto.exception import S3ResponseError, S3DataError
from boto.exception import NoAuthHandlerFound, S3DataError, S3ResponseError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from control_env import *
from VEDA.utils import extract_course_org, get_config
from veda_file_ingest import VedaIngest, VideoProto
from VEDA_OS01.models import TranscriptCredentials
from veda_utils import ErrorObject
from veda_val import VALAPICall
try:
boto.config.add_section('Boto')
......@@ -17,21 +31,6 @@ except:
pass
boto.config.set('Boto', 'http_socket_timeout', '100')
"""
multi-point videofile discovery
Currently:
FTP
Amazon S3 (studio-ingest as well as about/marketing
video ingest
)
Local (watchfolder w/o edit priv.)
"""
from control_env import *
from veda_utils import ErrorObject
from veda_file_ingest import VideoProto, VedaIngest
from veda_val import VALAPICall
LOGGER = logging.getLogger(__name__)
......@@ -39,18 +38,8 @@ class FileDiscovery(object):
def __init__(self, **kwargs):
self.video_info = {}
self.auth_dict = get_config()
self.bucket = None
"""
FTP Server Vars
"""
self.ftp_key = None
self.ftp_follow_delay = str(5000)
self.ftp_log = "/Users/Shared/edX1/LG/Transfers.log"
self.wfm_log = "/Users/Shared/edX1/LG/WFM.log"
self.ftp_faillog = "/Users/Shared/edX1/LG/FailedTransfers.log"
self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
def about_video_ingest(self):
......@@ -58,12 +47,13 @@ class FileDiscovery(object):
Crawl VEDA Upload bucket
"""
if self.node_work_directory is None:
ErrorObject().print_error(
message='No Workdir'
)
print '[Discovery Error] No Workdir'
return
try:
conn = boto.connect_s3()
except NoAuthHandlerFound:
print '[Discovery Error] BOTO Auth Handler'
return
conn = boto.connect_s3()
try:
self.bucket = conn.get_bucket(self.auth_dict['veda_s3_upload_bucket'])
except S3ResponseError:
......@@ -217,7 +207,6 @@ class FileDiscovery(object):
"""
if len(file_extension) == 3:
file_name = u'{file_name}.{ext}'.format(file_name=file_name, ext=file_extension)
try:
key.get_contents_to_filename(os.path.join(self.node_work_directory, file_name))
file_ingested = True
......@@ -263,6 +252,8 @@ class FileDiscovery(object):
self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
except S3ResponseError:
ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
except NoAuthHandlerFound:
ErrorObject.print_error(message='[Discovery Error] BOTO Auth Handler')
else:
ErrorObject.print_error(message='[File Ingest] No Working Node directory')
......
control/veda_file_ingest.py
......@@ -162,7 +162,7 @@ class VedaIngest:
jobid = uuid.uuid1().hex[0:10]
celeryapp.worker_task_fire.apply_async(
(veda_id, encode_profile, jobid),
queue=self.auth_dict['celery_worker_queue']
queue=self.auth_dict['celery_worker_queue'].split(',')[0]
)
"""
......
control/veda_heal.py
......@@ -85,7 +85,7 @@ class VedaHeal(object):
jobid = uuid.uuid1().hex[0:10]
celeryapp.worker_task_fire.apply_async(
(veda_id, encode_profile, jobid),
queue=self.auth_dict['celery_worker_queue']
queue=self.auth_dict['celery_worker_queue'].split(',')[0]
)
def determine_fault(self, video_object):
......@@ -95,15 +95,15 @@ class VedaHeal(object):
LOGGER.info('[HEAL] : {id}'.format(id=video_object.edx_id))
if self.freezing_bug is True:
if video_object.video_trans_status == 'Corrupt File':
self.val_status = 'file_corrupt'
return []
if video_object.video_trans_status == 'Review Reject':
if video_object.video_trans_status == 'Review Reject' or video_object.video_trans_status == 'Review Hold' or \
video_object.video_trans_status == 'Review Hold':
return []
if video_object.video_trans_status == 'Review Hold':
return []
if video_object.video_active is False:
if video_object.video_trans_status == 'Youtube Duplicate':
self.val_status = 'duplicate'
return []
"""
......
youtube_callback/sftp_id_retrieve.py
......@@ -3,16 +3,23 @@ Check SFTP dropboxes for YT Video ID XML information
"""
import datetime
from datetime import timedelta
import django
from django.utils.timezone import utc
import fnmatch
import os
from os.path import expanduser
import pysftp
import shutil
import sys
import xml.etree.ElementTree as ET
from datetime import timedelta
from os.path import expanduser
import django
import pysftp
from django.utils.timezone import utc
from control.veda_utils import ErrorObject, Metadata, VideoProto
from control.veda_val import VALAPICall
from frontend.abvid_reporting import report_status
from VEDA_OS01.models import URL, Encode, Video
from youtube_callback.daemon import get_course
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
......@@ -21,11 +28,6 @@ if project_path not in sys.path:
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
django.setup()
from VEDA_OS01.models import Video, Encode, URL
from frontend.abvid_reporting import report_status
from control.veda_val import VALAPICall
from control.veda_utils import ErrorObject, Metadata, VideoProto
from youtube_callback.daemon import get_course
"""
Defaults:
......@@ -72,19 +74,16 @@ def xml_downloader(course):
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
try:
with pysftp.Connection(
'partnerupload.google.com',
username=course.yt_logon,
private_key=private_key,
port=19321,
cnopts=cnopts
) as s1:
for d in s1.listdir_attr():
crawl_sftp(d=d, s1=s1)
except:
ErrorObject.print_error("Failed Auth: Youtube SFTP")
return None
with pysftp.Connection(
'partnerupload.google.com',
username=course.yt_logon,
private_key=private_key,
port=19321,
cnopts=cnopts
) as s1:
s1.timeout = 60.0
for d in s1.listdir_attr():
crawl_sftp(d=d, s1=s1)
def crawl_sftp(d, s1):
......@@ -284,30 +283,25 @@ def urlpatch(upload_data):
return None
if len(encode_list) == 0:
Video.objects.filter(
edx_id=upload_data['edx_id']
).update(
video_trans_status='Complete'
)
Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Complete')
val_status = 'file_complete'
else:
val_status = 'transcode_active'
VAC = VALAPICall(
ApiConn = VALAPICall(
video_proto=video_proto,
val_status=val_status,
endpoint_url=upload_data['youtube_id'],
encode_profile='youtube'
)
VAC.call()
ApiConn.call()
elif upload_data['status'] == 'Duplicate' and \
upload_data['file_suffix'] == '100':
url_query = URL.objects.filter(
videoID=Video.objects.filter(
edx_id=test_id.edx_id
edx_id=upload_data['edx_id']
).latest(),
encode_profile=Encode.objects.get(
encode_suffix=upload_data['file_suffix']
......@@ -322,12 +316,7 @@ def urlpatch(upload_data):
youtube_id=''
)
Video.objects.filter(
edx_id=upload_data['edx_id']
).update(
video_trans_status='Youtube Duplicate'
)
Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Youtube Duplicate')
video_proto = VideoProto(
veda_id=test_id.edx_id,
val_id=test_id.studio_id,
......@@ -336,11 +325,10 @@ def urlpatch(upload_data):
bitrate='0',
s3_filename=test_id.studio_id
)
VAC = VALAPICall(
ApiConn = VALAPICall(
video_proto=video_proto,
val_status="duplicate",
endpoint_url="DUPLICATE",
encode_profile='youtube'
)
VAC.call()
ApiConn.call()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment