Skip to content

E
edx-video-pipeline
Unverified Commit 5404599e by M. Rehan Committed by GitHub
Options

Merge pull request #103 from edx/mrehan/revert-old-transcripts-workflow 

[EDUCATOR-2768] Revert 3rd party transcripts custom workflow
parents c881f199 2b143c25
Showing with 0 additions and 248 deletions
control/old_veda_deliver_cielo.py deleted 100644 → 0
"""
Cielo24 API Job Start and Download
Options (reflected in Course.models):
transcription_fidelity =
Mechanical (75%),
Premium (95%)(3-72h),
Professional (99+%)(3-72h)
priority =
standard (24h),
priority (48h)
turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently
"""
import logging
import requests
import ast
import urllib
from control_env import *
requests.packages.urllib3.disable_warnings()
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class Cielo24TranscriptOld(object):
def __init__(self, veda_id):
self.veda_id = veda_id
'''Defaults'''
self.c24_site = 'https://api.cielo24.com/api'
self.c24_login = '/account/login'
self.c24_joblist = '/job/list'
self.c24_newjob = '/job/new'
self.add_media = '/job/add_media'
self.transcribe = '/job/perform_transcription'
'''Retreive C24 Course-based defaults'''
self.c24_defaults = self.retrieve_defaults()
def perform_transcription(self):
if self.c24_defaults['c24_user'] is None:
return None
'''
GET /api/job/perform_transcription?v=1 HTTP/1.1
&api_token=xxxx
&job_id=xxxx
&transcription_fidelity=PREMIUM&priority=STANDARD
Host: api.cielo24.com
'''
api_token = self.tokengenerator()
if api_token is None:
return None
job_id = self.generate_jobs(api_token)
task_id = self.embed_url(api_token, job_id)
r5 = requests.get(
''.join((
self.c24_site,
self.transcribe,
'?v=1&api_token=',
api_token,
'&job_id=',
job_id,
'&transcription_fidelity=',
self.c24_defaults['c24_fidelity'],
'&priority=',
self.c24_defaults['c24_speed']
))
)
return ast.literal_eval(r5.text)['TaskId']
def retrieve_defaults(self):
video_query = Video.objects.filter(
edx_id=self.veda_id
).latest()
url_query = URL.objects.filter(
videoID=video_query,
encode_url__icontains='_DTH.mp4',
).latest()
if video_query.inst_class.c24_username is None:
LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API : Course record incomplete'.format(id=self.veda_id))
return None
c24_defaults = {
'c24_user': video_query.inst_class.c24_username,
'c24_pass': video_query.inst_class.c24_password,
'c24_speed': video_query.inst_class.c24_speed,
'c24_fidelity': video_query.inst_class.c24_fidelity,
'edx_id': self.veda_id,
'url': url_query.encode_url
}
return c24_defaults
def tokengenerator(self):
token_url = self.c24_site + self.c24_login + \
'?v=1&username=' + self.c24_defaults['c24_user'] + \
'&password=' + self.c24_defaults['c24_pass']
# Generate Token
r1 = requests.get(token_url)
if r1.status_code > 299:
LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API access'.format(id=self.veda_id))
return
api_token = ast.literal_eval(r1.text)["ApiToken"]
return api_token
def listjobs(self):
"""List Jobs"""
api_token = self.tokengenerator()
r2 = requests.get(
''.join((
self.c24_site,
self.c24_joblist,
'?v=1&api_token=',
api_token
))
)
job_list = r2.text
return job_list
def generate_jobs(self, api_token):
"""
'https://api.cielo24.com/job/new?v=1&\
api_token=xxx&job_name=xxx&language=en'
"""
r3 = requests.get(
''.join((
self.c24_site,
self.c24_newjob,
'?v=1&api_token=',
api_token,
'&job_name=',
self.c24_defaults['edx_id'],
'&language=en'
))
)
job_id = ast.literal_eval(r3.text)['JobId']
return job_id
def embed_url(self, api_token, job_id):
"""
GET /api/job/add_media?v=1&api_token=xxxx
&job_id=xxxxx
&media_url=http%3A%2F%2Fwww.domain.com%2Fvideo.mp4 HTTP/1.1
Host: api.cielo24.com
"""
r4 = requests.get(
''.join((
self.c24_site,
self.add_media,
'?v=1&api_token=',
api_token,
'&job_id=',
job_id,
'&media_url=',
urllib.quote_plus(self.c24_defaults['url'])
))
)
return ast.literal_eval(r4.text)['TaskId']
def main():
pass
if __name__ == "__main__":
sys.exit(main())
control/veda_deliver.py
......@@ -5,7 +5,6 @@ endpoint via the custom methods
"""
import datetime
import ftplib
import logging
import shutil
from os.path import expanduser
......@@ -19,7 +18,6 @@ from boto.exception import S3ResponseError, NoAuthHandlerFound
from boto.s3.key import Key
from django.core.urlresolvers import reverse
from control.old_veda_deliver_cielo import Cielo24TranscriptOld
from control_env import *
from veda_deliver_cielo import Cielo24Transcript
from veda_deliver_youtube import DeliverYoutube
......@@ -129,11 +127,6 @@ class VedaDelivery(object):
u1.encode_size = self.video_proto.filesize
u1.save()
# TODO: Warning! this shall be removed once 3rd party credentials
# for existing courses are migrated according to new flow.
self._THREEPLAY_UPLOAD()
self._CIELO24_UPLOAD()
self.status = self._DETERMINE_STATUS()
self._UPDATE_DATA()
self._CLEANUP()
......@@ -624,68 +617,6 @@ class VedaDelivery(object):
self.video_query.studio_id,
)
def _CIELO24_UPLOAD(self):
"""
Note: This is part of the old flow which was deprecated when transcript phase 1 was reased.
"""
# TODO: this must be removed once existing 3rd party credentials are migrated according
# to the new workflow.
if self.video_query.inst_class.c24_proc is False:
return None
if self.encode_profile != 'desktop_mp4':
return None
cielojob = Cielo24TranscriptOld(
veda_id=self.video_query.edx_id
)
cielojob.perform_transcription()
LOGGER.info('[DELIVERY] {id} : Cielo24 job sent '.format(id=self.video_query.edx_id))
def _THREEPLAY_UPLOAD(self):
"""
Note: This is part of the old flow which was deprecated when transcript phase 1 was reased.
"""
# TODO: this must be removed once existing 3rd party credentials are migrated according
# to the new workflow.
if self.video_query.inst_class.tp_proc is False:
return None
if self.encode_profile != 'desktop_mp4':
return None
ftp1 = ftplib.FTP(
self.auth_dict['threeplay_ftphost']
)
user = self.video_query.inst_class.tp_username.strip()
passwd = self.video_query.inst_class.tp_password.strip()
try:
ftp1.login(user, passwd)
except:
LOGGER.error('[DELIVERY] {file} : 3Play Authentication Failure'.format(file=self.encoded_file))
try:
ftp1.cwd(
self.video_query.inst_class.tp_speed
)
except:
ftp1.mkd(
self.video_query.inst_class.tp_speed
)
ftp1.cwd(
self.video_query.inst_class.tp_speed
)
os.chdir(self.node_work_directory)
ftp1.storbinary(
'STOR ' + self.encoded_file,
open(os.path.join(
self.node_work_directory,
self.encoded_file
), 'rb')
)
os.chdir(homedir)
def YOUTUBE_SFTP(self, review=False):
if self.video_query.inst_class.yt_proc is False:
if self.video_query.inst_class.review_proc is False:
......
instance_config.yaml
......@@ -114,9 +114,4 @@ sg_server_path:
sg_script_name:
sg_script_key:
# ---
# Endpoints
# ---
threeplay_ftphost:
...
static_config.yaml
---
# This configuration should only have static settings.
# 3PlayMedia FTP host for old workflow
threeplay_ftphost: ftp.3playmedia.com
# s3 bucket static prefixes
edx_s3_processed_prefix: prod-edx/processed/
edx_s3_rejected_prefix: prod-edx/rejected/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment