Skip to content

E
edx-video-pipeline
Commit ee2de74e by Muhammad Ammar Committed by GitHub
Options

Merge pull request #52 from edx/ammar/refactor-settings-read 

refactor config setting file read
parents 543fe663 9742d75d
Showing with 296 additions and 434 deletions
VEDA/django_secrets.py
......@@ -5,25 +5,16 @@ This acts as a django-secret shimmer until we can finish pushing all changes to
"""
import os
import yaml
from VEDA.utils import get_config
read_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
with open(read_yaml, 'r') as stream:
try:
return_dict = yaml.load(stream)
except yaml.YAMLError as exc:
return_dict = None
CONFIG_DATA = get_config()
DJANGO_SECRET_KEY = return_dict['django_secret_key'] or 'test_secret_key'
DJANGO_SECRET_KEY = CONFIG_DATA['django_secret_key'] or 'test_secret_key'
DJANGO_ADMIN = ('', '')
DJANGO_DEBUG = return_dict['debug'] if 'debug' in return_dict else False
DATABASES = return_dict['DATABASES']
STATIC_ROOT_PATH = return_dict.get(
DJANGO_DEBUG = CONFIG_DATA['debug'] if 'debug' in CONFIG_DATA else False
DATABASES = CONFIG_DATA['DATABASES']
STATIC_ROOT_PATH = CONFIG_DATA.get(
'STATIC_ROOT_PATH',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
......
VEDA/tests/test_utils.py 0 → 100644
"""
Tests common utils
"""
import os
import tempfile
from unittest import TestCase
import yaml
from ddt import data, ddt, unpack
from mock import MagicMock, Mock, patch
from VEDA import utils
TEST_CONFIG = {
'var1': 123,
'var2': 999,
'sub': {
'sub_var': 'http://example.com'
}
}
@ddt
class UtilTests(TestCase):
"""
Common util tests.
"""
def setUp(self):
"""
Tests setup.
"""
self._orig_environ = dict(os.environ)
_, self.file_path = tempfile.mkstemp(
suffix='.yml',
dir=tempfile.tempdir
)
with open(self.file_path, 'w') as outfile:
yaml.dump(TEST_CONFIG, outfile, default_flow_style=False)
os.environ['VIDEO_PIPELINE_CFG'] = self.file_path
def tearDown(self):
"""
Reverse the setup
"""
# Reset Environment back to original state
os.environ.clear()
os.environ.update(self._orig_environ)
@data(
{
'urls': ('http://api.cielo24/', '/add/job'),
'params': {},
'expected_url': 'http://api.cielo24/add/job'
},
{
'urls': ('http://api.cielo24', '/add/job'),
'params': {'a': 1, 'b': 2},
'expected_url': 'http://api.cielo24/add/job?a=1&b=2'
},
{
'urls': ('http://api.cielo24/', 'add/job'),
'params': {'c': 3, 'd': 4},
'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
},
{
'urls': ('http://api.cielo24','add/job'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job?p=100'
},
{
'urls': ('http://api.cielo24', 'add/job', 'media'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job/media?p=100'
}
)
@unpack
def test_build_url(self, urls, params, expected_url):
"""
Tests that utils.build_url works as expected.
"""
url = utils.build_url(
*urls,
**params
)
self.assertEqual(
url,
expected_url
)
@data(
{
'course_id': 'course-v1:MITx+4.605x+3T2017',
'expected_org': 'MITx'
},
{
'course_id': 'WestonHS/PFLC1x/3T2015',
'expected_org': 'WestonHS'
},
{
'course_id': '',
'expected_org': None
},
)
@unpack
def test_extract_course_org(self, course_id, expected_org):
"""
Tests that utils.extract_course_org works as expected.
"""
org = utils.extract_course_org(course_id)
self.assertEqual(
org,
expected_org
)
def test_get_config_does_not_exist(self):
"""
Tests that utils.get_config if file does not exist.
"""
del os.environ['VIDEO_PIPELINE_CFG']
with self.assertRaises(IOError):
utils.get_config(yaml_config_file='does_not_exist')
def test_get_config_with_default(self):
"""
Tests that utils.get_config works as expected when reading default config.
"""
del os.environ['VIDEO_PIPELINE_CFG']
instance_config = utils.get_config()
self.assertNotEqual(instance_config, {})
# read the default config file to verify that correct config is loaded
default_yaml_config_file = os.path.join(
utils.DEFAULT_CONFIG_FILE_PATH,
utils.DEFAULT_CONFIG_FILE_NAME
)
with open(default_yaml_config_file, 'r') as config:
config_dict = yaml.load(config)
self.assertDictEqual(instance_config, config_dict)
def test_get_config_with_path(self):
"""
Tests that utils.get_config works as expected when reading config from environment path.
"""
instance_config = utils.get_config()
self.assertDictEqual(instance_config, TEST_CONFIG)
VEDA/utils.py 0 → 100644
"""
Common utils.
"""
import os
import yaml
import urllib
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
DEFAULT_CONFIG_FILE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DEFAULT_CONFIG_FILE_NAME = 'instance_config.yaml'
def extract_course_org(course_id):
"""
Extract video organization from course url.
"""
org = None
try:
org = CourseKey.from_string(course_id).org
except InvalidKeyError:
pass
return org
def build_url(*urls, **query_params):
"""
Build a url from specified params.
Arguments:
base_url (str): base url
relative_url (str): endpoint
query_params (dict): query params
Returns:
absolute url
"""
url = '/'.join(item.strip('/') for item in urls)
if query_params:
url = '{}?{}'.format(url, urllib.urlencode(query_params))
return url
def get_config(yaml_config_file=DEFAULT_CONFIG_FILE_NAME):
"""
Read yaml config file.
Arguments:
yaml_config_file (str): yaml config file name
Returns:
dict: yaml config
"""
config_dict = {}
try:
yaml_config_file = os.environ['VIDEO_PIPELINE_CFG']
except KeyError:
yaml_config_file = os.path.join(
DEFAULT_CONFIG_FILE_PATH,
yaml_config_file
)
with open(yaml_config_file, 'r') as config:
config_dict = yaml.load(config)
return config_dict
VEDA_OS01/tests/test_transcripts.py
......@@ -17,12 +17,13 @@ from moto import mock_s3_deprecated
from rest_framework import status
from rest_framework.test import APITestCase
from VEDA.utils import get_config, build_url
from VEDA_OS01 import transcripts, utils
from VEDA_OS01.models import (Course, TranscriptCredentials,
TranscriptProcessMetadata, TranscriptProvider,
TranscriptStatus, Video)
CONFIG_DATA = utils.get_config('test_config.yaml')
CONFIG_DATA = get_config('test_config.yaml')
VIDEO_DATA = {
'studio_id': '12345',
......@@ -87,7 +88,7 @@ TRANSCRIPT_SJSON_DATA = {
@ddt
@patch.dict('VEDA_OS01.transcripts.CONFIG', CONFIG_DATA)
@patch('VEDA_OS01.utils.get_config', Mock(return_value=CONFIG_DATA))
@patch('VEDA.utils.get_config', Mock(return_value=CONFIG_DATA))
class Cielo24TranscriptTests(APITestCase):
"""
Cielo24 Transcript Tests
......@@ -401,7 +402,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
"""
response = self.client.post(
# `build_url` strips `/`, putting it back and add necessary query params.
'/{}'.format(utils.build_url(
'/{}'.format(build_url(
self.url, edx_video_id=self.video.studio_id,
org=self.org, lang_code=self.video_source_language
)),
......@@ -483,7 +484,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
Test the callback in case of missing attributes.
"""
response = self.client.post(
'/{}'.format(utils.build_url(self.url, **request_data['query_params'])),
'/{}'.format(build_url(self.url, **request_data['query_params'])),
content_type='application/x-www-form-urlencoded',
data=urllib.urlencode(request_data['data']),
)
......@@ -519,7 +520,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
"""
Tests the callback for all the non-success statuses.
"""
self.url = '/{}'.format(utils.build_url(
self.url = '/{}'.format(build_url(
self.url, edx_video_id='12345', org='MAx', lang_code=self.video_source_language
))
self.client.post(self.url, content_type='application/x-www-form-urlencoded', data=urllib.urlencode({
......@@ -579,7 +580,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
expected_requests = [
# request - 1
{
'url': utils.build_url(
'url': build_url(
transcripts.THREE_PLAY_TRANSCRIPT_URL.format(file_id=self.file_id),
apikey=self.transcript_prefs.api_key
)
......@@ -742,7 +743,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
expected_requests = [
# request - 1
{
'url': utils.build_url(
'url': build_url(
transcripts.THREE_PLAY_TRANSCRIPT_URL.format(file_id=self.file_id),
apikey=self.transcript_prefs.api_key
)
......@@ -779,7 +780,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
},
# request - 4
{
'url': utils.build_url(
'url': build_url(
transcripts.THREE_PLAY_TRANSLATION_SERVICES_URL,
apikey=self.transcript_prefs.api_key
)
......@@ -1165,7 +1166,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
# Assert that the first request was made for getting translations metadata from 3Play Media.
expected_video_status_update_request = {
'url': utils.build_url(
'url': build_url(
transcripts.THREE_PLAY_TRANSLATIONS_METADATA_URL.format(file_id=self.file_id),
apikey=self.transcript_prefs.api_key
)
......@@ -1180,7 +1181,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
expected_requests = [
# request - 1
{
'url': utils.build_url(transcripts.THREE_PLAY_TRANSLATION_DOWNLOAD_URL.format(
'url': build_url(transcripts.THREE_PLAY_TRANSLATION_DOWNLOAD_URL.format(
file_id=self.file_id, translation_id=translation_id
), apikey=self.transcript_prefs.api_key)
},
......
VEDA_OS01/tests/test_utils.py
......@@ -14,80 +14,6 @@ class UtilTests(TestCase):
Common util tests.
"""
@data(
{
'urls': ('http://api.cielo24/', '/add/job'),
'params': {},
'expected_url': 'http://api.cielo24/add/job'
},
{
'urls': ('http://api.cielo24', '/add/job'),
'params': {'a': 1, 'b': 2},
'expected_url': 'http://api.cielo24/add/job?a=1&b=2'
},
{
'urls': ('http://api.cielo24/', 'add/job'),
'params': {'c': 3, 'd': 4},
'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
},
{
'urls': ('http://api.cielo24','add/job'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job?p=100'
},
{
'urls': ('http://api.cielo24', 'add/job', 'media'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job/media?p=100'
}
)
@unpack
def test_build_url(self, urls, params, expected_url):
"""
Tests that utils.build_url works as expected.
"""
url = utils.build_url(
*urls,
**params
)
self.assertEqual(
url,
expected_url
)
@data(
{
'course_id': 'course-v1:MITx+4.605x+3T2017',
'expected_org': 'MITx'
},
{
'course_id': 'WestonHS/PFLC1x/3T2015',
'expected_org': 'WestonHS'
},
{
'course_id': '',
'expected_org': None
},
)
@unpack
def test_extract_course_org(self, course_id, expected_org):
"""
Tests that utils.extract_course_org works as expected.
"""
org = utils.extract_course_org(course_id)
self.assertEqual(
org,
expected_org
)
def test_get_config(self):
"""
Tests that utils.get_config works as expected.
"""
config = utils.get_config()
self.assertNotEqual(config, {})
@data(
('IN PROGRESS', True),
('FAILED', False)
)
......
VEDA_OS01/transcripts.py
......@@ -19,6 +19,7 @@ from rest_framework.response import Response
from rest_framework.views import APIView
from control.veda_val import VALAPICall
from VEDA.utils import get_config, build_url
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
TranscriptProvider, TranscriptStatus, Video)
......@@ -37,10 +38,10 @@ TRANSCRIPT_SJSON = 'sjson'
CIELO24_TRANSCRIPT_COMPLETED = django.dispatch.Signal(providing_args=[
'job_id', 'iwp_name', 'lang_code', 'org', 'video_id'
])
CONFIG = utils.get_config()
CONFIG = get_config()
# Cielo24 API URLs
CIELO24_GET_CAPTION_URL = utils.build_url(
CIELO24_GET_CAPTION_URL = build_url(
CONFIG['cielo24_api_base_url'],
'job/get_caption'
)
......@@ -50,23 +51,23 @@ THREE_PLAY_TRANSCRIPTION_DONE = django.dispatch.Signal(
providing_args=['org', 'lang_code', 'edx_video_id', 'file_id', 'status', 'error_description']
)
# 3PlayMedia API URLs.
THREE_PLAY_TRANSCRIPT_URL = utils.build_url(
THREE_PLAY_TRANSCRIPT_URL = build_url(
CONFIG['three_play_api_transcript_url'],
'files/{file_id}/transcript.srt'
)
THREE_PLAY_TRANSLATION_SERVICES_URL = utils.build_url(
THREE_PLAY_TRANSLATION_SERVICES_URL = build_url(
CONFIG['three_play_api_transcript_url'],
'translation_services'
)
THREE_PLAY_ORDER_TRANSLATION_URL = utils.build_url(
THREE_PLAY_ORDER_TRANSLATION_URL = build_url(
CONFIG['three_play_api_base_url'],
'files/{file_id}/translations/order'
)
THREE_PLAY_TRANSLATIONS_METADATA_URL = utils.build_url(
THREE_PLAY_TRANSLATIONS_METADATA_URL = build_url(
CONFIG['three_play_api_transcript_url'],
'files/{file_id}/translations'
)
THREE_PLAY_TRANSLATION_DOWNLOAD_URL = utils.build_url(
THREE_PLAY_TRANSLATION_DOWNLOAD_URL = build_url(
CONFIG['three_play_api_transcript_url'],
'files/{file_id}/translations/{translation_id}/captions.srt'
)
......@@ -267,7 +268,7 @@ def fetch_srt_data(url, **request_params):
"""
# return TRANSCRIPT_SRT_DATA
response = requests.get(
utils.build_url(url, **request_params)
build_url(url, **request_params)
)
if not response.ok:
......@@ -376,7 +377,7 @@ def get_translation_services(api_key):
Returns:
Available 3Play Media Translation services.
"""
response = requests.get(utils.build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
response = requests.get(build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
if not response.ok:
raise TranscriptTranslationError(
u'[3PlayMedia Callback] Error while fetching the translation services -- {status}, {response}'.format(
......@@ -824,7 +825,7 @@ def get_translations_metadata(api_key, file_id, edx_video_id):
}
]
"""
translations_metadata_url = utils.build_url(
translations_metadata_url = build_url(
THREE_PLAY_TRANSLATIONS_METADATA_URL.format(
file_id=file_id,
),
......
VEDA_OS01/utils.py
"""
Common utils.
"""
import os
import urllib
import yaml
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from VEDA_OS01.models import TranscriptStatus
......@@ -26,65 +20,6 @@ VAL_TRANSCRIPT_STATUS_MAP = {
}
def get_config(yaml_config_file='instance_config.yaml'):
"""
Read yaml config file.
Arguments:
yaml_config_file (str): yaml config file name
Returns:
dict: yaml conifg
"""
config_dict = {}
yaml_config_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
yaml_config_file
)
with open(yaml_config_file, 'r') as config:
try:
config_dict = yaml.load(config)
except yaml.YAMLError:
pass
return config_dict
def extract_course_org(course_id):
"""
Extract video organization from course url.
"""
org = None
try:
org = CourseKey.from_string(course_id).org
except InvalidKeyError:
pass
return org
def build_url(*urls, **query_params):
"""
Build a url from specified params.
Arguments:
base_url (str): base url
relative_url (str): endpoint
query_params (dict): query params
Returns:
absolute url
"""
url = '/'.join(item.strip('/') for item in urls)
if query_params:
url = '{}?{}'.format(url, urllib.urlencode(query_params))
return url
def update_video_status(val_api_client, video, status):
"""
Updates video status both in edx-val and edx-video-pipeline.
......
bin/deliver
......@@ -2,7 +2,7 @@
import os
import sys
import argparse
import yaml
from VEDA.utils import get_config
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
......@@ -24,7 +24,6 @@ class DeliverCli:
self.ROOT_DIR = os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
))
self.auth_yaml = os.path.join(self.ROOT_DIR, 'instance_config.yaml')
self.celery_daemon = os.path.join(self.ROOT_DIR, 'control', 'celeryapp.py')
def get_args(self):
......@@ -52,11 +51,7 @@ class DeliverCli:
Launch Celery Delivery Worker
"""
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
auth_dict = None
auth_dict = get_config()
if auth_dict is not None:
os.system(
......
control/celeryapp.py
......@@ -6,7 +6,7 @@ from __future__ import absolute_import
import os
from celery import Celery
import yaml
from VEDA.utils import get_config
try:
from control.veda_deliver import VedaDelivery
......@@ -14,15 +14,7 @@ except ImportError:
from veda_deliver import VedaDelivery
auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
with open(auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
auth_dict = None
auth_dict = get_config()
CEL_BROKER = 'amqp://{rabbitmq_user}:{rabbitmq_pass}@{rabbitmq_broker}:5672//'.format(
rabbitmq_user=auth_dict['rabbitmq_user'],
......
control/tests/test_deliver_3play.py
......@@ -19,7 +19,7 @@ from VEDA_OS01.models import (
Video,
ThreePlayTurnaround,
)
from VEDA_OS01.utils import build_url
from VEDA.utils import build_url
VIDEO_DATA = {
'studio_id': '12345',
......
control/tests/test_deliver_cielo.py
......@@ -11,7 +11,7 @@ from control.veda_deliver_cielo import Cielo24Transcript
from VEDA_OS01.models import (Cielo24Fidelity, Cielo24Turnaround, Course,
TranscriptProcessMetadata, TranscriptStatus,
Video)
from VEDA_OS01.utils import build_url
from VEDA.utils import build_url
VIDEO_DATA = {
'studio_id': '12345',
......
control/tests/test_file_ingest.py
......@@ -6,7 +6,6 @@ import unittest
from django.test import TestCase
import requests
import yaml
from veda_file_ingest import VedaIngest, VideoProto
......
control/tests/test_heal.py
......@@ -14,7 +14,8 @@ from mock import PropertyMock, patch
from control.veda_heal import VedaHeal
from VEDA_OS01.models import URL, Course, Destination, Encode, Video, TranscriptStatus
from VEDA_OS01.utils import build_url, get_config, ValTranscriptStatus
from VEDA_OS01.utils import ValTranscriptStatus
from VEDA.utils import build_url, get_config
sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
......
control/tests/test_hotstore.py
......@@ -9,7 +9,7 @@ from django.test import TestCase
from boto.s3.connection import S3Connection
from mock import PropertyMock, patch
from moto import mock_s3_deprecated
from VEDA_OS01 import utils
from VEDA import utils
from control.veda_file_ingest import VideoProto
from control.veda_hotstore import Hotstore
......
control/tests/test_val.py
......@@ -11,7 +11,7 @@ import responses
from control.veda_val import VALAPICall
from veda_file_ingest import VideoProto
from VEDA_OS01 import utils
from VEDA import utils
requests.packages.urllib3.disable_warnings()
......
control/veda_deliver.py
......@@ -7,7 +7,6 @@ from os.path import expanduser
import boto
import boto.s3
import requests
import yaml
from boto.exception import S3ResponseError
from boto.s3.key import Key
from django.core.urlresolvers import reverse
......@@ -19,7 +18,7 @@ from veda_deliver_youtube import DeliverYoutube
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus)
from VEDA_OS01.utils import build_url
from VEDA.utils import build_url, extract_course_org, get_config
from veda_utils import ErrorObject, Metadata, Output, VideoProto
from veda_val import VALAPICall
from veda_video_validation import Validation
......@@ -60,14 +59,7 @@ class VedaDelivery:
self.veda_id = veda_id
self.encode_profile = encode_profile
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
),
)
self.auth_dict = self._READ_YAML(self.auth_yaml)
self.auth_dict = get_config()
# Internal Methods
self.video_query = None
self.encode_query = None
......@@ -78,19 +70,6 @@ class VedaDelivery:
self.endpoint_url = None
self.video_proto = None
def _READ_YAML(self, read_yaml):
if read_yaml is None:
return None
if not os.path.exists(read_yaml):
return None
with open(read_yaml, 'r') as stream:
try:
return_dict = yaml.load(stream)
return return_dict
except yaml.YAMLError as exc:
return None
def run(self):
"""
Check the destination, route via available methods,
......@@ -536,7 +515,7 @@ class VedaDelivery:
"""
Cielo24 transcription flow.
"""
org = utils.extract_course_org(self.video_proto.platform_course_url[0])
org = extract_course_org(self.video_proto.platform_course_url[0])
try:
api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key
......@@ -586,7 +565,7 @@ class VedaDelivery:
try:
# Picks the first course from the list as there may be multiple
# course runs in that list (i.e. all having the same org).
org = utils.extract_course_org(self.video_proto.platform_course_url[0])
org = extract_course_org(self.video_proto.platform_course_url[0])
transcript_secrets = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider)
# update transcript status for video.
......
control/veda_deliver_3play.py
......@@ -8,7 +8,7 @@ import sys
from requests.packages.urllib3.exceptions import InsecurePlatformWarning
from VEDA_OS01.models import TranscriptProcessMetadata, TranscriptProvider, TranscriptStatus
from VEDA_OS01.utils import build_url
from VEDA.utils import build_url
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
......
control/veda_deliver_cielo.py
......@@ -10,7 +10,7 @@ from requests.packages.urllib3.exceptions import InsecurePlatformWarning
from VEDA_OS01.models import (TranscriptProcessMetadata, TranscriptProvider,
TranscriptStatus)
from VEDA_OS01.utils import build_url
from VEDA.utils import build_url
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
......
control/veda_deliver_xuetang.py
......@@ -8,7 +8,7 @@ import requests
import json
import time
from time import strftime
import yaml
from VEDA.utils import get_config
"""
Authored by Ed Zarecor / edx DevOps
......@@ -18,20 +18,13 @@ included by request
Some adaptations for VEDA:
-auth yaml
**VEDA Note: since this isn't a real CDN, and represents the
'least effort' response to getting video into china,
**VEDA Note: since this isn't a real CDN, and represents the
'least effort' response to getting video into china,
we shan't monitor for success**
"""
read_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
if os.path.exists(read_yaml):
with open(read_yaml, 'r') as stream:
auth_dict = yaml.load(stream)
auth_dict = get_config()
API_SHARED_SECRET = auth_dict['xuetang_api_shared_secret']
API_ENDPOINT = auth_dict['xuetang_api_url']
......@@ -48,7 +41,7 @@ This script provides a functions for accessing the Xuetang CDN API
It expects that an environment variable name XUETANG_SHARED_SECRET is
available and refers to a valid secret provided by the Xuetang CDN team.
Running this script will cause a video hosted in cloudfront to be
Running this script will cause a video hosted in cloudfront to be
uploaded to the CDN via the API.
The status of the video will be monitored in a loop, exiting when
......@@ -77,10 +70,10 @@ def build_message(verb, uri, query_string, date, payload_hash):
format is defined in their CDN API specification document.
"""
return os.linesep.join([verb, uri, query_string, date, payload_hash])
def sign_message(message, secret):
"""
Returns a hexdigest of HMAC generated using sha256. The value is included in
Returns a hexdigest of HMAC generated using sha256. The value is included in
the HTTP headers and used for mutual authentication via a shared secret.
"""
return hmac.new(bytes(secret), bytes(message), digestmod=hashlib.sha256).hexdigest()
......@@ -125,7 +118,7 @@ def prepare_check_task_status(edx_url):
def _prepare_api_request(http_verb, api_target, payload):
"""
General convenience function for creating prepared HTTP requests that conform the
General convenience function for creating prepared HTTP requests that conform the
Xuetang API specificiation.
"""
payload_json = json.dumps(payload)
......@@ -188,7 +181,7 @@ if __name__ == '__main__':
# if res.json()['status'] == 'available':
# break
# time.sleep(5)
# delete file
......
control/veda_encode.py
......@@ -4,11 +4,10 @@ import sys
import uuid
import django
import yaml
from control_env import *
from dependencies.shotgun_api3 import Shotgun
from VEDA.utils import get_config
"""
Get a list of needed encodes from VEDA
......@@ -31,30 +30,11 @@ class VedaEncode(object):
self.overencode = kwargs.get('overencode', False)
self.veda_id = kwargs.get('veda_id', None)
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
),
)
self.encode_dict = self._READ_AUTH()['encode_dict']
self.sg_server_path = self._READ_AUTH()['sg_server_path']
self.sg_script_name = self._READ_AUTH()['sg_script_name']
self.sg_script_key = self._READ_AUTH()['sg_script_key']
def _READ_AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
config_data = get_config()
self.encode_dict = config_data['encode_dict']
self.sg_server_path = config_data['sg_server_path']
self.sg_script_name = config_data['sg_script_name']
self.sg_script_key = config_data['sg_script_key']
def determine_encodes(self):
"""
......
control/veda_file_discovery.py
......@@ -5,10 +5,9 @@ import os.path
import boto
import boto.s3
from boto.exception import S3ResponseError, S3DataError
import yaml
from VEDA_OS01.models import TranscriptCredentials
from VEDA_OS01.utils import extract_course_org
from VEDA.utils import extract_course_org, get_config
try:
boto.config.add_section('Boto')
......@@ -39,16 +38,7 @@ class FileDiscovery(object):
def __init__(self, **kwargs):
self.video_info = {}
self.auth_dict = {}
self.auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
with open(self.auth_yaml, 'r') as stream:
try:
self.auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
pass
self.auth_dict = get_config()
self.bucket = None
"""
......
control/veda_file_ingest.py
......@@ -11,8 +11,8 @@ from django.db.utils import DatabaseError
from django.utils.timezone import utc
from django.db import reset_queries
import uuid
import yaml
import hashlib
from VEDA.utils import get_config
"""
......@@ -89,14 +89,7 @@ class VedaIngest:
def __init__(self, course_object, video_proto, **kwargs):
self.course_object = course_object
self.video_proto = video_proto
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
),
)
self.auth_dict = self._READ_AUTH()
self.auth_dict = get_config()
# --- #
self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
......@@ -104,19 +97,6 @@ class VedaIngest:
self.complete = False
self.archived = False
def _READ_AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
def insert(self):
"""
NOTE:
......
control/veda_heal.py
......@@ -23,6 +23,7 @@ import celeryapp
from control_env import WORK_DIRECTORY
from veda_encode import VedaEncode
from veda_val import VALAPICall
from VEDA.utils import get_config
time_safetygap = datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=1)
......@@ -34,33 +35,13 @@ class VedaHeal(object):
"""
def __init__(self, **kwargs):
self.current_time = datetime.datetime.utcnow().replace(tzinfo=utc)
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
),
)
self.auth_dict = self._READ_AUTH()
self.auth_dict = get_config()
# for individuals
self.video_query = kwargs.get('video_query', None)
self.freezing_bug = kwargs.get('freezing_bug', True)
self.val_status = None
self.retry_barrier_hours = 24
def _READ_AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
def discovery(self):
self.video_query = Video.objects.filter(
video_trans_start__lt=self.current_time - timedelta(
......
control/veda_hotstore.py
......@@ -11,6 +11,7 @@ from boto.exception import S3ResponseError
from os.path import expanduser
from veda_utils import ErrorObject
from VEDA.utils import get_config
try:
boto.config.add_section('Boto')
......@@ -32,28 +33,11 @@ class Hotstore(object):
# is this a final/encode file?
self.endpoint = kwargs.get('endpoint', False)
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
),
)
self.auth_dict = self._READ_AUTH()
self.endpoint_url = None
def _READ_AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
return get_config()
def upload(self):
if self.auth_dict is None:
......
control/veda_utils.py
import os
import sys
import yaml
import datetime
import boto.ses
import hashlib
......@@ -14,35 +13,15 @@ Let's do some quick and dirty error handling & logging
"""
from control.control_env import *
from control.veda_encode import VedaEncode
from VEDA.utils import get_config
class EmailAlert():
def __init__(self, **kwargs):
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
)
self.auth_dict = self._AUTH()
self.auth_dict = get_config()
self.message = kwargs.get('message', None)
self.subject = kwargs.get('subject', None)
def _AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
def email(self):
email_subject = '[ VEDA ALERTING ]'
email_subject += ' : ' + self.subject
......@@ -132,31 +111,11 @@ class Output(object):
class Report():
def __init__(self, **kwargs):
self.auth_yaml = kwargs.get(
'auth_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
)
self.auth_dict = self._AUTH()
self.auth_dict = get_config()
self.status = kwargs.get('status', None)
self.upload_serial = kwargs.get('upload_serial', None)
self.youtube_id = kwargs.get('youtube_id', None)
def _AUTH(self):
if self.auth_yaml is None:
return None
if not os.path.exists(self.auth_yaml):
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
return None
def upload_status(self):
if self.upload_serial is None:
return None
......
control/veda_val.py
......@@ -7,6 +7,7 @@ import ast
import json
import datetime
import yaml
from VEDA.utils import get_config
LOGGER = logging.getLogger(__name__)
......@@ -59,10 +60,6 @@ class VALAPICall():
self.headers = None
"""Credentials"""
self.auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
self.auth_dict = self._AUTH()
def call(self):
......@@ -81,21 +78,7 @@ class VALAPICall():
self.send_val_data()
def _AUTH(self):
if not os.path.exists(self.auth_yaml):
ErrorObject.print_error(
message='No Auth YAML'
)
return None
with open(self.auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
return auth_dict
except yaml.YAMLError as exc:
ErrorObject.print_error(
message='YAML READ ERROR'
)
return None
return get_config()
def val_tokengen(self):
"""
......
frontend/abvid_reporting.py
......@@ -3,7 +3,7 @@ import sys
from email.mime.text import MIMEText
from datetime import date
import boto.ses
import yaml
from VEDA.utils import get_config
'''
ABVID REPORTING - email / etc.
......@@ -26,16 +26,7 @@ if v1.abvid_serial != None:
get auth keys from instance yaml
"""
auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
with open(auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
print 'AUTH ERROR'
auth_dict = get_config()
def report_status(status, abvid_serial, youtube_id):
......
frontend/course_validate.py
......@@ -14,16 +14,6 @@ from frontend_env import *
"""
Import Django Shit
"""
auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
with open(auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
print 'AUTH ERROR'
class VEDACat():
......
frontend/views.py
......@@ -3,7 +3,6 @@
import json
import datetime
from datetime import timedelta
import yaml
import base64
import hmac
import hashlib
......@@ -16,11 +15,7 @@ from django.http import HttpResponseRedirect
from frontend_env import *
from course_validate import VEDACat
from abvid_validate import validate_incoming, create_record, send_to_pipeline
auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
from VEDA.utils import get_config
"""
Here's the links for the main Page
......@@ -169,11 +164,7 @@ def upload_alpha_1(request):
Generate metadata From Fields
Auth?
"""
with open(auth_yaml, 'r') as stream:
try:
auth_dict = yaml.load(stream)
except yaml.YAMLError as exc:
print 'AUTH ERROR'
auth_dict = get_config()
policy_expiration = datetime.datetime.utcnow() + timedelta(hours=24)
policy_exp = str(policy_expiration).replace(' ', 'T').split('.')[0] + 'Z'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment