Skip to content

E
edx-video-pipeline
Unverified Commit 4bac14c7 by Muhammad Ammar Committed by GitHub
Options

Merge pull request #57 from edx/ammar/add-urls-to-logs 

add urls to logs
parents 3191dbd4 279275fc
Showing with 116 additions and 42 deletions
VEDA/tests/test_utils.py
......@@ -147,3 +147,30 @@ class UtilTests(TestCase):
"""
instance_config = utils.get_config()
self.assertDictEqual(instance_config, TEST_CONFIG)
@data(
{
'url': 'http://sandbox.edx.org/do?aaa=11&vvv=234',
'params_to_scrub': ['aaa'],
'expected_url': 'http://sandbox.edx.org/do?vvv=234&aaa=XX'
},
{
'url': 'http://sandbox.edx.org/do?aaa=1&vvv=234',
'params_to_scrub': ['aaa', 'vvv'],
'expected_url': 'http://sandbox.edx.org/do?vvv=XXX&aaa=X'
},
{
'url': 'http://sandbox.edx.org/do?aaa=1&vvv=234',
'params_to_scrub': ['zzzz'],
'expected_url': 'http://sandbox.edx.org/do?vvv=234&aaa=1'
},
)
@unpack
def test_scrub_query_params(self, url, params_to_scrub, expected_url):
"""
Tests that utils.scrub_query_params works as expected.
"""
self.assertEqual(
utils.scrub_query_params(url, params_to_scrub),
expected_url
)
VEDA/utils.py
......@@ -3,7 +3,7 @@ Common utils.
"""
import os
import urllib
import urlparse
import yaml
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
......@@ -69,3 +69,33 @@ def get_config(yaml_config_file=DEFAULT_CONFIG_FILE_NAME):
config_dict = yaml.load(config)
return config_dict
def scrub_query_params(url, params_to_scrub):
"""
Scrub query params present in `params_to_scrub` from `url`
Arguments:
url (str): url
params_to_scrub (list): name of query params to be scrubbed from url
Returns:
url with query params scrubbed
>>> old_url = https://sandbox.veda.com/api/do?api_token=veda_api_key&job_name=12345&language=en&v=1
>>> new_url = https://sandbox.veda.com/api/do?v=1&job_name=12345&language=en&api_token=XXXXXXXXXXXX
"""
parsed = urlparse.urlparse(urllib.unquote(url))
# query_params will be in the form of [('v', '1'), ('job_name', '12345')]
query_params = urlparse.parse_qsl(parsed.query)
new_query_params = {}
for key, value in query_params:
new_query_params[key] = len(value) * 'X' if key in params_to_scrub else value
return build_url(
'{scheme}://{netloc}'.format(scheme=parsed.scheme, netloc=parsed.netloc),
parsed.path,
**new_query_params
)
VEDA_OS01/tests/test_transcripts.py
......@@ -17,7 +17,7 @@ from moto import mock_s3_deprecated
from rest_framework import status
from rest_framework.test import APITestCase
from VEDA.utils import get_config, build_url
from VEDA.utils import get_config, build_url, scrub_query_params
from VEDA_OS01 import transcripts, utils
from VEDA_OS01.models import (Course, TranscriptCredentials,
TranscriptProcessMetadata, TranscriptProvider,
......@@ -1279,7 +1279,16 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
{
'method': 'error',
'args': (
'[3PlayMedia Task] Translations metadata request failed for video=%s -- process_id=%s -- status=%s',
'[3PlayMedia Task] Translations metadata request failed, url=%s -- video=%s -- process_id=%s -- status=%s', # pylint: disable=line-too-long
scrub_query_params(
build_url(
transcripts.THREE_PLAY_TRANSLATIONS_METADATA_URL.format(
file_id='112233',
),
apikey='insecure_api_key'
),
['apikey']
),
VIDEO_DATA['studio_id'],
'112233',
400,
......
VEDA_OS01/transcripts.py
......@@ -19,7 +19,7 @@ from rest_framework.response import Response
from rest_framework.views import APIView
from control.veda_val import VALAPICall
from VEDA.utils import get_config, build_url
from VEDA.utils import get_config, build_url, scrub_query_params
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
TranscriptProvider, TranscriptStatus, Video)
......@@ -269,13 +269,13 @@ def fetch_srt_data(url, **request_params):
Fetch srt data from transcript provider.
"""
# return TRANSCRIPT_SRT_DATA
response = requests.get(
build_url(url, **request_params)
)
fetch_srt_data_url = build_url(url, **request_params)
response = requests.get(fetch_srt_data_url)
if not response.ok:
raise TranscriptFetchError(
'[TRANSCRIPT FETCH ERROR] status={} -- text={}'.format(
'[TRANSCRIPT FETCH ERROR] url={} -- status={} -- text={}'.format(
scrub_query_params(fetch_srt_data_url, ['apikey', 'api_token']),
response.status_code,
response.text
)
......@@ -379,10 +379,12 @@ def get_translation_services(api_key):
Returns:
Available 3Play Media Translation services.
"""
response = requests.get(build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
get_translation_services_url = build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key)
response = requests.get(get_translation_services_url)
if not response.ok:
raise TranscriptTranslationError(
u'[3PlayMedia Callback] Error while fetching the translation services -- {status}, {response}'.format(
u'[3PlayMedia Callback] Error fetching the translation services -- url={url}, {status}, {response}'.format(
url=scrub_query_params(get_translation_services_url, ['apikey']),
status=response.status_code,
response=response.text,
)
......@@ -836,7 +838,8 @@ def get_translations_metadata(api_key, file_id, edx_video_id):
translations_metadata_response = requests.get(translations_metadata_url)
if not translations_metadata_response.ok:
LOGGER.error(
u'[3PlayMedia Task] Translations metadata request failed for video=%s -- process_id=%s -- status=%s',
u'[3PlayMedia Task] Translations metadata request failed, url=%s -- video=%s -- process_id=%s -- status=%s',
scrub_query_params(translations_metadata_url, ['apikey']),
edx_video_id,
file_id,
translations_metadata_response.status_code,
......
control/veda_deliver_3play.py
......@@ -8,7 +8,7 @@ import sys
from requests.packages.urllib3.exceptions import InsecurePlatformWarning
from VEDA_OS01.models import TranscriptProcessMetadata, TranscriptProvider, TranscriptStatus
from VEDA.utils import build_url
from VEDA.utils import build_url, scrub_query_params
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
......@@ -94,7 +94,7 @@ class ThreePlayMediaClient(object):
raise ThreePlayMediaUrlError('The URL "{media_url}" is not Accessible.'.format(media_url=self.media_url))
elif response.headers['Content-Type'] != self.allowed_content_type:
raise ThreePlayMediaUrlError(
'Media content-type should be "{allowed_type}". URL was "{media_url}", content-type was "{type}"'.format(
'Media content-type should be "{allowed_type}". URL was "{media_url}", content-type was "{type}"'.format( # pylint: disable=line-too-long
allowed_type=self.allowed_content_type,
media_url=self.media_url,
type=response.headers['Content-Type'],
......@@ -105,11 +105,16 @@ class ThreePlayMediaClient(object):
"""
Gets all the 3Play Media supported languages
"""
response = requests.get(url=build_url(self.base_url, self.available_languages_url, apikey=self.api_key))
available_languages_url = build_url(self.base_url, self.available_languages_url, apikey=self.api_key)
response = requests.get(
url=available_languages_url
)
if not response.ok:
raise ThreePlayMediaLanguagesRetrievalError(
'Error while retrieving available languages: {response} -- {status}'.format(
response=response.text, status=response.status_code
'Error while retrieving available languages: url={url} -- {response} -- {status}'.format(
url=scrub_query_params(available_languages_url, ['apikey']),
response=response.text,
status=response.status_code
)
)
......
control/veda_deliver_cielo.py
......@@ -10,7 +10,7 @@ from requests.packages.urllib3.exceptions import InsecurePlatformWarning
from VEDA_OS01.models import (TranscriptProcessMetadata, TranscriptProvider,
TranscriptStatus)
from VEDA.utils import build_url
from VEDA.utils import build_url, scrub_query_params
from VEDA_OS01.transcripts import CIELO24_API_VERSION
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
......@@ -126,24 +126,24 @@ class Cielo24Transcript(object):
video_id=self.video.studio_id
)
response = requests.get(
build_url(
self.cielo24_api_base_url,
self.cielo24_perform_transcription,
v=CIELO24_API_VERSION,
job_id=job_id,
target_language=lang_code,
callback_url=callback_url,
api_token=self.api_key,
priority=self.turnaround,
transcription_fidelity=self.fidelity,
options=json.dumps({"return_iwp": ["FINAL"]})
)
perform_transcript_url = build_url(
self.cielo24_api_base_url,
self.cielo24_perform_transcription,
v=CIELO24_API_VERSION,
job_id=job_id,
target_language=lang_code,
callback_url=callback_url,
api_token=self.api_key,
priority=self.turnaround,
transcription_fidelity=self.fidelity,
options=json.dumps({"return_iwp": ["FINAL"]})
)
response = requests.get(perform_transcript_url)
if not response.ok:
raise Cielo24PerformTranscriptError(
'[PERFORM TRANSCRIPT ERROR] status={} -- text={}'.format(
'[PERFORM TRANSCRIPT ERROR] url={} -- status={} -- text={}'.format(
scrub_query_params(perform_transcript_url, ['api_token']),
response.status_code,
response.text
)
......@@ -168,20 +168,20 @@ class Cielo24Transcript(object):
Returns:
cielo24 task id
"""
response = requests.get(
build_url(
self.cielo24_api_base_url,
self.cielo24_add_media,
v=CIELO24_API_VERSION,
job_id=job_id,
api_token=self.api_key,
media_url=self.s3_video_url
)
media_url = build_url(
self.cielo24_api_base_url,
self.cielo24_add_media,
v=CIELO24_API_VERSION,
job_id=job_id,
api_token=self.api_key,
media_url=self.s3_video_url
)
response = requests.get(media_url)
if not response.ok:
raise Cielo24AddMediaError(
'[ADD MEDIA ERROR] status={} -- text={}'.format(
'[ADD MEDIA ERROR] url={} -- status={} -- text={}'.format(
scrub_query_params(media_url, ['api_token']),
response.status_code,
response.text
)
......@@ -216,7 +216,7 @@ class Cielo24Transcript(object):
if not response.ok:
raise Cielo24CreateJobError(
'[CREATE JOB ERROR] url={} -- status={} -- text={}'.format(
create_job_url,
scrub_query_params(create_job_url, ['api_token']),
response.status_code,
response.text,
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment