Merge pull request #57 from edx/ammar/add-urls-to-logs

add urls to logs

VEDA/tests/test_utils.py

@@ -147,3 +147,30 @@ class UtilTests(TestCase):
         """
         instance_config = utils.get_config()
         self.assertDictEqual(instance_config, TEST_CONFIG)
+
+    @data(
+        {
+            'url': 'http://sandbox.edx.org/do?aaa=11&vvv=234',
+            'params_to_scrub': ['aaa'],
+            'expected_url': 'http://sandbox.edx.org/do?vvv=234&aaa=XX'
+        },
+        {
+            'url': 'http://sandbox.edx.org/do?aaa=1&vvv=234',
+            'params_to_scrub': ['aaa', 'vvv'],
+            'expected_url': 'http://sandbox.edx.org/do?vvv=XXX&aaa=X'
+        },
+        {
+            'url': 'http://sandbox.edx.org/do?aaa=1&vvv=234',
+            'params_to_scrub': ['zzzz'],
+            'expected_url': 'http://sandbox.edx.org/do?vvv=234&aaa=1'
+        },
+    )
+    @unpack
+    def test_scrub_query_params(self, url, params_to_scrub, expected_url):
+        """
+        Tests that utils.scrub_query_params works as expected.
+        """
+        self.assertEqual(
+            utils.scrub_query_params(url, params_to_scrub),
+            expected_url
+        )

VEDA/utils.py

@@ -3,7 +3,7 @@ Common utils.
 """
 import os
 import urllib
-
+import urlparse
 import yaml
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
@@ -69,3 +69,33 @@ def get_config(yaml_config_file=DEFAULT_CONFIG_FILE_NAME):
         config_dict = yaml.load(config)
 
     return config_dict
+
+
+def scrub_query_params(url, params_to_scrub):
+    """
+    Scrub query params present in `params_to_scrub` from `url`
+
+    Arguments:
+        url (str): url
+        params_to_scrub (list): name of query params to be scrubbed from url
+
+    Returns:
+        url with query params scrubbed
+
+    >>> old_url = https://sandbox.veda.com/api/do?api_token=veda_api_key&job_name=12345&language=en&v=1
+    >>> new_url = https://sandbox.veda.com/api/do?v=1&job_name=12345&language=en&api_token=XXXXXXXXXXXX
+    """
+    parsed = urlparse.urlparse(urllib.unquote(url))
+
+    # query_params will be in the form of [('v', '1'), ('job_name', '12345')]
+    query_params = urlparse.parse_qsl(parsed.query)
+
+    new_query_params = {}
+    for key, value in query_params:
+        new_query_params[key] = len(value) * 'X' if key in params_to_scrub else value
+
+    return build_url(
+        '{scheme}://{netloc}'.format(scheme=parsed.scheme, netloc=parsed.netloc),
+        parsed.path,
+        **new_query_params
+    )

VEDA_OS01/tests/test_transcripts.py

@@ -17,7 +17,7 @@ from moto import mock_s3_deprecated
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from VEDA.utils import get_config, build_url
+from VEDA.utils import get_config, build_url, scrub_query_params
 from VEDA_OS01 import transcripts, utils
 from VEDA_OS01.models import (Course, TranscriptCredentials,
                               TranscriptProcessMetadata, TranscriptProvider,
@@ -1279,7 +1279,16 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             {
                 'method': 'error',
                 'args': (
-                    '[3PlayMedia Task] Translations metadata request failed for video=%s -- process_id=%s -- status=%s',
+                    '[3PlayMedia Task] Translations metadata request failed, url=%s -- video=%s -- process_id=%s -- status=%s',  # pylint: disable=line-too-long
+                    scrub_query_params(
+                        build_url(
+                            transcripts.THREE_PLAY_TRANSLATIONS_METADATA_URL.format(
+                                file_id='112233',
+                            ),
+                            apikey='insecure_api_key'
+                        ),
+                        ['apikey']
+                    ),
                     VIDEO_DATA['studio_id'],
                     '112233',
                     400,

VEDA_OS01/transcripts.py

@@ -19,7 +19,7 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from control.veda_val import VALAPICall
-from VEDA.utils import get_config, build_url
+from VEDA.utils import get_config, build_url, scrub_query_params
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
                               TranscriptProvider, TranscriptStatus, Video)
@@ -269,13 +269,13 @@ def fetch_srt_data(url, **request_params):
     Fetch srt data from transcript provider.
     """
     # return TRANSCRIPT_SRT_DATA
-    response = requests.get(
-        build_url(url, **request_params)
-    )
+    fetch_srt_data_url = build_url(url, **request_params)
+    response = requests.get(fetch_srt_data_url)
 
     if not response.ok:
         raise TranscriptFetchError(
-            '[TRANSCRIPT FETCH ERROR] status={} -- text={}'.format(
+            '[TRANSCRIPT FETCH ERROR] url={} -- status={} -- text={}'.format(
+                scrub_query_params(fetch_srt_data_url, ['apikey', 'api_token']),
                 response.status_code,
                 response.text
             )
@@ -379,10 +379,12 @@ def get_translation_services(api_key):
     Returns:
         Available 3Play Media Translation services.
     """
-    response = requests.get(build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
+    get_translation_services_url = build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key)
+    response = requests.get(get_translation_services_url)
     if not response.ok:
         raise TranscriptTranslationError(
-            u'[3PlayMedia Callback] Error while fetching the translation services -- {status}, {response}'.format(
+            u'[3PlayMedia Callback] Error fetching the translation services -- url={url}, {status}, {response}'.format(
+                url=scrub_query_params(get_translation_services_url, ['apikey']),
                 status=response.status_code,
                 response=response.text,
             )
@@ -836,7 +838,8 @@ def get_translations_metadata(api_key, file_id, edx_video_id):
     translations_metadata_response = requests.get(translations_metadata_url)
     if not translations_metadata_response.ok:
         LOGGER.error(
-            u'[3PlayMedia Task] Translations metadata request failed for video=%s -- process_id=%s -- status=%s',
+            u'[3PlayMedia Task] Translations metadata request failed, url=%s -- video=%s -- process_id=%s -- status=%s',
+            scrub_query_params(translations_metadata_url, ['apikey']),
             edx_video_id,
             file_id,
             translations_metadata_response.status_code,

control/veda_deliver_3play.py

@@ -8,7 +8,7 @@ import sys
 
 from requests.packages.urllib3.exceptions import InsecurePlatformWarning
 from VEDA_OS01.models import TranscriptProcessMetadata, TranscriptProvider, TranscriptStatus
-from VEDA.utils import build_url
+from VEDA.utils import build_url, scrub_query_params
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 
@@ -94,7 +94,7 @@ class ThreePlayMediaClient(object):
             raise ThreePlayMediaUrlError('The URL "{media_url}" is not Accessible.'.format(media_url=self.media_url))
         elif response.headers['Content-Type'] != self.allowed_content_type:
             raise ThreePlayMediaUrlError(
-                'Media content-type should be "{allowed_type}". URL was "{media_url}", content-type was "{type}"'.format(
+                'Media content-type should be "{allowed_type}". URL was "{media_url}", content-type was "{type}"'.format(  # pylint: disable=line-too-long
                     allowed_type=self.allowed_content_type,
                     media_url=self.media_url,
                     type=response.headers['Content-Type'],
@@ -105,11 +105,16 @@ class ThreePlayMediaClient(object):
         """
         Gets all the 3Play Media supported languages
         """
-        response = requests.get(url=build_url(self.base_url, self.available_languages_url, apikey=self.api_key))
+        available_languages_url = build_url(self.base_url, self.available_languages_url, apikey=self.api_key)
+        response = requests.get(
+            url=available_languages_url
+        )
         if not response.ok:
             raise ThreePlayMediaLanguagesRetrievalError(
-                'Error while retrieving available languages: {response} -- {status}'.format(
-                    response=response.text, status=response.status_code
+                'Error while retrieving available languages: url={url} -- {response} -- {status}'.format(
+                    url=scrub_query_params(available_languages_url, ['apikey']),
+                    response=response.text,
+                    status=response.status_code
                 )
             )
 

control/veda_deliver_cielo.py

@@ -10,7 +10,7 @@ from requests.packages.urllib3.exceptions import InsecurePlatformWarning
 
 from VEDA_OS01.models import (TranscriptProcessMetadata, TranscriptProvider,
                               TranscriptStatus)
-from VEDA.utils import build_url
+from VEDA.utils import build_url, scrub_query_params
 from VEDA_OS01.transcripts import CIELO24_API_VERSION
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
@@ -126,24 +126,24 @@ class Cielo24Transcript(object):
             video_id=self.video.studio_id
         )
 
-        response = requests.get(
-            build_url(
-                self.cielo24_api_base_url,
-                self.cielo24_perform_transcription,
-                v=CIELO24_API_VERSION,
-                job_id=job_id,
-                target_language=lang_code,
-                callback_url=callback_url,
-                api_token=self.api_key,
-                priority=self.turnaround,
-                transcription_fidelity=self.fidelity,
-                options=json.dumps({"return_iwp": ["FINAL"]})
-            )
+        perform_transcript_url = build_url(
+            self.cielo24_api_base_url,
+            self.cielo24_perform_transcription,
+            v=CIELO24_API_VERSION,
+            job_id=job_id,
+            target_language=lang_code,
+            callback_url=callback_url,
+            api_token=self.api_key,
+            priority=self.turnaround,
+            transcription_fidelity=self.fidelity,
+            options=json.dumps({"return_iwp": ["FINAL"]})
         )
+        response = requests.get(perform_transcript_url)
 
         if not response.ok:
             raise Cielo24PerformTranscriptError(
-                '[PERFORM TRANSCRIPT ERROR] status={} -- text={}'.format(
+                '[PERFORM TRANSCRIPT ERROR] url={} -- status={} -- text={}'.format(
+                    scrub_query_params(perform_transcript_url, ['api_token']),
                     response.status_code,
                     response.text
                 )
@@ -168,20 +168,20 @@ class Cielo24Transcript(object):
         Returns:
             cielo24 task id
         """
-        response = requests.get(
-            build_url(
-                self.cielo24_api_base_url,
-                self.cielo24_add_media,
-                v=CIELO24_API_VERSION,
-                job_id=job_id,
-                api_token=self.api_key,
-                media_url=self.s3_video_url
-            )
+        media_url = build_url(
+            self.cielo24_api_base_url,
+            self.cielo24_add_media,
+            v=CIELO24_API_VERSION,
+            job_id=job_id,
+            api_token=self.api_key,
+            media_url=self.s3_video_url
         )
+        response = requests.get(media_url)
 
         if not response.ok:
             raise Cielo24AddMediaError(
-                '[ADD MEDIA ERROR] status={} -- text={}'.format(
+                '[ADD MEDIA ERROR] url={} -- status={} -- text={}'.format(
+                    scrub_query_params(media_url, ['api_token']),
                     response.status_code,
                     response.text
                 )
@@ -216,7 +216,7 @@ class Cielo24Transcript(object):
         if not response.ok:
             raise Cielo24CreateJobError(
                 '[CREATE JOB ERROR] url={} -- status={} -- text={}'.format(
-                    create_job_url,
+                    scrub_query_params(create_job_url, ['api_token']),
                     response.status_code,
                     response.text,
                 )