Commit 3a983295 by Clinton Blackburn

Logging Drupal errors (#115)

Logging data integrity errors for the Drupal data loader. These errors will no longer stop execution of the data loader, so valid data will continue to be written to the database.

ECOM-4528
parent fab8ab61
......@@ -276,9 +276,13 @@ class DrupalApiDataLoader(AbstractDataLoader):
if not body:
continue
cleaned_body = self.clean_strings(body)
course = self.update_course(cleaned_body)
self.update_course_run(course, cleaned_body)
course_run_id = body['course_id']
try:
cleaned_body = self.clean_strings(body)
course = self.update_course(cleaned_body)
self.update_course_run(course, cleaned_body)
except: # pylint: disable=bare-except
logger.exception('An error occurred while updating [%s] from [%s]!', course_run_id, self.api_url)
# Clean Organizations separately from other orphaned instances to avoid removing all orgnaziations
# after an initial data load on an empty table.
......
......@@ -5,6 +5,7 @@ from decimal import Decimal
from urllib.parse import parse_qs, urlparse
import ddt
import mock
import responses
from django.conf import settings
from django.test import TestCase, override_settings
......@@ -665,6 +666,25 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
self.assertFalse(Person.objects.filter(key__startswith='orphan_staff_').exists())
self.assertFalse(Organization.objects.filter(key__startswith='orphan_org_').exists())
@responses.activate
def test_ingest_exception_handling(self):
""" Verify the data loader properly handles exceptions during processing of the data from the API. """
data = self.mock_api()
# Include all data, except the empty array.
# TODO: Remove the -1 after ECOM-4493 is in production.
expected_call_count = len(data) - 1
with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
with mock.patch('course_discovery.apps.course_metadata.data_loaders.logger') as mock_logger:
self.loader.ingest()
print(mock_logger.exception.call_args)
self.assertEqual(mock_logger.exception.call_count, expected_call_count)
# TODO: Change the -2 to -1 after ECOM-4493 is in production.
mock_logger.exception.call_args(
'An error occurred while updating [%s] from [%s]!', data[-2]['course_id'], MARKETING_API_URL
)
@ddt.data(
('', ''),
('<h1>foo</h1>', '# foo'),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment