Logging Drupal errors (#115)

Logging data integrity errors for the Drupal data loader. These errors will no longer stop execution of the data loader, so valid data will continue to be written to the database.

ECOM-4528

course_discovery/apps/course_metadata/data_loaders.py

@@ -276,9 +276,13 @@ class DrupalApiDataLoader(AbstractDataLoader):
             if not body:
                 continue
 
-            cleaned_body = self.clean_strings(body)
-            course = self.update_course(cleaned_body)
-            self.update_course_run(course, cleaned_body)
+            course_run_id = body['course_id']
+            try:
+                cleaned_body = self.clean_strings(body)
+                course = self.update_course(cleaned_body)
+                self.update_course_run(course, cleaned_body)
+            except:  # pylint: disable=bare-except
+                logger.exception('An error occurred while updating [%s] from [%s]!', course_run_id, self.api_url)
 
         # Clean Organizations separately from other orphaned instances to avoid removing all orgnaziations
         # after an initial data load on an empty table.

course_discovery/apps/course_metadata/tests/test_data_loaders.py

@@ -5,6 +5,7 @@ from decimal import Decimal
 from urllib.parse import parse_qs, urlparse
 
 import ddt
+import mock
 import responses
 from django.conf import settings
 from django.test import TestCase, override_settings
@@ -665,6 +666,25 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
         self.assertFalse(Person.objects.filter(key__startswith='orphan_staff_').exists())
         self.assertFalse(Organization.objects.filter(key__startswith='orphan_org_').exists())
 
+    @responses.activate
+    def test_ingest_exception_handling(self):
+        """ Verify the data loader properly handles exceptions during processing of the data from the API. """
+        data = self.mock_api()
+        # Include all data, except the empty array.
+        # TODO: Remove the -1 after ECOM-4493 is in production.
+        expected_call_count = len(data) - 1
+
+        with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
+            with mock.patch('course_discovery.apps.course_metadata.data_loaders.logger') as mock_logger:
+                self.loader.ingest()
+                print(mock_logger.exception.call_args)
+                self.assertEqual(mock_logger.exception.call_count, expected_call_count)
+
+                # TODO: Change the -2 to -1 after ECOM-4493 is in production.
+                mock_logger.exception.call_args(
+                    'An error occurred while updating [%s] from [%s]!', data[-2]['course_id'], MARKETING_API_URL
+                )
+
     @ddt.data(
         ('', ''),
         ('<h1>foo</h1>', '# foo'),