Added exception handling to programs data loader (#185)

Loading will continue despite a single program failing to load.

ECOM-5078

course_discovery/apps/course_metadata/data_loaders.py

@@ -532,6 +532,9 @@ class ProgramsApiDataLoader(AbstractDataLoader):
         logger.info('Retrieved %d programs from %s.', count, api_url)
 
     def update_program(self, body):
+        uuid = body['uuid']
+
+        try:
             defaults = {
                 'title': body['name'],
                 'subtitle': body['subtitle'],
@@ -541,7 +544,8 @@ class ProgramsApiDataLoader(AbstractDataLoader):
                 'image': self._get_image(body),
                 'partner': self.partner,
             }
-        program, __ = Program.objects.update_or_create(uuid=body['uuid'], defaults=defaults)
+
+            program, __ = Program.objects.update_or_create(uuid=uuid, defaults=defaults)
 
             organizations = []
             for org in body['organizations']:
@@ -552,6 +556,8 @@ class ProgramsApiDataLoader(AbstractDataLoader):
 
             program.organizations.clear()
             program.organizations.add(*organizations)
+        except Exception:  # pylint: disable=broad-except
+            logger.exception('Failed to load program %s', uuid)
 
     def _get_image(self, body):
         image = None

course_discovery/apps/course_metadata/tests/mock_data.py

@@ -19,13 +19,10 @@ EXISTING_COURSE_AND_RUN_DATA = (
     }
 )
 
-
 ORPHAN_ORGANIZATION_KEY = 'orphan_org'
 
-
 ORPHAN_STAFF_KEY = 'orphan_staff'
 
-
 COURSES_API_BODIES = [
     {
         'end': '2015-08-08T00:00:00Z',
@@ -89,7 +86,6 @@ COURSES_API_BODIES = [
     },
 ]
 
-
 ECOMMERCE_API_BODIES = [
     {
         "id": "audit/course/run",
@@ -308,7 +304,6 @@ ECOMMERCE_API_BODIES = [
     }
 ]
 
-
 MARKETING_API_BODY = {
     'items': [
         {
@@ -424,7 +419,6 @@ MARKETING_API_BODY = {
     ]
 }
 
-
 ORGANIZATIONS_API_BODIES = [
     {
         'name': 'edX',
@@ -440,7 +434,6 @@ ORGANIZATIONS_API_BODIES = [
     }
 ]
 
-
 PROGRAMS_API_BODIES = [
     {
         'uuid': 'd9ee1a73-d82d-4ed7-8eb1-80ea2b142ad6',
@@ -479,4 +472,21 @@ PROGRAMS_API_BODIES = [
         ],
         'banner_image_urls': {},
     },
+    {
+        'uuid': '01bc3a40-fa9d-4076-8885-660b2f7a594e',
+        'id': 3,
+        'name': 'Data Science and Analytics in Context',
+        'subtitle': 'Learn the foundations of statistical thinking, the power of machine learning, and enabling '
+                    'technologies for data science.',
+        'category': 'xseries',
+        'status': 'active',
+        'marketing_slug': None,
+        'organizations': [
+            {
+                'display_name': 'Columbia University',
+                'key': 'ColumbiaX'
+            }
+        ],
+        'banner_image_urls': {},
+    },
 ]

course_discovery/apps/course_metadata/tests/test_data_loaders.py

@@ -594,7 +594,9 @@ class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
             callback=mock_api_callback(url, bodies),
             content_type=JSON
         )
-        return bodies
+
+        # We exclude the one invalid item
+        return bodies[:-1]
 
     def assert_program_loaded(self, body):
         """ Assert a Program corresponding to the specified data body was properly loaded into the database. """