Ignoring invalid data from the Drupal API

ECOM-4414

course_discovery/apps/course_metadata/data_loaders.py

@@ -271,6 +271,11 @@ class DrupalApiDataLoader(AbstractDataLoader):
         logger.info('Retrieved %d course runs...', len(data))
 
         for body in data:
+            # NOTE (CCB): Some of the entries are empty arrays. We will fix this on the Drupal side of things
+            # later (ECOM-4493). For now, ignore them.
+            if not body:
+                continue
+
             cleaned_body = self.clean_strings(body)
             course = self.update_course(cleaned_body)
             self.update_course_run(course, cleaned_body)

course_discovery/apps/course_metadata/tests/test_data_loaders.py

@@ -462,7 +462,8 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     def mock_api(self):
         """Mock out the Drupal API. Returns a list of mocked-out course runs."""
         body = {
-            'items': [{
+            'items': [
+                {
                     'title': self.EXISTING_COURSE_AND_RUN_DATA[0]['title'],
                     'level': {
                         'title': 'Introductory',
@@ -474,7 +475,7 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                     }],
                     'current_language': self.EXISTING_COURSE_AND_RUN_DATA[0]['current_language'],
                     'subtitle': 'Learn about Bread',
-                'description': '<p><b>Bread</b> is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.',
+                    'description': '<p>Bread is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.',
                     'sponsors': [{
                         'uuid': 'abc123',
                         'title': 'Tatte',
@@ -496,7 +497,8 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                             'title': 'King'
                         }
                     }]
-            }, {
+                },
+                {
                     'title': self.EXISTING_COURSE_AND_RUN_DATA[1]['title'],
                     'level': {
                         'title': 'Intermediate',
@@ -518,7 +520,8 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                             'title': 'Tester'
                         }
                     }]
-            }, {  # Create a course which exists in LMS/Otto, but without course runs
+                },
+                {  # Create a course which exists in LMS/Otto, but without course runs
                     'title': self.EXISTING_COURSE['title'],
                     'level': {
                         'title': 'Advanced',
@@ -543,7 +546,8 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                         'uri': 'sponsor/faux'
                     }],
                     'staff': [],
-            }, {  # Create a fake course run which doesn't exist in LMS/Otto
+                },
+                {  # Create a fake course run which doesn't exist in LMS/Otto
                     'title': 'A partial course',
                     'level': {
                         'title': 'Advanced',
@@ -558,7 +562,10 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                     'description': 'what is real?',
                     'sponsors': [],
                     'staff': [],
-            }]
+                },
+                # NOTE (CCB): Some of the entries are empty arrays. Remove this as part of ECOM-4493.
+                [],
+            ]
         }
 
         responses.add(
@@ -633,8 +640,9 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     def test_ingest(self):
         """Verify the data loader ingests data from Drupal."""
         data = self.mock_api()
-        # The faked course should not be loaded from Drupal
-        loaded_data = data[:-2]
+        # Neither the faked course, nor the empty array, should not be loaded from Drupal.
+        # Change this back to -2 as part of ECOM-4493.
+        loaded_data = data[:-3]
 
         self.loader.ingest()