Skip to content

C
course-discovery
Commit 248ec350 by Clinton Blackburn
Options

Merge pull request #107 from edx/clintonb/drupal-data-loader-fix 

Ignoring invalid data from the Drupal API
parents a0699349 63afe95d
Showing with 108 additions and 95 deletions
course_discovery/apps/course_metadata/data_loaders.py
...@@ -271,6 +271,11 @@ class DrupalApiDataLoader(AbstractDataLoader): ...@@ -271,6 +271,11 @@ class DrupalApiDataLoader(AbstractDataLoader):
logger.info('Retrieved %d course runs...', len(data)) logger.info('Retrieved %d course runs...', len(data))
for body in data: for body in data:
# NOTE (CCB): Some of the entries are empty arrays. We will fix this on the Drupal side of things
# later (ECOM-4493). For now, ignore them.
if not body:
continue
cleaned_body = self.clean_strings(body) cleaned_body = self.clean_strings(body)
course = self.update_course(cleaned_body) course = self.update_course(cleaned_body)
self.update_course_run(course, cleaned_body) self.update_course_run(course, cleaned_body)
......
course_discovery/apps/course_metadata/tests/test_data_loaders.py
...@@ -462,103 +462,110 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -462,103 +462,110 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
def mock_api(self): def mock_api(self):
"""Mock out the Drupal API. Returns a list of mocked-out course runs.""" """Mock out the Drupal API. Returns a list of mocked-out course runs."""
body = { body = {
'items': [{ 'items': [
'title': self.EXISTING_COURSE_AND_RUN_DATA[0]['title'], {
'level': { 'title': self.EXISTING_COURSE_AND_RUN_DATA[0]['title'],
'title': 'Introductory', 'level': {
'title': 'Introductory',
},
'course_about_uri': '/course/bread-baking-101',
'course_id': self.EXISTING_COURSE_AND_RUN_DATA[0]['course_run_key'],
'subjects': [{
'title': 'Bread baking',
}],
'current_language': self.EXISTING_COURSE_AND_RUN_DATA[0]['current_language'],
'subtitle': 'Learn about Bread',
'description': '<p>Bread is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.',
'sponsors': [{
'uuid': 'abc123',
'title': 'Tatte',
'image': 'http://example.com/tatte.jpg',
'uri': 'sponsor/tatte'
}],
'staff': [{
'uuid': 'staff123',
'title': 'The Muffin Man',
'image': 'http://example.com/muffinman.jpg',
'display_position': {
'title': 'Baker'
}
}, {
'uuid': 'staffZYX',
'title': 'Arthur',
'image': 'http://example.com/kingarthur.jpg',
'display_position': {
'title': 'King'
}
}]
}, },
'course_about_uri': '/course/bread-baking-101', {
'course_id': self.EXISTING_COURSE_AND_RUN_DATA[0]['course_run_key'], 'title': self.EXISTING_COURSE_AND_RUN_DATA[1]['title'],
'subjects': [{ 'level': {
'title': 'Bread baking', 'title': 'Intermediate',
}], },
'current_language': self.EXISTING_COURSE_AND_RUN_DATA[0]['current_language'], 'course_about_uri': '/course/testing-201',
'subtitle': 'Learn about Bread', 'course_id': self.EXISTING_COURSE_AND_RUN_DATA[1]['course_run_key'],
'description': '<p><b>Bread</b> is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.', 'subjects': [{
'sponsors': [{ 'title': 'testing',
'uuid': 'abc123', }],
'title': 'Tatte', 'current_language': self.EXISTING_COURSE_AND_RUN_DATA[1]['current_language'],
'image': 'http://example.com/tatte.jpg', 'subtitle': 'Testing 201',
'uri': 'sponsor/tatte' 'description': "how to test better",
}], 'sponsors': [],
'staff': [{ 'staff': [{
'uuid': 'staff123', 'uuid': '432staff',
'title': 'The Muffin Man', 'title': 'Test',
'image': 'http://example.com/muffinman.jpg', 'image': 'http://example.com/test.jpg',
'display_position': { 'display_position': {
'title': 'Baker' 'title': 'Tester'
} }
}, { }]
'uuid': 'staffZYX',
'title': 'Arthur',
'image': 'http://example.com/kingarthur.jpg',
'display_position': {
'title': 'King'
}
}]
}, {
'title': self.EXISTING_COURSE_AND_RUN_DATA[1]['title'],
'level': {
'title': 'Intermediate',
}, },
'course_about_uri': '/course/testing-201', { # Create a course which exists in LMS/Otto, but without course runs
'course_id': self.EXISTING_COURSE_AND_RUN_DATA[1]['course_run_key'], 'title': self.EXISTING_COURSE['title'],
'subjects': [{ 'level': {
'title': 'testing', 'title': 'Advanced',
}], },
'current_language': self.EXISTING_COURSE_AND_RUN_DATA[1]['current_language'], 'course_about_uri': '/course/partial-101',
'subtitle': 'Testing 201', 'course_id': 'course-v1:{course_key}+run'.format(course_key=self.EXISTING_COURSE['course_key']),
'description': "how to test better", 'subjects': [{
'sponsors': [], 'title': 'partially fake',
'staff': [{ }],
'uuid': '432staff', 'current_language': 'en-us',
'title': 'Test', 'subtitle': 'Nope',
'image': 'http://example.com/test.jpg', 'description': 'what is fake?',
'display_position': { 'sponsors': [{
'title': 'Tester' 'uuid': '123abc',
} 'title': 'Fake',
}] 'image': 'http://example.com/fake.jpg',
}, { # Create a course which exists in LMS/Otto, but without course runs 'uri': 'sponsor/fake'
'title': self.EXISTING_COURSE['title'], }, {
'level': { 'uuid': 'qwertyuiop',
'title': 'Advanced', 'title': 'Faux',
'image': 'http://example.com/faux.jpg',
'uri': 'sponsor/faux'
}],
'staff': [],
}, },
'course_about_uri': '/course/partial-101', { # Create a fake course run which doesn't exist in LMS/Otto
'course_id': 'course-v1:{course_key}+run'.format(course_key=self.EXISTING_COURSE['course_key']), 'title': 'A partial course',
'subjects': [{ 'level': {
'title': 'partially fake', 'title': 'Advanced',
}], },
'current_language': 'en-us', 'course_about_uri': '/course/partial-101',
'subtitle': 'Nope', 'course_id': 'course-v1:fakeX+fake+reallyfake',
'description': 'what is fake?', 'subjects': [{
'sponsors': [{ 'title': 'seriously fake',
'uuid': '123abc', }],
'title': 'Fake', 'current_language': 'en-us',
'image': 'http://example.com/fake.jpg', 'subtitle': 'Nope',
'uri': 'sponsor/fake' 'description': 'what is real?',
}, { 'sponsors': [],
'uuid': 'qwertyuiop', 'staff': [],
'title': 'Faux',
'image': 'http://example.com/faux.jpg',
'uri': 'sponsor/faux'
}],
'staff': [],
}, { # Create a fake course run which doesn't exist in LMS/Otto
'title': 'A partial course',
'level': {
'title': 'Advanced',
}, },
'course_about_uri': '/course/partial-101', # NOTE (CCB): Some of the entries are empty arrays. Remove this as part of ECOM-4493.
'course_id': 'course-v1:fakeX+fake+reallyfake', [],
'subjects': [{ ]
'title': 'seriously fake',
}],
'current_language': 'en-us',
'subtitle': 'Nope',
'description': 'what is real?',
'sponsors': [],
'staff': [],
}]
} }
responses.add( responses.add(
...@@ -633,8 +640,9 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -633,8 +640,9 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
def test_ingest(self): def test_ingest(self):
"""Verify the data loader ingests data from Drupal.""" """Verify the data loader ingests data from Drupal."""
data = self.mock_api() data = self.mock_api()
# The faked course should not be loaded from Drupal # Neither the faked course, nor the empty array, should not be loaded from Drupal.
loaded_data = data[:-2] # Change this back to -2 as part of ECOM-4493.
loaded_data = data[:-3]
self.loader.ingest() self.loader.ingest()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment