Added programs data loader (#153)

ECOM-4786

course_discovery/apps/course_metadata/data_loaders.py

@@ -14,7 +14,8 @@ from opaque_keys.edx.keys import CourseKey
 from course_discovery.apps.core.models import Currency
 from course_discovery.apps.core.utils import delete_orphans
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseOrganization, CourseRun, Image, LanguageTag, LevelType, Organization, Person, Seat, Subject, Video
+    Course, CourseOrganization, CourseRun, Image, LanguageTag, LevelType, Organization, Person, Seat, Subject, Video,
+    Program,
 )
 
 logger = logging.getLogger(__name__)
@@ -486,3 +487,51 @@ class EcommerceApiDataLoader(AbstractDataLoader):
             (att['value'] for att in product['attribute_values'] if att['name'] == 'certificate_type'),
             Seat.AUDIT
         )
+
+
+class ProgramsApiDataLoader(AbstractDataLoader):
+    """ Loads programs from the Programs API. """
+
+    def ingest(self):
+        client = self.api_client
+        count = None
+        page = 1
+
+        logger.info('Refreshing programs from %s...', self.api_url)
+
+        while page:
+            response = client.programs.get(page=page, page_size=self.PAGE_SIZE)
+            count = response['count']
+            results = response['results']
+            logger.info('Retrieved %d programs...', len(results))
+
+            if response['next']:
+                page += 1
+            else:
+                page = None
+
+            for program in results:
+                program = self.clean_strings(program)
+                self.update_program(program)
+
+        logger.info('Retrieved %d programs from %s.', count, self.api_url)
+
+    def update_program(self, body):
+        defaults = {
+            'name': body['name'],
+            'subtitle': body['subtitle'],
+            'category': body['category'],
+            'status': body['status'],
+            'marketing_slug': body['marketing_slug'],
+        }
+        program, __ = Program.objects.update_or_create(uuid=body['uuid'], defaults=defaults)
+
+        organizations = []
+        for org in body['organizations']:
+            organization, __ = Organization.objects.get_or_create(
+                key=org['key'], defaults={'name': org['display_name']}
+            )
+            organizations.append(organization)
+
+        program.organizations.clear()
+        program.organizations.add(*organizations)

course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py

@@ -5,7 +5,7 @@ from django.core.management import BaseCommand, CommandError
 from edx_rest_api_client.client import EdxRestApiClient
 
 from course_discovery.apps.course_metadata.data_loaders import (
-    CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader
+    CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader
 )
 
 logger = logging.getLogger(__name__)
@@ -58,6 +58,7 @@ class Command(BaseCommand):
             (CoursesApiDataLoader, settings.COURSES_API_URL,),
             (EcommerceApiDataLoader, settings.ECOMMERCE_API_URL,),
             (DrupalApiDataLoader, settings.MARKETING_API_URL,),
+            (ProgramsApiDataLoader, settings.PROGRAMS_API_URL,),
         )
 
         for loader_class, api_url in loaders:

course_discovery/apps/course_metadata/tests/test_data_loaders.py

@@ -15,11 +15,11 @@ from opaque_keys.edx.keys import CourseKey
 from pytz import UTC
 
 from course_discovery.apps.course_metadata.data_loaders import (
-    OrganizationsApiDataLoader, CoursesApiDataLoader, DrupalApiDataLoader, EcommerceApiDataLoader, AbstractDataLoader
-)
+    OrganizationsApiDataLoader, CoursesApiDataLoader, DrupalApiDataLoader, EcommerceApiDataLoader, AbstractDataLoader,
+    ProgramsApiDataLoader)
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseOrganization, CourseRun, Image, LanguageTag, Organization, Person, Seat, Subject
-)
+    Course, CourseOrganization, CourseRun, Image, LanguageTag, Organization, Person, Seat, Subject,
+    Program)
 from course_discovery.apps.course_metadata.tests.factories import (
     CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory
 )
@@ -32,6 +32,7 @@ ENGLISH_LANGUAGE_TAG = LanguageTag(code='en-us', name='English - United States')
 JSON = 'application/json'
 MARKETING_API_URL = 'https://example.com/api/catalog/v2/'
 ORGANIZATIONS_API_URL = 'https://lms.example.com/api/organizations/v0'
+PROGRAMS_API_URL = 'https://programs.example.com/api/v1'
 
 
 class AbstractDataLoaderTest(TestCase):
@@ -1081,3 +1082,103 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     def test_get_certificate_type(self, product, expected_certificate_type):
         """ Verify the method returns the correct certificate type"""
         self.assertEqual(self.loader.get_certificate_type(product), expected_certificate_type)
+
+
+@ddt.ddt
+@override_settings(PROGRAMS_API_URL=PROGRAMS_API_URL)
+class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
+    api_url = PROGRAMS_API_URL
+    loader_class = ProgramsApiDataLoader
+
+    def mock_api(self):
+        bodies = [
+            {
+                'uuid': 'd9ee1a73-d82d-4ed7-8eb1-80ea2b142ad6',
+                'id': 1,
+                'name': 'Water Management',
+                'subtitle': 'Explore water management concepts and technologies',
+                'category': 'xseries',
+                'status': 'active',
+                'marketing_slug': 'water-management',
+                'organizations': [
+                    {
+                        'display_name': 'Delft University of Technology',
+                        'key': 'DelftX'
+                    }
+                ]
+            },
+            {
+                'uuid': 'b043f467-5e80-4225-93d2-248a93a8556a',
+                'id': 2,
+                'name': 'Supply Chain Management',
+                'subtitle': 'Learn how to design and optimize the supply chain to enhance business performance.',
+                'category': 'xseries',
+                'status': 'active',
+                'marketing_slug': 'supply-chain-management-0',
+                'organizations': [
+                    {
+                        'display_name': 'Massachusetts Institute of Technology',
+                        'key': 'MITx'
+                    }
+                ]
+            },
+        ]
+
+        def programs_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(bodies)
+
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'count': count,
+                    'next': next,
+                    'previous': None,
+                    'results': [data[page - 1]]
+                }
+
+                return 200, {}, json.dumps(body)
+
+            return request_callback
+
+        url = '{host}/programs/'.format(host=self.api_url)
+        responses.add_callback(responses.GET, url, callback=programs_api_callback(url, bodies), content_type=JSON)
+
+        return bodies
+
+    def assert_program_loaded(self, body):
+        """ Assert a Program corresponding to the specified data body was properly loaded into the database. """
+        program = Program.objects.get(uuid=AbstractDataLoader.clean_string(body['uuid']))
+
+        for attr in ('name', 'subtitle', 'category', 'status', 'marketing_slug',):
+            self.assertEqual(getattr(program, attr), AbstractDataLoader.clean_string(body[attr]))
+
+        keys = [org['key'] for org in body['organizations']]
+        expected_organizations = list(Organization.objects.filter(key__in=keys))
+        self.assertEqual(keys, [org.key for org in expected_organizations])
+        self.assertListEqual(list(program.organizations.all()), expected_organizations)
+
+    @responses.activate
+    def test_ingest(self):
+        """ Verify the method ingests data from the Organizations API. """
+        data = self.mock_api()
+        self.assertEqual(Program.objects.count(), 0)
+
+        self.loader.ingest()
+
+        expected_num_programs = len(data)
+        self.assert_api_called(expected_num_programs)
+        self.assertEqual(Program.objects.count(), expected_num_programs)
+
+        for datum in data:
+            self.assert_program_loaded(datum)
+
+        self.loader.ingest()

course_discovery/settings/base.py

@@ -334,5 +334,6 @@ HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 COURSES_API_URL = 'http://127.0.0.1:8000/api/courses/v1/'
 ECOMMERCE_API_URL = 'http://127.0.0.1:8002/api/v2/'
 ORGANIZATIONS_API_URL = 'http://127.0.0.1:8000/api/organizations/v0/'
+PROGRAMS_API_URL = 'http://127.0.0.1:8003/api/v1/'
 MARKETING_API_URL = 'http://example.org/api/catalog/v2/'
 MARKETING_URL_ROOT = 'http://example.org/'