Added Organizations API data loader

This commit adds support for loading organizations from the Organizations API.

ECOM-3982

course_discovery/apps/course_metadata/data_loaders.py

+""" Data loaders. """
+import abc
+import logging
+
+from edx_rest_api_client.client import EdxRestApiClient
+
+from course_discovery.apps.course_metadata.models import Organization, Image
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractDataLoader(metaclass=abc.ABCMeta):
+    """ Base class for all data loaders.
+
+    Attributes:
+        api_url (str): URL of the API from which data is loaded
+        access_token (str): OAuth2 access token
+        PAGE_SIZE (int): Number of items to load per API call
+    """
+
+    PAGE_SIZE = 50
+
+    def __init__(self, api_url, access_token):
+        """
+        Arguments:
+            api_url (str): URL of the API from which data is loaded
+            access_token (str): OAuth2 access token
+        """
+        self.access_token = access_token
+        self.api_url = api_url
+
+    @abc.abstractmethod
+    def ingest(self):  # pragma: no cover
+        """ Load data for all supported objects (e.g. courses, runs). """
+        pass
+
+    def clean_strings(self, data):
+        """ Iterates over all string values, removing leading and trailing spaces,
+        and replacing empty strings with None. """
+        return {k: v.strip() or None for k, v in data.items() if isinstance(v, str)}
+
+
+class OrganizationsApiDataLoader(AbstractDataLoader):
+    """ Loads organizations from the Organizations API. """
+
+    def ingest(self):
+        client = EdxRestApiClient(self.api_url, oauth_access_token=self.access_token)
+        count = None
+        page = 1
+
+        logger.info('Refreshing Organizations from %s....', self.api_url)
+
+        while page:
+            response = client.organizations().get(page=page, page_size=self.PAGE_SIZE)
+            count = response['count']
+            results = response['results']
+            logger.info('Retrieved %d organizations...', len(results))
+
+            if response['next']:
+                page += 1
+            else:
+                page = None
+
+            for body in results:
+                body = self.clean_strings(body)
+                self.update_organization(body)
+
+        logger.info('Retrieved %d organizations from %s.', count, self.api_url)
+
+    def update_organization(self, body):
+        image = None
+        image_url = body['logo']
+
+        if image_url:
+            image_url = image_url.lower()
+            image, __ = Image.objects.get_or_create(src=image_url)
+
+        defaults = {
+            'name': body['name'],
+            'description': body['description'],
+            'logo_image': image,
+        }
+
+        Organization.objects.update_or_create(key=body['short_name'], defaults=defaults)

course_discovery/apps/course_metadata/tests/test_data_loaders.py

+""" Tests for data loaders. """
+import json
+from urllib.parse import parse_qs, urlparse
+
+import responses
+from django.conf import settings
+from django.test import TestCase, override_settings
+
+from course_discovery.apps.course_metadata.data_loaders import OrganizationsApiDataLoader
+from course_discovery.apps.course_metadata.models import Organization, Image
+
+ACCESS_TOKEN = 'secret'
+ORGANIZATIONS_API_URL = 'https://lms.example.com/api/organizations/v0'
+JSON = 'application/json'
+
+
+@override_settings(ORGANIZATIONS_API_URL=ORGANIZATIONS_API_URL)
+class OrganizationsApiDataLoaderTests(TestCase):
+    def setUp(self):
+        super(OrganizationsApiDataLoaderTests, self).setUp()
+        self.loader = OrganizationsApiDataLoader(ORGANIZATIONS_API_URL, ACCESS_TOKEN)
+
+    def test_init(self):
+        """ Verify the constructor sets the appropriate attributes. """
+        self.assertEqual(self.loader.api_url, ORGANIZATIONS_API_URL)
+        self.assertEqual(self.loader.access_token, ACCESS_TOKEN)
+
+    def mock_api(self):
+        bodies = [
+            {
+                'name': 'edX',
+                'short_name': ' edX ',
+                'description': 'edX',
+                'logo': 'https://example.com/edx.jpg',
+            },
+            {
+                'name': 'Massachusetts Institute of Technology ',
+                'short_name': 'MITx',
+                'description': ' ',
+                'logo': '',
+            }
+        ]
+
+        def organizations_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(bodies)
+
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'count': count,
+                    'next': next,
+                    'previous': None,
+                    'results': [data[page - 1]]
+                }
+
+                return 200, {}, json.dumps(body)
+
+            return request_callback
+
+        url = '{host}/organizations/'.format(host=settings.ORGANIZATIONS_API_URL)
+        responses.add_callback(responses.GET, url, callback=organizations_api_callback(url, bodies), content_type=JSON)
+
+        return bodies
+
+    def assert_organization_loaded(self, body):
+        """ Assert an Organization corresponding to the specified data body was properly loaded into the database. """
+        organization = Organization.objects.get(key=body['short_name'].strip())
+        self.assertEqual(organization.name, body['name'].strip() or None)
+        self.assertEqual(organization.description, body['description'].strip() or None)
+
+        image = None
+        image_url = body['logo'].strip() or None
+        if image_url:
+            image = Image.objects.get(src=image_url)
+
+        self.assertEqual(organization.logo_image, image)
+
+    @responses.activate
+    def test_ingest(self):
+        """ Verify the method ingests data from the Organizations API. """
+        data = self.mock_api()
+        self.assertEqual(Organization.objects.count(), 0)
+
+        self.loader.ingest()
+
+        # Verify the API was called with the correct authorization header
+        self.assertEqual(len(responses.calls), len(data))
+        self.assertEqual(responses.calls[0].request.headers['Authorization'], 'Bearer {}'.format(ACCESS_TOKEN))
+
+        # Verify the Organizations were created correctly
+        self.assertEqual(Organization.objects.count(), len(data))
+
+        for datum in data:
+            self.assert_organization_loaded(datum)

course_discovery/settings/base.py

@@ -307,9 +307,8 @@ HAYSTACK_CONNECTIONS = {
 
 HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
-# TODO Replace with None and document.
-ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
-COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'
+ECOMMERCE_API_URL = 'http://127.0.0.1:8002/api/v2/'
+ORGANIZATIONS_API_URL = 'http://127.0.0.1:8000/api/organizations/v0/'
 
 EDX_DRF_EXTENSIONS = {
     'OAUTH2_USER_INFO_URL': 'http://localhost:8000/oauth2/user_info',

course_discovery/settings/local.py

@@ -58,8 +58,6 @@ ENABLE_AUTO_AUTH = True
 
 JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
 
-ECOMMERCE_API_URL = 'http://localhost:8002/api/v2/'
-COURSES_API_URL = 'http://localhost:8000/api/courses/v1/'
 
 #####################################################################
 # Lastly, see if the developer has any local overrides.