Retrieving data from Courses API

course_discovery/apps/courses/models.py

@@ -146,12 +146,18 @@ class Course(object):
         Returns:
             None
         """
-        client = EdxRestApiClient(settings.ECOMMERCE_API_URL, oauth_access_token=access_token)
-
-        logger.info('Refreshing course data from %s....', settings.ECOMMERCE_API_URL)
+        cls.refresh_all_ecommerce_data(access_token)
+        cls.refresh_all_course_api_data(access_token)
 
+    @classmethod
+    def refresh_all_ecommerce_data(cls, access_token):
+        ecommerce_api_url = settings.ECOMMERCE_API_URL
+        client = EdxRestApiClient(ecommerce_api_url, oauth_access_token=access_token)
         count = None
         page = 1
+
+        logger.info('Refreshing ecommerce data from %s....', ecommerce_api_url)
+
         while page:
             response = client.courses().get(include_products=True, page=page, page_size=50)
             count = response['count']
@@ -164,9 +170,36 @@ class Course(object):
                 page = None
 
             for body in results:
-                Course(body['id'], body).save()
+                Course(body['id']).update(body)
+
+        logger.info('Retrieved %d courses from %s.', count, ecommerce_api_url)
+
+    @classmethod
+    def refresh_all_course_api_data(cls, access_token):
+        course_api_url = settings.COURSES_API_URL
+        client = EdxRestApiClient(course_api_url, oauth_access_token=access_token)
+
+        count = None
+        page = 1
+
+        logger.info('Refreshing course api data from %s....', course_api_url)
+
+        while page:
+            # TODO Update API to not require username?
+            response = client.courses().get(page=page, page_size=50, username='ecommerce_worker')
+            count = response['pagination']['count']
+            results = response['results']
+            logger.info('Retrieved %d courses...', len(results))
+
+            if response['pagination']['next']:
+                page += 1
+            else:
+                page = None
+
+            for body in results:
+                Course(body['id']).update(body)
 
-        logger.info('Retrieved %d courses.', count)
+        logger.info('Retrieved %d courses from %s.', count, course_api_url)
 
     def __init__(self, id, body=None):  # pylint: disable=redefined-builtin
         if not id:
@@ -202,3 +235,20 @@ class Course(object):
         logger.info('Indexing course %s...', self.id)
         self._es_client().index(index=self._index, doc_type=self.doc_type, id=self.id, body=self.body)
         logger.info('Finished indexing course %s.', self.id)
+
+    def update(self, body):
+        """ Updates (merges) the data in the index with the provided data.
+
+        Args:
+            body (dict): Data to be merged into the index.
+
+        Returns:
+            None
+        """
+        body = {
+            'doc': body,
+            'doc_as_upsert': True,
+        }
+        logger.info('Updating course %s...', self.id)
+        self._es_client().update(index=self._index, doc_type=self.doc_type, id=self.id, body=body)
+        logger.info('Finished updating course %s.', self.id)

course_discovery/apps/courses/tests/test_models.py

@@ -10,11 +10,12 @@ from course_discovery.apps.courses.models import Course
 from course_discovery.apps.courses.tests.factories import CourseFactory
 
 ACCESS_TOKEN = 'secret'
+COURSES_API_URL = 'https://lms.example.com/api/courses/v1'
 ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
 JSON = 'application/json'
 
 
-@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL)
+@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL, COURSES_API_URL=COURSES_API_URL)
 class CourseTests(ElasticsearchTestMixin, TestCase):
     def assert_course_attrs(self, course, attrs):
         """
@@ -30,14 +31,12 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
     @responses.activate  # pylint: disable=no-member
     def mock_refresh_all(self):
         """
-        Mock the E-Commerce API and refresh all course data.
+        Mock the external APIs and refresh all course data.
 
         Returns:
             [dict]: List of dictionaries representing course content bodies.
         """
 
-        # Mock the call to the E-Commerce API, simulating multiple pages of data
-        url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
         course_bodies = [
             {
                 'id': 'a/b/c',
@@ -57,30 +56,64 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
             }
         ]
 
-        def request_callback(request):
-            # pylint: disable=redefined-builtin
-            next = None
-            count = len(course_bodies)
+        def ecommerce_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(course_bodies)
+
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'count': count,
+                    'next': next,
+                    'previous': None,
+                    'results': [data[page - 1]]
+                }
 
-            # Use the querystring to determine which page should be returned. Default to page 1.
-            # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-            qs = parse_qs(urlparse(request.path_url).query)
-            page = int(qs.get('page', [1])[0])
+                return 200, {}, json.dumps(body)
 
-            if page < count:
-                next = '{}?page={}'.format(url, page)
+            return request_callback
 
-            body = {
-                'count': count,
-                'next': next,
-                'previous': None,
-                'results': [course_bodies[page - 1]]
-            }
+        def courses_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(course_bodies)
 
-            return 200, {}, json.dumps(body)
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'pagination': {
+                        'count': count,
+                        'next': next,
+                        'previous': None,
+                    },
+                    'results': [data[page - 1]]
+                }
+
+                return 200, {}, json.dumps(body)
+
+            return request_callback
 
         # pylint: disable=no-member
-        responses.add_callback(responses.GET, url, callback=request_callback, content_type=JSON)
+        url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
+        responses.add_callback(responses.GET, url, callback=ecommerce_api_callback(url, course_bodies),
+                               content_type=JSON)
+        url = '{host}/courses/'.format(host=COURSES_API_URL)
+        responses.add_callback(responses.GET, url, callback=courses_api_callback(url, course_bodies), content_type=JSON)
 
         # Refresh all course data
         Course.refresh_all(ACCESS_TOKEN)

course_discovery/settings/base.py

@@ -258,9 +258,10 @@ SWAGGER_SETTINGS = {
 }
 
 ELASTICSEARCH = {
-    'host': '',
+    'host': 'localhost:9200',
     'index': 'course_discovery',
 }
 
 # TODO Replace with None and document.
 ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
+COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'

course_discovery/settings/local.py

@@ -57,6 +57,9 @@ ENABLE_AUTO_AUTH = True
 
 JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
 
+ECOMMERCE_API_URL = 'http://localhost:8002/api/v2/'
+COURSES_API_URL = 'http://localhost:8000/api/courses/v1/'
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):