Skip to content

C
course-discovery
Commit 40945eb9 by Clinton Blackburn
Options

Retrieving data from Courses API

parent 8c175563
Showing with 115 additions and 28 deletions
course_discovery/apps/courses/models.py
......@@ -146,12 +146,18 @@ class Course(object):
Returns:
None
"""
client = EdxRestApiClient(settings.ECOMMERCE_API_URL, oauth_access_token=access_token)
logger.info('Refreshing course data from %s....', settings.ECOMMERCE_API_URL)
cls.refresh_all_ecommerce_data(access_token)
cls.refresh_all_course_api_data(access_token)
@classmethod
def refresh_all_ecommerce_data(cls, access_token):
ecommerce_api_url = settings.ECOMMERCE_API_URL
client = EdxRestApiClient(ecommerce_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing ecommerce data from %s....', ecommerce_api_url)
while page:
response = client.courses().get(include_products=True, page=page, page_size=50)
count = response['count']
......@@ -164,9 +170,36 @@ class Course(object):
page = None
for body in results:
Course(body['id'], body).save()
Course(body['id']).update(body)
logger.info('Retrieved %d courses from %s.', count, ecommerce_api_url)
@classmethod
def refresh_all_course_api_data(cls, access_token):
course_api_url = settings.COURSES_API_URL
client = EdxRestApiClient(course_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing course api data from %s....', course_api_url)
while page:
# TODO Update API to not require username?
response = client.courses().get(page=page, page_size=50, username='ecommerce_worker')
count = response['pagination']['count']
results = response['results']
logger.info('Retrieved %d courses...', len(results))
if response['pagination']['next']:
page += 1
else:
page = None
for body in results:
Course(body['id']).update(body)
logger.info('Retrieved %d courses.', count)
logger.info('Retrieved %d courses from %s.', count, course_api_url)
def __init__(self, id, body=None): # pylint: disable=redefined-builtin
if not id:
......@@ -202,3 +235,20 @@ class Course(object):
logger.info('Indexing course %s...', self.id)
self._es_client().index(index=self._index, doc_type=self.doc_type, id=self.id, body=self.body)
logger.info('Finished indexing course %s.', self.id)
def update(self, body):
""" Updates (merges) the data in the index with the provided data.
Args:
body (dict): Data to be merged into the index.
Returns:
None
"""
body = {
'doc': body,
'doc_as_upsert': True,
}
logger.info('Updating course %s...', self.id)
self._es_client().update(index=self._index, doc_type=self.doc_type, id=self.id, body=body)
logger.info('Finished updating course %s.', self.id)
course_discovery/apps/courses/tests/test_models.py
......@@ -10,11 +10,12 @@ from course_discovery.apps.courses.models import Course
from course_discovery.apps.courses.tests.factories import CourseFactory
ACCESS_TOKEN = 'secret'
COURSES_API_URL = 'https://lms.example.com/api/courses/v1'
ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
JSON = 'application/json'
@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL)
@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL, COURSES_API_URL=COURSES_API_URL)
class CourseTests(ElasticsearchTestMixin, TestCase):
def assert_course_attrs(self, course, attrs):
"""
......@@ -30,14 +31,12 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
@responses.activate # pylint: disable=no-member
def mock_refresh_all(self):
"""
Mock the E-Commerce API and refresh all course data.
Mock the external APIs and refresh all course data.
Returns:
[dict]: List of dictionaries representing course content bodies.
"""
# Mock the call to the E-Commerce API, simulating multiple pages of data
url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
course_bodies = [
{
'id': 'a/b/c',
......@@ -57,30 +56,64 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
}
]
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
def ecommerce_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'count': count,
'next': next,
'previous': None,
'results': [data[page - 1]]
}
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
return 200, {}, json.dumps(body)
if page < count:
next = '{}?page={}'.format(url, page)
return request_callback
body = {
'count': count,
'next': next,
'previous': None,
'results': [course_bodies[page - 1]]
}
def courses_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
return 200, {}, json.dumps(body)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'pagination': {
'count': count,
'next': next,
'previous': None,
},
'results': [data[page - 1]]
}
return 200, {}, json.dumps(body)
return request_callback
# pylint: disable=no-member
responses.add_callback(responses.GET, url, callback=request_callback, content_type=JSON)
url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
responses.add_callback(responses.GET, url, callback=ecommerce_api_callback(url, course_bodies),
content_type=JSON)
url = '{host}/courses/'.format(host=COURSES_API_URL)
responses.add_callback(responses.GET, url, callback=courses_api_callback(url, course_bodies), content_type=JSON)
# Refresh all course data
Course.refresh_all(ACCESS_TOKEN)
......
course_discovery/settings/base.py
......@@ -258,9 +258,10 @@ SWAGGER_SETTINGS = {
}
ELASTICSEARCH = {
'host': '',
'host': 'localhost:9200',
'index': 'course_discovery',
}
# TODO Replace with None and document.
ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'
course_discovery/settings/local.py
......@@ -57,6 +57,9 @@ ENABLE_AUTO_AUTH = True
JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
ECOMMERCE_API_URL = 'http://localhost:8002/api/v2/'
COURSES_API_URL = 'http://localhost:8000/api/courses/v1/'
#####################################################################
# Lastly, see if the developer has any local overrides.
if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment