Commit bb36ab01 by Giovanni Di Milia

Added CCXCon app

 The CCXCon app is used to push course updated to the CCXCon
 externale service.
parent 2c9783c8
......@@ -92,6 +92,7 @@ class CourseMetadata(object):
# Do not show enable_ccx if feature is not enabled.
if not settings.FEATURES.get('CUSTOM_COURSES_EDX'):
filtered_list.append('enable_ccx')
filtered_list.append('ccx_connector')
return filtered_list
......
......@@ -379,3 +379,7 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get(
# OpenID Connect issuer ID. Normally the URL of the authentication endpoint.
OAUTH_OIDC_ISSUER = ENV_TOKENS['OAUTH_OIDC_ISSUER']
######################## CUSTOM COURSES for EDX CONNECTOR ######################
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)
......@@ -313,3 +313,7 @@ FEATURES['ENABLE_TEAMS'] = True
# Dummy secret key for dev/test
SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
######### custom courses #########
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)
FEATURES['CUSTOM_COURSES_EDX'] = True
......@@ -247,3 +247,7 @@ BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT,
CELERY_BROKER_PASSWORD,
CELERY_BROKER_HOSTNAME,
CELERY_BROKER_VHOST)
######################## CUSTOM COURSES for EDX CONNECTOR ######################
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)
......@@ -395,6 +395,16 @@ class CourseFields(object):
default=False,
scope=Scope.settings
)
ccx_connector = String(
# Translators: Custom Courses for edX (CCX) is an edX feature for re-using course content.
display_name=_("CCX Connector URL"),
# Translators: Custom Courses for edX (CCX) is an edX feature for re-using course content.
help=_(
"URL for CCX Connector application for managing creation of CCXs. (optional)."
" Ignored unless 'Enable CCX' is set to 'true'."
),
scope=Scope.settings, default=""
)
allow_anonymous = Boolean(
display_name=_("Allow Anonymous Discussion Posts"),
help=_("Enter true or false. If true, students can create discussion posts that are anonymous to all users."),
......
......@@ -675,7 +675,7 @@ ECOMMERCE_API_TIMEOUT = ENV_TOKENS.get('ECOMMERCE_API_TIMEOUT', ECOMMERCE_API_TI
##### Custom Courses for EdX #####
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FIELD_OVERRIDE_PROVIDERS += (
'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
)
......
......@@ -535,7 +535,7 @@ FACEBOOK_APP_ID = "Test"
FACEBOOK_API_VERSION = "v2.2"
######### custom courses #########
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FEATURES['CUSTOM_COURSES_EDX'] = True
# Set dummy values for profile image settings.
......
......@@ -298,7 +298,7 @@ GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE
##### Custom Courses for EdX #####
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FIELD_OVERRIDE_PROVIDERS += (
'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
)
......
"""
The ccxcon app contains the models and the APIs to interact
with the `CCX Connector`, an application external to openedx
that is used to interact with the CCX and their master courses.
The ccxcon app needs to be placed in `openedx.core.djangoapps`
because it will be used both in CMS and LMS.
"""
import openedx.core.djangoapps.ccxcon.signals
"""
Admin site bindings for ccxcon
"""
from django.contrib import admin
from .models import CCXCon
admin.site.register(CCXCon)
"""
Module containing API functions for the CCXCon
"""
import logging
import urlparse
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from django.http import Http404
from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
from rest_framework.status import (
HTTP_200_OK,
HTTP_201_CREATED,
)
from lms.djangoapps.courseware.courses import get_course_by_id
from lms.djangoapps.instructor.access import list_with_level
from openedx.core.djangoapps.models.course_details import CourseDetails
from student.models import anonymous_id_for_user
from .models import CCXCon
log = logging.getLogger(__name__)
CCXCON_COURSEXS_URL = '/api/v1/coursexs/'
CCXCON_TOKEN_URL = '/o/token/'
CCXCON_REQUEST_TIMEOUT = 30
class CCXConnServerError(Exception):
"""
Custom exception to be raised in case there is any
issue with the request to the server
"""
def is_valid_url(url):
"""
Helper function used to check if a string is a valid url.
Args:
url (str): the url string to be validated
Returns:
bool: whether the url is valid or not
"""
validate = URLValidator()
try:
validate(url)
return True
except ValidationError:
return False
def get_oauth_client(server_token_url, client_id, client_secret):
"""
Function that creates an oauth client and fetches a token.
It intentionally doesn't handle errors.
Args:
server_token_url (str): server URL where to get an authentication token
client_id (str): oauth client ID
client_secret (str): oauth client secret
Returns:
OAuth2Session: an instance of OAuth2Session with a token
"""
if not is_valid_url(server_token_url):
return
client = BackendApplicationClient(client_id=client_id)
oauth_ccxcon = OAuth2Session(client=client)
oauth_ccxcon.fetch_token(
token_url=server_token_url,
client_id=client_id,
client_secret=client_secret,
timeout=CCXCON_REQUEST_TIMEOUT
)
return oauth_ccxcon
def course_info_to_ccxcon(course_key):
"""
Function that gathers informations about the course and
makes a post request to a CCXCon with the data.
Args:
course_key (CourseLocator): the master course key
"""
try:
course = get_course_by_id(course_key)
except Http404:
log.error('Master Course with key "%s" not found', unicode(course_key))
return
if not course.enable_ccx:
log.debug('ccx not enabled for course key "%s"', unicode(course_key))
return
if not course.ccx_connector:
log.debug('ccx connector not defined for course key "%s"', unicode(course_key))
return
if not is_valid_url(course.ccx_connector):
log.error(
'ccx connector URL "%s" for course key "%s" is not a valid URL.',
course.ccx_connector, unicode(course_key)
)
return
# get the oauth credential for this URL
try:
ccxcon = CCXCon.objects.get(url=course.ccx_connector)
except CCXCon.DoesNotExist:
log.error('ccx connector Oauth credentials not configured for URL "%s".', course.ccx_connector)
return
# get an oauth client with a valid token
oauth_ccxcon = get_oauth_client(
server_token_url=urlparse.urljoin(course.ccx_connector, CCXCON_TOKEN_URL),
client_id=ccxcon.oauth_client_id,
client_secret=ccxcon.oauth_client_secret
)
# get the entire list of instructors
course_instructors = list_with_level(course, 'instructor')
# get anonymous ids for each of them
course_instructors_ids = [anonymous_id_for_user(user, course_key) for user in course_instructors]
# extract the course details
course_details = CourseDetails.fetch(course_key)
payload = {
'course_id': unicode(course_key),
'title': course.display_name,
'author_name': None,
'overview': course_details.overview,
'description': course_details.short_description,
'image_url': course_details.course_image_asset_path,
'instructors': course_instructors_ids
}
headers = {'content-type': 'application/json'}
# make the POST request
add_course_url = urlparse.urljoin(course.ccx_connector, CCXCON_COURSEXS_URL)
resp = oauth_ccxcon.post(
url=add_course_url,
json=payload,
headers=headers,
timeout=CCXCON_REQUEST_TIMEOUT
)
if resp.status_code >= 500:
raise CCXConnServerError('Server returned error Status: %s, Content: %s', resp.status_code, resp.content)
if resp.status_code >= 400:
log.error("Error creating course on ccxcon. Status: %s, Content: %s", resp.status_code, resp.content)
# this API performs a POST request both for POST and PATCH, but the POST returns 201 and the PATCH returns 200
elif resp.status_code != HTTP_200_OK and resp.status_code != HTTP_201_CREATED:
log.error('Server returned unexpected status code %s', resp.status_code)
else:
log.debug('Request successful. Status: %s, Content: %s', resp.status_code, resp.content)
"""
Initial migration
"""
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
"""
Initial migration for CCXCon model
"""
dependencies = [
]
operations = [
migrations.CreateModel(
name='CCXCon',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('url', models.URLField(unique=True, db_index=True)),
('oauth_client_id', models.CharField(max_length=255)),
('oauth_client_secret', models.CharField(max_length=255)),
('title', models.CharField(max_length=255)),
],
),
]
"""
Models for the ccxcon
"""
from django.db import models
class CCXCon(models.Model):
"""
The definition of the CCXCon model.
This will store the url and the oauth key to access the REST APIs
on the CCX Connector.
"""
url = models.URLField(unique=True, db_index=True)
oauth_client_id = models.CharField(max_length=255)
oauth_client_secret = models.CharField(max_length=255)
title = models.CharField(max_length=255)
class Meta(object):
app_label = 'ccxcon'
verbose_name = 'CCX Connector'
verbose_name_plural = 'CCX Connectors'
def __repr__(self):
return '<CCXCon {}>'.format(self.title)
def __str__(self):
return self.title
def __unicode__(self):
return unicode(self.__str__())
"""
Signal handler for posting course updated to CCXCon
"""
from django.dispatch.dispatcher import receiver
from xmodule.modulestore.django import SignalHandler
@receiver(SignalHandler.course_published, dispatch_uid='ccxcon_course_publish_handler')
def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Listener for course_plublish events.
This listener takes care of submitting a task to update CCXCon
"""
# update the course information on ccxcon using celery
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from openedx.core.djangoapps.ccxcon import tasks
tasks.update_ccxcon.delay(unicode(course_key))
"""
This file contains celery tasks for ccxcon
"""
from celery.task import task # pylint: disable=no-name-in-module, import-error
from celery.utils.log import get_task_logger # pylint: disable=no-name-in-module, import-error
from requests.exceptions import (
ConnectionError,
HTTPError,
RequestException,
TooManyRedirects
)
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.ccxcon import api
log = get_task_logger(__name__)
@task()
def update_ccxcon(course_id, cur_retry=0):
"""
Pass through function to update course information on CCXCon.
Takes care of retries in case of some specific exceptions.
Args:
course_id (str): string representing a course key
cur_retry (int): integer representing the current task retry
"""
course_key = CourseKey.from_string(course_id)
try:
api.course_info_to_ccxcon(course_key)
log.info('Course update to CCXCon returned no errors. Course key: %s', course_id)
except (ConnectionError, HTTPError, RequestException, TooManyRedirects, api.CCXConnServerError) as exp:
log.error('Course update to CCXCon failed for course_id %s with error: %s', course_id, exp)
# in case the maximum amount of retries has not been reached,
# insert another task delayed exponentially up to 5 retries
if cur_retry < 5:
update_ccxcon.apply_async(
kwargs={'course_id': course_id, 'cur_retry': cur_retry + 1},
countdown=10 ** cur_retry # number of seconds the task should be delayed
)
log.info('Requeued celery task for course key %s ; retry # %s', course_id, cur_retry + 1)
"""
Dummy factories for tests
"""
from factory.django import DjangoModelFactory
from openedx.core.djangoapps.ccxcon.models import CCXCon
class CcxConFactory(DjangoModelFactory):
"""
Model factory for the CCXCon model
"""
class Meta(object):
model = CCXCon
oauth_client_id = 'asdfjasdljfasdkjffsdfjksd98fsd8y24fdsiuhsfdsf'
oauth_client_secret = '19123084091238901912308409123890'
title = 'title for test ccxcon'
"""
Unit tests for the API module
"""
import datetime
import mock
import pytz
import urlparse
from nose.plugins.attrib import attr
from opaque_keys.edx.keys import CourseKey
from student.tests.factories import AdminFactory
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import (
SharedModuleStoreTestCase,
TEST_DATA_SPLIT_MODULESTORE
)
from xmodule.modulestore.tests.factories import (
CourseFactory,
ItemFactory,
)
from openedx.core.djangoapps.ccxcon import api as ccxconapi
from .factories import CcxConFactory
def flatten(seq):
"""
For [[1, 2], [3, 4]] returns [1, 2, 3, 4]. Does not recurse.
"""
return [x for sub in seq for x in sub]
def fetch_token_mock(*args, **kwargs): # pylint: disable=unused-argument
"""
Mock function used to bypass the oauth fetch token
"""
return
@attr('shard_1')
class APIsTestCase(SharedModuleStoreTestCase):
"""
Unit tests for the API module functions
"""
MODULESTORE = TEST_DATA_SPLIT_MODULESTORE
@classmethod
def setUpClass(cls):
super(APIsTestCase, cls).setUpClass()
cls.course = course = CourseFactory.create()
cls.course_key = cls.course.location.course_key
# Create a course outline
start = datetime.datetime(
2010, 5, 12, 2, 42, tzinfo=pytz.UTC
)
due = datetime.datetime(
2010, 7, 7, 0, 0, tzinfo=pytz.UTC
)
cls.chapters = [
ItemFactory.create(start=start, parent=course) for _ in xrange(2)
]
cls.sequentials = flatten([
[
ItemFactory.create(parent=chapter) for _ in xrange(2)
] for chapter in cls.chapters
])
cls.verticals = flatten([
[
ItemFactory.create(
start=start, due=due, parent=sequential, graded=True, format='Homework', category=u'vertical'
) for _ in xrange(2)
] for sequential in cls.sequentials
])
# Trying to wrap the whole thing in a bulk operation fails because it
# doesn't find the parents. But we can at least wrap this part...
with cls.store.bulk_operations(course.id, emit_signals=False):
blocks = flatten([ # pylint: disable=unused-variable
[
ItemFactory.create(parent=vertical) for _ in xrange(2)
] for vertical in cls.verticals
])
def setUp(self):
"""
Set up tests
"""
super(APIsTestCase, self).setUp()
# Create instructor account
self.instructor = AdminFactory.create()
# create an instance of modulestore
self.mstore = modulestore()
# enable ccx
self.course.enable_ccx = True
# setup CCX connector
self.course.ccx_connector = 'https://url.to.cxx.connector.mit.edu'
# save the changes
self.mstore.update_item(self.course, self.instructor.id)
# create a configuration for the ccx connector: this must match the one in the course
self.ccxcon_conf = CcxConFactory(url=self.course.ccx_connector)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_valid_course_key(self, mock_post):
"""
Test for an invalid course key
"""
missing_course_key = CourseKey.from_string('course-v1:FakeOrganization+CN999+CR-FALL99')
self.assertIsNone(ccxconapi.course_info_to_ccxcon(missing_course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_ccx_enabled(self, mock_post):
"""
Test for a course without CCX enabled
"""
self.course.enable_ccx = False
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_invalid_ccx_connector(self, mock_post):
"""
Test for a course with invalid CCX connector URL
"""
# no connector at all
self.course.ccx_connector = ""
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
# invalid url
self.course.ccx_connector = "www.foo"
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_config(self, mock_post):
"""
Test for course with ccx connector credentials not configured
"""
self.course.ccx_connector = "https://www.foo.com"
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_ok(self, mock_post):
"""
Test for happy path
"""
mock_response = mock.Mock()
mock_response.status_code = 201
mock_post.return_value = mock_response
ccxconapi.course_info_to_ccxcon(self.course_key)
self.assertEqual(mock_post.call_count, 1)
k_args, k_kwargs = mock_post.call_args
# no args used for the call
self.assertEqual(k_args, tuple())
self.assertEqual(
k_kwargs.get('url'),
urlparse.urljoin(self.course.ccx_connector, ccxconapi.CCXCON_COURSEXS_URL)
)
# second call with different status code
mock_response.status_code = 200
mock_post.return_value = mock_response
ccxconapi.course_info_to_ccxcon(self.course_key)
self.assertEqual(mock_post.call_count, 2)
k_args, k_kwargs = mock_post.call_args
# no args used for the call
self.assertEqual(k_args, tuple())
self.assertEqual(
k_kwargs.get('url'),
urlparse.urljoin(self.course.ccx_connector, ccxconapi.CCXCON_COURSEXS_URL)
)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_500_error(self, mock_post):
"""
Test for 500 error: a CCXConnServerError exception is raised
"""
mock_response = mock.Mock()
mock_response.status_code = 500
mock_post.return_value = mock_response
with self.assertRaises(ccxconapi.CCXConnServerError):
ccxconapi.course_info_to_ccxcon(self.course_key)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_other_status_codes(self, mock_post):
"""
Test for status codes different from >= 500 and 201:
The called function doesn't raise any exception and simply returns None.
"""
mock_response = mock.Mock()
for status_code in (204, 300, 304, 400, 404):
mock_response.status_code = status_code
mock_post.return_value = mock_response
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
"""
Test for contentstore signals receiver
"""
import mock
from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.django import modulestore, SignalHandler
class CCXConSignalTestCase(TestCase):
"""
The only tests currently implemented are for verifying that
the call for the ccxcon update are performed correctly by the
course_published signal handler
"""
@mock.patch('openedx.core.djangoapps.ccxcon.tasks.update_ccxcon.delay')
def test_course_published_ccxcon_call(self, mock_upc):
"""
Tests the async call to the ccxcon task.
It bypasses all the other calls.
"""
mock_response = mock.MagicMock(return_value=None)
mock_upc.return_value = mock_response
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
course_key = CourseKey.from_string(course_id)
signal_handler = SignalHandler(modulestore())
signal_handler.send('course_published', course_key=course_key)
mock_upc.assert_called_once_with(course_id)
"""
Tests for the CCXCon celery tasks
"""
import mock
from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.ccxcon import api, tasks
class CCXConTaskTestCase(TestCase):
"""
Tests for CCXCon tasks.
"""
@mock.patch('openedx.core.djangoapps.ccxcon.api.course_info_to_ccxcon')
def test_update_ccxcon_task_ok(self, mock_citc):
"""
Test task with no problems
"""
mock_response = mock.Mock()
mock_citc.return_value = mock_response
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
tasks.update_ccxcon.delay(course_id)
mock_citc.assert_called_once_with(CourseKey.from_string(course_id))
@mock.patch('openedx.core.djangoapps.ccxcon.api.course_info_to_ccxcon')
def test_update_ccxcon_task_retry(self, mock_citc):
"""
Test task with exception that triggers a retry
"""
mock_citc.side_effect = api.CCXConnServerError()
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
tasks.update_ccxcon.delay(course_id)
self.assertEqual(mock_citc.call_count, 6)
course_key = CourseKey.from_string(course_id)
for call in mock_citc.call_args_list:
c_args, c_kwargs = call
self.assertEqual(c_kwargs, {})
self.assertEqual(len(c_args), 1)
self.assertEqual(c_args[0], course_key)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment