Merge pull request #8942 from mitocw/bdero/import-export-reprise

Public Course Import/Export API (continued)

cms/djangoapps/contentstore/tests/tests.py

@@ -345,7 +345,7 @@ class CourseKeyVerificationTestCase(CourseTestCase):
         resp = self.client.get_html(url)
         self.assertEqual(resp.status_code, status_code)
 
-        url = '/import_status/{course_key}/{filename}'.format(
+        url = '/api/import_export/v1/courses/{course_key}/import_status/{filename}'.format(
             course_key=course_key,
             filename='xyz.tar.gz'
         )

cms/envs/aws.py

@@ -319,6 +319,19 @@ SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIME
 ##### X-Frame-Options response header settings #####
 X_FRAME_OPTIONS = ENV_TOKENS.get('X_FRAME_OPTIONS', X_FRAME_OPTIONS)
 
+##### OAUTH2 Provider ##############
+if FEATURES.get('ENABLE_OAUTH2_PROVIDER'):
+    OAUTH_OIDC_ISSUER_PATH = ENV_TOKENS.get('OAUTH_OIDC_ISSUER_PATH', 'oauth2')
+    OAUTH_OIDC_ISSUER = ENV_TOKENS.get(
+        'OAUTH_OIDC_ISSUER',
+        'https://{0}/{1}'.format(
+            SITE_NAME,
+            OAUTH_OIDC_ISSUER_PATH
+        )
+    )
+    OAUTH_ENFORCE_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_SECURE', True)
+    OAUTH_ENFORCE_CLIENT_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_CLIENT_SECURE', True)
+
 ##### ADVANCED_SECURITY_CONFIG #####
 ADVANCED_SECURITY_CONFIG = ENV_TOKENS.get('ADVANCED_SECURITY_CONFIG', {})
 

cms/envs/common.py

@@ -72,6 +72,9 @@ FEATURES = {
 
     'AUTH_USE_CERTIFICATES': False,
 
+    # Toggles OAuth2 authentication provider
+    'ENABLE_OAUTH2_PROVIDER': False,
+
     # email address for studio staff (eg to request course creation)
     'STUDIO_REQUEST_EMAIL': '',
 
@@ -209,6 +212,29 @@ sys.path.append(COMMON_ROOT / 'djangoapps')
 GEOIP_PATH = REPO_ROOT / "common/static/data/geoip/GeoIP.dat"
 GEOIPV6_PATH = REPO_ROOT / "common/static/data/geoip/GeoIPv6.dat"
 
+############################ OAUTH2 Provider ###################################
+
+# OpenID Connect issuer ID. Normally the URL of the authentication endpoint.
+
+OAUTH_OIDC_ISSUER_PATH = 'oauth2'
+OAUTH_OIDC_ISSUER = 'https:/example.com/oauth2'
+
+# OpenID Connect claim handlers
+
+OAUTH_OIDC_ID_TOKEN_HANDLERS = (
+    'oauth2_provider.oidc.handlers.BasicIDTokenHandler',
+    'oauth2_provider.oidc.handlers.ProfileHandler',
+    'oauth2_provider.oidc.handlers.EmailHandler',
+    'oauth2_handler.IDTokenHandler'
+)
+
+OAUTH_OIDC_USERINFO_HANDLERS = (
+    'oauth2_provider.oidc.handlers.BasicUserInfoHandler',
+    'oauth2_provider.oidc.handlers.ProfileHandler',
+    'oauth2_provider.oidc.handlers.EmailHandler',
+    'oauth2_handler.UserInfoHandler'
+)
+
 ############################# WEB CONFIGURATION #############################
 # This is where we stick our compiled template files.
 import tempfile
@@ -254,7 +280,8 @@ LMS_BASE = None
 # These are standard regexes for pulling out info like course_ids, usage_ids, etc.
 # They are used so that URLs with deprecated-format strings still work.
 from lms.envs.common import (
-    COURSE_KEY_PATTERN, COURSE_ID_PATTERN, USAGE_KEY_PATTERN, ASSET_KEY_PATTERN
+    COURSE_KEY_PATTERN, COURSELIKE_KEY_PATTERN, COURSE_ID_PATTERN,
+    USAGE_KEY_PATTERN, ASSET_KEY_PATTERN
 )
 
 ######################### CSRF #########################################
@@ -750,6 +777,11 @@ INSTALLED_APPS = (
     # Theming
     'openedx.core.djangoapps.theming',
 
+    # OAuth2 Provider
+    'provider',
+    'provider.oauth2',
+    'oauth2_provider',
+
     # comment common
     'django_comment_common',
 
@@ -786,6 +818,10 @@ INSTALLED_APPS = (
     # Credit courses
     'openedx.core.djangoapps.credit',
 
+    # Import/Export API
+    'rest_framework',
+    'openedx.core.djangoapps.import_export',
+
     'xblock_django',
 
     # edX Proctoring

cms/templates/export.html

@@ -33,6 +33,30 @@ else:
   require(["js/factories/export"], function(ExportFactory) {
       ExportFactory(hasUnit, editUnitUrl, courselikeHomeUrl, is_library, errMsg);
   });
+
+## Even though there isn't an export error, we should still show contextual
+## error popups if supplied.
+%elif raw_err_msg:
+  var errMsg = ${json.dumps(raw_err_msg)};
+
+  require(['gettext', 'js/views/feedback_prompt'], function(gettext, PromptView) {
+    dialog = new PromptView({
+      title: gettext('There has been an error.'),
+      message: errMsg,
+      intent: 'error',
+      actions: {
+        primary: {
+          text: gettext('Continue'),
+          click: function(view) {
+            view.hide();
+          }
+        }
+      }
+    });
+
+    $('body').addClass('js');
+    dialog.show();
+  });
 %endif
 </%block>
 

cms/templates/import.html

@@ -53,7 +53,7 @@ else:
 
       </div>
 
-      <form id="fileupload" method="post" enctype="multipart/form-data" class="import-form">
+      <form id="fileupload" method="post" action="${import_url}" enctype="multipart/form-data" class="import-form">
 
         ## Translators: ".tar.gz" is a file extension, and files with that extension are called "gzipped tar files": these terms should not be translated
         <h2 class="title">

cms/urls.py

@@ -7,10 +7,6 @@ admin.autodiscover()
 
 # pylint: disable=bad-continuation
 
-# Pattern to match a course key or a library key
-COURSELIKE_KEY_PATTERN = r'(?P<course_key_string>({}|{}))'.format(
-    r'[^/]+/[^/]+/[^/]+', r'[^/:]+:[^/+]+\+[^/+]+(\+[^/]+)?'
-)
 # Pattern to match a library key only
 LIBRARY_KEY_PATTERN = r'(?P<library_key_string>library-v1:[^/+]+\+[^/+]+)'
 
@@ -74,7 +70,7 @@ urlpatterns += patterns(
     url(r'^signin$', 'login_page', name='login'),
     url(r'^request_course_creator$', 'request_course_creator'),
 
-    url(r'^course_team/{}(?:/(?P<email>.+))?$'.format(COURSELIKE_KEY_PATTERN), 'course_team_handler'),
+    url(r'^course_team/{}(?:/(?P<email>.+))?$'.format(settings.COURSELIKE_KEY_PATTERN), 'course_team_handler'),
     url(r'^course_info/{}$'.format(settings.COURSE_KEY_PATTERN), 'course_info_handler'),
     url(
         r'^course_info_update/{}/(?P<provided_id>\d+)?$'.format(settings.COURSE_KEY_PATTERN),
@@ -94,9 +90,8 @@ urlpatterns += patterns(
     url(r'^checklists/{}/(?P<checklist_index>\d+)?$'.format(settings.COURSE_KEY_PATTERN), 'checklists_handler'),
     url(r'^orphan/{}$'.format(settings.COURSE_KEY_PATTERN), 'orphan_handler'),
     url(r'^assets/{}/{}?$'.format(settings.COURSE_KEY_PATTERN, settings.ASSET_KEY_PATTERN), 'assets_handler'),
-    url(r'^import/{}$'.format(COURSELIKE_KEY_PATTERN), 'import_handler'),
-    url(r'^import_status/{}/(?P<filename>.+)$'.format(COURSELIKE_KEY_PATTERN), 'import_status_handler'),
-    url(r'^export/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_handler'),
+    url(r'^import/{}$'.format(settings.COURSELIKE_KEY_PATTERN), 'import_handler'),
+    url(r'^export/{}$'.format(settings.COURSELIKE_KEY_PATTERN), 'export_handler'),
     url(r'^xblock/outline/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_outline_handler'),
     url(r'^xblock/container/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_container_handler'),
     url(r'^xblock/{}/(?P<view_name>[^/]+)$'.format(settings.USAGE_KEY_PATTERN), 'xblock_view_handler'),
@@ -112,7 +107,11 @@ urlpatterns += patterns(
     url(r'^group_configurations/{}$'.format(settings.COURSE_KEY_PATTERN), 'group_configurations_list_handler'),
     url(r'^group_configurations/{}/(?P<group_configuration_id>\d+)(/)?(?P<group_id>\d+)?$'.format(
         settings.COURSE_KEY_PATTERN), 'group_configurations_detail_handler'),
+
     url(r'^api/val/v0/', include('edxval.urls')),
+
+    # Import/Export API
+    url(r'^api/import_export/v1/', include('openedx.core.djangoapps.import_export.urls')),
 )
 
 JS_INFO_DICT = {
@@ -156,6 +155,12 @@ if settings.FEATURES.get('AUTH_USE_CAS'):
         url(r'^cas-auth/logout/$', 'django_cas.views.logout', {'next_page': '/'}, name="cas-logout"),
     )
 
+
+if settings.FEATURES.get('ENABLE_OAUTH2_PROVIDER'):
+    urlpatterns += (
+        url(r'^oauth2/', include('oauth2_provider.urls', namespace='oauth2')),
+    )
+
 urlpatterns += patterns('', url(r'^admin/', include(admin.site.urls)),)
 
 # enable automatic login

lms/envs/aws.py

@@ -138,6 +138,10 @@ DEFAULT_COURSE_ABOUT_IMAGE_URL = ENV_TOKENS.get('DEFAULT_COURSE_ABOUT_IMAGE_URL'
 MEDIA_ROOT = ENV_TOKENS.get('MEDIA_ROOT', MEDIA_ROOT)
 MEDIA_URL = ENV_TOKENS.get('MEDIA_URL', MEDIA_URL)
 
+# GITHUB_REPO_ROOT is the base directory
+# for course data
+GITHUB_REPO_ROOT = ENV_TOKENS.get('GITHUB_REPO_ROOT', GITHUB_REPO_ROOT)
+
 PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', PLATFORM_NAME)
 # For displaying on the receipt. At Stanford PLATFORM_NAME != MERCHANT_NAME, but PLATFORM_NAME is a fine default
 PLATFORM_TWITTER_ACCOUNT = ENV_TOKENS.get('PLATFORM_TWITTER_ACCOUNT', PLATFORM_TWITTER_ACCOUNT)
@@ -585,7 +589,14 @@ if FEATURES.get('ENABLE_THIRD_PARTY_AUTH'):
 
 ##### OAUTH2 Provider ##############
 if FEATURES.get('ENABLE_OAUTH2_PROVIDER'):
-    OAUTH_OIDC_ISSUER = ENV_TOKENS['OAUTH_OIDC_ISSUER']
+    OAUTH_OIDC_ISSUER_PATH = ENV_TOKENS.get('OAUTH_OIDC_ISSUER_PATH', 'oauth2')
+    OAUTH_OIDC_ISSUER = ENV_TOKENS.get(
+        'OAUTH_OIDC_ISSUER',
+        'https://{0}/{1}'.format(
+            SITE_NAME,
+            OAUTH_OIDC_ISSUER_PATH
+        )
+    )
     OAUTH_ENFORCE_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_SECURE', True)
     OAUTH_ENFORCE_CLIENT_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_CLIENT_SECURE', True)
 

lms/envs/common.py

@@ -400,6 +400,9 @@ FEATURES = {
     # Credit course API
     'ENABLE_CREDIT_API': True,
 
+    # Full Course/Library Import/Export API
+    'ENABLE_IMPORT_EXPORT_LMS': False,
+
     # The block types to disable need to be specified in "x block disable config" in django admin.
     'ENABLE_DISABLING_XBLOCK_TYPES': True,
 
@@ -428,6 +431,7 @@ PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /edx-platform/lms
 REPO_ROOT = PROJECT_ROOT.dirname()
 COMMON_ROOT = REPO_ROOT / "common"
 ENV_ROOT = REPO_ROOT.dirname()  # virtualenv dir /edx-platform is in
+GITHUB_REPO_ROOT = ENV_ROOT / "data"
 COURSES_ROOT = ENV_ROOT / "data"
 
 DATA_DIR = COURSES_ROOT
@@ -465,6 +469,7 @@ OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net']
 
 # OpenID Connect issuer ID. Normally the URL of the authentication endpoint.
 
+OAUTH_OIDC_ISSUER_PATH = 'oauth2'
 OAUTH_OIDC_ISSUER = 'https:/example.com/oauth2'
 
 # OpenID Connect claim handlers
@@ -587,6 +592,12 @@ COURSE_KEY_PATTERN = r'(?P<course_key_string>[^/+]+(/|\+)[^/+]+(/|\+)[^/?]+)'
 COURSE_ID_PATTERN = COURSE_KEY_PATTERN.replace('course_key_string', 'course_id')
 COURSE_KEY_REGEX = COURSE_KEY_PATTERN.replace('P<course_key_string>', ':')
 
+# Pattern to match a course key or a library key
+COURSELIKE_KEY_PATTERN = r'(?P<course_key_string>({}|{}))'.format(
+    r'[^/:+]+/[^/:+]+/[^/:+]+',
+    r'[^/:]+:[^/+]+\+[^/+]+(\+[^/]+)?',
+)
+
 USAGE_KEY_PATTERN = r'(?P<usage_key_string>(?:i4x://?[^/]+/[^/]+/[^/]+/[^@]+(?:@[^/]+)?)|(?:[^/]+))'
 ASSET_KEY_PATTERN = r'(?P<asset_key_string>(?:/?c4x(:/)?/[^/]+/[^/]+/[^/]+/[^@]+(?:@[^/]+)?)|(?:[^/]+))'
 USAGE_ID_PATTERN = r'(?P<usage_id>(?:i4x://?[^/]+/[^/]+/[^/]+/[^@]+(?:@[^/]+)?)|(?:[^/]+))'
@@ -1963,6 +1974,9 @@ INSTALLED_APPS = (
     # Course teams
     'teams',
 
+    # Import/Export API
+    'openedx.core.djangoapps.import_export',
+
     'xblock_django',
 )
 

lms/envs/test.py

@@ -110,7 +110,7 @@ DATA_DIR = COURSES_ROOT
 
 COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"
 # Where the content data is checked out.  This may not exist on jenkins.
-GITHUB_REPO_ROOT = ENV_ROOT / "data"
+GITHUB_REPO_ROOT = TEST_ROOT / "data"
 
 USE_I18N = True
 LANGUAGE_CODE = 'en'  # tests assume they will get English.
@@ -532,3 +532,6 @@ AUTHENTICATION_BACKENDS += ('lti_provider.users.LtiBackend',)
 
 # ORGANIZATIONS
 FEATURES['ORGANIZATIONS_APP'] = True
+
+# Enable the Full Course/Library Import/Export API
+FEATURES['ENABLE_IMPORT_EXPORT_LMS'] = True

lms/urls.py

@@ -94,6 +94,12 @@ urlpatterns = (
     url(r'^api/commerce/', include('commerce.api.urls', namespace='commerce_api')),
 )
 
+# Full Course/Library Import/Export API
+if settings.FEATURES["ENABLE_IMPORT_EXPORT_LMS"]:
+    urlpatterns += (
+        url(r'^api/import_export/v1/', include('openedx.core.djangoapps.import_export.urls')),
+    )
+
 if settings.FEATURES["ENABLE_COMBINED_LOGIN_REGISTRATION"]:
     # Backwards compatibility with old URL structure, but serve the new views
     urlpatterns += (

openedx/core/djangoapps/import_export/__init__.py

+"""
+Publishing API
+"""

openedx/core/djangoapps/import_export/courses/__init__.py

+"""
+Course publishing API
+"""

openedx/core/djangoapps/import_export/courses/models.py

+"""
+A models.py is required to make this an app (until we move to Django 1.7)
+"""

openedx/core/djangoapps/import_export/courses/tests/__init__.py

+"""
+Tests for course publishing API
+"""

openedx/core/djangoapps/import_export/courses/tests/utils.py

+'''
+Utilities for contentstore tests
+'''
+
+from datetime import timedelta
+from django.conf import settings
+from django.utils import timezone
+
+from provider.oauth2.models import AccessToken, Client as OAuth2Client
+from provider import constants
+from rest_framework.test import APIClient
+
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory
+
+
+TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
+
+
+def create_oauth2_client(user):
+    """
+    Create an OAuth2 client associated with the given user and generate an
+    access token for said client.
+
+    :param user:
+    :return: a Client (provider.oauth2) and an AccessToken
+    """
+    # Register an OAuth2 Client
+    client = OAuth2Client(
+        user=user,
+        name=user.username,
+        url="http://127.0.0.1/",
+        redirect_uri="http://127.0.0.1/",
+        client_type=constants.CONFIDENTIAL
+    )
+    client.save()
+
+    # Generate an access token for the client
+    access_token = AccessToken(
+        user=user,
+        client=client,
+
+        # Set the access token to expire one day from now
+        expires=timezone.now() + timedelta(1, 0),
+        scope=constants.READ_WRITE
+    )
+    access_token.save()
+
+    return client, access_token
+
+
+def use_access_token(client, access_token):
+    """
+    Make an APIClient pass an access token for all requests
+
+    :param client: an APIClient
+    :param access_token: an AccessToken
+    """
+    client.credentials(
+        HTTP_AUTHORIZATION="Bearer {}".format(access_token.token)
+    )
+
+    return client
+
+
+class CourseTestCase(ModuleStoreTestCase):
+    """
+    Extendable base for test cases dealing with courses
+    """
+    def setUp(self):
+        """
+        These tests need a user in the DB so that the django Test Client can
+        log them in.
+        The test user is created in the ModuleStoreTestCase setUp method.
+        They inherit from the ModuleStoreTestCase class so that the mongodb
+        collection will be cleared out before each test case execution and
+        deleted afterwards.
+        """
+        self.user_password = super(CourseTestCase, self).setUp()
+
+        # Create an APIClient to simulate requests (like the Django Client, but
+        # without CSRF)
+        api_client = APIClient()
+
+        # Register an OAuth2 Client
+        _oauth2_client, access_token = create_oauth2_client(self.user)
+        self.client = use_access_token(api_client, access_token)
+
+        self.course = CourseFactory.create()
+
+    def create_non_staff_authed_user_client(self):
+        """
+        Create a non-staff user, log them in (if authenticate=True), and return
+        the client, user to use for testing.
+        """
+        nonstaff, _password = self.create_non_staff_user()
+
+        client = APIClient()
+
+        return client, nonstaff

openedx/core/djangoapps/import_export/courses/urls.py

+"""
+URLs for course publishing API
+"""
+from django.conf.urls import patterns, url
+from django.conf import settings
+
+from .views import FullCourseImportExport, FullCourseImportStatus
+
+
+urlpatterns = patterns(
+    'api.courses.views',
+    url(
+        r'^{}$'.format(settings.COURSELIKE_KEY_PATTERN),
+        FullCourseImportExport.as_view(),
+        name='course_import_export_handler',
+    ),
+    url(
+        r'^{}/import_status/(?P<filename>.+)$'.format(
+            settings.COURSELIKE_KEY_PATTERN
+        ),
+        FullCourseImportStatus.as_view(),
+        name='course_import_status_handler',
+    ),
+)

openedx/core/djangoapps/import_export/models.py

+"""
+A models.py is required to make this an app (until we move to Django 1.7)
+"""

openedx/core/djangoapps/import_export/urls.py

+"""
+URLs for the public API
+"""
+from django.conf.urls import patterns, url, include
+
+urlpatterns = patterns(
+    '',
+    # Import/Export API
+    url(
+        r'^courses/',
+        include('openedx.core.djangoapps.import_export.courses.urls')
+    ),
+)