Merge pull request #14552 from edx/jmbowman/async_course_import

PLAT-1104 Import courses asynchronously

.gitignore

@@ -75,6 +75,10 @@ jscover.log.*
 .tddium*
 common/test/data/test_unicode/static/
 test_root/courses/
+test_root/data/test_bare.git/
+test_root/export_course_repos/
+test_root/paver_logs/
+test_root/uploads/
 django-pyfs
 
 ### Installation artifacts

cms/djangoapps/contentstore/storage.py

+"""
+Storage backend for course import and export.
+"""
+from __future__ import absolute_import
+
+from django.conf import settings
+from django.core.files.storage import get_storage_class
+
+from storages.backends.s3boto import S3BotoStorage
+from storages.utils import setting
+
+
+class ImportExportS3Storage(S3BotoStorage):  # pylint: disable=abstract-method
+    """
+    S3 backend for course import and export OLX files.
+    """
+
+    def __init__(self):
+        bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME)
+        super(ImportExportS3Storage, self).__init__(bucket=bucket, querystring_auth=True)
+
+# pylint: disable=invalid-name
+course_import_export_storage = get_storage_class(settings.COURSE_IMPORT_EXPORT_STORAGE)()

cms/djangoapps/contentstore/views/tests/test_import_export.py

@@ -184,7 +184,7 @@ class ImportTestCase(CourseTestCase):
                     "name": self.bad_tar,
                     "course-data": [btar]
                 })
-        self.assertEquals(resp.status_code, 415)
+        self.assertEquals(resp.status_code, 200)
         # Check that `import_status` returns the appropriate stage (i.e., the
         # stage at which import failed).
         resp_status = self.client.get(
@@ -336,8 +336,16 @@ class ImportTestCase(CourseTestCase):
             with open(tarpath) as tar:
                 args = {"name": tarpath, "course-data": [tar]}
                 resp = self.client.post(self.url, args)
-            self.assertEquals(resp.status_code, 400)
-            self.assertIn("SuspiciousFileOperation", resp.content)
+            self.assertEquals(resp.status_code, 200)
+            resp = self.client.get(
+                reverse_course_url(
+                    'import_status_handler',
+                    self.course.id,
+                    kwargs={'filename': os.path.split(tarpath)[1]}
+                )
+            )
+            status = json.loads(resp.content)["ImportStatus"]
+            self.assertEqual(status, -1)
 
         try_tar(self._fifo_tar())
         try_tar(self._symlink_tar())

cms/envs/aws.py

@@ -299,10 +299,17 @@ AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"]
 if AWS_SECRET_ACCESS_KEY == "":
     AWS_SECRET_ACCESS_KEY = None
 
+AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads')
+
 # Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it
 # normally appends to every returned URL.
 AWS_QUERYSTRING_AUTH = AUTH_TOKENS.get('AWS_QUERYSTRING_AUTH', True)
 
+AWS_DEFAULT_ACL = 'private'
+AWS_BUCKET_ACL = AWS_DEFAULT_ACL
+AWS_QUERYSTRING_EXPIRE = 7 * 24 * 60 * 60  # 7 days
+AWS_S3_CUSTOM_DOMAIN = AUTH_TOKENS.get('AWS_S3_CUSTOM_DOMAIN', 'edxuploads.s3.amazonaws.com')
+
 if AUTH_TOKENS.get('DEFAULT_FILE_STORAGE'):
     DEFAULT_FILE_STORAGE = AUTH_TOKENS.get('DEFAULT_FILE_STORAGE')
 elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
@@ -310,6 +317,15 @@ elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
 else:
     DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
+COURSE_IMPORT_EXPORT_BUCKET = ENV_TOKENS.get('COURSE_IMPORT_EXPORT_BUCKET', '')
+
+if COURSE_IMPORT_EXPORT_BUCKET:
+    COURSE_IMPORT_EXPORT_STORAGE = 'contentstore.storage.ImportExportS3Storage'
+else:
+    COURSE_IMPORT_EXPORT_STORAGE = DEFAULT_FILE_STORAGE
+
+USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE
+
 DATABASES = AUTH_TOKENS['DATABASES']
 
 # The normal database user does not have enough permissions to run migrations.

cms/envs/common.py

@@ -555,6 +555,8 @@ LOCALE_PATHS = (REPO_ROOT + '/conf/locale',)  # edx-platform/conf/locale/
 # Messages
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
+COURSE_IMPORT_EXPORT_STORAGE = 'django.core.files.storage.FileSystemStorage'
+
 ##### EMBARGO #####
 EMBARGO_SITE_REDIRECT_URL = None
 

cms/envs/devstack.py

@@ -8,6 +8,8 @@ from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import
 
 # Don't use S3 in devstack, fall back to filesystem
 del DEFAULT_FILE_STORAGE
+COURSE_IMPORT_EXPORT_STORAGE = 'django.core.files.storage.FileSystemStorage'
+USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE
 MEDIA_ROOT = "/edx/var/edxapp/uploads"
 
 DEBUG = True

cms/static/js/views/import.js

@@ -81,7 +81,7 @@ define(
          */
         var initEventListeners = function() {
             $(window).on('beforeunload.import', function() {
-                if (current.stage <= STAGE.UNPACKING) {
+                if (current.stage < STAGE.UNPACKING) {
                     return gettext('Your import is in progress; navigating away will abort it.');
                 }
             });

cms/templates/import.html

@@ -118,7 +118,7 @@ else:
             <li class="item-progresspoint item-progresspoint-unpack is-started">
               <span class="deco status-visual">
                 <span class="icon fa fa-cog" aria-hidden="true"></span>
-                <span class="icon fa fa-warning" aria-hidden="true"v></span>
+                <span class="icon fa fa-warning" aria-hidden="true"></span>
               </span>
 
               <div class="status-detail">

lms/envs/common.py

@@ -2176,6 +2176,9 @@ CSRF_COOKIE_SECURE = False
 
 REST_FRAMEWORK = {
     'DEFAULT_PAGINATION_CLASS': 'openedx.core.lib.api.paginators.DefaultPagination',
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+    ),
     'PAGE_SIZE': 10,
     'URL_FORMAT_OVERRIDE': None,
     'DEFAULT_THROTTLE_RATES': {

pavelib/paver_tests/test_servers.py

@@ -141,7 +141,7 @@ class TestPaverServerTasks(PaverTestCase):
         """
         Test the "celery" task.
         """
-        settings = options.get("settings", "dev_with_worker")
+        settings = options.get("settings", "devstack_with_worker")
         call_task("pavelib.servers.celery", options=options)
         self.assertEquals(self.task_messages, [EXPECTED_CELERY_COMMAND.format(settings=settings)])
 
@@ -292,7 +292,7 @@ class TestPaverServerTasks(PaverTestCase):
                 port=8001,
             )
         )
-        expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="dev_with_worker"))
+        expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="devstack_with_worker"))
         self.assertEquals(self.task_messages, expected_messages)
 
     def expected_sass_commands(self, system=None, asset_settings=u"test_static_optimized"):

pavelib/servers.py

@@ -157,7 +157,7 @@ def celery(options):
     """
     Runs Celery workers.
     """
-    settings = getattr(options, 'settings', 'dev_with_worker')
+    settings = getattr(options, 'settings', 'devstack_with_worker')
     run_process(django_cmd('lms', settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.'))
 
 
@@ -187,7 +187,7 @@ def run_all_servers(options):
     """
     settings = getattr(options, 'settings', DEFAULT_SETTINGS)
     asset_settings = getattr(options, 'asset_settings', settings)
-    worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
+    worker_settings = getattr(options, 'worker_settings', 'devstack_with_worker')
     fast = getattr(options, 'fast', False)
     optimized = getattr(options, 'optimized', False)
 

requirements/edx/base.txt

@@ -34,7 +34,7 @@ django-simple-history==1.6.3
 django-statici18n==1.1.5
 django-storages==1.4.1
 django-method-override==0.1.0
-django-user-tasks==0.1.2
+django-user-tasks==0.1.4
 # We need a fix to DRF 3.2.x, for now use it from our own cherry-picked repo
 #djangorestframework>=3.1,<3.2
 git+https://github.com/edx/django-rest-framework.git@3c72cb5ee5baebc4328947371195eae2077197b0#egg=djangorestframework==3.2.3