Skip to content

E
edx-platform
Commit 87e292ba by zubair-arbi
Options

Merge pull request #4560 from edx/zub/story/updatestudioimportstatusui 

show upload progress on import course view
parents 84c8d653 fc7d491c
Showing with 278 additions and 166 deletions
cms/djangoapps/contentstore/views/import_export.py
......@@ -66,187 +66,212 @@ def import_handler(request, course_key_string):
if not has_course_access(request.user, course_key):
raise PermissionDenied()
if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
if request.method == 'GET':
raise NotImplementedError('coming soon')
else:
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1
},
status=415
)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
# Do everything in a try-except block to make sure everything is properly cleaned up.
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = unicode(course_key) + filename
_save_request_status(request, key, 0)
if not filename.endswith('.tar.gz'):
_save_request_status(request, key, -1)
return JsonResponse(
{
'ErrMsg': _('File upload corrupted. Please try again'),
'Stage': 1
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': -1
},
status=409
status=415
)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse_course_url('import_handler', course_key),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = unicode(course_key) + filename
session_status[key] = 1
request.session.modified = True
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
# Do everything from now on in a try-finally block to make sure
# everything is properly cleaned up.
try:
logging.debug('importing course to {0}'.format(temp_filepath))
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc:
return JsonResponse(
{
'ErrMsg': 'Unsafe tar file. Aborting import.',
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': 1
},
status=400
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
_save_request_status(request, key, -1)
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': 2
'ErrMsg': _('File upload corrupted. Please try again'),
'Stage': -1
},
status=415
status=409
)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
dirpath = os.path.relpath(dirpath, data_root)
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
logging.debug('found course.xml at {0}'.format(dirpath))
size = os.path.getsize(temp_filepath)
course_items = import_from_xml(
modulestore(),
request.user.id,
settings.GITHUB_REPO_ROOT,
[dirpath],
load_error_modules=False,
static_content_store=contentstore(),
target_course_id=course_key,
)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse_course_url('import_handler', course_key),
"thumbnailUrl": ""
}]
})
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
_save_request_status(request, key, -1)
if course_dir.isdir():
shutil.rmtree(course_dir)
log.info("Course import {0}: Temp data cleared".format(course_key))
new_location = course_items[0].location
logging.debug('new course at {0}'.format(new_location))
log.exception(
"error importing course"
)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': -1
},
status=400
)
session_status[key] = 3
request.session.modified = True
# try-finally block for proper clean up after receiving last chunk.
try:
# This was the last chunk.
log.info("Course import {0}: Upload complete".format(course_key))
_save_request_status(request, key, 1)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
log.exception(
"error importing course"
)
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc:
_save_request_status(request, key, -1)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': session_status[key]
'ErrMsg': 'Unsafe tar file. Aborting import.',
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': -1
},
status=400
)
finally:
tar_file.close()
log.info("Course import {0}: Uploaded file extracted".format(course_key))
_save_request_status(request, key, 2)
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
_save_request_status(request, key, -2)
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': -2
},
status=415
)
dirpath = os.path.relpath(dirpath, data_root)
logging.debug('found course.xml at {0}'.format(dirpath))
log.info("Course import {0}: Extracted file verified".format(course_key))
_save_request_status(request, key, 3)
course_items = import_from_xml(
modulestore(),
request.user.id,
settings.GITHUB_REPO_ROOT,
[dirpath],
load_error_modules=False,
static_content_store=contentstore(),
target_course_id=course_key,
)
new_location = course_items[0].location
logging.debug('new course at {0}'.format(new_location))
log.info("Course import {0}: Course import successful".format(course_key))
_save_request_status(request, key, 4)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
log.exception(
"error importing course"
)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': -session_status[key]
},
status=400
)
finally:
if course_dir.isdir():
shutil.rmtree(course_dir)
log.info("Course import {0}: Temp data cleared".format(course_key))
# set failed stage number with negative sign in case of unsuccessful import
if session_status[key] != 4:
_save_request_status(request, key, -abs(session_status[key]))
return JsonResponse({'Status': 'OK'})
return JsonResponse({'Status': 'OK'})
elif request.method == 'GET': # assume html
course_module = modulestore().get_course(course_key)
return render_to_response('import.html', {
......@@ -258,6 +283,18 @@ def import_handler(request, course_key_string):
return HttpResponseNotFound()
def _save_request_status(request, key, status):
"""
Save import status for a course in request session
"""
session_status = request.session.get('import_status')
if session_status is None:
session_status = request.session.setdefault("import_status", {})
session_status[key] = status
request.session.save()
# pylint: disable=unused-argument
@require_GET
@ensure_csrf_cookie
......@@ -266,10 +303,12 @@ def import_status_handler(request, course_key_string, filename=None):
"""
Returns an integer corresponding to the status of a file import. These are:
-X : Import unsuccessful due to some error with X as stage [0-3]
0 : No status info found (import done or upload still in progress)
1 : Extracting file
2 : Validating.
3 : Importing to mongo
4 : Import successful
"""
course_key = CourseKey.from_string(course_key_string)
......
cms/djangoapps/contentstore/views/tests/test_import_export.py
......@@ -93,7 +93,7 @@ class ImportTestCase(CourseTestCase):
)
)
self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2)
self.assertEquals(json.loads(resp_status.content)["ImportStatus"], -2)
def test_with_coursexml(self):
"""
......
cms/static/js/views/import.js
......@@ -50,8 +50,22 @@ define(
var getStatus = function (url, timeout, stage) {
var currentStage = stage || 0;
if (CourseImport.stopGetStatus) { return ;}
updateStage(currentStage);
if (currentStage == 3 ) { return ;}
if (currentStage === 4) {
// Succeeded
CourseImport.displayFinishedImport();
$('.view-import .choose-file-button').html(gettext("Choose new file")).show();
} else if (currentStage < 0) {
// Failed
var errMsg = gettext("Error importing course");
var failedStage = Math.abs(currentStage);
CourseImport.stageError(failedStage, errMsg);
$('.view-import .choose-file-button').html(gettext("Choose new file")).show();
} else {
// In progress
updateStage(currentStage);
}
var time = timeout || 1000;
$.getJSON(url,
function (data) {
......@@ -109,18 +123,58 @@ define(
},
/**
* Make Import feedback status list visible.
*/
displayFeedbackList: function (){
this.stopGetStatus = false;
$('div.wrapper-status').removeClass('is-hidden');
$('.status-info').show();
},
/**
* Start upload feedback. Makes status list visible and starts
* showing upload progress.
*/
startUploadFeedback: function (){
this.displayFeedbackList();
updateStage(0);
},
/**
* Show last import status from server and start sending requests to the server for status updates.
*/
getAndStartUploadFeedback: function (url, fileName){
var self = this;
$.getJSON(url,
function (data) {
if (data.ImportStatus != 0) {
$('.file-name').html(fileName);
$('.file-name-block').show();
self.displayFeedbackList();
if (data.ImportStatus === 4){
self.displayFinishedImport();
} else {
$('.view-import .choose-file-button').hide();
var time = 1000;
setTimeout(function () {
getStatus(url, time, data.ImportStatus);
}, time);
}
}
}
);
},
/**
* Entry point for server feedback. Makes status list visible and starts
* sending requests to the server for status updates.
* @param {string} url The url to send Ajax GET requests for updates.
*/
startServerFeedback: function (url){
this.stopGetStatus = false;
$('div.wrapper-status').removeClass('is-hidden');
$('.status-info').show();
getStatus(url, 500, 0);
getStatus(url, 1000, 0);
},
/**
* Give error message at the list element that corresponds to the stage
* where the error occurred.
......@@ -128,6 +182,7 @@ define(
* @param {string} msg Error message to display.
*/
stageError: function (stageNo, msg) {
this.stopGetStatus = true;
var all = $('ol.status-progress').children();
// Make all stages up to, and including, the error stage 'complete'.
var prevList = all.slice(0, stageNo + 1);
......@@ -140,8 +195,10 @@ define(
});
var message = msg || gettext("There was an error with the upload");
var elem = $('ol.status-progress').children().eq(stageNo);
elem.removeClass('is-started').addClass('has-error');
elem.find('p.copy').hide().after("<p class='copy error'>" + message + "</p>");
if (!elem.hasClass('has-error')) {
elem.removeClass('is-started').addClass('has-error');
elem.find('p.copy').hide().after("<p class='copy error'>" + message + "</p>");
}
}
};
......
cms/templates/import.html
......@@ -63,6 +63,9 @@
<div class="status-detail">
<h3 class="title">${_("Uploading")}</h3>
<div class="progress-bar">
<div class="progress-fill"></div>
</div>
<p class="copy">${_("Transferring your file to our servers")}</p>
</div>
</li>
......@@ -147,7 +150,7 @@
<%block name="jsextra">
<script>
require(
["js/views/import", "jquery", "gettext", "jquery.fileupload"],
["js/views/import", "jquery", "gettext", "jquery.fileupload", "jquery.cookie"],
function(CourseImport, $, gettext) {
var file;
......@@ -170,6 +173,12 @@ var defaults = [
"${_("There was an error while importing the new course to our database.")}\n"
];
// Display the status of last file upload on page load
var lastfileupload = $.cookie('lastfileupload');
if (lastfileupload){
CourseImport.getAndStartUploadFeedback(feedbackUrl.replace('fillerName', lastfileupload), lastfileupload);
}
$('#fileupload').fileupload({
dataType: 'json',
......@@ -185,20 +194,22 @@ $('#fileupload').fileupload({
file = data.files[0];
if (file.name.match(/tar\.gz$/)) {
submitBtn.click(function(e){
$.cookie('lastfileupload', file.name);
e.preventDefault();
submitBtn.hide();
CourseImport.startUploadFeedback();
data.submit().complete(function(result, textStatus, xhr) {
CourseImport.stopGetStatus = true;
window.onbeforeunload = null;
if (xhr.status != 200) {
if (!result.responseText) {
alert(gettext("Your browser has timed out, but the server is still processing your import. Please wait 5 minutes and verify that the new content has appeared."));
try{
var serverMsg = $.parseJSON(result.responseText);
} catch (e) {
return;
}
var serverMsg = $.parseJSON(result.responseText);
var errMsg = serverMsg.hasOwnProperty("ErrMsg") ? serverMsg.ErrMsg : "" ;
if (serverMsg.hasOwnProperty("Stage")) {
var stage = serverMsg.Stage;
var stage = Math.abs(serverMsg.Stage);
CourseImport.stageError(stage, defaults[stage] + errMsg);
}
else {
......@@ -207,6 +218,7 @@ $('#fileupload').fileupload({
chooseBtn.html("${_("Choose new file")}").show();
bar.hide();
}
CourseImport.stopGetStatus = true;
chooseBtn.html("${_("Choose new file")}").show();
bar.hide();
});
......@@ -230,11 +242,15 @@ $('#fileupload').fileupload({
}
if (percentInt >= doneAt) {
bar.hide();
CourseImport.startServerFeedback(feedbackUrl.replace("fillerName", file.name));
// Start feedback with delay so that current stage of import properly updates in session
setTimeout(
function() { CourseImport.startServerFeedback(feedbackUrl.replace('fillerName', file.name)) },
3000
);
} else {
bar.show();
fill.width(percentVal);
percent.html(percentVal);
fill.html(percentVal);
}
},
done: function(e, data){
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment