Skip to content

E
edx-platform
Commit fc7d491c by zubair-arbi
Options

show upload progress on import course view + display last import status on import page load 

STUD-2017
parent 5364508e
Showing with 278 additions and 166 deletions
cms/djangoapps/contentstore/views/import_export.py
...@@ -66,187 +66,212 @@ def import_handler(request, course_key_string): ...@@ -66,187 +66,212 @@ def import_handler(request, course_key_string):
if not has_course_access(request.user, course_key): if not has_course_access(request.user, course_key):
raise PermissionDenied() raise PermissionDenied()
if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
if request.method == 'GET': if request.method == 'GET':
raise NotImplementedError('coming soon') raise NotImplementedError('coming soon')
else: else:
data_root = path(settings.GITHUB_REPO_ROOT) # Do everything in a try-except block to make sure everything is properly cleaned up.
course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1
},
status=415
)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try: try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) data_root = path(settings.GITHUB_REPO_ROOT)
content_range = matches.groupdict() course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
except KeyError: # Single chunk course_dir = data_root / course_subdir
# no Content-Range header, so make one that will work filename = request.FILES['course-data'].name
content_range = {'start': 0, 'stop': 1, 'end': 2}
# Use sessions to keep info about import progress
# stream out the uploaded files in chunks to disk session_status = request.session.setdefault("import_status", {})
if int(content_range['start']) == 0: key = unicode(course_key) + filename
mode = "wb+" _save_request_status(request, key, 0)
else: if not filename.endswith('.tar.gz'):
mode = "ab+" _save_request_status(request, key, -1)
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': _('File upload corrupted. Please try again'), 'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1 'Stage': -1
}, },
status=409 status=415
) )
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse_course_url('import_handler', course_key),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress temp_filepath = course_dir / filename
session_status = request.session.setdefault("import_status", {}) if not course_dir.isdir():
key = unicode(course_key) + filename os.mkdir(course_dir)
session_status[key] = 1
request.session.modified = True
# Do everything from now on in a try-finally block to make sure logging.debug('importing course to {0}'.format(temp_filepath))
# everything is properly cleaned up.
try:
tar_file = tarfile.open(temp_filepath) # Get upload chunks byte ranges
try: try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8')) matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
except SuspiciousOperation as exc: content_range = matches.groupdict()
return JsonResponse( except KeyError: # Single chunk
{ # no Content-Range header, so make one that will work
'ErrMsg': 'Unsafe tar file. Aborting import.', content_range = {'start': 0, 'stop': 1, 'end': 2}
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': 1 # stream out the uploaded files in chunks to disk
}, if int(content_range['start']) == 0:
status=400 mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
_save_request_status(request, key, -1)
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
) )
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': _('File upload corrupted. Please try again'),
'ErrMsg': _('Could not find the course.xml file in the package.'), 'Stage': -1
'Stage': 2
}, },
status=415 status=409
) )
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
dirpath = os.path.relpath(dirpath, data_root) with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
logging.debug('found course.xml at {0}'.format(dirpath)) size = os.path.getsize(temp_filepath)
course_items = import_from_xml( if int(content_range['stop']) != int(content_range['end']) - 1:
modulestore(), # More chunks coming
request.user.id, return JsonResponse({
settings.GITHUB_REPO_ROOT, "files": [{
[dirpath], "name": filename,
load_error_modules=False, "size": size,
static_content_store=contentstore(), "deleteUrl": "",
target_course_id=course_key, "deleteType": "",
) "url": reverse_course_url('import_handler', course_key),
"thumbnailUrl": ""
}]
})
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
_save_request_status(request, key, -1)
if course_dir.isdir():
shutil.rmtree(course_dir)
log.info("Course import {0}: Temp data cleared".format(course_key))
new_location = course_items[0].location log.exception(
logging.debug('new course at {0}'.format(new_location)) "error importing course"
)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': -1
},
status=400
)
session_status[key] = 3 # try-finally block for proper clean up after receiving last chunk.
request.session.modified = True try:
# This was the last chunk.
log.info("Course import {0}: Upload complete".format(course_key))
_save_request_status(request, key, 1)
# Send errors to client with stage at which error occurred. tar_file = tarfile.open(temp_filepath)
except Exception as exception: # pylint: disable=W0703 try:
log.exception( safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
"error importing course" except SuspiciousOperation as exc:
) _save_request_status(request, key, -1)
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': str(exception), 'ErrMsg': 'Unsafe tar file. Aborting import.',
'Stage': session_status[key] 'SuspiciousFileOperationMsg': exc.args[0],
'Stage': -1
}, },
status=400 status=400
) )
finally: finally:
tar_file.close()
log.info("Course import {0}: Uploaded file extracted".format(course_key))
_save_request_status(request, key, 2)
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
_save_request_status(request, key, -2)
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': -2
},
status=415
)
dirpath = os.path.relpath(dirpath, data_root)
logging.debug('found course.xml at {0}'.format(dirpath))
log.info("Course import {0}: Extracted file verified".format(course_key))
_save_request_status(request, key, 3)
course_items = import_from_xml(
modulestore(),
request.user.id,
settings.GITHUB_REPO_ROOT,
[dirpath],
load_error_modules=False,
static_content_store=contentstore(),
target_course_id=course_key,
)
new_location = course_items[0].location
logging.debug('new course at {0}'.format(new_location))
log.info("Course import {0}: Course import successful".format(course_key))
_save_request_status(request, key, 4)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
log.exception(
"error importing course"
)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': -session_status[key]
},
status=400
)
finally:
if course_dir.isdir():
shutil.rmtree(course_dir) shutil.rmtree(course_dir)
log.info("Course import {0}: Temp data cleared".format(course_key))
# set failed stage number with negative sign in case of unsuccessful import
if session_status[key] != 4:
_save_request_status(request, key, -abs(session_status[key]))
return JsonResponse({'Status': 'OK'}) return JsonResponse({'Status': 'OK'})
elif request.method == 'GET': # assume html elif request.method == 'GET': # assume html
course_module = modulestore().get_course(course_key) course_module = modulestore().get_course(course_key)
return render_to_response('import.html', { return render_to_response('import.html', {
...@@ -258,6 +283,18 @@ def import_handler(request, course_key_string): ...@@ -258,6 +283,18 @@ def import_handler(request, course_key_string):
return HttpResponseNotFound() return HttpResponseNotFound()
def _save_request_status(request, key, status):
"""
Save import status for a course in request session
"""
session_status = request.session.get('import_status')
if session_status is None:
session_status = request.session.setdefault("import_status", {})
session_status[key] = status
request.session.save()
# pylint: disable=unused-argument # pylint: disable=unused-argument
@require_GET @require_GET
@ensure_csrf_cookie @ensure_csrf_cookie
...@@ -266,10 +303,12 @@ def import_status_handler(request, course_key_string, filename=None): ...@@ -266,10 +303,12 @@ def import_status_handler(request, course_key_string, filename=None):
""" """
Returns an integer corresponding to the status of a file import. These are: Returns an integer corresponding to the status of a file import. These are:
-X : Import unsuccessful due to some error with X as stage [0-3]
0 : No status info found (import done or upload still in progress) 0 : No status info found (import done or upload still in progress)
1 : Extracting file 1 : Extracting file
2 : Validating. 2 : Validating.
3 : Importing to mongo 3 : Importing to mongo
4 : Import successful
""" """
course_key = CourseKey.from_string(course_key_string) course_key = CourseKey.from_string(course_key_string)
......
cms/djangoapps/contentstore/views/tests/test_import_export.py
...@@ -93,7 +93,7 @@ class ImportTestCase(CourseTestCase): ...@@ -93,7 +93,7 @@ class ImportTestCase(CourseTestCase):
) )
) )
self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2) self.assertEquals(json.loads(resp_status.content)["ImportStatus"], -2)
def test_with_coursexml(self): def test_with_coursexml(self):
""" """
......
cms/static/js/views/import.js
...@@ -50,8 +50,22 @@ define( ...@@ -50,8 +50,22 @@ define(
var getStatus = function (url, timeout, stage) { var getStatus = function (url, timeout, stage) {
var currentStage = stage || 0; var currentStage = stage || 0;
if (CourseImport.stopGetStatus) { return ;} if (CourseImport.stopGetStatus) { return ;}
updateStage(currentStage);
if (currentStage == 3 ) { return ;} if (currentStage === 4) {
// Succeeded
CourseImport.displayFinishedImport();
$('.view-import .choose-file-button').html(gettext("Choose new file")).show();
} else if (currentStage < 0) {
// Failed
var errMsg = gettext("Error importing course");
var failedStage = Math.abs(currentStage);
CourseImport.stageError(failedStage, errMsg);
$('.view-import .choose-file-button').html(gettext("Choose new file")).show();
} else {
// In progress
updateStage(currentStage);
}
var time = timeout || 1000; var time = timeout || 1000;
$.getJSON(url, $.getJSON(url,
function (data) { function (data) {
...@@ -109,18 +123,58 @@ define( ...@@ -109,18 +123,58 @@ define(
}, },
/** /**
* Make Import feedback status list visible.
*/
displayFeedbackList: function (){
this.stopGetStatus = false;
$('div.wrapper-status').removeClass('is-hidden');
$('.status-info').show();
},
/**
* Start upload feedback. Makes status list visible and starts
* showing upload progress.
*/
startUploadFeedback: function (){
this.displayFeedbackList();
updateStage(0);
},
/**
* Show last import status from server and start sending requests to the server for status updates.
*/
getAndStartUploadFeedback: function (url, fileName){
var self = this;
$.getJSON(url,
function (data) {
if (data.ImportStatus != 0) {
$('.file-name').html(fileName);
$('.file-name-block').show();
self.displayFeedbackList();
if (data.ImportStatus === 4){
self.displayFinishedImport();
} else {
$('.view-import .choose-file-button').hide();
var time = 1000;
setTimeout(function () {
getStatus(url, time, data.ImportStatus);
}, time);
}
}
}
);
},
/**
* Entry point for server feedback. Makes status list visible and starts * Entry point for server feedback. Makes status list visible and starts
* sending requests to the server for status updates. * sending requests to the server for status updates.
* @param {string} url The url to send Ajax GET requests for updates. * @param {string} url The url to send Ajax GET requests for updates.
*/ */
startServerFeedback: function (url){ startServerFeedback: function (url){
this.stopGetStatus = false; this.stopGetStatus = false;
$('div.wrapper-status').removeClass('is-hidden'); getStatus(url, 1000, 0);
$('.status-info').show();
getStatus(url, 500, 0);
}, },
/** /**
* Give error message at the list element that corresponds to the stage * Give error message at the list element that corresponds to the stage
* where the error occurred. * where the error occurred.
...@@ -128,6 +182,7 @@ define( ...@@ -128,6 +182,7 @@ define(
* @param {string} msg Error message to display. * @param {string} msg Error message to display.
*/ */
stageError: function (stageNo, msg) { stageError: function (stageNo, msg) {
this.stopGetStatus = true;
var all = $('ol.status-progress').children(); var all = $('ol.status-progress').children();
// Make all stages up to, and including, the error stage 'complete'. // Make all stages up to, and including, the error stage 'complete'.
var prevList = all.slice(0, stageNo + 1); var prevList = all.slice(0, stageNo + 1);
...@@ -140,8 +195,10 @@ define( ...@@ -140,8 +195,10 @@ define(
}); });
var message = msg || gettext("There was an error with the upload"); var message = msg || gettext("There was an error with the upload");
var elem = $('ol.status-progress').children().eq(stageNo); var elem = $('ol.status-progress').children().eq(stageNo);
elem.removeClass('is-started').addClass('has-error'); if (!elem.hasClass('has-error')) {
elem.find('p.copy').hide().after("<p class='copy error'>" + message + "</p>"); elem.removeClass('is-started').addClass('has-error');
elem.find('p.copy').hide().after("<p class='copy error'>" + message + "</p>");
}
} }
}; };
......
cms/templates/import.html
...@@ -63,6 +63,9 @@ ...@@ -63,6 +63,9 @@
<div class="status-detail"> <div class="status-detail">
<h3 class="title">${_("Uploading")}</h3> <h3 class="title">${_("Uploading")}</h3>
<div class="progress-bar">
<div class="progress-fill"></div>
</div>
<p class="copy">${_("Transferring your file to our servers")}</p> <p class="copy">${_("Transferring your file to our servers")}</p>
</div> </div>
</li> </li>
...@@ -147,7 +150,7 @@ ...@@ -147,7 +150,7 @@
<%block name="jsextra"> <%block name="jsextra">
<script> <script>
require( require(
["js/views/import", "jquery", "gettext", "jquery.fileupload"], ["js/views/import", "jquery", "gettext", "jquery.fileupload", "jquery.cookie"],
function(CourseImport, $, gettext) { function(CourseImport, $, gettext) {
var file; var file;
...@@ -170,6 +173,12 @@ var defaults = [ ...@@ -170,6 +173,12 @@ var defaults = [
"${_("There was an error while importing the new course to our database.")}\n" "${_("There was an error while importing the new course to our database.")}\n"
]; ];
// Display the status of last file upload on page load
var lastfileupload = $.cookie('lastfileupload');
if (lastfileupload){
CourseImport.getAndStartUploadFeedback(feedbackUrl.replace('fillerName', lastfileupload), lastfileupload);
}
$('#fileupload').fileupload({ $('#fileupload').fileupload({
dataType: 'json', dataType: 'json',
...@@ -185,20 +194,22 @@ $('#fileupload').fileupload({ ...@@ -185,20 +194,22 @@ $('#fileupload').fileupload({
file = data.files[0]; file = data.files[0];
if (file.name.match(/tar\.gz$/)) { if (file.name.match(/tar\.gz$/)) {
submitBtn.click(function(e){ submitBtn.click(function(e){
$.cookie('lastfileupload', file.name);
e.preventDefault(); e.preventDefault();
submitBtn.hide(); submitBtn.hide();
CourseImport.startUploadFeedback();
data.submit().complete(function(result, textStatus, xhr) { data.submit().complete(function(result, textStatus, xhr) {
CourseImport.stopGetStatus = true;
window.onbeforeunload = null; window.onbeforeunload = null;
if (xhr.status != 200) { if (xhr.status != 200) {
if (!result.responseText) { try{
alert(gettext("Your browser has timed out, but the server is still processing your import. Please wait 5 minutes and verify that the new content has appeared.")); var serverMsg = $.parseJSON(result.responseText);
} catch (e) {
return; return;
} }
var serverMsg = $.parseJSON(result.responseText); var serverMsg = $.parseJSON(result.responseText);
var errMsg = serverMsg.hasOwnProperty("ErrMsg") ? serverMsg.ErrMsg : "" ; var errMsg = serverMsg.hasOwnProperty("ErrMsg") ? serverMsg.ErrMsg : "" ;
if (serverMsg.hasOwnProperty("Stage")) { if (serverMsg.hasOwnProperty("Stage")) {
var stage = serverMsg.Stage; var stage = Math.abs(serverMsg.Stage);
CourseImport.stageError(stage, defaults[stage] + errMsg); CourseImport.stageError(stage, defaults[stage] + errMsg);
} }
else { else {
...@@ -207,6 +218,7 @@ $('#fileupload').fileupload({ ...@@ -207,6 +218,7 @@ $('#fileupload').fileupload({
chooseBtn.html("${_("Choose new file")}").show(); chooseBtn.html("${_("Choose new file")}").show();
bar.hide(); bar.hide();
} }
CourseImport.stopGetStatus = true;
chooseBtn.html("${_("Choose new file")}").show(); chooseBtn.html("${_("Choose new file")}").show();
bar.hide(); bar.hide();
}); });
...@@ -230,11 +242,15 @@ $('#fileupload').fileupload({ ...@@ -230,11 +242,15 @@ $('#fileupload').fileupload({
} }
if (percentInt >= doneAt) { if (percentInt >= doneAt) {
bar.hide(); bar.hide();
CourseImport.startServerFeedback(feedbackUrl.replace("fillerName", file.name)); // Start feedback with delay so that current stage of import properly updates in session
setTimeout(
function() { CourseImport.startServerFeedback(feedbackUrl.replace('fillerName', file.name)) },
3000
);
} else { } else {
bar.show(); bar.show();
fill.width(percentVal); fill.width(percentVal);
percent.html(percentVal); fill.html(percentVal);
} }
}, },
done: function(e, data){ done: function(e, data){
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment