Commit 5c433ec9 by Sarina Canelake

Merge pull request #5731 from Stanford-Online/ataki/upstream

Limit Upload File Sizes to GridFS
parents b1e5002b fb9320af
......@@ -180,4 +180,5 @@ Eugeny Kolpakov <eugeny.kolpakov@gmail.com>
Omar Al-Ithawi <oithawi@qrf.org>
Louis Pilfold <louis@lpil.uk>
Akiva Leffert <akiva@edx.org>
Mike Bifulco <mbifulco@aquent.com>
\ No newline at end of file
Mike Bifulco <mbifulco@aquent.com>
Jim Zheng <jimzheng@stanford.edu>
......@@ -83,6 +83,9 @@ def _asset_index(request, course_key):
return render_to_response('asset_index.html', {
'context_course': course_module,
'max_file_size_in_mbs': settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB,
'chunk_size_in_mbs': settings.UPLOAD_CHUNK_SIZE_IN_MB,
'max_file_size_redirect_url': settings.MAX_ASSET_UPLOAD_FILE_SIZE_URL,
'asset_callback_url': reverse_course_url('assets_handler', course_key)
})
......@@ -152,6 +155,14 @@ def _get_assets_for_page(request, course_key, current_page, page_size, sort):
)
def get_file_size(upload_file):
"""
Helper method for getting file size of an upload file.
Can be used for mocking test file sizes.
"""
return upload_file.size
@require_POST
@ensure_csrf_cookie
@login_required
......@@ -176,6 +187,26 @@ def _upload_asset(request, course_key):
upload_file = request.FILES['file']
filename = upload_file.name
mime_type = upload_file.content_type
size = get_file_size(upload_file)
# If file is greater than a specified size, reject the upload
# request and send a message to the user. Note that since
# the front-end may batch large file uploads in smaller chunks,
# we validate the file-size on the front-end in addition to
# validating on the backend. (see cms/static/js/views/assets.js)
max_file_size_in_bytes = settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB * 1000 ** 2
if size > max_file_size_in_bytes:
return JsonResponse({
'error': _(
'File {filename} exceeds maximum size of '
'{size_mb} MB. Please follow the instructions here '
'to upload a file elsewhere and link to it instead: '
'{faq_url}').format(
filename=filename,
size_mb=settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB,
faq_url=settings.MAX_ASSET_UPLOAD_FILE_SIZE_URL,
)
}, status=413)
content_loc = StaticContent.compute_location(course_key, filename)
......
......@@ -5,6 +5,7 @@ from datetime import datetime
from io import BytesIO
from pytz import UTC
import json
from django.conf import settings
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
......@@ -16,10 +17,14 @@ from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import import_from_xml
from django.test.utils import override_settings
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from django.conf import settings
import mock
from ddt import ddt
from ddt import data
TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
MAX_FILE_SIZE = settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB * 1000 ** 2
class AssetsTestCase(CourseTestCase):
"""
......@@ -33,9 +38,14 @@ class AssetsTestCase(CourseTestCase):
"""
Post to the asset upload url
"""
f = self.get_sample_asset(name)
return self.client.post(self.url, {"name": name, "file": f})
def get_sample_asset(self, name):
"""Returns an in-memory file with the given name for testing"""
f = BytesIO(name)
f.name = name + ".txt"
return self.client.post(self.url, {"name": name, "file": f})
return f
class BasicAssetsTestCase(AssetsTestCase):
......@@ -132,6 +142,7 @@ class PaginationTestCase(AssetsTestCase):
self.assertGreaterEqual(name2, name3)
@ddt
class UploadTestCase(AssetsTestCase):
"""
Unit tests for uploading a file
......@@ -148,6 +159,24 @@ class UploadTestCase(AssetsTestCase):
resp = self.client.post(self.url, {"name": "file.txt"}, "application/json")
self.assertEquals(resp.status_code, 400)
@data(
(int(MAX_FILE_SIZE / 2.0), "small.file.test", 200),
(MAX_FILE_SIZE, "justequals.file.test", 200),
(MAX_FILE_SIZE + 90, "large.file.test", 413),
)
@mock.patch('contentstore.views.assets.get_file_size')
def test_file_size(self, case, get_file_size):
max_file_size, name, status_code = case
get_file_size.return_value = max_file_size
f = self.get_sample_asset(name=name)
resp = self.client.post(self.url, {
"name": name,
"file": f
})
self.assertEquals(resp.status_code, status_code)
class DownloadTestCase(AssetsTestCase):
"""
......
......@@ -718,6 +718,16 @@ ADVANCED_SECURITY_CONFIG = {}
SHIBBOLETH_DOMAIN_PREFIX = 'shib:'
OPENID_DOMAIN_PREFIX = 'openid:'
### Size of chunks into which asset uploads will be divided
UPLOAD_CHUNK_SIZE_IN_MB = 10
### Max size of asset uploads to GridFS
MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB = 10
# FAQ url to direct users to if they upload
# a file that exceeds the above size
MAX_ASSET_UPLOAD_FILE_SIZE_URL = ""
################ ADVANCED_COMPONENT_TYPES ###############
ADVANCED_COMPONENT_TYPES = [
......
......@@ -15,6 +15,8 @@ requirejs.config({
"jquery.cookie": "xmodule_js/common_static/js/vendor/jquery.cookie",
"jquery.qtip": "xmodule_js/common_static/js/vendor/jquery.qtip.min",
"jquery.fileupload": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload",
"jquery.fileupload-process": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-process",
"jquery.fileupload-validate": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-validate",
"jquery.iframe-transport": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.iframe-transport",
"jquery.inputnumber": "xmodule_js/common_static/js/vendor/html5-input-polyfills/number-polyfill",
"jquery.immediateDescendents": "xmodule_js/common_static/coffee/src/jquery.immediateDescendents",
......@@ -94,9 +96,15 @@ requirejs.config({
exports: "jQuery.fn.qtip"
},
"jquery.fileupload": {
deps: ["jquery.iframe-transport"],
deps: ["jquery.ui", "jquery.iframe-transport"],
exports: "jQuery.fn.fileupload"
},
"jquery.fileupload-process": {
deps: ["jquery.fileupload"]
},
"jquery.fileupload-validate": {
deps: ["jquery.fileupload"]
},
"jquery.inputnumber": {
deps: ["jquery"],
exports: "jQuery.fn.inputNumber"
......
......@@ -14,6 +14,8 @@ requirejs.config({
"jquery.cookie": "xmodule_js/common_static/js/vendor/jquery.cookie",
"jquery.qtip": "xmodule_js/common_static/js/vendor/jquery.qtip.min",
"jquery.fileupload": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload",
"jquery.fileupload-process": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-process",
"jquery.fileupload-validate": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-validate",
"jquery.iframe-transport": "xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.iframe-transport",
"jquery.inputnumber": "xmodule_js/common_static/js/vendor/html5-input-polyfills/number-polyfill",
"jquery.immediateDescendents": "xmodule_js/common_static/coffee/src/jquery.immediateDescendents",
......@@ -84,9 +86,15 @@ requirejs.config({
exports: "jQuery.fn.qtip"
},
"jquery.fileupload": {
deps: ["jquery.iframe-transport"],
deps: ["jquery.ui", "jquery.iframe-transport"],
exports: "jQuery.fn.fileupload"
},
"jquery.fileupload-process": {
deps: ["jquery.fileupload"]
},
"jquery.fileupload-validate": {
deps: ["jquery.fileupload"]
},
"jquery.inputnumber": {
deps: ["jquery"],
exports: "jQuery.fn.inputNumber"
......
......@@ -2,12 +2,18 @@ define([
'jquery', 'js/collections/asset', 'js/views/assets', 'jquery.fileupload'
], function($, AssetCollection, AssetsView) {
'use strict';
return function (assetCallbackUrl) {
return function (config) {
var assets = new AssetCollection(),
assetsView;
assets.url = assetCallbackUrl;
assetsView = new AssetsView({collection: assets, el: $('.assets-wrapper')});
assets.url = config.assetCallbackUrl;
assetsView = new AssetsView({
collection: assets,
el: $('.assets-wrapper'),
uploadChunkSizeInMBs: config.uploadChunkSizeInMBs,
maxFileSizeInMBs: config.maxFileSizeInMBs,
maxFileSizeRedirectUrl: config.maxFileSizeRedirectUrl
});
assetsView.render();
};
});
define([ "jquery", "js/common_helpers/ajax_helpers", "js/views/asset", "js/views/assets",
"js/models/asset", "js/collections/asset", "js/spec_helpers/view_helpers" ],
"js/models/asset", "js/collections/asset", "js/spec_helpers/view_helpers"],
function ($, AjaxHelpers, AssetView, AssetsView, AssetModel, AssetCollection, ViewHelpers) {
describe("Assets", function() {
var assetsView, mockEmptyAssetsResponse, mockAssetUploadResponse,
var assetsView, mockEmptyAssetsResponse, mockAssetUploadResponse, mockFileUpload,
assetLibraryTpl, assetTpl, pagingFooterTpl, pagingHeaderTpl, uploadModalTpl;
assetLibraryTpl = readFixtures('asset-library.underscore');
......@@ -53,6 +53,10 @@ define([ "jquery", "js/common_helpers/ajax_helpers", "js/views/asset", "js/views
msg: "Upload completed"
};
mockFileUpload = {
files: [{name: 'largefile', size: 0}]
};
$.fn.fileupload = function() {
return '';
};
......@@ -95,6 +99,15 @@ define([ "jquery", "js/common_helpers/ajax_helpers", "js/views/asset", "js/views
expect($('.upload-modal').is(':visible')).toBe(false);
});
it('has properly initialized constants for handling upload file errors', function() {
expect(assetsView).toBeDefined();
expect(assetsView.uploadChunkSizeInMBs).toBeDefined();
expect(assetsView.maxFileSizeInMBs).toBeDefined();
expect(assetsView.uploadChunkSizeInBytes).toBeDefined();
expect(assetsView.maxFileSizeInBytes).toBeDefined();
expect(assetsView.largeFileErrorMsg).toBeNull();
});
it('uploads file properly', function () {
var requests = setup.call(this);
expect(assetsView).toBeDefined();
......@@ -122,6 +135,42 @@ define([ "jquery", "js/common_helpers/ajax_helpers", "js/views/asset", "js/views
expect($('#asset_table_body').html()).toContain("dummy.jpg");
expect(assetsView.collection.length).toBe(1);
});
it('blocks file uploads larger than the max file size', function() {
expect(assetsView).toBeDefined();
mockFileUpload.files[0].size = assetsView.maxFileSize * 10;
$('.choose-file-button').click();
$(".upload-modal .file-chooser").fileupload('add', mockFileUpload);
expect($('.upload-modal h1').text()).not.toContain("Uploading");
expect(assetsView.largeFileErrorMsg).toBeDefined();
expect($('div.progress-bar').text()).not.toContain("Upload completed");
expect($('div.progress-fill').width()).toBe(0);
});
it('allows file uploads equal in size to the max file size', function() {
expect(assetsView).toBeDefined();
mockFileUpload.files[0].size = assetsView.maxFileSize;
$('.choose-file-button').click();
$(".upload-modal .file-chooser").fileupload('add', mockFileUpload);
expect(assetsView.largeFileErrorMsg).toBeNull();
});
it('allows file uploads smaller than the max file size', function() {
expect(assetsView).toBeDefined();
mockFileUpload.files[0].size = assetsView.maxFileSize / 100;
$('.choose-file-button').click();
$(".upload-modal .file-chooser").fileupload('add', mockFileUpload);
expect(assetsView.largeFileErrorMsg).toBeNull();
});
});
});
});
define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging", "js/views/asset",
"js/views/paging_header", "js/views/paging_footer", "js/utils/modal", "js/views/utils/view_utils"],
function($, _, gettext, AssetModel, PagingView, AssetView, PagingHeader, PagingFooter, ModalUtils, ViewUtils) {
"js/views/paging_header", "js/views/paging_footer", "js/utils/modal", "js/views/utils/view_utils",
"js/views/feedback_notification", "jquery.fileupload-process", "jquery.fileupload-validate"],
function($, _, gettext, AssetModel, PagingView, AssetView, PagingHeader, PagingFooter, ModalUtils, ViewUtils, NotificationView) {
var CONVERSION_FACTOR_MBS_TO_BYTES = 1000 * 1000;
var AssetsView = PagingView.extend({
// takes AssetCollection as model
......@@ -10,7 +13,9 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
"click .upload-button": "showUploadModal"
},
initialize : function() {
initialize : function(options) {
options = options || {};
PagingView.prototype.initialize.call(this);
var collection = this.collection;
this.template = this.loadTemplate("asset-library");
......@@ -20,7 +25,16 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
this.setInitialSortColumn('js-asset-date-col');
ViewUtils.showLoadingIndicator();
this.setPage(0);
// set default file size for uploads via template var,
// and default to static old value if none exists
this.uploadChunkSizeInMBs = options.uploadChunkSizeInMBs || 10;
this.maxFileSizeInMBs = options.maxFileSizeInMBs || 10;
this.uploadChunkSizeInBytes = this.uploadChunkSizeInMBs * CONVERSION_FACTOR_MBS_TO_BYTES;
this.maxFileSizeInBytes = this.maxFileSizeInMBs * CONVERSION_FACTOR_MBS_TO_BYTES;
this.maxFileSizeRedirectUrl = options.maxFileSizeRedirectUrl || '';
assetsView = this;
// error message modal for large file uploads
this.largeFileErrorMsg = null;
},
render: function() {
......@@ -111,6 +125,9 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
}
$('.file-input').unbind('change.startUpload');
ModalUtils.hideModal();
if (assetsView.largeFileErrorMsg) {
assetsView.largeFileErrorMsg.hide();
}
},
showUploadModal: function (event) {
......@@ -122,23 +139,44 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
$('.upload-modal .file-chooser').fileupload({
dataType: 'json',
type: 'POST',
maxChunkSize: 100 * 1000 * 1000, // 100 MB
maxChunkSize: self.uploadChunkSizeInBytes,
autoUpload: true,
progressall: function(event, data) {
var percentComplete = parseInt((100 * data.loaded) / data.total, 10);
self.showUploadFeedback(event, percentComplete);
},
maxFileSize: 100 * 1000 * 1000, // 100 MB
maxFileSize: self.maxFileSizeInBytes,
maxNumberofFiles: 100,
add: function(event, data) {
data.process().done(function () {
data.submit();
});
},
done: function(event, data) {
self.displayFinishedUpload(data.result);
}
},
processfail: function(event, data) {
var filename = data.files[data.index].name;
var error = gettext("File {filename} exceeds maximum size of {maxFileSizeInMBs} MB")
.replace("{filename}", filename)
.replace("{maxFileSizeInMBs}", self.maxFileSizeInMBs)
// disable second part of message for any falsy value,
// which can be null or an empty string
if(self.maxFileSizeRedirectUrl) {
var instructions = gettext("Please follow the instructions here to upload a file elsewhere and link to it: {maxFileSizeRedirectUrl}")
.replace("{maxFileSizeRedirectUrl}", self.maxFileSizeRedirectUrl);
error = error + " " + instructions;
}
assetsView.largeFileErrorMsg = new NotificationView.Error({
"title": gettext("Your file could not be uploaded"),
"message": error
});
assetsView.largeFileErrorMsg.show();
assetsView.displayFailedUpload({
"msg": gettext("Max file size exceeded")
});
},
processdone: function(event, data) {
assetsView.largeFileErrorMsg = null;
}
});
},
......@@ -149,11 +187,12 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
startUpload: function (event) {
var file = event.target.value;
$('.upload-modal h1').text(gettext('Uploading…'));
$('.upload-modal .file-name').html(file.substring(file.lastIndexOf("\\") + 1));
$('.upload-modal .choose-file-button').hide();
$('.upload-modal .progress-bar').removeClass('loaded').show();
if (!assetsView.largeFileErrorMsg) {
$('.upload-modal h1').text(gettext('Uploading'));
$('.upload-modal .file-name').html(file.substring(file.lastIndexOf("\\") + 1));
$('.upload-modal .choose-file-button').hide();
$('.upload-modal .progress-bar').removeClass('loaded').show();
}
},
resetUploadModal: function () {
......@@ -169,6 +208,8 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
$('.upload-modal .choose-file-button').text(gettext('Choose File'));
$('.upload-modal .embeddable-xml-input').val('');
$('.upload-modal .embeddable').hide();
assetsView.largeFileErrorMsg = null;
},
showUploadFeedback: function (event, percentComplete) {
......@@ -181,7 +222,7 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
var asset = resp.asset;
$('.upload-modal h1').text(gettext('Upload New File'));
$('.upload-modal .embeddable-xml-input').val(asset.portable_url);
$('.upload-modal .embeddable-xml-input').val(asset.portable_url).show();
$('.upload-modal .embeddable').show();
$('.upload-modal .file-name').hide();
$('.upload-modal .progress-fill').html(resp.msg);
......@@ -189,6 +230,16 @@ define(["jquery", "underscore", "gettext", "js/models/asset", "js/views/paging",
$('.upload-modal .progress-fill').width('100%');
assetsView.addAsset(new AssetModel(asset));
},
displayFailedUpload: function (resp) {
$('.upload-modal h1').text(gettext('Upload New File'));
$('.upload-modal .embeddable-xml-input').hide();
$('.upload-modal .embeddable').hide();
$('.upload-modal .file-name').hide();
$('.upload-modal .progress-fill').html(resp.msg);
$('.upload-modal .choose-file-button').text(gettext('Load Another File')).show();
$('.upload-modal .progress-fill').width('0%');
}
});
......
......@@ -62,6 +62,10 @@ lib_paths:
- xmodule_js/common_static/coffee/src/jquery.immediateDescendents.js
- xmodule_js/common_static/coffee/src/xblock/
- xmodule_js/common_static/js/vendor/URI.min.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.iframe-transport.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-process.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-validate.js
# Paths to source JavaScript files
src_paths:
......
......@@ -57,6 +57,10 @@ lib_paths:
- xmodule_js/common_static/js/test/i18n.js
- xmodule_js/common_static/coffee/src/xblock/
- xmodule_js/common_static/js/vendor/URI.min.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.iframe-transport.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-process.js
- xmodule_js/common_static/js/vendor/jQuery-File-Upload/js/jquery.fileupload-validate.js
# Paths to source JavaScript files
src_paths:
......
......@@ -20,6 +20,8 @@ require.config({
"jquery.scrollTo": "js/vendor/jquery.scrollTo-1.4.2-min",
"jquery.flot": "js/vendor/flot/jquery.flot.min",
"jquery.fileupload": "js/vendor/jQuery-File-Upload/js/jquery.fileupload",
"jquery.fileupload-process": "js/vendor/jQuery-File-Upload/js/jquery.fileupload-process",
"jquery.fileupload-validate": "js/vendor/jQuery-File-Upload/js/jquery.fileupload-validate",
"jquery.iframe-transport": "js/vendor/jQuery-File-Upload/js/jquery.iframe-transport",
"jquery.inputnumber": "js/vendor/html5-input-polyfills/number-polyfill",
"jquery.immediateDescendents": "coffee/src/jquery.immediateDescendents",
......@@ -128,9 +130,15 @@ require.config({
exports: "jQuery.fn.plot"
},
"jquery.fileupload": {
deps: ["jquery.iframe-transport"],
deps: ["jquery.ui", "jquery.iframe-transport"],
exports: "jQuery.fn.fileupload"
},
"jquery.fileupload-process": {
deps: ["jquery.fileupload"]
},
"jquery.fileupload-validate": {
deps: ["jquery.fileupload"]
},
"jquery.inputnumber": {
deps: ["jquery"],
exports: "jQuery.fn.inputNumber"
......
......@@ -19,7 +19,12 @@
<%block name="requirejs">
require(["js/factories/asset_index"], function (AssetIndexFactory) {
AssetIndexFactory("${asset_callback_url}");
AssetIndexFactory({
assetCallbackUrl: "${asset_callback_url}",
uploadChunkSizeInMBs: ${chunk_size_in_mbs},
maxFileSizeInMBs: ${max_file_size_in_mbs},
maxFileSizeRedirectUrl: "${max_file_size_redirect_url}"
});
});
</%block>
......@@ -82,6 +87,7 @@
<a href="#" class="close-button"><i class="icon-remove-sign"></i> <span class="sr">${_('close')}</span></a>
<div class="modal-body">
<h1 class="title">${_("Upload New File")}</h1>
<h2>${_("Max per-file size: {max_filesize}MB").format(max_filesize=max_file_size_in_mbs)}</h2>
<p class="file-name">
<div class="progress-bar">
<div class="progress-fill"></div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment