Skip to content

E
edx-platform
Commit a9d1d4ee by cahrens
Options

Convert import to new URL pattern.

parent d0f659d6
Showing with 213 additions and 209 deletions
cms/djangoapps/contentstore/tests/test_contentstore.py
......@@ -1592,10 +1592,7 @@ class ContentStoreTest(ModuleStoreTestCase):
# go to various pages
# import page
resp = self.client.get(reverse('import_course',
kwargs={'org': loc.org,
'course': loc.course,
'name': loc.name}))
resp = self.client.get(new_location.url_reverse('import/', ''), HTTP_ACCEPT='text/html')
self.assertEqual(resp.status_code, 200)
# export page
......@@ -1632,9 +1629,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertEqual(resp.status_code, 200)
# assets_handler (HTML for full page content)
new_location = loc_mapper().translate_location(loc.course_id, loc, False, True)
url = new_location.url_reverse('assets/', '')
resp = self.client.get(url, HTTP_ACCEPT='text/html')
self.assertEqual(resp.status_code, 200)
......
cms/djangoapps/contentstore/tests/test_import_export.py
......@@ -13,9 +13,9 @@ from uuid import uuid4
from pymongo import MongoClient
from .utils import CourseTestCase
from django.core.urlresolvers import reverse
from django.test.utils import override_settings
from django.conf import settings
from xmodule.modulestore.django import loc_mapper
from xmodule.contentstore.django import _CONTENTSTORE
......@@ -32,11 +32,10 @@ class ImportTestCase(CourseTestCase):
def setUp(self):
super(ImportTestCase, self).setUp()
self.url = reverse("import_course", kwargs={
'org': self.course.location.org,
'course': self.course.location.course,
'name': self.course.location.name,
})
self.new_location = loc_mapper().translate_location(
self.course.location.course_id, self.course.location, False, True
)
self.url = self.new_location.url_reverse('import/', '')
self.content_dir = path(tempfile.mkdtemp())
def touch(name):
......@@ -89,13 +88,13 @@ class ImportTestCase(CourseTestCase):
self.assertEquals(resp.status_code, 415)
# Check that `import_status` returns the appropriate stage (i.e., the
# stage at which import failed).
status_url = reverse("import_status", kwargs={
'org': self.course.location.org,
'course': self.course.location.course,
'name': os.path.split(self.bad_tar)[1],
})
resp_status = self.client.get(status_url)
log.debug(str(self.client.session["import_status"]))
resp_status = self.client.get(
self.new_location.url_reverse(
'import_status',
os.path.split(self.bad_tar)[1]
)
)
self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2)
......@@ -200,11 +199,11 @@ class ImportTestCase(CourseTestCase):
# Check that `import_status` returns the appropriate stage (i.e.,
# either 3, indicating all previous steps are completed, or 0,
# indicating no upload in progress)
status_url = reverse("import_status", kwargs={
'org': self.course.location.org,
'course': self.course.location.course,
'name': os.path.split(self.good_tar)[1],
})
resp_status = self.client.get(status_url)
resp_status = self.client.get(
self.new_location.url_reverse(
'import_status',
os.path.split(self.good_tar)[1]
)
)
import_status = json.loads(resp_status.content)["ImportStatus"]
self.assertIn(import_status, (0, 3))
cms/djangoapps/contentstore/views/import_export.py
......@@ -17,7 +17,8 @@ from django_future.csrf import ensure_csrf_cookie
from django.core.urlresolvers import reverse
from django.core.servers.basehttp import FileWrapper
from django.core.files.temp import NamedTemporaryFile
from django.core.exceptions import SuspiciousOperation
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods, require_GET
from django.utils.translation import ugettext as _
......@@ -30,13 +31,15 @@ from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore import Location
from xmodule.exceptions import SerializationError
from xmodule.modulestore.locator import BlockUsageLocator
from .access import has_access
from .access import get_location_and_verify_access
from util.json_request import JsonResponse
from extract_tar import safetar_extractall
__all__ = ['import_course', 'import_status', 'generate_export_course', 'export_course']
__all__ = ['import_handler', 'import_status_handler', 'generate_export_course', 'export_course']
log = logging.getLogger(__name__)
......@@ -45,212 +48,221 @@ log = logging.getLogger(__name__)
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
@login_required
def import_course(request, org, course, name):
"""
This method will handle a POST request to upload and import a .tar.gz file
into a specified course
def import_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None):
"""
location = get_location_and_verify_access(request, org, course, name)
if request.method == 'POST':
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(org, course, name)
course_dir = data_root / course_subdir
The restful handler for importing a course.
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1
},
status=415
)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
GET
html: return html page for import page
json: not supported
POST or PUT
json: import a course via the .tar.gz file specified in request.FILES
"""
location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block)
if not has_access(request.user, location):
raise PermissionDenied()
logging.debug('importing course to {0}'.format(temp_filepath))
old_location = loc_mapper().translate_locator_to_location(location)
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
if request.method == 'GET':
raise NotImplementedError('coming soon')
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(old_location.org, old_location.course, old_location.name)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{
'ErrMsg': _('File upload corrupted. Please try again'),
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1
},
status=409
status=415
)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse('import_course', kwargs={
'org': location.org,
'course': location.course,
'name': location.name
}),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = org + course + filename
session_status[key] = 1
request.session.modified = True
# Do everything from now on in a try-finally block to make sure
# everything is properly cleaned up.
try:
temp_filepath = course_dir / filename
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc:
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
return JsonResponse(
{
'ErrMsg': 'Unsafe tar file. Aborting import.',
'SuspiciousFileOperationMsg': exc.args[0],
'ErrMsg': _('File upload corrupted. Please try again'),
'Stage': 1
},
status=400
status=409
)
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': 2
},
status=415
)
size = os.path.getsize(temp_filepath)
logging.debug('found course.xml at {0}'.format(dirpath))
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": location.url_reverse('import/', ''),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = location.course_id + filename
session_status[key] = 1
request.session.modified = True
if dirpath != course_dir:
for fname in os.listdir(dirpath):
shutil.move(dirpath / fname, course_dir)
# Do everything from now on in a try-finally block to make sure
# everything is properly cleaned up.
try:
_module_store, course_items = import_from_xml(
modulestore('direct'),
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_location_namespace=location,
draft_store=modulestore()
)
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc:
return JsonResponse(
{
'ErrMsg': 'Unsafe tar file. Aborting import.',
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': 1
},
status=400
)
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': 2
},
status=415
)
logging.debug('found course.xml at {0}'.format(dirpath))
if dirpath != course_dir:
for fname in os.listdir(dirpath):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_location_namespace=old_location,
draft_store=modulestore()
)
logging.debug('new course at {0}'.format(course_items[0].location))
logging.debug('new course at {0}'.format(course_items[0].location))
session_status[key] = 3
request.session.modified = True
session_status[key] = 3
request.session.modified = True
create_all_course_groups(request.user, course_items[0].location)
logging.debug('created all course groups at {0}'.format(course_items[0].location))
create_all_course_groups(request.user, course_items[0].location)
logging.debug('created all course groups at {0}'.format(course_items[0].location))
# Send errors to client with stage at which error occured.
except Exception as exception: # pylint: disable=W0703
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': session_status[key]
},
status=400
)
# Send errors to client with stage at which error occured.
except Exception as exception: # pylint: disable=W0703
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': session_status[key]
},
status=400
)
finally:
shutil.rmtree(course_dir)
finally:
shutil.rmtree(course_dir)
return JsonResponse({'Status': 'OK'})
else:
course_module = modulestore().get_item(location)
new_location = loc_mapper().translate_location(course_module.location.course_id, course_module.location, False, True)
return JsonResponse({'Status': 'OK'})
elif request.method == 'GET': # assume html
course_module = modulestore().get_item(old_location)
return render_to_response('import.html', {
'context_course': course_module,
'successful_import_redirect_url': new_location.url_reverse("course/", "")
'successful_import_redirect_url': location.url_reverse("course/", ""),
'import_status_url': location.url_reverse("import_status/", "fillerName"),
})
else:
return HttpResponseNotFound()
@require_GET
@ensure_csrf_cookie
@login_required
def import_status(request, org, course, name):
def import_status_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None, filename=None):
"""
Returns an integer corresponding to the status of a file import. These are:
......@@ -260,10 +272,13 @@ def import_status(request, org, course, name):
3 : Importing to mongo
"""
location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block)
if not has_access(request.user, location):
raise PermissionDenied()
try:
session_status = request.session["import_status"]
status = session_status[org + course + name]
status = session_status[location.course_id + filename]
except KeyError:
status = 0
......
cms/templates/import.html
<%inherit file="base.html" />
<%namespace name='static' file='static_content.html'/>
<%!
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
%>
<%block name="title">${_("Course Import")}</%block>
......@@ -28,8 +27,7 @@
<p>${_("During the initial stages of the import process, please do not navigate away from this page.")}</p>
</div>
<form id="fileupload" method="post" enctype="multipart/form-data"
class="import-form" url="${reverse('import_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}">
<form id="fileupload" method="post" enctype="multipart/form-data" class="import-form">
<input type="hidden" name="csrfmiddlewaretoken" value="${csrf_token}" />
......@@ -158,7 +156,7 @@ var chooseBtn = $('.choose-file-button');
var allStats = $('#status-infos');
var feedbackUrl = "${reverse('import_status', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name='fillerName'))}"
var feedbackUrl = "${import_status_url}";
var defaults = [
'${_("There was an error during the upload process.")}\n',
......
cms/templates/widgets/header.html
......@@ -20,6 +20,7 @@
checklists_url = location.url_reverse('checklists/', '')
course_team_url = location.url_reverse('course_team/', '')
assets_url = location.url_reverse('assets/', '')
import_url = location.url_reverse('import/', '')
%>
<h2 class="info-course">
<span class="sr">${_("Current Course:")}</span>
......@@ -91,7 +92,7 @@
<a href="${checklists_url}">${_("Checklists")}</a>
</li>
<li class="nav-item nav-course-tools-import">
<a href="${reverse('import_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}">${_("Import")}</a>
<a href="${import_url}">${_("Import")}</a>
</li>
<li class="nav-item nav-course-tools-export">
<a href="${reverse('export_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}">${_("Export")}</a>
......
cms/urls.py
import re
from django.conf import settings
from django.conf.urls import patterns, include, url
......@@ -34,11 +33,6 @@ urlpatterns = patterns('', # nopep8
url(r'^create_new_course', 'contentstore.views.create_new_course', name='create_new_course'),
url(r'^reorder_static_tabs', 'contentstore.views.reorder_static_tabs', name='reorder_static_tabs'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import/(?P<name>[^/]+)$',
'contentstore.views.import_course', name='import_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import_status/(?P<name>[^/]+)$',
'contentstore.views.import_status', name='import_status'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/export/(?P<name>[^/]+)$',
'contentstore.views.export_course', name='export_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/generate_export/(?P<name>[^/]+)$',
......@@ -129,7 +123,9 @@ urlpatterns += patterns(
url(r'(?ix)^checklists/{}(/)?(?P<checklist_index>\d+)?$'.format(parsers.URL_RE_SOURCE), 'checklists_handler'),
url(r'(?ix)^course_team/{}(/)?(?P<email>.+)?$'.format(parsers.URL_RE_SOURCE), 'course_team_handler'),
url(r'(?ix)^orphan/{}$'.format(parsers.URL_RE_SOURCE), 'orphan'),
url(r'(?ix)^assets/{}(/)?(?P<asset_id>.+)?$'.format(parsers.URL_RE_SOURCE), 'assets_handler')
url(r'(?ix)^assets/{}(/)?(?P<asset_id>.+)?$'.format(parsers.URL_RE_SOURCE), 'assets_handler'),
url(r'(?ix)^import/{}$'.format(parsers.URL_RE_SOURCE), 'import_handler'),
url(r'(?ix)^import_status/{}/(?P<filename>.+)$'.format(parsers.URL_RE_SOURCE), 'import_status_handler'),
)
js_info_dict = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment