Commit a9d1d4ee by cahrens

Convert import to new URL pattern.

parent d0f659d6
...@@ -1592,10 +1592,7 @@ class ContentStoreTest(ModuleStoreTestCase): ...@@ -1592,10 +1592,7 @@ class ContentStoreTest(ModuleStoreTestCase):
# go to various pages # go to various pages
# import page # import page
resp = self.client.get(reverse('import_course', resp = self.client.get(new_location.url_reverse('import/', ''), HTTP_ACCEPT='text/html')
kwargs={'org': loc.org,
'course': loc.course,
'name': loc.name}))
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
# export page # export page
...@@ -1632,9 +1629,7 @@ class ContentStoreTest(ModuleStoreTestCase): ...@@ -1632,9 +1629,7 @@ class ContentStoreTest(ModuleStoreTestCase):
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
# assets_handler (HTML for full page content) # assets_handler (HTML for full page content)
new_location = loc_mapper().translate_location(loc.course_id, loc, False, True)
url = new_location.url_reverse('assets/', '') url = new_location.url_reverse('assets/', '')
resp = self.client.get(url, HTTP_ACCEPT='text/html') resp = self.client.get(url, HTTP_ACCEPT='text/html')
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
......
...@@ -13,9 +13,9 @@ from uuid import uuid4 ...@@ -13,9 +13,9 @@ from uuid import uuid4
from pymongo import MongoClient from pymongo import MongoClient
from .utils import CourseTestCase from .utils import CourseTestCase
from django.core.urlresolvers import reverse
from django.test.utils import override_settings from django.test.utils import override_settings
from django.conf import settings from django.conf import settings
from xmodule.modulestore.django import loc_mapper
from xmodule.contentstore.django import _CONTENTSTORE from xmodule.contentstore.django import _CONTENTSTORE
...@@ -32,11 +32,10 @@ class ImportTestCase(CourseTestCase): ...@@ -32,11 +32,10 @@ class ImportTestCase(CourseTestCase):
def setUp(self): def setUp(self):
super(ImportTestCase, self).setUp() super(ImportTestCase, self).setUp()
self.url = reverse("import_course", kwargs={ self.new_location = loc_mapper().translate_location(
'org': self.course.location.org, self.course.location.course_id, self.course.location, False, True
'course': self.course.location.course, )
'name': self.course.location.name, self.url = self.new_location.url_reverse('import/', '')
})
self.content_dir = path(tempfile.mkdtemp()) self.content_dir = path(tempfile.mkdtemp())
def touch(name): def touch(name):
...@@ -89,13 +88,13 @@ class ImportTestCase(CourseTestCase): ...@@ -89,13 +88,13 @@ class ImportTestCase(CourseTestCase):
self.assertEquals(resp.status_code, 415) self.assertEquals(resp.status_code, 415)
# Check that `import_status` returns the appropriate stage (i.e., the # Check that `import_status` returns the appropriate stage (i.e., the
# stage at which import failed). # stage at which import failed).
status_url = reverse("import_status", kwargs={ resp_status = self.client.get(
'org': self.course.location.org, self.new_location.url_reverse(
'course': self.course.location.course, 'import_status',
'name': os.path.split(self.bad_tar)[1], os.path.split(self.bad_tar)[1]
}) )
resp_status = self.client.get(status_url) )
log.debug(str(self.client.session["import_status"]))
self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2) self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2)
...@@ -200,11 +199,11 @@ class ImportTestCase(CourseTestCase): ...@@ -200,11 +199,11 @@ class ImportTestCase(CourseTestCase):
# Check that `import_status` returns the appropriate stage (i.e., # Check that `import_status` returns the appropriate stage (i.e.,
# either 3, indicating all previous steps are completed, or 0, # either 3, indicating all previous steps are completed, or 0,
# indicating no upload in progress) # indicating no upload in progress)
status_url = reverse("import_status", kwargs={ resp_status = self.client.get(
'org': self.course.location.org, self.new_location.url_reverse(
'course': self.course.location.course, 'import_status',
'name': os.path.split(self.good_tar)[1], os.path.split(self.good_tar)[1]
}) )
resp_status = self.client.get(status_url) )
import_status = json.loads(resp_status.content)["ImportStatus"] import_status = json.loads(resp_status.content)["ImportStatus"]
self.assertIn(import_status, (0, 3)) self.assertIn(import_status, (0, 3))
...@@ -17,7 +17,8 @@ from django_future.csrf import ensure_csrf_cookie ...@@ -17,7 +17,8 @@ from django_future.csrf import ensure_csrf_cookie
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
from django.core.servers.basehttp import FileWrapper from django.core.servers.basehttp import FileWrapper
from django.core.files.temp import NamedTemporaryFile from django.core.files.temp import NamedTemporaryFile
from django.core.exceptions import SuspiciousOperation from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods, require_GET from django.views.decorators.http import require_http_methods, require_GET
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
...@@ -30,13 +31,15 @@ from xmodule.modulestore.xml_exporter import export_to_xml ...@@ -30,13 +31,15 @@ from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore, loc_mapper from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore import Location from xmodule.modulestore import Location
from xmodule.exceptions import SerializationError from xmodule.exceptions import SerializationError
from xmodule.modulestore.locator import BlockUsageLocator
from .access import has_access
from .access import get_location_and_verify_access from .access import get_location_and_verify_access
from util.json_request import JsonResponse from util.json_request import JsonResponse
from extract_tar import safetar_extractall from extract_tar import safetar_extractall
__all__ = ['import_course', 'import_status', 'generate_export_course', 'export_course'] __all__ = ['import_handler', 'import_status_handler', 'generate_export_course', 'export_course']
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -45,212 +48,221 @@ log = logging.getLogger(__name__) ...@@ -45,212 +48,221 @@ log = logging.getLogger(__name__)
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})") CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")
@login_required
@ensure_csrf_cookie @ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT")) @require_http_methods(("GET", "POST", "PUT"))
@login_required def import_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None):
def import_course(request, org, course, name):
"""
This method will handle a POST request to upload and import a .tar.gz file
into a specified course
""" """
location = get_location_and_verify_access(request, org, course, name) The restful handler for importing a course.
if request.method == 'POST':
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(org, course, name)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name GET
if not filename.endswith('.tar.gz'): html: return html page for import page
return JsonResponse( json: not supported
{ POST or PUT
'ErrMsg': _('We only support uploading a .tar.gz file.'), json: import a course via the .tar.gz file specified in request.FILES
'Stage': 1 """
}, location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block)
status=415 if not has_access(request.user, location):
) raise PermissionDenied()
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath)) old_location = loc_mapper().translate_locator_to_location(location)
# Get upload chunks byte ranges if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
try: if request.method == 'GET':
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) raise NotImplementedError('coming soon')
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else: else:
mode = "ab+" data_root = path(settings.GITHUB_REPO_ROOT)
size = os.path.getsize(temp_filepath) course_subdir = "{0}-{1}-{2}".format(old_location.org, old_location.course, old_location.name)
# Check to make sure we haven't missed a chunk course_dir = data_root / course_subdir
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier. filename = request.FILES['course-data'].name
if size < int(content_range['start']): if not filename.endswith('.tar.gz'):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': _('File upload corrupted. Please try again'), 'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1 'Stage': 1
}, },
status=409 status=415
) )
# The last request sometimes comes twice. This happens because temp_filepath = course_dir / filename
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse('import_course', kwargs={
'org': location.org,
'course': location.course,
'name': location.name
}),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = org + course + filename
session_status[key] = 1
request.session.modified = True
# Do everything from now on in a try-finally block to make sure
# everything is properly cleaned up.
try:
tar_file = tarfile.open(temp_filepath) if not course_dir.isdir():
try: os.mkdir(course_dir)
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc: logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': 'Unsafe tar file. Aborting import.', 'ErrMsg': _('File upload corrupted. Please try again'),
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': 1 'Stage': 1
}, },
status=400 status=409
) )
finally: # The last request sometimes comes twice. This happens because
tar_file.close() # nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
session_status[key] = 2 return JsonResponse({'ImportStatus': 1})
request.session.modified = True
# find the 'course.xml' file with open(temp_filepath, mode) as temp_file:
def get_all_files(directory): for chunk in request.FILES['course-data'].chunks():
""" temp_file.write(chunk)
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'), size = os.path.getsize(temp_filepath)
'Stage': 2
},
status=415
)
logging.debug('found course.xml at {0}'.format(dirpath)) if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": location.url_reverse('import/', ''),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = location.course_id + filename
session_status[key] = 1
request.session.modified = True
if dirpath != course_dir: # Do everything from now on in a try-finally block to make sure
for fname in os.listdir(dirpath): # everything is properly cleaned up.
shutil.move(dirpath / fname, course_dir) try:
_module_store, course_items = import_from_xml( tar_file = tarfile.open(temp_filepath)
modulestore('direct'), try:
settings.GITHUB_REPO_ROOT, safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
[course_subdir], except SuspiciousOperation as exc:
load_error_modules=False, return JsonResponse(
static_content_store=contentstore(), {
target_location_namespace=location, 'ErrMsg': 'Unsafe tar file. Aborting import.',
draft_store=modulestore() 'SuspiciousFileOperationMsg': exc.args[0],
) 'Stage': 1
},
status=400
)
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': 2
},
status=415
)
logging.debug('found course.xml at {0}'.format(dirpath))
if dirpath != course_dir:
for fname in os.listdir(dirpath):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_location_namespace=old_location,
draft_store=modulestore()
)
logging.debug('new course at {0}'.format(course_items[0].location)) logging.debug('new course at {0}'.format(course_items[0].location))
session_status[key] = 3 session_status[key] = 3
request.session.modified = True request.session.modified = True
create_all_course_groups(request.user, course_items[0].location) create_all_course_groups(request.user, course_items[0].location)
logging.debug('created all course groups at {0}'.format(course_items[0].location)) logging.debug('created all course groups at {0}'.format(course_items[0].location))
# Send errors to client with stage at which error occured. # Send errors to client with stage at which error occured.
except Exception as exception: # pylint: disable=W0703 except Exception as exception: # pylint: disable=W0703
return JsonResponse( return JsonResponse(
{ {
'ErrMsg': str(exception), 'ErrMsg': str(exception),
'Stage': session_status[key] 'Stage': session_status[key]
}, },
status=400 status=400
) )
finally: finally:
shutil.rmtree(course_dir) shutil.rmtree(course_dir)
return JsonResponse({'Status': 'OK'}) return JsonResponse({'Status': 'OK'})
else: elif request.method == 'GET': # assume html
course_module = modulestore().get_item(location) course_module = modulestore().get_item(old_location)
new_location = loc_mapper().translate_location(course_module.location.course_id, course_module.location, False, True)
return render_to_response('import.html', { return render_to_response('import.html', {
'context_course': course_module, 'context_course': course_module,
'successful_import_redirect_url': new_location.url_reverse("course/", "") 'successful_import_redirect_url': location.url_reverse("course/", ""),
'import_status_url': location.url_reverse("import_status/", "fillerName"),
}) })
else:
return HttpResponseNotFound()
@require_GET @require_GET
@ensure_csrf_cookie @ensure_csrf_cookie
@login_required @login_required
def import_status(request, org, course, name): def import_status_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None, filename=None):
""" """
Returns an integer corresponding to the status of a file import. These are: Returns an integer corresponding to the status of a file import. These are:
...@@ -260,10 +272,13 @@ def import_status(request, org, course, name): ...@@ -260,10 +272,13 @@ def import_status(request, org, course, name):
3 : Importing to mongo 3 : Importing to mongo
""" """
location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block)
if not has_access(request.user, location):
raise PermissionDenied()
try: try:
session_status = request.session["import_status"] session_status = request.session["import_status"]
status = session_status[org + course + name] status = session_status[location.course_id + filename]
except KeyError: except KeyError:
status = 0 status = 0
......
<%inherit file="base.html" /> <%inherit file="base.html" />
<%namespace name='static' file='static_content.html'/> <%namespace name='static' file='static_content.html'/>
<%! <%!
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
%> %>
<%block name="title">${_("Course Import")}</%block> <%block name="title">${_("Course Import")}</%block>
...@@ -28,8 +27,7 @@ ...@@ -28,8 +27,7 @@
<p>${_("During the initial stages of the import process, please do not navigate away from this page.")}</p> <p>${_("During the initial stages of the import process, please do not navigate away from this page.")}</p>
</div> </div>
<form id="fileupload" method="post" enctype="multipart/form-data" <form id="fileupload" method="post" enctype="multipart/form-data" class="import-form">
class="import-form" url="${reverse('import_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}">
<input type="hidden" name="csrfmiddlewaretoken" value="${csrf_token}" /> <input type="hidden" name="csrfmiddlewaretoken" value="${csrf_token}" />
...@@ -158,7 +156,7 @@ var chooseBtn = $('.choose-file-button'); ...@@ -158,7 +156,7 @@ var chooseBtn = $('.choose-file-button');
var allStats = $('#status-infos'); var allStats = $('#status-infos');
var feedbackUrl = "${reverse('import_status', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name='fillerName'))}" var feedbackUrl = "${import_status_url}";
var defaults = [ var defaults = [
'${_("There was an error during the upload process.")}\n', '${_("There was an error during the upload process.")}\n',
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
checklists_url = location.url_reverse('checklists/', '') checklists_url = location.url_reverse('checklists/', '')
course_team_url = location.url_reverse('course_team/', '') course_team_url = location.url_reverse('course_team/', '')
assets_url = location.url_reverse('assets/', '') assets_url = location.url_reverse('assets/', '')
import_url = location.url_reverse('import/', '')
%> %>
<h2 class="info-course"> <h2 class="info-course">
<span class="sr">${_("Current Course:")}</span> <span class="sr">${_("Current Course:")}</span>
...@@ -91,7 +92,7 @@ ...@@ -91,7 +92,7 @@
<a href="${checklists_url}">${_("Checklists")}</a> <a href="${checklists_url}">${_("Checklists")}</a>
</li> </li>
<li class="nav-item nav-course-tools-import"> <li class="nav-item nav-course-tools-import">
<a href="${reverse('import_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}">${_("Import")}</a> <a href="${import_url}">${_("Import")}</a>
</li> </li>
<li class="nav-item nav-course-tools-export"> <li class="nav-item nav-course-tools-export">
<a href="${reverse('export_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}">${_("Export")}</a> <a href="${reverse('export_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}">${_("Export")}</a>
......
import re
from django.conf import settings from django.conf import settings
from django.conf.urls import patterns, include, url from django.conf.urls import patterns, include, url
...@@ -34,11 +33,6 @@ urlpatterns = patterns('', # nopep8 ...@@ -34,11 +33,6 @@ urlpatterns = patterns('', # nopep8
url(r'^create_new_course', 'contentstore.views.create_new_course', name='create_new_course'), url(r'^create_new_course', 'contentstore.views.create_new_course', name='create_new_course'),
url(r'^reorder_static_tabs', 'contentstore.views.reorder_static_tabs', name='reorder_static_tabs'), url(r'^reorder_static_tabs', 'contentstore.views.reorder_static_tabs', name='reorder_static_tabs'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import/(?P<name>[^/]+)$',
'contentstore.views.import_course', name='import_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import_status/(?P<name>[^/]+)$',
'contentstore.views.import_status', name='import_status'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/export/(?P<name>[^/]+)$', url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/export/(?P<name>[^/]+)$',
'contentstore.views.export_course', name='export_course'), 'contentstore.views.export_course', name='export_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/generate_export/(?P<name>[^/]+)$', url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/generate_export/(?P<name>[^/]+)$',
...@@ -129,7 +123,9 @@ urlpatterns += patterns( ...@@ -129,7 +123,9 @@ urlpatterns += patterns(
url(r'(?ix)^checklists/{}(/)?(?P<checklist_index>\d+)?$'.format(parsers.URL_RE_SOURCE), 'checklists_handler'), url(r'(?ix)^checklists/{}(/)?(?P<checklist_index>\d+)?$'.format(parsers.URL_RE_SOURCE), 'checklists_handler'),
url(r'(?ix)^course_team/{}(/)?(?P<email>.+)?$'.format(parsers.URL_RE_SOURCE), 'course_team_handler'), url(r'(?ix)^course_team/{}(/)?(?P<email>.+)?$'.format(parsers.URL_RE_SOURCE), 'course_team_handler'),
url(r'(?ix)^orphan/{}$'.format(parsers.URL_RE_SOURCE), 'orphan'), url(r'(?ix)^orphan/{}$'.format(parsers.URL_RE_SOURCE), 'orphan'),
url(r'(?ix)^assets/{}(/)?(?P<asset_id>.+)?$'.format(parsers.URL_RE_SOURCE), 'assets_handler') url(r'(?ix)^assets/{}(/)?(?P<asset_id>.+)?$'.format(parsers.URL_RE_SOURCE), 'assets_handler'),
url(r'(?ix)^import/{}$'.format(parsers.URL_RE_SOURCE), 'import_handler'),
url(r'(?ix)^import_status/{}/(?P<filename>.+)$'.format(parsers.URL_RE_SOURCE), 'import_status_handler'),
) )
js_info_dict = { js_info_dict = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment