Skip to content

E
edx-platform
Commit d991595e by Julian Arni
Options

Split import-export into new file

parent 993b92bc
Showing with 87 additions and 319 deletions
cms/djangoapps/contentstore/tests/test_assets.py
......@@ -75,78 +75,6 @@ class UploadTestCase(CourseTestCase):
resp = self.client.get(self.url)
self.assertEquals(resp.status_code, 405)
class ImportTestCase(CourseTestCase):
"""
Unit tests for importing a course
"""
def setUp(self):
super(ImportTestCase, self).setUp()
self.url = reverse("import_course", kwargs={
'org': self.course.location.org,
'course': self.course.location.course,
'name': self.course.location.name,
})
self.content_dir = tempfile.mkdtemp()
def touch(name):
""" Equivalent to shell's 'touch'"""
with file(name, 'a'):
os.utime(name, None)
# Create tar test files
good_dir = tempfile.mkdtemp(dir=self.content_dir)
os.makedirs(os.path.join(good_dir, "course"))
with open(os.path.join(good_dir, "course.xml") , "w+") as f:
f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')
with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
f.write('<course></course>')
self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
with tarfile.open(self.good_tar, "w:gz") as gtar:
gtar.add(good_dir)
bad_dir = tempfile.mkdtemp(dir=self.content_dir)
touch(os.path.join(bad_dir, "bad.xml"))
self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
with tarfile.open(self.bad_tar, "w:gz") as btar:
btar.add(bad_dir)
def tearDown(self):
shutil.rmtree(self.content_dir)
def test_no_coursexml(self):
"""
Check that the response for a tar.gz import without a course.xml is
correct.
"""
with open(self.bad_tar) as btar:
resp = self.client.post(
self.url,
{
"name": self.bad_tar,
"course-data": [btar]
})
self.assertEquals(resp.status_code, 415)
def test_with_coursexml(self):
"""
Check that the response for a tar.gz import with a course.xml is
correct.
"""
with open(self.good_tar) as gtar:
resp = self.client.post(
self.url,
{
"name": self.good_tar,
"course-data": [gtar]
})
self.assert2XX(resp.status_code)
class AssetsToJsonTestCase(TestCase):
"""
......
cms/djangoapps/contentstore/tests/test_import_export.py 0 → 100644
"""
Unit tests for course import and export
"""
import os
import shutil
import tarfile
import tempfile
from .utils import CourseTestCase
from django.core.urlresolvers import reverse
from xmodule.modulestore import Location
from contentstore.views import import_export
class ImportTestCase(CourseTestCase):
"""
Unit tests for importing a course
"""
def setUp(self):
super(ImportTestCase, self).setUp()
self.url = reverse("import_course", kwargs={
'org': self.course.location.org,
'course': self.course.location.course,
'name': self.course.location.name,
})
self.content_dir = tempfile.mkdtemp()
def touch(name):
""" Equivalent to shell's 'touch'"""
with file(name, 'a'):
os.utime(name, None)
# Create tar test files -----------------------------------------------
# OK course:
good_dir = tempfile.mkdtemp(dir=self.content_dir)
os.makedirs(os.path.join(good_dir, "course"))
with open(os.path.join(good_dir, "course.xml") , "w+") as f:
f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')
with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
f.write('<course></course>')
self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
with tarfile.open(self.good_tar, "w:gz") as gtar:
gtar.add(good_dir)
# Bad course (no 'course.xml' file):
bad_dir = tempfile.mkdtemp(dir=self.content_dir)
touch(os.path.join(bad_dir, "bad.xml"))
self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
with tarfile.open(self.bad_tar, "w:gz") as btar:
btar.add(bad_dir)
def tearDown(self):
shutil.rmtree(self.content_dir)
def test_no_coursexml(self):
"""
Check that the response for a tar.gz import without a course.xml is
correct.
"""
with open(self.bad_tar) as btar:
resp = self.client.post(
self.url,
{
"name": self.bad_tar,
"course-data": [btar]
})
self.assertEquals(resp.status_code, 415)
def test_with_coursexml(self):
"""
Check that the response for a tar.gz import with a course.xml is
correct.
"""
with open(self.good_tar) as gtar:
resp = self.client.post(
self.url,
{
"name": self.good_tar,
"course-data": [gtar]
})
self.assert2XX(resp.status_code)
cms/djangoapps/contentstore/views/__init__.py
......@@ -10,6 +10,7 @@ from .component import *
from .course import *
from .error import *
from .item import *
from .import_export import *
from .preview import *
from .public import *
from .user import *
......
cms/djangoapps/contentstore/views/assets.py
......@@ -36,8 +36,7 @@ from .access import get_location_and_verify_access
from util.json_request import JsonResponse
__all__ = ['asset_index', 'upload_asset', 'import_course',
'generate_export_course', 'export_course']
__all__ = ['asset_index', 'upload_asset']
MAX_UP_LENGTH = 20000352 # Max chunk size
......@@ -265,248 +264,3 @@ def remove_asset(request, org, course, name):
return HttpResponse()
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
@login_required
def import_course(request, org, course, name):
"""
This method will handle a POST request to upload and import a .tar.gz file
into a specified course
"""
location = get_location_and_verify_access(request, org, course, name)
if request.method == 'POST':
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(org, course, name)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{ 'ErrMsg': 'We only support uploading a .tar.gz file.' },
status=415
)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk - no Content-Range header
content_range = {'start': 0, 'stop': 9, 'end': 10}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size != int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
size,
content_range['start']
)
return JsonResponse(
{ 'ErrMsg': 'File upload corrupted. Please try again' },
status=409
)
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse('import_course', kwargs={
'org': location.org,
'course': location.course,
'name': location.name
}),
"thumbnailUrl": ""
}]
})
else: #This was the last chunk.
# 'Lock' with status info.
lock_filepath = data_root / (filename + ".lock")
with open(lock_filepath, 'w+') as lf:
lf.write("Extracting")
tar_file = tarfile.open(temp_filepath)
tar_file.extractall(course_dir + '/')
with open(lock_filepath, 'w+') as lf:
lf.write("Verifying")
# find the 'course.xml' file
dirpath = None
coursexmls = ((d, f) for d, _, f in os.walk(course_dir)
if f.count('course.xml') > 0)
try:
(dirpath, fname) = coursexmls.next()
except StopIteration:
return JsonResponse(
{'ErrMsg': 'Could not find the course.xml file in the package.' },
status=415
)
logging.debug('found course.xml at {0}'.format(dirpath))
if dirpath != course_dir:
for fname in os.listdir(dirpath):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_location_namespace=location,
draft_store=modulestore()
)
# we can blow this away when we're done importing.
shutil.rmtree(course_dir)
logging.debug('new course at {0}'.format(course_items[0].location))
with open(lock_filepath, 'w') as lf:
lf.write("Updating course")
create_all_course_groups(request.user, course_items[0].location)
logging.debug('created all course groups at {0}'.format(course_items[0].location))
os.remove(lock_filepath)
return JsonResponse({'Status': 'OK'})
else:
course_module = modulestore().get_item(location)
return render_to_response('import.html', {
'context_course': course_module,
'successful_import_redirect_url': reverse('course_index', kwargs={
'org': location.org,
'course': location.course,
'name': location.name,
})
})
@ensure_csrf_cookie
@login_required
def generate_export_course(request, org, course, name):
"""
This method will serialize out a course to a .tar.gz file which contains a
XML-based representation of the course
"""
location = get_location_and_verify_access(request, org, course, name)
course_module = modulestore().get_instance(location.course_id, location)
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
except SerializationError, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
return render_to_response('export.html', {
'context_course': course_module,
'successful_import_redirect_url': '',
'in_err': True,
'raw_err_msg': str(e),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse('edit_unit', kwargs={
'location': parent.location
}) if parent else '',
'course_home_url': reverse('course_index', kwargs={
'org': org,
'course': course,
'name': name
})
})
except Exception, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
return render_to_response('export.html', {
'context_course': course_module,
'successful_import_redirect_url': '',
'in_err': True,
'unit': None,
'raw_err_msg': str(e),
'course_home_url': reverse('course_index', kwargs={
'org': org,
'course': course,
'name': name
})
})
logging.debug('tar file being generated at {0}'.format(export_file.name))
tar_file = tarfile.open(name=export_file.name, mode='w:gz')
tar_file.add(root_dir / name, arcname=name)
tar_file.close()
# remove temp dir
shutil.rmtree(root_dir / name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
@ensure_csrf_cookie
@login_required
def export_course(request, org, course, name):
"""
This method serves up the 'Export Course' page
"""
location = get_location_and_verify_access(request, org, course, name)
course_module = modulestore().get_item(location)
return render_to_response('export.html', {
'context_course': course_module,
'successful_import_redirect_url': ''
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment