Commit f6434ed4 by Don Mitchell

Merge pull request #1840 from MITx/fix/cdodge/export-draft-modules

Fix/cdodge/export draft modules
parents 4d9dd402 0a293731
...@@ -11,6 +11,7 @@ import json ...@@ -11,6 +11,7 @@ import json
from fs.osfs import OSFS from fs.osfs import OSFS
import copy import copy
from json import loads from json import loads
import traceback
from django.contrib.auth.models import User from django.contrib.auth.models import User
from contentstore.utils import get_modulestore from contentstore.utils import get_modulestore
...@@ -215,13 +216,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -215,13 +216,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
module_store = modulestore('direct') module_store = modulestore('direct')
found = False found = False
item = None
items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None]) items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None])
found = len(items) > 0 found = len(items) > 0
self.assertTrue(found) self.assertTrue(found)
# check that there's actually content in the 'question' field # check that there's actually content in the 'question' field
self.assertGreater(len(items[0].question),0) self.assertGreater(len(items[0].question), 0)
def test_xlint_fails(self): def test_xlint_fails(self):
err_cnt = perform_xlint('common/test/data', ['full']) err_cnt = perform_xlint('common/test/data', ['full'])
...@@ -234,7 +234,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -234,7 +234,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None])) sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None]))
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None])) chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
# make sure the parent no longer points to the child object which was deleted # make sure the parent no longer points to the child object which was deleted
self.assertTrue(sequential.location.url() in chapter.children) self.assertTrue(sequential.location.url() in chapter.children)
...@@ -252,7 +252,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -252,7 +252,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertFalse(found) self.assertFalse(found)
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None])) chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
# make sure the parent no longer points to the child object which was deleted # make sure the parent no longer points to the child object which was deleted
self.assertFalse(sequential.location.url() in chapter.children) self.assertFalse(sequential.location.url() in chapter.children)
...@@ -275,7 +275,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -275,7 +275,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
import_from_xml(modulestore(), 'common/test/data/', ['full']) import_from_xml(modulestore(), 'common/test/data/', ['full'])
module_store = modulestore('direct') module_store = modulestore('direct')
content_store = contentstore()
source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
course = module_store.get_item(source_location) course = module_store.get_item(source_location)
...@@ -347,17 +346,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -347,17 +346,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
def test_export_course(self): def test_export_course(self):
module_store = modulestore('direct') module_store = modulestore('direct')
draft_store = modulestore('draft')
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['full']) import_from_xml(module_store, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
# get a vertical (and components in it) to put into 'draft'
vertical = module_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
draft_store.clone_item(vertical.location, vertical.location)
for child in vertical.get_children():
draft_store.clone_item(child.location, child.location)
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
# now create a private vertical
private_vertical = draft_store.clone_item(vertical.location,
Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None]))
# add private to list of children
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
private_location_no_draft = private_vertical.location._replace(revision=None)
module_store.update_children(sequential.location, sequential.children +
[private_location_no_draft.url()])
# read back the sequential, to make sure we have a pointer to
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
self.assertIn(private_location_no_draft.url(), sequential.children)
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir # export out to a tempdir
export_to_xml(module_store, content_store, location, root_dir, 'test_export') export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store)
# check for static tabs # check for static tabs
self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html') self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html')
...@@ -391,20 +417,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -391,20 +417,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
delete_course(module_store, content_store, location) delete_course(module_store, content_store, location)
# reimport # reimport
import_from_xml(module_store, root_dir, ['test_export']) import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store)
items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None])) items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None]))
self.assertGreater(len(items), 0) self.assertGreater(len(items), 0)
for descriptor in items: for descriptor in items:
# don't try to look at private verticals. Right now we're running
# the service in non-draft aware
if getattr(descriptor, 'is_draft', False):
print "Checking {0}....".format(descriptor.location.url()) print "Checking {0}....".format(descriptor.location.url())
resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
# verify that we have the content in the draft store as well
vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
self.assertTrue(getattr(vertical, 'is_draft', False))
for child in vertical.get_children():
self.assertTrue(getattr(child, 'is_draft', False))
# verify that we have the private vertical
test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]))
self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
shutil.rmtree(root_dir) shutil.rmtree(root_dir)
def test_course_handouts_rewrites(self): def test_course_handouts_rewrites(self):
module_store = modulestore('direct') module_store = modulestore('direct')
content_store = contentstore()
# import a test course # import a test course
import_from_xml(module_store, 'common/test/data/', ['full']) import_from_xml(module_store, 'common/test/data/', ['full'])
...@@ -440,8 +482,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -440,8 +482,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
'Administrivia_and_Circuit_Elements', None]) in course.system.module_data) 'Administrivia_and_Circuit_Elements', None]) in course.system.module_data)
# make sure we don't have a specific vertical which should be at depth=3 # make sure we don't have a specific vertical which should be at depth=3
self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', None])
None]) in course.system.module_data) in course.system.module_data)
def test_export_course_with_unknown_metadata(self): def test_export_course_with_unknown_metadata(self):
module_store = modulestore('direct') module_store = modulestore('direct')
...@@ -468,10 +510,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -468,10 +510,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
export_to_xml(module_store, content_store, location, root_dir, 'test_export') export_to_xml(module_store, content_store, location, root_dir, 'test_export')
exported = True exported = True
except Exception: except Exception:
print 'Exception thrown: {0}'.format(traceback.format_exc())
pass pass
self.assertTrue(exported) self.assertTrue(exported)
class ContentStoreTest(ModuleStoreTestCase): class ContentStoreTest(ModuleStoreTestCase):
""" """
Tests for the CMS ContentStore application. Tests for the CMS ContentStore application.
......
...@@ -1586,7 +1586,8 @@ def import_course(request, org, course, name): ...@@ -1586,7 +1586,8 @@ def import_course(request, org, course, name):
shutil.move(r / fname, course_dir) shutil.move(r / fname, course_dir)
module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT, module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT,
[course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location)) [course_subdir], load_error_modules=False, static_content_store=contentstore(),
target_location_namespace=Location(location), draft_store=modulestore())
# we can blow this away when we're done importing. # we can blow this away when we're done importing.
shutil.rmtree(course_dir) shutil.rmtree(course_dir)
...@@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name): ...@@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name):
logging.debug('root = {0}'.format(root_dir)) logging.debug('root = {0}'.format(root_dir))
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name) export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
# filename = root_dir / name + '.tar.gz' #filename = root_dir / name + '.tar.gz'
logging.debug('tar file being generated at {0}'.format(export_file.name)) logging.debug('tar file being generated at {0}'.format(export_file.name))
tf = tarfile.open(name=export_file.name, mode='w:gz') tf = tarfile.open(name=export_file.name, mode='w:gz')
......
...@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor): ...@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
with system.resources_fs.open(filepath) as file: with system.resources_fs.open(filepath) as file:
html = file.read().decode('utf-8') html = file.read().decode('utf-8')
# Log a warning if we can't parse the file, but don't error # Log a warning if we can't parse the file, but don't error
if not check_html(html): if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}.".format(filepath) msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
log.warning(msg) log.warning(msg)
system.error_tracker("Warning: " + msg) system.error_tracker("Warning: " + msg)
...@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor): ...@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True) resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as file: with resource_fs.open(filepath, 'w') as file:
file.write(self.data.encode('utf-8')) html_data = self.data.encode('utf-8')
file.write(html_data)
# write out the relative name # write out the relative name
relname = path(pathname).basename() relname = path(pathname).basename()
......
...@@ -3,7 +3,6 @@ from datetime import datetime ...@@ -3,7 +3,6 @@ from datetime import datetime
from . import ModuleStoreBase, Location, namedtuple_to_son from . import ModuleStoreBase, Location, namedtuple_to_son
from .exceptions import ItemNotFoundError from .exceptions import ItemNotFoundError
from .inheritance import own_metadata from .inheritance import own_metadata
import logging
DRAFT = 'draft' DRAFT = 'draft'
...@@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase): ...@@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase):
""" """
return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location))) return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
def update_item(self, location, data): def update_item(self, location, data, allow_not_found=False):
""" """
Set the data in the item specified by the location to Set the data in the item specified by the location to
data data
...@@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase): ...@@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase):
data: A nested dictionary of problem data data: A nested dictionary of problem data
""" """
draft_loc = as_draft(location) draft_loc = as_draft(location)
try:
draft_item = self.get_item(location) draft_item = self.get_item(location)
if not getattr(draft_item, 'is_draft', False): if not getattr(draft_item, 'is_draft', False):
self.clone_item(location, draft_loc) self.clone_item(location, draft_loc)
except ItemNotFoundError, e:
if not allow_not_found:
raise e
return super(DraftModuleStore, self).update_item(draft_loc, data) return super(DraftModuleStore, self).update_item(draft_loc, data)
...@@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase): ...@@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase):
""" """
return super(DraftModuleStore, self).delete_item(as_draft(location)) return super(DraftModuleStore, self).delete_item(as_draft(location))
def get_parent_locations(self, location, course_id): def get_parent_locations(self, location, course_id):
'''Find all locations that are the parents of this location. Needed '''Find all locations that are the parents of this location. Needed
for path_to_location(). for path_to_location().
...@@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase): ...@@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase):
Save a current draft to the underlying modulestore Save a current draft to the underlying modulestore
""" """
draft = self.get_item(location) draft = self.get_item(location)
draft.cms.published_date = datetime.utcnow() draft.cms.published_date = datetime.utcnow()
draft.cms.published_by = published_by_id draft.cms.published_by = published_by_id
super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data) super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data)
......
import logging import logging
from xmodule.modulestore import Location from xmodule.modulestore import Location
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.inheritance import own_metadata from xmodule.modulestore.inheritance import own_metadata
from fs.osfs import OSFS from fs.osfs import OSFS
from json import dumps from json import dumps
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir): def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
course = modulestore.get_item(course_location) course = modulestore.get_item(course_location)
...@@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d ...@@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
policy = {'course/' + course.location.name: own_metadata(course)} policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy)) course_policy.write(dumps(policy))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir('drafts')
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''): def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''):
query_loc = Location('i4x', course_location.org, course_location.course, category_type, None) query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)
......
...@@ -6,11 +6,11 @@ from path import path ...@@ -6,11 +6,11 @@ from path import path
from xblock.core import Scope from xblock.core import Scope
from .xml import XMLModuleStore from .xml import XMLModuleStore, ImportSystem, ParentTracker
from .exceptions import DuplicateItemError
from xmodule.modulestore import Location from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX from xmodule.contentstore.content import StaticContent
from .inheritance import own_metadata from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -139,8 +139,7 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path, ...@@ -139,8 +139,7 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path,
# Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's
# no good, so we have to do this kludge # no good, so we have to do this kludge
if isinstance(module.data, str) or isinstance(module.data, unicode): # some module 'data' fields are non strings which blows up the link traversal code if isinstance(module.data, str) or isinstance(module.data, unicode): # some module 'data' fields are non strings which blows up the link traversal code
lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path, lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
static_content_store, link, remap_dict))
for key in remap_dict.keys(): for key in remap_dict.keys():
module.data = module.data.replace(key, remap_dict[key]) module.data = module.data.replace(key, remap_dict[key])
...@@ -175,7 +174,8 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path, ...@@ -175,7 +174,8 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path,
def import_from_xml(store, data_dir, course_dirs=None, def import_from_xml(store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor', default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None, target_location_namespace=None, verbose=False): load_error_modules=True, static_content_store=None, target_location_namespace=None,
verbose=False, draft_store=None):
""" """
Import the specified xml data_dir into the "store" modulestore, Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course. using org and course as the location org and course.
...@@ -190,7 +190,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -190,7 +190,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
""" """
module_store = XMLModuleStore( xml_module_store = XMLModuleStore(
data_dir, data_dir,
default_class=default_class, default_class=default_class,
course_dirs=course_dirs, course_dirs=course_dirs,
...@@ -201,7 +201,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -201,7 +201,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# to enumerate the entire collection of course modules. It will be left as a TBD to implement that # to enumerate the entire collection of course modules. It will be left as a TBD to implement that
# method on XmlModuleStore. # method on XmlModuleStore.
course_items = [] course_items = []
for course_id in module_store.modules.keys(): for course_id in xml_module_store.modules.keys():
if target_location_namespace is not None: if target_location_namespace is not None:
pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course]) pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course])
...@@ -222,7 +222,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -222,7 +222,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Quick scan to get course module as we need some info from there. Also we need to make sure that the # Quick scan to get course module as we need some info from there. Also we need to make sure that the
# course module is committed first into the store # course module is committed first into the store
for module in module_store.modules[course_id].itervalues(): for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course': if module.category == 'course':
course_data_path = path(data_dir) / module.data_dir course_data_path = path(data_dir) / module.data_dir
course_location = module.location course_location = module.location
...@@ -239,11 +239,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -239,11 +239,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
{"type": "discussion", "name": "Discussion"}, {"type": "discussion", "name": "Discussion"},
{"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge
import_module(module, store, course_data_path, static_content_store)
if hasattr(module, 'data'):
store.update_item(module.location, module.data)
store.update_children(module.location, module.children)
store.update_metadata(module.location, dict(own_metadata(module)))
# a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg # a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg
# so let's make sure we import in case there are no other references to it in the modules # so let's make sure we import in case there are no other references to it in the modules
...@@ -251,17 +247,16 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -251,17 +247,16 @@ def import_from_xml(store, data_dir, course_dirs=None,
course_items.append(module) course_items.append(module)
# then import all the static content # then import all the static content
if static_content_store is not None: if static_content_store is not None:
_namespace_rename = target_location_namespace if target_location_namespace is not None else course_location _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
# first pass to find everything in /static/ # first pass to find everything in /static/
import_static_content(module_store.modules[course_id], course_location, course_data_path, static_content_store, import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
_namespace_rename, subpath='static', verbose=verbose) _namespace_rename, subpath='static', verbose=verbose)
# finally loop through all the modules # finally loop through all the modules
for module in module_store.modules[course_id].itervalues(): for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course': if module.category == 'course':
# we've already saved the course module up at the top of the loop # we've already saved the course module up at the top of the loop
...@@ -275,6 +270,25 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -275,6 +270,25 @@ def import_from_xml(store, data_dir, course_dirs=None,
if verbose: if verbose:
log.debug('importing module location {0}'.format(module.location)) log.debug('importing module location {0}'.format(module.location))
import_module(module, store, course_data_path, static_content_store)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(xml_module_store, draft_store, course_data_path,
static_content_store, target_location_namespace if target_location_namespace is not None
else course_location)
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
return xml_module_store, course_items
def import_module(module, store, course_data_path, static_content_store, allow_not_found=False):
content = {} content = {}
for field in module.fields: for field in module.fields:
if field.scope != Scope.content: if field.scope != Scope.content:
...@@ -285,6 +299,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -285,6 +299,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Ignore any missing keys in _model_data # Ignore any missing keys in _model_data
pass pass
module_data = {}
if 'data' in content: if 'data' in content:
module_data = content['data'] module_data = content['data']
...@@ -301,8 +316,7 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -301,8 +316,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's
# no good, so we have to do this kludge # no good, so we have to do this kludge
if isinstance(module_data, str) or isinstance(module_data, unicode): # some module 'data' fields are non strings which blows up the link traversal code if isinstance(module_data, str) or isinstance(module_data, unicode): # some module 'data' fields are non strings which blows up the link traversal code
lxml_rewrite_links(module_data, lxml_rewrite_links(module_data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
for key in remap_dict.keys(): for key in remap_dict.keys():
module_data = module_data.replace(key, remap_dict[key]) module_data = module_data.replace(key, remap_dict[key])
...@@ -312,6 +326,9 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -312,6 +326,9 @@ def import_from_xml(store, data_dir, course_dirs=None,
else: else:
module_data = content module_data = content
if allow_not_found:
store.update_item(module.location, module_data, allow_not_found=allow_not_found)
else:
store.update_item(module.location, module_data) store.update_item(module.location, module_data)
if hasattr(module, 'children') and module.children != []: if hasattr(module, 'children') and module.children != []:
...@@ -320,14 +337,82 @@ def import_from_xml(store, data_dir, course_dirs=None, ...@@ -320,14 +337,82 @@ def import_from_xml(store, data_dir, course_dirs=None,
# NOTE: It's important to use own_metadata here to avoid writing # NOTE: It's important to use own_metadata here to avoid writing
# inherited metadata everywhere. # inherited metadata everywhere.
store.update_metadata(module.location, dict(own_metadata(module))) store.update_metadata(module.location, dict(own_metadata(module)))
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
return module_store, course_items
def import_course_draft(xml_module_store, store, course_data_path, static_content_store, target_location_namespace):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current application only verticals (and downwards)
can be in draft. Therefore, we need to use slightly different call points into the import process_xml
as we can't simply call XMLModuleStore() constructor (like we do for importing public content)
'''
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
system = ImportSystem(
xml_module_store,
target_location_namespace.course_id,
draft_dir,
{},
errorlog.tracker,
ParentTracker(),
None,
)
# now walk the /vertical directory where each file in there will be a draft copy of the Vertical
for dirname, dirnames, filenames in os.walk(draft_dir + "/vertical"):
for filename in filenames:
module_path = os.path.join(dirname, filename)
with open(module_path) as f:
try:
xml = f.read().decode('utf-8')
descriptor = system.process_xml(xml)
def _import_module(module):
module.location = module.location._replace(revision='draft')
# make sure our parent has us in its list of children
# this is to make sure private only verticals show up in the list of children since
# they would have been filtered out from the non-draft store export
if module.location.category == 'vertical':
module.location = module.location._replace(revision=None)
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
seq_location = Location(sequential_url)
# IMPORTANT: Be sure to update the sequential in the NEW namespace
seq_location = seq_location._replace(org=target_location_namespace.org,
course=target_location_namespace.course
)
sequential = store.get_item(seq_location)
if module.location.url() not in sequential.children:
sequential.children.insert(index, module.location.url())
store.update_children(sequential.location, sequential.children)
del module.xml_attributes['parent_sequential_url']
del module.xml_attributes['index_in_children_list']
import_module(module, store, course_data_path, static_content_store, allow_not_found=True)
for child in module.get_children():
_import_module(child)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the attributes (they are normally in the parent
# object, aka sequential), so we have to replace the location.name with the XML filename
# that is part of the pack
fn, fileExtension = os.path.splitext(filename)
descriptor.location = descriptor.location._replace(name=fn)
_import_module(descriptor)
except Exception, e:
logging.exception('There was an error. {0}'.format(unicode(e)))
pass
def remap_namespace(module, target_location_namespace): def remap_namespace(module, target_location_namespace):
if target_location_namespace is None: if target_location_namespace is None:
...@@ -343,7 +428,7 @@ def remap_namespace(module, target_location_namespace): ...@@ -343,7 +428,7 @@ def remap_namespace(module, target_location_namespace):
course=target_location_namespace.course, name=target_location_namespace.name) course=target_location_namespace.course, name=target_location_namespace.name)
# then remap children pointers since they too will be re-namespaced # then remap children pointers since they too will be re-namespaced
if hasattr(module,'children'): if hasattr(module, 'children'):
children_locs = module.children children_locs = module.children
if children_locs is not None and children_locs != []: if children_locs is not None and children_locs != []:
new_locs = [] new_locs = []
...@@ -365,7 +450,7 @@ def allowed_metadata_by_category(category): ...@@ -365,7 +450,7 @@ def allowed_metadata_by_category(category):
'vertical': [], 'vertical': [],
'chapter': ['start'], 'chapter': ['start'],
'sequential': ['due', 'format', 'start', 'graded'] 'sequential': ['due', 'format', 'start', 'graded']
}.get(category,['*']) }.get(category, ['*'])
def check_module_metadata_editability(module): def check_module_metadata_editability(module):
...@@ -380,7 +465,6 @@ def check_module_metadata_editability(module): ...@@ -380,7 +465,6 @@ def check_module_metadata_editability(module):
allowed = allowed + ['xml_attributes', 'display_name'] allowed = allowed + ['xml_attributes', 'display_name']
err_cnt = 0 err_cnt = 0
my_metadata = dict(own_metadata(module))
illegal_keys = set(own_metadata(module).keys()) - set(allowed) illegal_keys = set(own_metadata(module).keys()) - set(allowed)
if len(illegal_keys) > 0: if len(illegal_keys) > 0:
...@@ -497,7 +581,6 @@ def perform_xlint(data_dir, course_dirs, ...@@ -497,7 +581,6 @@ def perform_xlint(data_dir, course_dirs,
print "WARN: Missing course marketing video. It is recommended that every course have a marketing video." print "WARN: Missing course marketing video. It is recommended that every course have a marketing video."
warn_cnt += 1 warn_cnt += 1
print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt) print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt)
if err_cnt > 0: if err_cnt > 0:
......
...@@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor): ...@@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
'name', 'slug') 'name', 'slug')
metadata_to_strip = ('data_dir', metadata_to_strip = ('data_dir',
# cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course 'tabs', 'grading_policy', 'published_by', 'published_date',
'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
'discussion_blackouts', 'testcenter_info', 'discussion_blackouts', 'testcenter_info',
# VS[compat] -- remove the below attrs once everything is in the CMS # VS[compat] -- remove the below attrs once everything is in the CMS
'course', 'org', 'url_name', 'filename', 'course', 'org', 'url_name', 'filename',
...@@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor): ...@@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor):
'graded': bool_map, 'graded': bool_map,
'hide_progress_tab': bool_map, 'hide_progress_tab': bool_map,
'allow_anonymous': bool_map, 'allow_anonymous': bool_map,
'allow_anonymous_to_peers': bool_map 'allow_anonymous_to_peers': bool_map,
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment