Skip to content

E
edx-platform
Commit 1789814c by Chris Dodge
Options

wip

parent 63085ec6
Showing with 100 additions and 35 deletions
cms/djangoapps/contentstore/views.py
......@@ -1580,7 +1580,8 @@ def import_course(request, org, course, name):
shutil.move(r / fname, course_dir)
module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT,
[course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location))
[course_subdir], load_error_modules=False, static_content_store=contentstore(),
target_location_namespace=Location(location), draft_store=modulestore())
# we can blow this away when we're done importing.
shutil.rmtree(course_dir)
......
common/lib/xmodule/xmodule/html_module.py
......@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
with system.resources_fs.open(filepath) as file:
html = file.read().decode('utf-8')
# Log a warning if we can't parse the file, but don't error
if not check_html(html):
msg = "Couldn't parse html in {0}.".format(filepath)
if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
log.warning(msg)
system.error_tracker("Warning: " + msg)
......@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.data.encode('utf-8'))
html_data = self.data.encode('utf-8')
file.write(html_data)
# write out the relative name
relname = path(pathname).basename()
......
common/lib/xmodule/xmodule/modulestore/draft.py
......@@ -106,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase):
"""
return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
def update_item(self, location, data):
def update_item(self, location, data, allow_not_found=False):
"""
Set the data in the item specified by the location to
data
......@@ -115,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase):
data: A nested dictionary of problem data
"""
draft_loc = as_draft(location)
try:
draft_item = self.get_item(location)
if not getattr(draft_item, 'is_draft', False):
self.clone_item(location, draft_loc)
except ItemNotFoundError, e:
if not allow_not_found:
raise e
return super(DraftModuleStore, self).update_item(draft_loc, data)
......
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
......@@ -41,22 +41,26 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
course_policy.write(dumps(policy))
# export everything from the draft store, unfortunately this will create lots of duplicates
'''
if draft_modulestore is not None:
draft_course = draft_modulestore.get_item(course_location)
draft_course_dir = export_fs.makeopendir('drafts')
xml = draft_course.export_to_xml(draft_course_dir)
with draft_course_dir.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
'''
draft_items = modulestore.get_items([None, None, None, 'vertical', None, 'draft'])
logging.debug('draft_items = {0}'.format(draft_items))
if len(draft_items) > 0:
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
draft_items = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
if len(draft_items)>0:
draft_course_dir = export_fs.makeopendir('drafts')
for draft_item in draft_items:
draft_item.export_to_xml(draft_items_dir)
#with draft_items_dir.open(draft_item.location.name + '.xml', 'w'):
'''
draft_item.export_to_xml(draft_course_dir)
def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''):
......
common/lib/xmodule/xmodule/modulestore/xml_importer.py
......@@ -6,11 +6,13 @@ from path import path
from xblock.core import Scope
from .xml import XMLModuleStore
from .xml import XMLModuleStore, ImportSystem, ParentTracker
from .exceptions import DuplicateItemError
from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
from collections import defaultdict
log = logging.getLogger(__name__)
......@@ -175,7 +177,8 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path,
def import_from_xml(store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None, target_location_namespace=None, verbose=False):
load_error_modules=True, static_content_store=None, target_location_namespace=None,
verbose=False, draft_store=None):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
......@@ -190,7 +193,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
"""
module_store = XMLModuleStore(
xml_module_store = XMLModuleStore(
data_dir,
default_class=default_class,
course_dirs=course_dirs,
......@@ -201,7 +204,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# to enumerate the entire collection of course modules. It will be left as a TBD to implement that
# method on XmlModuleStore.
course_items = []
for course_id in module_store.modules.keys():
for course_id in xml_module_store.modules.keys():
if target_location_namespace is not None:
pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course])
......@@ -222,7 +225,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Quick scan to get course module as we need some info from there. Also we need to make sure that the
# course module is committed first into the store
for module in module_store.modules[course_id].itervalues():
for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course':
course_data_path = path(data_dir) / module.data_dir
course_location = module.location
......@@ -239,11 +242,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
{"type": "discussion", "name": "Discussion"},
{"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge
if hasattr(module, 'data'):
store.update_item(module.location, module.data)
store.update_children(module.location, module.children)
store.update_metadata(module.location, dict(own_metadata(module)))
import_module(module, store, course_data_path, static_content_store)
# a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg
# so let's make sure we import in case there are no other references to it in the modules
......@@ -257,11 +256,11 @@ def import_from_xml(store, data_dir, course_dirs=None,
_namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
# first pass to find everything in /static/
import_static_content(module_store.modules[course_id], course_location, course_data_path, static_content_store,
import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
_namespace_rename, subpath='static', verbose=verbose)
# finally loop through all the modules
for module in module_store.modules[course_id].itervalues():
for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course':
# we've already saved the course module up at the top of the loop
......@@ -275,6 +274,22 @@ def import_from_xml(store, data_dir, course_dirs=None,
if verbose:
log.debug('importing module location {0}'.format(module.location))
import_module(module, store, course_data_path, static_content_store)
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(xml_module_store, draft_store, course_data_path, static_content_store, target_location_namespace)
return xml_module_store, course_items
def import_module(module, store, course_data_path, static_content_store, allow_not_found=False):
content = {}
for field in module.fields:
if field.scope != Scope.content:
......@@ -310,6 +325,9 @@ def import_from_xml(store, data_dir, course_dirs=None,
except Exception, e:
logging.exception("failed to rewrite links on {0}. Continuing...".format(module.location))
if allow_not_found:
store.update_item(module.location, content, allow_not_found=allow_not_found)
else:
store.update_item(module.location, content)
if hasattr(module, 'children') and module.children != []:
......@@ -318,20 +336,57 @@ def import_from_xml(store, data_dir, course_dirs=None,
# NOTE: It's important to use own_metadata here to avoid writing
# inherited metadata everywhere.
store.update_metadata(module.location, dict(own_metadata(module)))
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
# now import any 'draft' items
import_course_draft(store, course_data_path, target_location_namespace)
return module_store, course_items
def import_course_draft(xml_module_store, store, course_data_path, static_content_store, target_location_namespace):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current application only verticals (and downwards)
can be in draft. Therefore, we need to use slightly different call points into the import process_xml
as we can't simply call XMLModuleStore() constructor (like we do for importing public content)
'''
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
system = ImportSystem(
xml_module_store,
target_location_namespace.course_id,
draft_dir,
{},
errorlog.tracker,
ParentTracker(),
None,
)
# now walk the /vertical directory where each file in there will be a draft copy of the Vertical
for dirname, dirnames, filenames in os.walk(draft_dir + "/vertical"):
for filename in filenames:
module_path = os.path.join(dirname, filename)
with open(module_path) as f:
try:
xml = f.read().decode('utf-8')
descriptor = system.process_xml(xml)
def import_course_draft(store, course_data_path, target_location_namespace):
def _import_module(module):
module.location = module.location._replace(revision='draft')
import_module(module, store, course_data_path, static_content_store, allow_not_found=True)
for child in module.get_children():
_import_module(child)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the attributes (they are normally in the parent
# object, aka sequential), so we have to replace the location.name with the XML filename
# that is part of the pack
fn, fileExtension = os.path.splitext(filename)
descriptor.location = descriptor.location._replace(name=fn)
_import_module(descriptor)
except Exception, e:
logging.exception('There was an error. {0}'.format(unicode(e)))
pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment