Skip to content

E
edx-platform
Commit f6434ed4 by Don Mitchell
Options

Merge pull request #1840 from MITx/fix/cdodge/export-draft-modules 

Fix/cdodge/export draft modules
parents 4d9dd402 0a293731
Showing with 202 additions and 54 deletions
cms/djangoapps/contentstore/tests/test_contentstore.py
......@@ -11,6 +11,7 @@ import json
from fs.osfs import OSFS
import copy
from json import loads
import traceback
from django.contrib.auth.models import User
from contentstore.utils import get_modulestore
......@@ -215,13 +216,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
module_store = modulestore('direct')
found = False
item = None
items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None])
found = len(items) > 0
self.assertTrue(found)
# check that there's actually content in the 'question' field
self.assertGreater(len(items[0].question),0)
self.assertGreater(len(items[0].question), 0)
def test_xlint_fails(self):
err_cnt = perform_xlint('common/test/data', ['full'])
......@@ -234,7 +234,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None]))
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
# make sure the parent no longer points to the child object which was deleted
self.assertTrue(sequential.location.url() in chapter.children)
......@@ -252,7 +252,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
self.assertFalse(found)
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
# make sure the parent no longer points to the child object which was deleted
self.assertFalse(sequential.location.url() in chapter.children)
......@@ -275,7 +275,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
import_from_xml(modulestore(), 'common/test/data/', ['full'])
module_store = modulestore('direct')
content_store = contentstore()
source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
course = module_store.get_item(source_location)
......@@ -347,17 +346,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
def test_export_course(self):
module_store = modulestore('direct')
draft_store = modulestore('draft')
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
# get a vertical (and components in it) to put into 'draft'
vertical = module_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
draft_store.clone_item(vertical.location, vertical.location)
for child in vertical.get_children():
draft_store.clone_item(child.location, child.location)
root_dir = path(mkdtemp_clean())
# now create a private vertical
private_vertical = draft_store.clone_item(vertical.location,
Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None]))
# add private to list of children
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
private_location_no_draft = private_vertical.location._replace(revision=None)
module_store.update_children(sequential.location, sequential.children +
[private_location_no_draft.url()])
# read back the sequential, to make sure we have a pointer to
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
self.assertIn(private_location_no_draft.url(), sequential.children)
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, location, root_dir, 'test_export')
export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store)
# check for static tabs
self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html')
......@@ -391,20 +417,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
delete_course(module_store, content_store, location)
# reimport
import_from_xml(module_store, root_dir, ['test_export'])
import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store)
items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None]))
self.assertGreater(len(items), 0)
for descriptor in items:
# don't try to look at private verticals. Right now we're running
# the service in non-draft aware
if getattr(descriptor, 'is_draft', False):
print "Checking {0}....".format(descriptor.location.url())
resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
self.assertEqual(resp.status_code, 200)
# verify that we have the content in the draft store as well
vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
self.assertTrue(getattr(vertical, 'is_draft', False))
for child in vertical.get_children():
self.assertTrue(getattr(child, 'is_draft', False))
# verify that we have the private vertical
test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]))
self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
shutil.rmtree(root_dir)
def test_course_handouts_rewrites(self):
module_store = modulestore('direct')
content_store = contentstore()
# import a test course
import_from_xml(module_store, 'common/test/data/', ['full'])
......@@ -440,8 +482,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
'Administrivia_and_Circuit_Elements', None]) in course.system.module_data)
# make sure we don't have a specific vertical which should be at depth=3
self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58',
None]) in course.system.module_data)
self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', None])
in course.system.module_data)
def test_export_course_with_unknown_metadata(self):
module_store = modulestore('direct')
......@@ -468,10 +510,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
export_to_xml(module_store, content_store, location, root_dir, 'test_export')
exported = True
except Exception:
print 'Exception thrown: {0}'.format(traceback.format_exc())
pass
self.assertTrue(exported)
class ContentStoreTest(ModuleStoreTestCase):
"""
Tests for the CMS ContentStore application.
......
cms/djangoapps/contentstore/views.py
......@@ -1586,7 +1586,8 @@ def import_course(request, org, course, name):
shutil.move(r / fname, course_dir)
module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT,
[course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location))
[course_subdir], load_error_modules=False, static_content_store=contentstore(),
target_location_namespace=Location(location), draft_store=modulestore())
# we can blow this away when we're done importing.
shutil.rmtree(course_dir)
......@@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name):
logging.debug('root = {0}'.format(root_dir))
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name)
# filename = root_dir / name + '.tar.gz'
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
#filename = root_dir / name + '.tar.gz'
logging.debug('tar file being generated at {0}'.format(export_file.name))
tf = tarfile.open(name=export_file.name, mode='w:gz')
......
common/lib/xmodule/xmodule/html_module.py
......@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
with system.resources_fs.open(filepath) as file:
html = file.read().decode('utf-8')
# Log a warning if we can't parse the file, but don't error
if not check_html(html):
msg = "Couldn't parse html in {0}.".format(filepath)
if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
log.warning(msg)
system.error_tracker("Warning: " + msg)
......@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.data.encode('utf-8'))
html_data = self.data.encode('utf-8')
file.write(html_data)
# write out the relative name
relname = path(pathname).basename()
......
common/lib/xmodule/xmodule/modulestore/draft.py
......@@ -3,7 +3,6 @@ from datetime import datetime
from . import ModuleStoreBase, Location, namedtuple_to_son
from .exceptions import ItemNotFoundError
from .inheritance import own_metadata
import logging
DRAFT = 'draft'
......@@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase):
"""
return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
def update_item(self, location, data):
def update_item(self, location, data, allow_not_found=False):
"""
Set the data in the item specified by the location to
data
......@@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase):
data: A nested dictionary of problem data
"""
draft_loc = as_draft(location)
try:
draft_item = self.get_item(location)
if not getattr(draft_item, 'is_draft', False):
self.clone_item(location, draft_loc)
except ItemNotFoundError, e:
if not allow_not_found:
raise e
return super(DraftModuleStore, self).update_item(draft_loc, data)
......@@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase):
"""
return super(DraftModuleStore, self).delete_item(as_draft(location))
def get_parent_locations(self, location, course_id):
'''Find all locations that are the parents of this location. Needed
for path_to_location().
......@@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase):
Save a current draft to the underlying modulestore
"""
draft = self.get_item(location)
draft.cms.published_date = datetime.utcnow()
draft.cms.published_by = published_by_id
super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data)
......
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
import logging
from xmodule.modulestore import Location
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.inheritance import own_metadata
from fs.osfs import OSFS
from json import dumps
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
course = modulestore.get_item(course_location)
......@@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir('drafts')
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''):
query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)
......
common/lib/xmodule/xmodule/modulestore/xml_importer.py
......@@ -6,11 +6,11 @@ from path import path
from xblock.core import Scope
from .xml import XMLModuleStore
from .exceptions import DuplicateItemError
from .xml import XMLModuleStore, ImportSystem, ParentTracker
from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
from xmodule.contentstore.content import StaticContent
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
log = logging.getLogger(__name__)
......@@ -139,8 +139,7 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path,
# Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's
# no good, so we have to do this kludge
if isinstance(module.data, str) or isinstance(module.data, unicode): # some module 'data' fields are non strings which blows up the link traversal code
lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path,
static_content_store, link, remap_dict))
lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
for key in remap_dict.keys():
module.data = module.data.replace(key, remap_dict[key])
......@@ -175,7 +174,8 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path,
def import_from_xml(store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None, target_location_namespace=None, verbose=False):
load_error_modules=True, static_content_store=None, target_location_namespace=None,
verbose=False, draft_store=None):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
......@@ -190,7 +190,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
"""
module_store = XMLModuleStore(
xml_module_store = XMLModuleStore(
data_dir,
default_class=default_class,
course_dirs=course_dirs,
......@@ -201,7 +201,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# to enumerate the entire collection of course modules. It will be left as a TBD to implement that
# method on XmlModuleStore.
course_items = []
for course_id in module_store.modules.keys():
for course_id in xml_module_store.modules.keys():
if target_location_namespace is not None:
pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course])
......@@ -222,7 +222,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Quick scan to get course module as we need some info from there. Also we need to make sure that the
# course module is committed first into the store
for module in module_store.modules[course_id].itervalues():
for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course':
course_data_path = path(data_dir) / module.data_dir
course_location = module.location
......@@ -239,11 +239,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
{"type": "discussion", "name": "Discussion"},
{"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge
if hasattr(module, 'data'):
store.update_item(module.location, module.data)
store.update_children(module.location, module.children)
store.update_metadata(module.location, dict(own_metadata(module)))
import_module(module, store, course_data_path, static_content_store)
# a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg
# so let's make sure we import in case there are no other references to it in the modules
......@@ -251,17 +247,16 @@ def import_from_xml(store, data_dir, course_dirs=None,
course_items.append(module)
# then import all the static content
if static_content_store is not None:
_namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
# first pass to find everything in /static/
import_static_content(module_store.modules[course_id], course_location, course_data_path, static_content_store,
import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
_namespace_rename, subpath='static', verbose=verbose)
# finally loop through all the modules
for module in module_store.modules[course_id].itervalues():
for module in xml_module_store.modules[course_id].itervalues():
if module.category == 'course':
# we've already saved the course module up at the top of the loop
......@@ -275,6 +270,25 @@ def import_from_xml(store, data_dir, course_dirs=None,
if verbose:
log.debug('importing module location {0}'.format(module.location))
import_module(module, store, course_data_path, static_content_store)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(xml_module_store, draft_store, course_data_path,
static_content_store, target_location_namespace if target_location_namespace is not None
else course_location)
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
return xml_module_store, course_items
def import_module(module, store, course_data_path, static_content_store, allow_not_found=False):
content = {}
for field in module.fields:
if field.scope != Scope.content:
......@@ -285,6 +299,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Ignore any missing keys in _model_data
pass
module_data = {}
if 'data' in content:
module_data = content['data']
......@@ -301,8 +316,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
# Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's
# no good, so we have to do this kludge
if isinstance(module_data, str) or isinstance(module_data, unicode): # some module 'data' fields are non strings which blows up the link traversal code
lxml_rewrite_links(module_data,
lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
lxml_rewrite_links(module_data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict))
for key in remap_dict.keys():
module_data = module_data.replace(key, remap_dict[key])
......@@ -312,6 +326,9 @@ def import_from_xml(store, data_dir, course_dirs=None,
else:
module_data = content
if allow_not_found:
store.update_item(module.location, module_data, allow_not_found=allow_not_found)
else:
store.update_item(module.location, module_data)
if hasattr(module, 'children') and module.children != []:
......@@ -320,14 +337,82 @@ def import_from_xml(store, data_dir, course_dirs=None,
# NOTE: It's important to use own_metadata here to avoid writing
# inherited metadata everywhere.
store.update_metadata(module.location, dict(own_metadata(module)))
finally:
# turn back on all write signalling
if pseudo_course_id in store.ignore_write_events_on_courses:
store.ignore_write_events_on_courses.remove(pseudo_course_id)
store.refresh_cached_metadata_inheritance_tree(target_location_namespace if
target_location_namespace is not None else course_location)
return module_store, course_items
def import_course_draft(xml_module_store, store, course_data_path, static_content_store, target_location_namespace):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current application only verticals (and downwards)
can be in draft. Therefore, we need to use slightly different call points into the import process_xml
as we can't simply call XMLModuleStore() constructor (like we do for importing public content)
'''
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
system = ImportSystem(
xml_module_store,
target_location_namespace.course_id,
draft_dir,
{},
errorlog.tracker,
ParentTracker(),
None,
)
# now walk the /vertical directory where each file in there will be a draft copy of the Vertical
for dirname, dirnames, filenames in os.walk(draft_dir + "/vertical"):
for filename in filenames:
module_path = os.path.join(dirname, filename)
with open(module_path) as f:
try:
xml = f.read().decode('utf-8')
descriptor = system.process_xml(xml)
def _import_module(module):
module.location = module.location._replace(revision='draft')
# make sure our parent has us in its list of children
# this is to make sure private only verticals show up in the list of children since
# they would have been filtered out from the non-draft store export
if module.location.category == 'vertical':
module.location = module.location._replace(revision=None)
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
seq_location = Location(sequential_url)
# IMPORTANT: Be sure to update the sequential in the NEW namespace
seq_location = seq_location._replace(org=target_location_namespace.org,
course=target_location_namespace.course
)
sequential = store.get_item(seq_location)
if module.location.url() not in sequential.children:
sequential.children.insert(index, module.location.url())
store.update_children(sequential.location, sequential.children)
del module.xml_attributes['parent_sequential_url']
del module.xml_attributes['index_in_children_list']
import_module(module, store, course_data_path, static_content_store, allow_not_found=True)
for child in module.get_children():
_import_module(child)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the attributes (they are normally in the parent
# object, aka sequential), so we have to replace the location.name with the XML filename
# that is part of the pack
fn, fileExtension = os.path.splitext(filename)
descriptor.location = descriptor.location._replace(name=fn)
_import_module(descriptor)
except Exception, e:
logging.exception('There was an error. {0}'.format(unicode(e)))
pass
def remap_namespace(module, target_location_namespace):
if target_location_namespace is None:
......@@ -343,7 +428,7 @@ def remap_namespace(module, target_location_namespace):
course=target_location_namespace.course, name=target_location_namespace.name)
# then remap children pointers since they too will be re-namespaced
if hasattr(module,'children'):
if hasattr(module, 'children'):
children_locs = module.children
if children_locs is not None and children_locs != []:
new_locs = []
......@@ -365,7 +450,7 @@ def allowed_metadata_by_category(category):
'vertical': [],
'chapter': ['start'],
'sequential': ['due', 'format', 'start', 'graded']
}.get(category,['*'])
}.get(category, ['*'])
def check_module_metadata_editability(module):
......@@ -380,7 +465,6 @@ def check_module_metadata_editability(module):
allowed = allowed + ['xml_attributes', 'display_name']
err_cnt = 0
my_metadata = dict(own_metadata(module))
illegal_keys = set(own_metadata(module).keys()) - set(allowed)
if len(illegal_keys) > 0:
......@@ -497,7 +581,6 @@ def perform_xlint(data_dir, course_dirs,
print "WARN: Missing course marketing video. It is recommended that every course have a marketing video."
warn_cnt += 1
print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt)
if err_cnt > 0:
......
common/lib/xmodule/xmodule/xml_module.py
......@@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
'name', 'slug')
metadata_to_strip = ('data_dir',
# cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
'tabs', 'grading_policy', 'published_by', 'published_date',
'discussion_blackouts', 'testcenter_info',
# VS[compat] -- remove the below attrs once everything is in the CMS
'course', 'org', 'url_name', 'filename',
......@@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor):
'graded': bool_map,
'hide_progress_tab': bool_map,
'allow_anonymous': bool_map,
'allow_anonymous_to_peers': bool_map
'allow_anonymous_to_peers': bool_map,
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment