Skip to content

E
edx-platform
Commit e9a00ffc by Calen Pennington
Options

Parse XModuleDescriptors on import using from_xml 

Also:
Render all XModuleDescriptors in the cms the same way
Default them to editing raw xml, if there is no specific module for them
parent 5b4c501a
Showing with 245 additions and 205 deletions
cms/djangoapps/contentstore/management/commands/import.py
......@@ -2,175 +2,33 @@
### One-off script for importing courseware form XML format
###
#import mitxmako.middleware
#from courseware import content_parser
#from django.contrib.auth.models import User
import os.path
from StringIO import StringIO
from mako.template import Template
from mako.lookup import TemplateLookup
from collections import defaultdict
from django.core.management.base import BaseCommand
from keystore.django import keystore
from raw_module import RawDescriptor
from path import path
from x_module import XModuleDescriptor, DescriptorSystem
from lxml import etree
class Command(BaseCommand):
help = \
''' Run FTP server.'''
def handle(self, *args, **options):
print args
data_dir = args[0]
parser = etree.XMLParser(remove_comments = True)
lookup = TemplateLookup(directories=[data_dir])
template = lookup.get_template("course.xml")
course_string = template.render(groups=[])
course = etree.parse(StringIO(course_string), parser=parser)
elements = list(course.iter())
tag_to_category = {
# Custom tags
'videodev': 'Custom',
'slides': 'Custom',
'book': 'Custom',
'image': 'Custom',
'discuss': 'Custom',
# Simple lists
'chapter': 'Week',
'course': 'Course',
'section': defaultdict(lambda: 'Section', {
'Lab': 'Lab',
'Lecture Sequence': 'LectureSequence',
'Homework': 'Homework',
'Tutorial Index': 'TutorialIndex',
'Video': 'VideoSegment',
'Midterm': 'Exam',
'Final': 'Exam',
'Problems': 'ProblemSet',
}),
'videosequence': 'VideoSequence',
'problemset': 'ProblemSet',
'vertical': 'Section',
'sequential': 'Section',
'tab': 'Section',
# True types
'video': 'VideoSegment',
'html': 'HTML',
'problem': 'Problem',
}
name_index = 0
for e in elements:
name = e.attrib.get('name', None)
for f in elements:
if f != e and f.attrib.get('name', None) == name:
name = None
if not name:
name = "{tag}_{index}".format(tag=e.tag, index=name_index)
name_index = name_index + 1
if e.tag in tag_to_category:
category = tag_to_category[e.tag]
if isinstance(category, dict):
category = category[e.get('format')]
category = category.replace('/', '-')
name = name.replace('/', '-')
e.set('url', 'i4x://mit.edu/6002xs12/{category}/{name}'.format(
category=category,
name=name))
else:
print "Skipping element with tag", e.tag
def handle_skip(e):
print "Skipping ", e
'''Import the specified data directory into the default keystore'''
results = {}
def handle_custom(e):
data = {'type':'i4x://mit.edu/6002xs12/tag/{tag}'.format(tag=e.tag),
'attrib':dict(e.attrib)}
results[e.attrib['url']] = {'data':data}
def handle_list(e):
if e.attrib.get("class", None) == "tutorials":
return
children = [le.attrib['url'] for le in e.getchildren()]
results[e.attrib['url']] = {'children':children}
def handle_video(e):
url = e.attrib['url']
clip_url = url.replace('VideoSegment', 'VideoClip')
# Take: 0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8
# Make: [(0.75, 'izygArpw-Qo'), (1.0, 'p2Q6BrNhdh8'), (1.25, '1EeWXzPdhSA'), (1.5, 'rABDYkeK0x8')]
youtube_str = e.attrib['youtube']
youtube_list = [(float(x), y) for x,y in map(lambda x:x.split(':'), youtube_str.split(','))]
clip_infos = [{ "status": "ready",
"format": "youtube",
"sane": True,
"location": "youtube",
"speed": speed,
"id": youtube_id,
"size": None} \
for (speed, youtube_id) \
in youtube_list]
results[clip_url] = {'data':{'clip_infos':clip_infos}}
results[url] = {'children' : [{'url':clip_url}]}
def handle_html(e):
if 'src' in e.attrib:
text = open(data_dir+'html/'+e.attrib['src']).read()
else:
textlist=[e.text]+[etree.tostring(elem) for elem in e]+[e.tail]
textlist=[i for i in textlist if type(i)==str]
text = "".join(textlist)
results[e.attrib['url']] = {'data':{'text':text}}
def handle_problem(e):
data = open(os.path.join(data_dir, 'problems', e.attrib['filename']+'.xml')).read()
results[e.attrib['url']] = {'data':{'statement':data}}
element_actions = {# Inside HTML ==> Skip these
'a': handle_skip,
'h1': handle_skip,
'h2': handle_skip,
'hr': handle_skip,
'strong': handle_skip,
'ul': handle_skip,
'li': handle_skip,
'p': handle_skip,
# Custom tags
'videodev': handle_custom,
'slides': handle_custom,
'book': handle_custom,
'image': handle_custom,
'discuss': handle_custom,
# Simple lists
'chapter': handle_list,
'course': handle_list,
'sequential': handle_list,
'vertical': handle_list,
'section': handle_list,
'videosequence': handle_list,
'problemset': handle_list,
'tab': handle_list,
# True types
'video': handle_video,
'html': handle_html,
'problem': handle_problem,
}
for e in elements:
element_actions[e.tag](e)
for k in results:
keystore().create_item(k, 'Piotr Mitros')
if 'data' in results[k]:
keystore().update_item(k, results[k]['data'])
if 'children' in results[k]:
keystore().update_children(k, results[k]['children'])
def handle(self, *args, **options):
org, course, data_dir = args
data_dir = path(data_dir)
with open(data_dir / "course.xml") as course_file:
system = DescriptorSystem(keystore().get_item)
def process_xml(xml):
module = XModuleDescriptor.load_from_xml(xml, system, org, course, RawDescriptor)
keystore().create_item(module.url)
if 'data' in module.definition:
keystore().update_item(module.url, module.definition['data'])
if 'children' in module.definition:
keystore().update_children(module.url, module.definition['children'])
return module.url
system.process_xml = process_xml
system.process_xml(course_file.read())
cms/djangoapps/contentstore/views.py
......@@ -11,7 +11,7 @@ def index(request):
org = 'mit.edu'
course = '6002xs12'
name = '6.002 Spring 2012'
course = keystore().get_item(['i4x', org, course, 'Course', name])
course = keystore().get_item(['i4x', org, course, 'course', name])
weeks = course.get_children()
return render_to_response('index.html', {'weeks': weeks})
......@@ -21,6 +21,7 @@ def edit_item(request):
item = keystore().get_item(item_id)
return render_to_response('unit.html', {
'contents': item.get_html(),
'js_module': item.js_module_name(),
'type': item.type,
'name': item.name,
})
......
cms/envs/common.py
......@@ -158,6 +158,7 @@ PIPELINE_CSS = {
PIPELINE_ALWAYS_RECOMPILE = ['sass/base-style.scss']
from x_module import XModuleDescriptor
from raw_module import RawDescriptor
js_file_dir = PROJECT_ROOT / "static" / "coffee" / "module"
try:
os.makedirs(js_file_dir)
......@@ -168,7 +169,7 @@ except OSError as exc:
raise
module_js_sources = []
for xmodule in XModuleDescriptor.load_classes():
for xmodule in XModuleDescriptor.load_classes() + [RawDescriptor]:
js = xmodule.get_javascript()
for filetype in ('coffee', 'js'):
for idx, fragment in enumerate(js.get(filetype, [])):
......
cms/static/coffee/main.coffee
class @CMS
@setHeight = =>
windowHeight = $(this).height()
@contentHeight = windowHeight - 29
@bind = =>
$('a.module-edit').click ->
CMS.edit_item($(this).attr('id'))
return false
$(window).bind('resize', CMS.setHeight)
@edit_item = (id) =>
$.get('/edit_item', {id: id}, (data) =>
$('#module-html').empty().append(data)
CMS.bind()
$('section.edit-pane').show()
$('body.content .cal').css('height', @contentHeight)
$('body').addClass('content')
$('section.edit-pane').show()
new Unit('unit-wrapper', id)
)
......@@ -78,6 +84,7 @@ $ ->
$('.problem-new a').click ->
$('section.edit-pane').show()
return false
CMS.setHeight()
CMS.bind()
cms/templates/unit.html
<section id="unit-wrapper" class="${type}">
<section id="unit-wrapper" class="${js_module}">
<header>
<section>
<h1 class="editable">${name}</h1>
......
cms/templates/widgets/navigation.html
......@@ -38,7 +38,7 @@
% for week in weeks:
<li>
<header>
<h1><a href="#" class="week-edit" id="${week.url}">${week.name}</a></h1>
<h1><a href="#" class="module-edit" id="${week.url}">${week.name}</a></h1>
<ul>
% if week.goals:
% for goal in week.goals:
......
cms/templates/widgets/raw-edit.html 0 → 100644
<section class="raw-edit">
<section class="meta wip">
<section class="status-settings">
<ul>
<li><a href="#" class="current">Scrap</a></li>
<li><a href="#">Draft</a></li>
<li><a href="#">Proofed</a></li>
<li><a href="#">Published</a></li>
</ul>
<a href="#" class="settings">Settings</a>
</section>
<section class="author">
<dl>
<dt>Last modified:</dt>
<dd>mm/dd/yy</dd>
<dt>By</dt>
<dd>Anant Agarwal</dd>
</dl>
</section>
<section class="tags">
<div>
<h2>Tags:</h2>
<p class="editable">Click to edit</p>
</div>
<div>
<h2>Goal</h2>
<p class="editable">Click to edit</p>
</div>
</section>
</section>
<textarea name="" class="edit-box" rows="8" cols="40">${data}</textarea>
<pre class="preview">${data | h}</pre>
<div class="actions wip">
<a href="" class="save-update">Save &amp; Update</a>
<a href="#" class="cancel">Cancel</a>
</div>
<%include file="notes.html"/>
</section>
common/lib/keystore/__init__.py
......@@ -125,7 +125,7 @@ class ModuleStore(object):
"""
An abstract interface for a database backend that stores XModuleDescriptor instances
"""
def get_item(self, location):
def get_item(self, location, default_class=None):
"""
Returns an XModuleDescriptor instance for the item at location.
If location.revision is None, returns the item with the most
......@@ -136,6 +136,8 @@ class ModuleStore(object):
If no object is found at that location, raises keystore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
default_class: An XModuleDescriptor subclass to use if no plugin matching the
location is found
"""
raise NotImplementedError
......
common/lib/keystore/django.py
......@@ -8,6 +8,7 @@ from __future__ import absolute_import
from django.conf import settings
from .mongo import MongoModuleStore
from raw_module import RawDescriptor
_KEYSTORES = {}
......@@ -16,6 +17,9 @@ def keystore(name='default'):
global _KEYSTORES
if name not in _KEYSTORES:
_KEYSTORES[name] = MongoModuleStore(**settings.KEYSTORE[name])
# TODO (cpennington): Load the default class from a string
_KEYSTORES[name] = MongoModuleStore(
default_class=RawDescriptor,
**settings.KEYSTORE[name])
return _KEYSTORES[name]
common/lib/keystore/mongo.py
......@@ -8,7 +8,7 @@ class MongoModuleStore(ModuleStore):
"""
A Mongodb backed ModuleStore
"""
def __init__(self, host, db, collection, port=27017):
def __init__(self, host, db, collection, port=27017, default_class=None):
self.collection = pymongo.connection.Connection(
host=host,
port=port
......@@ -16,6 +16,7 @@ class MongoModuleStore(ModuleStore):
# Force mongo to report errors, at the expense of performance
self.collection.safe = True
self.default_class = default_class
def get_item(self, location):
"""
......@@ -28,6 +29,8 @@ class MongoModuleStore(ModuleStore):
If no object is found at that location, raises keystore.exceptions.ItemNotFoundError
location: Something that can be passed to Location
default_class: An XModuleDescriptor subclass to use if no plugin matching the
location is found
"""
query = {}
......@@ -45,9 +48,10 @@ class MongoModuleStore(ModuleStore):
if item is None:
raise ItemNotFoundError(location)
return XModuleDescriptor.load_from_json(item, DescriptorSystem(self.get_item))
return XModuleDescriptor.load_from_json(
item, DescriptorSystem(self.get_item), self.default_class)
def create_item(self, location, editor):
def create_item(self, location):
"""
Create an empty item at the specified location with the supplied editor
......@@ -55,7 +59,6 @@ class MongoModuleStore(ModuleStore):
"""
self.collection.insert({
'location': Location(location).dict(),
'editor': editor
})
def update_item(self, location, data):
......
common/lib/xmodule/html_module.py
......@@ -16,8 +16,8 @@ class HtmlModuleDescriptor(MakoModuleDescriptor):
"""
mako_template = "widgets/html-edit.html"
# TODO (cpennington): Make this into a proper module
js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]}
js_module = 'HTML'
class Module(XModule):
......
common/lib/xmodule/js/module/raw.coffee 0 → 100644
class @Raw
constructor: (@id) ->
@edit_box = $("##{@id} .edit-box")
@preview = $("##{@id} .preview")
@edit_box.on('input', =>
@preview.empty().text(@edit_box.val())
)
save: -> @edit_box.val()
common/lib/xmodule/mako_module.py
......@@ -12,7 +12,11 @@ class MakoModuleDescriptor(XModuleDescriptor):
the descriptor as the `module` parameter to that template
"""
def get_context(self):
"""
Return the context to render the mako template with
"""
return {'module': self}
def get_html(self):
return render_to_string(self.mako_template, {
'module': self
})
return render_to_string(self.mako_template, self.get_context())
common/lib/xmodule/raw_module.py 0 → 100644
from pkg_resources import resource_string
from mako_module import MakoModuleDescriptor
from lxml import etree
class RawDescriptor(MakoModuleDescriptor):
"""
Module that provides a raw editing view of it's data and children
"""
mako_template = "widgets/raw-edit.html"
js = {'coffee': [resource_string(__name__, 'js/module/raw.coffee')]}
js_module = 'Raw'
def get_context(self):
return {
'module': self,
'data': self.definition['data'],
}
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None):
"""
Creates an instance of this descriptor from the supplied xml_data.
This may be overridden by subclasses
xml_data: A string of xml that will be translated into data and children for
this module
system: An XModuleSystem for interacting with external resources
org and course are optional strings that will be used in the generated modules
url identifiers
"""
xml_object = etree.fromstring(xml_data)
return cls(
system,
definition={'data': xml_data},
location=['i4x',
org,
course,
xml_object.tag,
xml_object.get('name')]
)
common/lib/xmodule/seq_module.py
......@@ -115,5 +115,23 @@ class Module(XModule):
self.rendered = False
class SectionDescriptor(MakoModuleDescriptor):
class SequenceDescriptor(MakoModuleDescriptor):
mako_template = 'widgets/sequence-edit.html'
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None):
xml_object = etree.fromstring(xml_data)
children = [
system.process_xml(etree.tostring(child_module))
for child_module in xml_object
]
return cls(
system, {'children': children},
location=['i4x',
org,
course,
xml_object.tag,
xml_object.get('name')]
)
common/lib/xmodule/setup.py
......@@ -13,18 +13,8 @@ setup(
# for a description of entry_points
entry_points={
'xmodule.v1': [
"Course = seq_module:SectionDescriptor",
"Week = seq_module:SectionDescriptor",
"Section = seq_module:SectionDescriptor",
"LectureSequence = seq_module:SectionDescriptor",
"Lab = seq_module:SectionDescriptor",
"Homework = seq_module:SectionDescriptor",
"TutorialIndex = seq_module:SectionDescriptor",
"Exam = seq_module:SectionDescriptor",
"VideoSegment = video_module:VideoSegmentDescriptor",
"ProblemSet = seq_module:SectionDescriptor",
"Problem = capa_module:CapaModuleDescriptor",
"HTML = html_module:HtmlModuleDescriptor",
"course = seq_module:SequenceDescriptor",
"html = html_module:HtmlModuleDescriptor",
]
}
)
common/lib/xmodule/x_module.py
......@@ -15,8 +15,24 @@ class ModuleMissingError(Exception):
class Plugin(object):
"""
Base class for a system that uses entry_points to load plugins.
Implementing classes are expected to have the following attributes:
entry_point: The name of the entry point to load plugins from
"""
@classmethod
def load_class(cls, identifier):
def load_class(cls, identifier, default=None):
"""
Loads a single class intance specified by identifier. If identifier
specifies more than a single class, then logs a warning and returns the first
class identified.
If default is not None, will return default if no entry_point matching identifier
is found. Otherwise, will raise a ModuleMissingError
"""
identifier = identifier.lower()
classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier))
if len(classes) > 1:
log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format(
......@@ -25,6 +41,8 @@ class Plugin(object):
classes=", ".join(class_.module_name for class_ in classes)))
if len(classes) == 0:
if default is not None:
return default
raise ModuleMissingError(identifier)
return classes[0].load()
......@@ -160,9 +178,10 @@ class XModuleDescriptor(Plugin):
"""
entry_point = "xmodule.v1"
js = {}
js_module = None
@staticmethod
def load_from_json(json_data, system):
def load_from_json(json_data, system, default_class=None):
"""
This method instantiates the correct subclass of XModuleDescriptor based
on the contents of json_data.
......@@ -170,7 +189,10 @@ class XModuleDescriptor(Plugin):
json_data must contain a 'location' element, and must be suitable to be
passed into the subclasses `from_json` method.
"""
class_ = XModuleDescriptor.load_class(json_data['location']['category'])
class_ = XModuleDescriptor.load_class(
json_data['location']['category'],
default_class
)
return class_.from_json(json_data, system)
@classmethod
......@@ -184,6 +206,36 @@ class XModuleDescriptor(Plugin):
"""
return cls(system=system, **json_data)
@staticmethod
def load_from_xml(xml_data, system, org=None, course=None, default_class=None):
"""
This method instantiates the correct subclass of XModuleDescriptor based
on the contents of xml_data.
xml_data must be a string containing valid xml
org and course are optional strings that will be used in the generated modules
url identifiers
"""
class_ = XModuleDescriptor.load_class(
etree.fromstring(xml_data).tag,
default_class
)
return class_.from_xml(xml_data, system, org, course)
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None):
"""
Creates an instance of this descriptor from the supplied xml_data.
This may be overridden by subclasses
xml_data: A string of xml that will be translated into data and children for
this module
system: An XModuleSystem for interacting with external resources
org and course are optional strings that will be used in the generated modules
url identifiers
"""
raise NotImplementedError('Modules must implement from_xml to be parsable from xml')
@classmethod
def get_javascript(cls):
"""
......@@ -196,6 +248,12 @@ class XModuleDescriptor(Plugin):
"""
return cls.js
def js_module_name(self):
"""
Return the name of the javascript class to instantiate when
this module descriptor is loaded for editing
"""
return self.js_module
def __init__(self,
system,
......@@ -230,15 +288,12 @@ class XModuleDescriptor(Plugin):
self._child_instances = None
def get_children(self, categories=None):
def get_children(self):
"""Returns a list of XModuleDescriptor instances for the children of this module"""
if self._child_instances is None:
self._child_instances = [self.system.load_item(child) for child in self.definition['children']]
self._child_instances = [self.system.load_item(child) for child in self.definition.get('children', [])]
if categories is None:
return self._child_instances
else:
return [child for child in self._child_instances if child.type in categories]
return self._child_instances
def get_html(self):
"""
......@@ -275,9 +330,11 @@ class XModuleDescriptor(Plugin):
class DescriptorSystem(object):
def __init__(self, load_item):
def __init__(self, load_item, process_xml=None):
"""
load_item: Takes a Location and returns and XModuleDescriptor
load_item: Takes a Location and returns an XModuleDescriptor
process_xml: Takes an xml string, and returns the url of the XModuleDescriptor created from that xml
"""
self.load_item = load_item
self.process_xml = process_xml
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment