Merge pull request #174 from MITx/cpennington/cms-backcompat

CMS import backwards compatibility

 - edx4edx and 8.01x course repo errors fixed and verified
 - moving forward, it would be nice if the LMS+CMS xml import failed more gracefully when encountering errors in the xml files, when in debug or authoring mode.  For example, an inadvertent xml error in one problem (eg created by a mistake in an import tool used in the CMS) should not bring down the whole LMS. Ditto if one problem gets deleted from the mongodb or filesystem.

common/lib/xmodule/xmodule/capa_module.py

@@ -6,6 +6,7 @@ import logging
 import traceback
 import re
 import StringIO
+import os
 
 from datetime import timedelta
 from lxml import etree
@@ -503,3 +504,12 @@ class CapaDescriptor(RawDescriptor):
     """
 
     module_class = CapaModule
+
+    # TODO (cpennington): Delete this method once all fall 2012 course are being
+    # edited in the cms
+    @classmethod
+    def backcompat_paths(cls, path):
+        return [
+            'problems/' + path[8:],
+            path[8:],
+        ]

common/lib/xmodule/xmodule/html_module.py

 import logging
+import os
 from lxml import etree
 
 from xmodule.x_module import XModule
@@ -28,6 +29,19 @@ class HtmlDescriptor(RawDescriptor):
     js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]}
     js_module = 'HTML'
 
+    # TODO (cpennington): Delete this method once all fall 2012 course are being
+    # edited in the cms
+    @classmethod
+    def backcompat_paths(cls, path):
+        if path.endswith('.html.html'):
+            path = path[:-5]
+        candidates = []
+        while os.sep in path:
+            candidates.append(path)
+            _, _, path = path.partition(os.sep)
+
+        return candidates
+
     @classmethod
     def file_to_xml(cls, file_object):
         parser = etree.HTMLParser()

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -5,7 +5,9 @@ from lxml import etree
 from path import path
 from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
 from xmodule.mako_module import MakoDescriptorSystem
+from cStringIO import StringIO
 import os
+import re
 
 from . import ModuleStore, Location
 from .exceptions import ItemNotFoundError
@@ -16,6 +18,13 @@ etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False,
 log = logging.getLogger('mitx.' + __name__)
 
 
+# TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml
+def clean_out_mako_templating(xml_string):
+    xml_string = xml_string.replace('%include', 'include')
+    xml_string = re.sub("(?m)^\s*%.*$", '', xml_string)
+    return xml_string
+
+
 class XMLModuleStore(ModuleStore):
     """
     An XML backed ModuleStore
@@ -54,8 +63,11 @@ class XMLModuleStore(ModuleStore):
             if not os.path.exists(self.data_dir / course_dir / "course.xml"):
                 continue
 
-            course_descriptor = self.load_course(course_dir)
-            self.courses[course_dir] = course_descriptor
+            try:
+                course_descriptor = self.load_course(course_dir)
+                self.courses[course_dir] = course_descriptor
+            except:
+                log.exception("Failed to load course %s" % course_dir)
 
     def load_course(self, course_dir):
         """
@@ -65,6 +77,9 @@ class XMLModuleStore(ModuleStore):
 
         with open(self.data_dir / course_dir / "course.xml") as course_file:
 
+            # TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml
+            course_file = StringIO(clean_out_mako_templating(course_file.read()))
+
             course_data = etree.parse(course_file).getroot()
             org = course_data.get('org')
 
@@ -91,6 +106,8 @@ class XMLModuleStore(ModuleStore):
 
                     def process_xml(xml):
                         try:
+                            # TODO (cpennington): Remove this once all fall 2012 courses have been imported into the cms from xml
+                            xml = clean_out_mako_templating(xml)
                             xml_data = etree.fromstring(xml)
                         except:
                             log.exception("Unable to parse xml: {xml}".format(xml=xml))

common/lib/xmodule/xmodule/xml_module.py

@@ -4,6 +4,8 @@ from lxml import etree
 import copy
 import logging
 from collections import namedtuple
+from fs.errors import ResourceNotFoundError
+import os
 
 log = logging.getLogger(__name__)
 
@@ -154,13 +156,30 @@ class XmlDescriptor(XModuleDescriptor):
                 definition_xml = copy.deepcopy(xml_object)
             else:
                 filepath = cls._format_filepath(xml_object.tag, filename)
-                log.debug('filepath=%s, resources_fs=%s' % (filepath,system.resources_fs))
-                with system.resources_fs.open(filepath) as file:
-                    try:
-                        definition_xml = cls.file_to_xml(file)
-                    except:
-                        log.exception("Failed to parse xml in file %s" % filepath)
-                        raise
+
+                # TODO (cpennington): If the file doesn't exist at the right path,
+                # give the class a chance to fix it up. The file will be written out again
+                # in the correct format.
+                # This should go away once the CMS is online and has imported all current (fall 2012)
+                # courses from xml
+                if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'):
+                    candidates = cls.backcompat_paths(filepath)
+                    for candidate in candidates:
+                        if system.resources_fs.exists(candidate):
+                            filepath = candidate
+                            break
+
+                log.debug('filepath=%s, resources_fs=%s' % (filepath, system.resources_fs))
+                try:
+                    with system.resources_fs.open(filepath) as file:
+                        try:
+                            definition_xml = cls.file_to_xml(file)
+                        except:
+                            log.exception("Failed to parse xml in file %s" % filepath)
+                            raise
+                except ResourceNotFoundError:
+                    log.exception('Unable to load file contents at path %s' % filepath)
+                    return {'data': 'Error loading file contents at path %s' % filepath}
 
             cls.clean_metadata_from_xml(definition_xml)
             return cls.definition_from_xml(definition_xml, system)
@@ -200,7 +219,7 @@ class XmlDescriptor(XModuleDescriptor):
         if len(list(xml_object.iter())) > 5:
 
             filepath = self.__class__._format_filepath(self.category, self.name)
-            resource_fs.makedir(self.category, allow_recreate=True)
+            resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))