Merge pull request #1048 from MITx/feature/cdodge/xlint

Feature/cdodge/xlint

cms/djangoapps/contentstore/management/commands/xlint.py

+from django.core.management.base import BaseCommand, CommandError
+from xmodule.modulestore.xml_importer import perform_xlint
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+
+
+unnamed_modules = 0
+
+
+class Command(BaseCommand):
+    help = \
+    '''
+    Verify the structure of courseware as to it's suitability for import
+    To run test: rake cms:xlint DATA_DIR=../data [COURSE_DIR=content-edx-101 (optional parameter)]
+    '''
+    def handle(self, *args, **options):
+        if len(args) == 0:
+            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")
+
+        data_dir = args[0]
+        if len(args) > 1:
+            course_dirs = args[1:]
+        else:
+            course_dirs = None
+        print "Importing.  Data_dir={data}, course_dirs={courses}".format(
+            data=data_dir,
+            courses=course_dirs)
+        perform_xlint(data_dir, course_dirs, load_error_modules=False)

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -303,7 +303,7 @@ class XMLModuleStore(ModuleStoreBase):
         try:
             course_descriptor = self.load_course(course_dir, errorlog.tracker)
         except Exception as e:
-            msg = "Failed to load course '{0}': {1}".format(course_dir, str(e))
+            msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e))
             log.exception(msg)
             errorlog.tracker(msg)
 
@@ -337,7 +337,7 @@ class XMLModuleStore(ModuleStoreBase):
             with open(policy_path) as f:
                 return json.load(f)
         except (IOError, ValueError) as err:
-            msg = "Error loading course policy from {0}".format(policy_path)
+            msg = "ERROR: loading course policy from {0}".format(policy_path)
             tracker(msg)
             log.warning(msg + " " + str(err))
         return {}
@@ -465,7 +465,8 @@ class XMLModuleStore(ModuleStoreBase):
                     module.metadata['data_dir'] = course_dir
                     self.modules[course_descriptor.id][module.location] = module   
                 except Exception, e:
-                    logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))          
+                    logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))   
+                    system.error_tracker("ERROR: " + str(e))
 
     def get_instance(self, course_id, location, depth=0):
         """

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -88,7 +88,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic
 
 def import_from_xml(store, data_dir, course_dirs=None, 
                     default_class='xmodule.raw_module.RawDescriptor',
-                    load_error_modules=True, static_content_store=None, target_location_namespace = None):
+                    load_error_modules=True, static_content_store=None, target_location_namespace=None):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
@@ -202,7 +202,6 @@ def import_from_xml(store, data_dir, course_dirs=None,
 
                 store.update_item(module.location, module_data)
 
-
             if 'children' in module.definition:
                 store.update_children(module.location, module.definition['children'])
 
@@ -210,6 +209,100 @@ def import_from_xml(store, data_dir, course_dirs=None,
             # inherited metadata everywhere.
             store.update_metadata(module.location, dict(module.own_metadata))
 
+    return module_store, course_items
 
 
-    return module_store, course_items
+def validate_category_hierarcy(module_store, course_id, parent_category, expected_child_category):
+    err_cnt = 0
+
+    parents = []
+    # get all modules of parent_category
+    for module in module_store.modules[course_id].itervalues():
+        if module.location.category == parent_category:
+            parents.append(module)
+
+    for parent in parents:
+        for child_loc in [Location(child) for child in parent.definition.get('children', [])]:
+            if child_loc.category != expected_child_category:
+                err_cnt += 1
+                print 'ERROR: child {0} of parent {1} was expected to be category of {2} but was {3}'.format(
+                    child_loc, parent.location, expected_child_category, child_loc.category)
+
+    return err_cnt
+
+def validate_data_source_path_existence(path, is_err = True, extra_msg = None):
+    _cnt = 0
+    if not os.path.exists(path):
+        print ("{0}: Expected folder at {1}. {2}".format('ERROR' if is_err == True else 'WARNING', path, extra_msg if 
+            extra_msg is not None else ''))
+        _cnt = 1
+    return _cnt
+
+def validate_data_source_paths(data_dir, course_dir):
+    # check that there is a '/static/' directory
+    course_path = data_dir / course_dir
+    err_cnt = 0
+    warn_cnt = 0
+    err_cnt += validate_data_source_path_existence(course_path / 'static')
+    warn_cnt += validate_data_source_path_existence(course_path / 'static/subs', is_err = False, 
+        extra_msg = 'Video captions (if they are used) will not work unless they are static/subs.')
+    return err_cnt, warn_cnt
+
+
+def perform_xlint(data_dir, course_dirs, 
+                    default_class='xmodule.raw_module.RawDescriptor',
+                    load_error_modules=True):
+    err_cnt = 0
+    warn_cnt = 0
+
+    module_store = XMLModuleStore(
+        data_dir,
+        default_class=default_class,
+        course_dirs=course_dirs,
+        load_error_modules=load_error_modules
+    )
+
+    # check all data source path information
+    for course_dir in course_dirs:
+        _err_cnt, _warn_cnt = validate_data_source_paths(path(data_dir), course_dir)
+        err_cnt += _err_cnt
+        warn_cnt += _warn_cnt
+
+    # first count all errors and warnings as part of the XMLModuleStore import
+    for err_log in module_store._location_errors.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
+    # then count outright all courses that failed to load at all
+    for err_log in module_store.errored_courses.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            print msg
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
+    for course_id in module_store.modules.keys():
+        # constrain that courses only have 'chapter' children
+        err_cnt += validate_category_hierarcy(module_store, course_id, "course", "chapter")
+        # constrain that chapters only have 'sequentials'
+        err_cnt += validate_category_hierarcy(module_store, course_id, "chapter", "sequential")
+        # constrain that sequentials only have 'verticals'
+        err_cnt += validate_category_hierarcy(module_store, course_id, "sequential", "vertical")
+
+    print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors   {1} Warnings\n".format(err_cnt, warn_cnt)
+
+    if err_cnt > 0:
+        print "This course is not suitable for importing. Please fix courseware according to specifications before importing."
+    elif warn_cnt > 0:
+        print "This course can be imported, but some errors may occur during the run of the course. It is recommend that you fix your courseware before importing"
+    else:
+        print "This course can be imported successfully."
+
+       
+

common/lib/xmodule/xmodule/seq_module.py

@@ -127,8 +127,10 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         for child in xml_object:
             try:
                 children.append(system.process_xml(etree.tostring(child)).location.url())
-            except:
+            except Exception, e:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
+                if system.error_tracker is not None:
+                    system.error_tracker("ERROR: " + str(e))
                 continue
         return {'children': children}
 

rakefile

@@ -408,6 +408,20 @@ namespace :cms do
   end
 end
 
+namespace :cms do
+  desc "Import course data within the given DATA_DIR variable"
+  task :xlint do
+    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
+      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
+    elsif ENV['DATA_DIR']
+      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
+    else
+      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
+        "Example: \`rake cms:import DATA_DIR=../data\`"
+    end
+  end
+end
+
 desc "Build a properties file used to trigger autodeploy builds"
 task :autodeploy_properties do
     File.open("autodeploy.properties", "w") do |file|