initial xlint implementation. Accumulate all import errors during XmlModuleStore…

initial xlint implementation. Accumulate all import errors during XmlModuleStore importing. Also do checks post XmlModuleStore import and assert that the structure (course->chapter->sequential->vertical) is present in the courses.

cms/djangoapps/contentstore/management/commands/xlint.py

@@ -9,8 +9,10 @@ unnamed_modules = 0
 
 class Command(BaseCommand):
     help = \
-'''Verify the structure of courseware as to it's suitability for import'''
-
+    '''
+    Verify the structure of courseware as to it's suitability for import
+    To run test: rake cms:xlint DATA_DIR=../data [COURSE_DIR=content-edx-101 (optional parameter)]
+    '''
     def handle(self, *args, **options):
         if len(args) == 0:
             raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -303,7 +303,7 @@ class XMLModuleStore(ModuleStoreBase):
         try:
             course_descriptor = self.load_course(course_dir, errorlog.tracker)
         except Exception as e:
-            msg = "Failed to load course '{0}': {1}".format(course_dir, str(e))
+            msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e))
             log.exception(msg)
             errorlog.tracker(msg)
 
@@ -337,7 +337,7 @@ class XMLModuleStore(ModuleStoreBase):
             with open(policy_path) as f:
                 return json.load(f)
         except (IOError, ValueError) as err:
-            msg = "Error loading course policy from {0}".format(policy_path)
+            msg = "ERROR: loading course policy from {0}".format(policy_path)
             tracker(msg)
             log.warning(msg + " " + str(err))
         return {}
@@ -458,7 +458,8 @@ class XMLModuleStore(ModuleStoreBase):
                     module.metadata['data_dir'] = course_dir
                     self.modules[course_descriptor.id][module.location] = module   
                 except Exception, e:
-                    logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))          
+                    logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))   
+                    system.error_tracker("ERROR: " + str(e))
 
     def get_instance(self, course_id, location, depth=0):
         """

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -227,6 +227,28 @@ def validate_category_hierarcy(module_store, course_id, parent_category, expecte
 def validate_module_structure(module_store):
     err_cnt = 0
     warn_cnt = 0
+
+    print module_store.errored_courses
+
+    # first count all errors and warnings as part of the XMLModuleStore import
+    for err_log in module_store._location_errors.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
+    # then count outright all courses that failed to load at all
+    for err_log in module_store.errored_courses.itervalues():
+        for err_log_entry in err_log.errors:
+            msg = err_log_entry[0]
+            print msg
+            if msg.startswith('ERROR:'):
+                err_cnt+=1
+            else:
+                warn_cnt+=1
+
     for course_id in module_store.modules.keys():
         # constrain that courses only have 'chapter' children
         err_cnt += validate_category_hierarcy(module_store, course_id, "course", "chapter")
@@ -235,6 +257,14 @@ def validate_module_structure(module_store):
         # constrain that sequentials only have 'verticals'
         err_cnt += validate_category_hierarcy(module_store, course_id, "sequential", "vertical")
 
-    print "SUMMARY: {0} Errors   {1} Warnings".format(err_cnt, warn_cnt)
+    print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors   {1} Warnings\n".format(err_cnt, warn_cnt)
+
+    if err_cnt > 0:
+        print "This course is not suitable for importing. Please fix courseware according to specifications before importing."
+    elif warn_cnt > 0:
+        print "This course can be imported, but some errors may occur during the run of the course. It is recommend that you fix your courseware before importing"
+    else:
+        print "This course can be imported successfully."
+
        
 

common/lib/xmodule/xmodule/seq_module.py

@@ -127,8 +127,10 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         for child in xml_object:
             try:
                 children.append(system.process_xml(etree.tostring(child)).location.url())
-            except:
+            except Exception, e:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
+                if system.error_tracker is not None:
+                    system.error_tracker("ERROR: " + str(e))
                 continue
         return {'children': children}
 

rakefile

@@ -367,7 +367,9 @@ end
 namespace :cms do
   desc "Import course data within the given DATA_DIR variable"
   task :xlint do
-    if ENV['DATA_DIR']
+    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
+      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
+    elsif ENV['DATA_DIR']
       sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
     else
       raise "Please specify a DATA_DIR variable that point to your data directory.\n" +