Merge pull request #1806 from MITx/feature/cdodge/integrity-check-mongo-course

Feature/cdodge/integrity check mongo course

cms/djangoapps/contentstore/management/commands/check_course.py

+from django.core.management.base import BaseCommand, CommandError
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.xml_importer import check_module_metadata_editability
+from xmodule.course_module import CourseDescriptor
+
+from request_cache.middleware import RequestCache
+
+from django.core.cache import get_cache, InvalidCacheBackendError
+
+class Command(BaseCommand):
+    help = '''Enumerates through the course and find common errors'''
+
+    def handle(self, *args, **options):
+        if len(args) != 1 :
+            raise CommandError("check_course requires one argument: <location>")
+
+        loc_str = args[0]
+
+        loc = CourseDescriptor.id_to_location(loc_str)
+     
+        store = modulestore()
+        
+        # setup a request cache so we don't throttle the DB with all the metadata inheritance requests
+        store.request_cache = RequestCache.get_request_cache()
+
+        course = store.get_item(loc, depth=3)
+
+        err_cnt = 0
+        def _xlint_metadata(module):
+            err_cnt = check_module_metadata_editability(module)
+            for child in module.get_children():
+                err_cnt = err_cnt + _xlint_metadata(child)
+            return err_cnt
+
+        _xlint_metadata(course)
+
+        # we've had a bug where the xml_attributes field can we rewritten as a string rather than a dict
+        def _check_xml_attributes_field(module):
+            err_cnt = 0
+            if hasattr(module, 'xml_attributes') and isinstance(module.xml_attributes, basestring):
+                print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location.url())
+                err_cnt = err_cnt + 1
+            for child in module.get_children():
+                err_cnt = err_cnt + _check_xml_attributes_field(child)
+            return err_cnt
+
+        _check_xml_attributes_field(course)
+
+        # check for dangling discussion items, this can cause errors in the forums
+        def _get_discussion_items(module):
+            discussion_items = []
+            if module.location.category == 'discussion':
+                discussion_items = discussion_items + [module.location.url()]
+
+            for child in module.get_children():
+                discussion_items = discussion_items + _get_discussion_items(child)
+
+            return discussion_items
+
+        discussion_items =_get_discussion_items(course)
+
+        # now query all discussion items via get_items() and compare with the tree-traversal
+        queried_discussion_items = store.get_items(['i4x', course.location.org, course.location.course, 
+            'discussion', None, None])
+
+        for item in queried_discussion_items:
+            if item.location.url() not in discussion_items:
+                print 'Found dangling discussion module = {0}'.format(item.location.url())
+
+
+
+
+

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -356,22 +356,44 @@ def remap_namespace(module, target_location_namespace):
 
     return module
 
-def validate_no_non_editable_metadata(module_store, course_id, category, allowed=None):
+
+def allowed_metadata_by_category(category):
+    # should this be in the descriptors?!?
+    return {
+        'vertical': [],
+        'chapter': ['start'],
+        'sequential': ['due', 'format', 'start', 'graded']
+    }.get(category,['*'])
+
+
+def check_module_metadata_editability(module):
     '''
-    Assert that there is no metadata within a particular category that we can't support editing
+    Assert that there is no metadata within a particular module that we can't support editing
     However we always allow 'display_name' and 'xml_attribtues'
     '''
-    _allowed = (allowed if allowed is not None else []) + ['xml_attributes', 'display_name']
+    allowed = allowed_metadata_by_category(module.location.category)
+    if '*' in allowed:
+        # everything is allowed
+        return 0
+
+    allowed = allowed + ['xml_attributes', 'display_name']
+    err_cnt = 0
+    my_metadata = dict(own_metadata(module))
+    illegal_keys = set(own_metadata(module).keys()) - set(allowed)
+
+    if len(illegal_keys) > 0:
+        err_cnt = err_cnt + 1
+        print ': found non-editable metadata on {0}. These metadata keys are not supported = {1}'. format(module.location.url(), illegal_keys)
+
+    return err_cnt
+
 
+def validate_no_non_editable_metadata(module_store, course_id, category):
     err_cnt = 0
     for module_loc in module_store.modules[course_id]:
         module = module_store.modules[course_id][module_loc]
         if module.location.category == category:
-            my_metadata = dict(own_metadata(module))
-            for key in my_metadata.keys():
-                if key not in _allowed:
-                    err_cnt = err_cnt + 1
-                    print ': found metadata on {0}. Studio will not support editing this piece of metadata, so it is not allowed. Metadata: {1} = {2}'. format(module.location.url(), key, my_metadata[key])
+            err_cnt = err_cnt + check_module_metadata_editability(module)
 
     return err_cnt
 
@@ -463,10 +485,9 @@ def perform_xlint(data_dir, course_dirs,
         # don't allow metadata on verticals, since we can't edit them in studio
         err_cnt += validate_no_non_editable_metadata(module_store, course_id, "vertical")
         # don't allow metadata on chapters, since we can't edit them in studio
-        err_cnt += validate_no_non_editable_metadata(module_store, course_id, "chapter",['start'])
+        err_cnt += validate_no_non_editable_metadata(module_store, course_id, "chapter")
         # don't allow metadata on sequences that we can't edit
-        err_cnt += validate_no_non_editable_metadata(module_store, course_id, "sequential",
-            ['due','format','start','graded'])
+        err_cnt += validate_no_non_editable_metadata(module_store, course_id, "sequential")
 
         # check for a presence of a course marketing video
         location_elements = course_id.split('/')