Merge pull request #4250 from edx/zub/story/listallexportfaildcourses

update export_all_courses management command to count and list all faile...

cms/djangoapps/contentstore/management/commands/export_all_courses.py

 """
-Script for exporting all courseware from Mongo to a directory
+Script for exporting all courseware from Mongo to a directory and listing the courses which failed to export
 """
 from django.core.management.base import BaseCommand, CommandError
 from xmodule.modulestore.xml_exporter import export_to_xml
@@ -8,35 +8,52 @@ from xmodule.contentstore.django import contentstore
 
 
 class Command(BaseCommand):
-    """Export all courses from mongo to the specified data directory"""
-    help = 'Export all courses from mongo to the specified data directory'
+    """
+    Export all courses from mongo to the specified data directory and list the courses which failed to export
+    """
+    help = 'Export all courses from mongo to the specified data directory and list the courses which failed to export'
 
     def handle(self, *args, **options):
-        "Execute the command"
+        """
+        Execute the command
+        """
         if len(args) != 1:
             raise CommandError("export requires one argument: <output path>")
 
         output_path = args[0]
-
-        cs = contentstore()
-        ms = modulestore()
-        root_dir = output_path
-        courses = ms.get_courses()
-
-        print("%d courses to export:" % len(courses))
-        cids = [x.id for x in courses]
-        print(cids)
-
-        for course_id in cids:
-
-            print("-"*77)
-            print("Exporting course id = {0} to {1}".format(course_id, output_path))
-
-            if 1:
-                try:
-                    course_dir = course_id.replace('/', '...')
-                    export_to_xml(ms, cs, course_id, root_dir, course_dir)
-                except Exception as err:
-                    print("="*30 + "> Oops, failed to export %s" % course_id)
-                    print("Error:")
-                    print(err)
+        courses, failed_export_courses = export_courses_to_output_path(output_path)
+
+        print("=" * 80)
+        print(u"=" * 30 + u"> Export summary")
+        print(u"Total number of courses to export: {0}".format(len(courses)))
+        print(u"Total number of courses which failed to export: {0}".format(len(failed_export_courses)))
+        print(u"List of export failed courses ids:")
+        print(u"\n".join(failed_export_courses))
+        print("=" * 80)
+
+
+def export_courses_to_output_path(output_path):
+    """
+    Export all courses to target directory and return the list of courses which failed to export
+    """
+    content_store = contentstore()
+    module_store = modulestore()
+    root_dir = output_path
+    courses = module_store.get_courses()
+
+    course_ids = [x.id for x in courses]
+    failed_export_courses = []
+
+    for course_id in course_ids:
+        print(u"-" * 80)
+        print(u"Exporting course id = {0} to {1}".format(course_id, output_path))
+        try:
+            course_dir = course_id.to_deprecated_string().replace('/', '...')
+            export_to_xml(module_store, content_store, course_id, root_dir, course_dir)
+        except Exception as err:  # pylint: disable=broad-except
+            failed_export_courses.append(unicode(course_id))
+            print(u"=" * 30 + u"> Oops, failed to export {0}".format(course_id))
+            print(u"Error:")
+            print(err)
+
+    return courses, failed_export_courses

cms/djangoapps/contentstore/management/commands/tests/test_export_all_courses.py

+"""
+Test for export all courses.
+"""
+import shutil
+from tempfile import mkdtemp
+
+from contentstore.management.commands.export_all_courses import export_courses_to_output_path
+
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory
+
+
+class ExportAllCourses(ModuleStoreTestCase):
+    """
+    Tests exporting all courses.
+    """
+    def setUp(self):
+        """ Common setup. """
+        self.store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
+        self.temp_dir = mkdtemp()
+        self.first_course = CourseFactory.create(org="test", course="course1", display_name="run1")
+        self.second_course = CourseFactory.create(org="test", course="course2", display_name="run2")
+
+    def test_export_all_courses(self):
+        """
+        Test exporting good and faulty courses
+        """
+        # check that both courses exported successfully
+        courses, failed_export_courses = export_courses_to_output_path(self.temp_dir)
+        self.assertEqual(len(courses), 2)
+        self.assertEqual(len(failed_export_courses), 0)
+
+        # manually make second course faulty and check that it fails on export
+        second_course_id = self.second_course.id
+        self.store.collection.update(
+            {'_id.org': second_course_id.org, '_id.course': second_course_id.course, '_id.name': second_course_id.run},
+            {'$set': {'metadata.tags': 'crash'}}
+        )
+        courses, failed_export_courses = export_courses_to_output_path(self.temp_dir)
+        self.assertEqual(len(courses), 2)
+        self.assertEqual(len(failed_export_courses), 1)
+        self.assertEqual(failed_export_courses[0], unicode(second_course_id))
+
+    def tearDown(self):
+        """ Common cleanup. """
+        shutil.rmtree(self.temp_dir)