Merge pull request #1289 from rocha/dump-course-command

Export course command

cms/djangoapps/contentstore/management/commands/dump_course_structure.py

-"""
-Script for dumping course dumping the course structure
-"""
-from django.core.management.base import BaseCommand, CommandError
-from xmodule.course_module import CourseDescriptor
-from xmodule.modulestore.django import modulestore
-from json import dumps
-from xmodule.modulestore.inheritance import own_metadata
-from django.conf import settings
-
-filter_list = ['xml_attributes', 'checklists']
-
-
-class Command(BaseCommand):
-    """
-    The Django command for dumping course structure
-    """
-    help = '''Write out to stdout a structural and metadata information about a course in a flat dictionary serialized
-              in a JSON format. This can be used for analytics.'''
-
-    def handle(self, *args, **options):
-        "Execute the command"
-        if len(args) < 2 or len(args) > 3:
-            raise CommandError("dump_course_structure requires two or more arguments: <location> <outfile> |<db>|")
-
-        course_id = args[0]
-        outfile = args[1]
-
-        # use a user-specified database name, if present
-        # this is useful for doing dumps from databases restored from prod backups
-        if len(args) == 3:
-            settings.MODULESTORE['direct']['OPTIONS']['db'] = args[2]
-
-        loc = CourseDescriptor.id_to_location(course_id)
-
-        store = modulestore()
-
-        course = None
-        try:
-            course = store.get_item(loc, depth=4)
-        except:
-            print('Could not find course at {0}'.format(course_id))
-            return
-
-        info = {}
-
-        def dump_into_dict(module, info):
-            filtered_metadata = dict((key, value) for key, value in own_metadata(module).iteritems()
-                                     if key not in filter_list)
-            info[module.location.url()] = {
-                'category': module.location.category,
-                'children': module.children if hasattr(module, 'children') else [],
-                'metadata': filtered_metadata
-            }
-
-            for child in module.get_children():
-                dump_into_dict(child, info)
-
-        dump_into_dict(course, info)
-
-        with open(outfile, 'w') as f:
-            f.write(dumps(info))

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -1273,6 +1273,47 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         # export out to a tempdir
         export_to_xml(module_store, content_store, location, root_dir, 'test_export')
 
+    def test_export_course_without_content_store(self):
+        module_store = modulestore('direct')
+        content_store = contentstore()
+
+        # Create toy course
+
+        import_from_xml(module_store, 'common/test/data/', ['toy'])
+        location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
+
+        # Add a sequence
+
+        stub_location = Location(['i4x', 'edX', 'toy', 'sequential', 'vertical_sequential'])
+        sequential = module_store.get_item(stub_location)
+        module_store.update_children(sequential.location, sequential.children)
+
+        # Get course and export it without a content_store
+
+        course = module_store.get_item(location)
+        course.save()
+
+        root_dir = path(mkdtemp_clean())
+
+        print 'Exporting to tempdir = {0}'.format(root_dir)
+        export_to_xml(module_store, None, location, root_dir, 'test_export_no_content_store')
+
+        # Delete the course from module store and reimport it
+
+        delete_course(module_store, content_store, location, commit=True)
+
+        import_from_xml(
+            module_store, root_dir, ['test_export_no_content_store'],
+            draft_store=None,
+            static_content_store=None,
+            target_location_namespace=course.location
+        )
+
+        # Verify reimported course
+
+        items = module_store.get_items(stub_location)
+        self.assertEqual(len(items), 1)
+
 
 @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
 class ContentStoreTest(ModuleStoreTestCase):

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

@@ -38,7 +38,7 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
     Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
 
     `modulestore`: A `ModuleStore` object that is the source of the modules to export
-    `contentstore`: A `ContentStore` object that is the source of the content to export
+    `contentstore`: A `ContentStore` object that is the source of the content to export, can be None
     `course_location`: The `Location` of the `CourseModuleDescriptor` to export
     `root_dir`: The directory to write the exported xml to
     `course_dir`: The name of the directory inside `root_dir` to write the course content to
@@ -46,7 +46,12 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
         alongside the public content in the course.
     """
 
-    course = modulestore.get_item(course_location)
+    # we use get_instance instead of get_item to support modulestores
+    # that can't guarantee that definitions are unique
+    course = modulestore.get_instance(
+        course_location.course_id,
+        course_location
+    )
 
     fs = OSFS(root_dir)
     export_fs = fs.makeopendir(course_dir)
@@ -55,13 +60,14 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
     with export_fs.open('course.xml', 'w') as course_xml:
         course_xml.write(xml)
 
-    policies_dir = export_fs.makeopendir('policies')
     # export the static assets
-    contentstore.export_all_for_course(
-        course_location,
-        root_dir + '/' + course_dir + '/static/',
-        root_dir + '/' + course_dir + '/policies/assets.json',
-    )
+    policies_dir = export_fs.makeopendir('policies')
+    if contentstore:
+        contentstore.export_all_for_course(
+            course_location,
+            root_dir + '/' + course_dir + '/static/',
+            root_dir + '/' + course_dir + '/policies/assets.json',
+        )
 
     # export the static tabs
     export_extra_content(export_fs, modulestore, course_location, 'static_tab', 'tabs', '.html')

lms/djangoapps/courseware/management/commands/dump_course_ids.py

+# pylint: disable=missing-docstring
+
+from optparse import make_option
+from textwrap import dedent
+
+from django.core.management.base import BaseCommand, CommandError
+
+from xmodule.modulestore.django import modulestore
+
+
+class Command(BaseCommand):
+    """
+    Simple command to dump the course_ids available to the lms.
+    """
+    help = dedent(__doc__).strip()
+    option_list = BaseCommand.option_list + (
+        make_option('--modulestore',
+                    action='store',
+                    default='default',
+                    help='Name of the modulestore to use'),
+    )
+
+    def handle(self, *args, **options):
+        output = []
+
+        try:
+            name = options['modulestore']
+            store = modulestore(name)
+        except KeyError:
+            raise CommandError("Unknown modulestore {}".format(name))
+
+        for course in store.get_courses():
+            course_id = course.location.course_id
+            output.append(course_id)
+
+        return '\n'.join(output) + '\n'

lms/djangoapps/courseware/management/commands/dump_course_structure.py

+"""
+A Django command that dumps the structure of a course as a JSON object.
+
+The resulting JSON object has one entry for each module in the course:
+
+{
+  "$module_url": {
+    "category": "$module_category",
+    "children": [$module_children_urls... ],
+    "metadata": {$module_metadata}
+  },
+
+  "$module_url": ....
+  ...
+}
+
+"""
+
+import json
+from optparse import make_option
+from textwrap import dedent
+
+from django.core.management.base import BaseCommand, CommandError
+
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+
+
+FILTER_LIST = ['xml_attributes', 'checklists']
+
+
+class Command(BaseCommand):
+    """
+    Write out to stdout a structural and metadata information for a
+    course as a JSON object
+    """
+    args = "<course_id>"
+    help = dedent(__doc__).strip()
+    option_list = BaseCommand.option_list + (
+        make_option('--modulestore',
+                    action='store',
+                    default='default',
+                    help='Name of the modulestore'),
+    )
+
+    def handle(self, *args, **options):
+        if len(args) != 1:
+            raise CommandError("course_id not specified")
+
+        # Get the modulestore
+
+        try:
+            name = options['modulestore']
+            store = modulestore(name)
+        except KeyError:
+            raise CommandError("Unknown modulestore {}".format(name))
+
+        # Get the course data
+
+        course_id = args[0]
+        course = store.get_course(course_id)
+        if course is None:
+            raise CommandError("Invalid course_id")
+
+        # Convert course data to dictionary and dump it as JSON to stdout
+
+        info = dump_module(course)
+
+        return json.dumps(info, indent=2, sort_keys=True)
+
+
+def dump_module(module, destination=None):
+    """
+    Add the module and all its children to the destination dictionary in
+    as a flat structure.
+    """
+
+    destination = destination if destination else {}
+
+    items = own_metadata(module).iteritems()
+    filtered_metadata = {k: v for k, v in items if k not in FILTER_LIST}
+
+    destination[module.location.url()] = {
+        'category': module.location.category,
+        'children': module.children if hasattr(module, 'children') else [],
+        'metadata': filtered_metadata
+    }
+
+    for child in module.get_children():
+        dump_module(child, destination)
+
+    return destination

lms/djangoapps/courseware/management/commands/export_course.py

+"""
+A Django command that exports a course to a tar.gz file.
+"""
+
+import shutil
+import tarfile
+from tempfile import mkdtemp
+from textwrap import dedent
+
+from path import path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.xml_exporter import export_to_xml
+
+
+class Command(BaseCommand):
+    """
+    Export a course to XML. The output is compressed as a tar.gz file
+
+    """
+    args = "<course_id> <output_filename>"
+    help = dedent(__doc__).strip()
+
+    def handle(self, *args, **options):
+        try:
+            course_id = args[0]
+            filename = args[1]
+        except IndexError:
+            raise CommandError("Insufficient arguments")
+
+        export_course_to_tarfile(course_id, filename)
+
+
+def export_course_to_tarfile(course_id, filename):
+    """Exports a course into a tar.gz file"""
+    tmp_dir = mkdtemp()
+    try:
+        course_dir = export_course_to_directory(course_id, tmp_dir)
+        compress_directory(course_dir, filename)
+    finally:
+        shutil.rmtree(tmp_dir)
+
+
+def export_course_to_directory(course_id, root_dir):
+    """Export course into a directory"""
+    store = modulestore()
+    course = store.get_course(course_id)
+    if course is None:
+        raise CommandError("Invalid course_id")
+
+    course_name = course.location.course_id.replace('/', '-')
+    export_to_xml(store, None, course.location, root_dir, course_name)
+
+    course_dir = path(root_dir) / course_name
+    return course_dir
+
+
+def compress_directory(directory, filename):
+    """Compress a directrory into a tar.gz file"""
+    mode = 'w:gz'
+    name = path(directory).name
+    with tarfile.open(filename, mode) as tar_file:
+        tar_file.add(directory, arcname=name)

lms/djangoapps/courseware/tests/test_commands.py

+"""Tests for Django management commands"""
+
+import json
+import shutil
+from StringIO import StringIO
+import tarfile
+from tempfile import mkdtemp
+
+from path import path
+
+from django.core.management import call_command
+from django.test.utils import override_settings
+
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.xml_importer import import_from_xml
+
+DATA_DIR = 'common/test/data/'
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class CommandTestCase(ModuleStoreTestCase):
+    """Parent class with helpers for testing management commands"""
+
+    def load_courses(self):
+        """Load test courses and return list of ids"""
+        store = modulestore()
+        import_from_xml(store, DATA_DIR, ['toy', 'simple'])
+        return [course.id for course in store.get_courses()]
+
+    def call_command(self, name, *args, **kwargs):
+        """Call management command and return output"""
+        out = StringIO()  # To Capture the output of the command
+        call_command(name, *args, stdout=out, **kwargs)
+        out.seek(0)
+        return out.read()
+
+
+class CommandsTestCase(CommandTestCase):
+    """Test case for management commands"""
+
+    def setUp(self):
+        self.loaded_courses = self.load_courses()
+
+    def test_dump_course_ids(self):
+        kwargs = {'modulestore': 'default'}
+        output = self.call_command('dump_course_ids', **kwargs)
+        dumped_courses = output.strip().split('\n')
+        self.assertEqual(self.loaded_courses, dumped_courses)
+
+    def test_dump_course_structure(self):
+        dumped_courses = self.call_command('dump_course_ids').split('\n')
+        self.assertEqual(self.loaded_courses, dumped_courses)
+
+    def test_dump_course_structure(self):
+        args = ['edX/simple/2012_Fall']
+        kwargs = {'modulestore': 'default'}
+        output = self.call_command('dump_course_structure', *args, **kwargs)
+
+        dump = json.loads(output)
+
+        # Check a few elements in the course dump
+
+        parent_id = 'i4x://edX/simple/chapter/Overview'
+        self.assertEqual(dump[parent_id]['category'], 'chapter')
+        self.assertEqual(len(dump[parent_id]['children']), 3)
+
+        child_id = dump[parent_id]['children'][1]
+        self.assertEqual(dump[child_id]['category'], 'videosequence')
+        self.assertEqual(len(dump[child_id]['children']), 2)
+
+        video_id = 'i4x://edX/simple/video/Welcome'
+        self.assertEqual(dump[video_id]['category'], 'video')
+        self.assertEqual(len(dump[video_id]['metadata']), 4)
+        self.assertIn('youtube_id_1_0', dump[video_id]['metadata'])
+
+        # Check if there is the right number of elements
+
+        self.assertEqual(len(dump), 16)
+
+    def test_export_course(self):
+        tmp_dir = path(mkdtemp())
+        filename = tmp_dir / 'test.tar.gz'
+        try:
+            self.run_export_course(filename)
+            with tarfile.open(filename) as tar_file:
+                self.check_export_file(tar_file)
+
+        finally:
+            shutil.rmtree(tmp_dir)
+
+    def run_export_course(self, filename):  # pylint: disable=missing-docstring
+        args = ['edX/simple/2012_Fall', filename]
+        kwargs = {'modulestore': 'default'}
+        self.call_command('export_course', *args, **kwargs)
+
+    def check_export_file(self, tar_file):  # pylint: disable=missing-docstring
+        names = tar_file.getnames()
+
+        # Check if some of the files are present.
+
+        # The rest is of the code should be covered by the tests for
+        # xmodule.modulestore.xml_exporter, used by the dump_course command
+
+        assert_in = self.assertIn
+        assert_in('edX-simple-2012_Fall', names)
+        assert_in('edX-simple-2012_Fall/policies/2012_Fall/policy.json', names)
+        assert_in('edX-simple-2012_Fall/html/toylab.html', names)
+        assert_in('edX-simple-2012_Fall/videosequence/A_simple_sequence.xml', names)
+        assert_in('edX-simple-2012_Fall/sequential/Lecture_2.xml', names)