Merge pull request #11490 from edx/jeskew/PLAT_736_fix_content_lib_export_import

Fix content library export with empty problems.

cms/djangoapps/contentstore/views/tests/test_import_export.py

@@ -2,6 +2,7 @@
 Unit tests for course import and export
 """
 import copy
+import ddt
 import json
 import logging
 import lxml
@@ -15,20 +16,24 @@ from uuid import uuid4
 from django.test.utils import override_settings
 from django.conf import settings
 from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml_exporter import export_library_to_xml
 from xmodule.modulestore.xml_importer import import_library_from_xml
-from xmodule.modulestore import LIBRARY_ROOT
+from xmodule.modulestore import LIBRARY_ROOT, ModuleStoreEnum
 from contentstore.utils import reverse_course_url
+from contentstore.tests.utils import CourseTestCase
 
 from xmodule.modulestore.tests.factories import ItemFactory, LibraryFactory
+from xmodule.modulestore.tests.utils import (
+    MongoContentstoreBuilder, SPLIT_MODULESTORE_SETUP, TEST_DATA_DIR
+)
+from opaque_keys.edx.locator import LibraryLocator
 
-from contentstore.tests.utils import CourseTestCase
 from openedx.core.lib.extract_tar import safetar_extractall
 from student import auth
 from student.roles import CourseInstructorRole, CourseStaffRole
 from models.settings.course_metadata import CourseMetadata
 from util import milestones_helpers
-from xmodule.modulestore.django import modulestore
 from milestones.tests.utils import MilestonesTestCaseMixin
 
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
@@ -123,6 +128,7 @@ class ImportEntranceExamTestCase(CourseTestCase, MilestonesTestCaseMixin):
         self.assertEquals(course.entrance_exam_minimum_score_pct, 0.7)
 
 
+@ddt.ddt
 @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class ImportTestCase(CourseTestCase):
     """
@@ -435,6 +441,73 @@ class ImportTestCase(CourseTestCase):
         self.assertIn(test_block3.url_name, children)
         self.assertIn(test_block4.url_name, children)
 
+    @ddt.data(
+        ModuleStoreEnum.Branch.draft_preferred,
+        ModuleStoreEnum.Branch.published_only,
+    )
+    def test_library_import_branch_settings(self, branch_setting):
+        """
+        Try importing a known good library archive under either branch setting.
+        The branch setting should have no effect on library import.
+        """
+        with self.store.branch_setting(branch_setting):
+            library = LibraryFactory.create(modulestore=self.store)
+            lib_key = library.location.library_key
+            extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
+            # the extract_dir needs to be passed as a relative dir to
+            # import_library_from_xml
+            extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
+
+            try:
+                with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar:
+                    safetar_extractall(tar, extract_dir)
+                import_library_from_xml(
+                    self.store,
+                    self.user.id,
+                    settings.GITHUB_REPO_ROOT,
+                    [extract_dir_relative / 'library'],
+                    load_error_modules=False,
+                    static_content_store=contentstore(),
+                    target_id=lib_key
+                )
+            finally:
+                shutil.rmtree(extract_dir)
+
+    @ddt.data(
+        ModuleStoreEnum.Branch.draft_preferred,
+        ModuleStoreEnum.Branch.published_only,
+    )
+    def test_library_import_branch_settings_again(self, branch_setting):
+        # Construct the contentstore for storing the import
+        with MongoContentstoreBuilder().build() as source_content:
+            # Construct the modulestore for storing the import (using the previously created contentstore)
+            with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
+                # Use the test branch setting.
+                with source_store.branch_setting(branch_setting):
+                    source_library_key = LibraryLocator(org='TestOrg', library='TestProbs')
+
+                    extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
+                    # the extract_dir needs to be passed as a relative dir to
+                    # import_library_from_xml
+                    extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
+
+                    try:
+                        with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar:
+                            safetar_extractall(tar, extract_dir)
+                        import_library_from_xml(
+                            source_store,
+                            self.user.id,
+                            settings.GITHUB_REPO_ROOT,
+                            [extract_dir_relative / 'library'],
+                            static_content_store=source_content,
+                            target_id=source_library_key,
+                            load_error_modules=False,
+                            raise_on_failure=True,
+                            create_if_not_present=True,
+                        )
+                    finally:
+                        shutil.rmtree(extract_dir)
+
 
 @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class ExportTestCase(CourseTestCase):
@@ -556,3 +629,59 @@ class ExportTestCase(CourseTestCase):
         )
 
         self.test_export_targz_urlparam()
+
+
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
+class TestLibraryImportExport(CourseTestCase):
+    """
+    Tests for importing content libraries from XML and exporting them to XML.
+    """
+    def setUp(self):
+        super(TestLibraryImportExport, self).setUp()
+        self.export_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.export_dir, ignore_errors=True)
+
+    def test_content_library_export_import(self):
+        library1 = LibraryFactory.create(modulestore=self.store)
+        source_library1_key = library1.location.library_key
+        library2 = LibraryFactory.create(modulestore=self.store)
+        source_library2_key = library2.location.library_key
+
+        import_library_from_xml(
+            self.store,
+            'test_user',
+            TEST_DATA_DIR,
+            ['library_empty_problem'],
+            static_content_store=contentstore(),
+            target_id=source_library1_key,
+            load_error_modules=False,
+            raise_on_failure=True,
+            create_if_not_present=True,
+        )
+
+        export_library_to_xml(
+            self.store,
+            contentstore(),
+            source_library1_key,
+            self.export_dir,
+            'exported_source_library',
+        )
+
+        source_library = self.store.get_library(source_library1_key)
+        self.assertEqual(source_library.url_name, 'library')
+
+        # Import the exported library into a different content library.
+        import_library_from_xml(
+            self.store,
+            'test_user',
+            self.export_dir,
+            ['exported_source_library'],
+            static_content_store=contentstore(),
+            target_id=source_library2_key,
+            load_error_modules=False,
+            raise_on_failure=True,
+            create_if_not_present=True,
+        )
+
+        # Compare the two content libraries for equality.
+        self.assertCoursesEqual(source_library1_key, source_library2_key)

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -68,6 +68,7 @@ from xblock.core import XBlock
 from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
 from xmodule.course_module import CourseSummary
 from xmodule.errortracker import null_error_tracker
+from opaque_keys.edx.keys import CourseKey
 from opaque_keys.edx.locator import (
     BlockUsageLocator, DefinitionLocator, CourseLocator, LibraryLocator, VersionTree, LocalId,
 )
@@ -1160,8 +1161,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
                 False - if we want only those items which are in the course tree. This would ensure no orphans are
                 fetched.
         """
-        if not isinstance(course_locator, CourseLocator) or course_locator.deprecated:
-            # The supplied CourseKey is of the wrong type, so it can't possibly be stored in this modulestore.
+        if not isinstance(course_locator, CourseKey) or course_locator.deprecated:
+            # The supplied courselike key is of the wrong type, so it can't possibly be stored in this modulestore.
             return []
 
         course = self._lookup_course(course_locator)

common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py

@@ -544,9 +544,11 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
                 block_id = self.DEFAULT_ROOT_LIBRARY_BLOCK_ID
             new_usage_key = course_key.make_usage_key(block_type, block_id)
 
-            # Only the course import process calls import_xblock(). If the branch setting is published_only,
-            # then the non-draft blocks are being imported.
-            if self.get_branch_setting() == ModuleStoreEnum.Branch.published_only:
+            # Both the course and library import process calls import_xblock().
+            # If importing a course -and- the branch setting is published_only,
+            # then the non-draft course blocks are being imported.
+            is_course = isinstance(course_key, CourseLocator)
+            if is_course and self.get_branch_setting() == ModuleStoreEnum.Branch.published_only:
                 # Override any existing drafts (PLAT-297, PLAT-299). This import/publish step removes
                 # any local changes during the course import.
                 draft_course = course_key.for_branch(ModuleStoreEnum.BranchName.draft)

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -607,6 +607,7 @@ class LibraryImportManager(ImportManager):
                     org=self.target_id.org,
                     library=self.target_id.library,
                     user_id=self.user_id,
+                    fields={"display_name": ""},
                 )
                 runtime = library.runtime
             except DuplicateCourseError:

common/test/data/library_empty_problem/library.xml

+<library xblock-family="xblock.v1" display_name="Test Problems" org="TestOrg" library="TestProbs100">
+  <problem url_name="afe9dbb29b724181944f56617c72b3e5"/>
+  <problem url_name="ba28f97e8f33414e9a5de0068508f7fa"/>
+</library>

common/test/data/library_empty_problem/policies/assets.json

+{}
\ No newline at end of file

common/test/data/library_empty_problem/problem/afe9dbb29b724181944f56617c72b3e5.xml

+<problem/>

common/test/data/library_empty_problem/problem/ba28f97e8f33414e9a5de0068508f7fa.xml

+<problem display_name="Multiple Choice" markdown="Multiple choice problems allow learners to select only one option. Learners can see all the options along with the problem text.&#10;&#10;When you add the problem, be sure to select Settings to specify a Display Name and other values that apply.&#10;&#10;You can use the following example problem as a model.&#10;&#10;&gt;&gt;Which of the following countries has the largest population?&lt;&lt;&#10;( ) Brazil {{ timely feedback -- explain why an almost correct answer is wrong }}&#10;( ) Germany&#10;(x) Indonesia&#10;( ) Russia&#10;&#10;[explanation]&#10;According to September 2014 estimates:&#10;The population of Indonesia is approximately 250 million.&#10;The population of Brazil  is approximately 200 million.&#10;The population of Russia is approximately 146 million.&#10;The population of Germany is approximately 81 million.&#10;[explanation]&#10;">
+<p>Multiple choice problems allow learners to select only one option.
+    Learners can see all the options along with the problem text.</p>
+<p>When you add the problem, be sure to select <strong>Settings</strong> 
+to specify a <strong>Display Name</strong> and other values that apply.</p>
+<p>You can use the following example problem as a model.</p>
+   <p>Which of the following countries has the largest population?</p>
+   <multiplechoiceresponse>
+    <choicegroup type="MultipleChoice">
+       <choice correct="false" name="brazil">Brazil
+         <choicehint>timely feedback -- explain why an almost correct answer is wrong</choicehint>
+       </choice>
+       <choice correct="false" name="germany">Germany</choice>
+       <choice correct="true" name="indonesia">Indonesia</choice>
+       <choice correct="false" name="russia">Russia</choice>
+    </choicegroup>
+   </multiplechoiceresponse>
+ <solution>
+   <div class="detailed-solution">
+     <p>Explanation</p>
+     <p>According to September 2014 estimates:</p>
+     <p>The population of Indonesia is approximately 250 million.</p>
+     <p>The population of Brazil  is approximately 200 million.</p>
+     <p>The population of Russia is approximately 146 million.</p>
+     <p>The population of Germany is approximately 81 million.</p>
+   </div>
+ </solution>
+ </problem>

requirements/edx/github.txt

@@ -70,7 +70,7 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
 git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
 
 # Our libraries:
-git+https://github.com/edx/XBlock.git@xblock-0.4.4#egg=XBlock==0.4.4
+git+https://github.com/edx/XBlock.git@xblock-0.4.5#egg=XBlock==0.4.5
 -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail
 -e git+https://github.com/edx/js-test-tool.git@v0.1.6#egg=js_test_tool
 -e git+https://github.com/edx/event-tracking.git@0.2.1#egg=event-tracking==0.2.1