Merge pull request #5940 from edx/jeskew/mixed_copy_assets_between_modulestores

One bok_choy video test is failing - but it passes locally and should be fine.

common/lib/xmodule/xmodule/assetstore/__init__.py

@@ -87,7 +87,7 @@ class AssetMetadata(object):
             else:
                 self.fields[attr] = val
 
-    def to_mongo(self):
+    def to_storable(self):
         """
         Converts metadata properties into a MongoDB-storable dict.
         """
@@ -106,7 +106,7 @@ class AssetMetadata(object):
         }
 
     @contract(asset_doc='dict|None')
-    def from_mongo(self, asset_doc):
+    def from_storable(self, asset_doc):
         """
         Fill in all metadata fields from a MongoDB document.
 

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -336,7 +336,7 @@ class ModuleStoreAssetInterface(object):
         info = asset_key.block_type
         mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
         all_assets = course_assets[info]
-        mdata.from_mongo(all_assets[asset_idx])
+        mdata.from_storable(all_assets[asset_idx])
         return mdata
 
     @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None',)
@@ -392,7 +392,7 @@ class ModuleStoreAssetInterface(object):
         for idx in xrange(start_idx, end_idx, step_incr):
             raw_asset = all_assets[idx]
             new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename']))
-            new_asset.from_mongo(raw_asset)
+            new_asset.from_storable(raw_asset)
             ret_assets.append(new_asset)
         return ret_assets
 

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -391,11 +391,21 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
             source_course_key (CourseKey): identifier of course to copy from
             dest_course_key (CourseKey): identifier of course to copy to
         """
-        # When implementing this in https://openedx.atlassian.net/browse/PLAT-78 , consider this:
-        #  Check the modulestores of both the source and dest course_keys. If in different modulestores,
-        #  export all asset data from one modulestore and import it into the dest one.
-        store = self._get_modulestore_for_courseid(source_course_key)
-        return store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id)
+        source_store = self._get_modulestore_for_courseid(source_course_key)
+        dest_store = self._get_modulestore_for_courseid(dest_course_key)
+        if source_store != dest_store:
+            with self.bulk_operations(dest_course_key):
+                # Get all the asset metadata in the source course.
+                all_assets = source_store.get_all_asset_metadata(source_course_key, 'asset')
+                # Store it all in the dest course.
+                for asset in all_assets:
+                    new_asset_key = dest_course_key.make_asset_key('asset', asset.asset_id.path)
+                    copied_asset = AssetMetadata(new_asset_key)
+                    copied_asset.from_storable(asset.to_storable())
+                    dest_store.save_asset_metadata(copied_asset, user_id)
+        else:
+            # Courses in the same modulestore can be handled by the modulestore itself.
+            source_store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id)
 
     @contract(asset_key='AssetKey', attr=str)
     def set_asset_metadata_attr(self, asset_key, attr, value, user_id):

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -1504,7 +1504,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
 
         # Translate metadata to Mongo format.
-        metadata_to_insert = asset_metadata.to_mongo()
+        metadata_to_insert = asset_metadata.to_storable()
         if asset_idx is None:
             # Add new metadata sorted into the list.
             all_assets.add(metadata_to_insert)
@@ -1562,11 +1562,11 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         # Form an AssetMetadata.
         all_assets = course_assets[asset_key.block_type]
         md = AssetMetadata(asset_key, asset_key.path)
-        md.from_mongo(all_assets[asset_idx])
+        md.from_storable(all_assets[asset_idx])
         md.update(attr_dict)
 
         # Generate a Mongo doc from the metadata and update the course asset info.
-        all_assets[asset_idx] = md.to_mongo()
+        all_assets[asset_idx] = md.to_storable()
 
         self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {asset_key.block_type: all_assets}})
 

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -2182,7 +2182,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         """
         The guts of saving a new or updated asset
         """
-        metadata_to_insert = asset_metadata.to_mongo()
+        metadata_to_insert = asset_metadata.to_storable()
 
         def _internal_method(all_assets, asset_idx):
             """
@@ -2218,11 +2218,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
 
             # Form an AssetMetadata.
             mdata = AssetMetadata(asset_key, asset_key.path)
-            mdata.from_mongo(all_assets[asset_idx])
+            mdata.from_storable(all_assets[asset_idx])
             mdata.update(attr_dict)
 
             # Generate a Mongo doc from the metadata and update the course asset info.
-            all_assets[asset_idx] = mdata.to_mongo()
+            all_assets[asset_idx] = mdata.to_storable()
             return all_assets
 
         self._update_course_assets(user_id, asset_key, _internal_method)

common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py

@@ -12,7 +12,8 @@ from xmodule.modulestore import ModuleStoreEnum
 
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
-    MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder
+    MIXED_MODULESTORE_BOTH_SETUP, MODULESTORE_SETUPS, MongoContentstoreBuilder,
+    XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder
 )
 
 
@@ -414,9 +415,9 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 store.get_all_asset_metadata(course_key, 'asset')
 
     @ddt.data(*MODULESTORE_SETUPS)
-    def test_copy_all_assets(self, storebuilder):
+    def test_copy_all_assets_same_modulestore(self, storebuilder):
         """
-        Create a course with assets and such, copy it all to another course, and check on it.
+        Create a course with assets, copy them all to another course in the same modulestore, and check on it.
         """
         with MongoContentstoreBuilder().build() as contentstore:
             with storebuilder.build(contentstore) as store:
@@ -433,3 +434,30 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 self.assertEquals(len(all_assets), 2)
                 self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
                 self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
+
+    @ddt.data(
+        ('mongo', 'split'),
+        ('split', 'mongo'),
+    )
+    @ddt.unpack
+    def test_copy_all_assets_cross_modulestore(self, from_store, to_store):
+        """
+        Create a course with assets, copy them all to another course in a different modulestore, and check on it.
+        """
+        mixed_builder = MIXED_MODULESTORE_BOTH_SETUP
+        with MongoContentstoreBuilder().build() as contentstore:
+            with mixed_builder.build(contentstore) as mixed_store:
+                with mixed_store.default_store(from_store):
+                    course1 = CourseFactory.create(modulestore=mixed_store)
+                with mixed_store.default_store(to_store):
+                    course2 = CourseFactory.create(modulestore=mixed_store)
+                self.setup_assets(course1.id, None, mixed_store)
+                self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
+                self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
+                mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102)
+                all_assets = mixed_store.get_all_asset_metadata(
+                    course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
+                )
+                self.assertEquals(len(all_assets), 2)
+                self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
+                self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')

common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py

@@ -265,13 +265,20 @@ class MongoContentstoreBuilder(object):
     def __repr__(self):
         return 'MongoContentstoreBuilder()'
 
-
-MODULESTORE_SETUPS = (
-    MongoModulestoreBuilder(),
-#     VersioningModulestoreBuilder(),  # FIXME LMS-11227
+MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
+    ('draft', MongoModulestoreBuilder()),
+    ('split', VersioningModulestoreBuilder())
+])
+MIXED_MODULESTORE_SETUPS = (
     MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())]),
     MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]),
 )
+DIRECT_MODULESTORE_SETUPS = (
+    MongoModulestoreBuilder(),
+#     VersioningModulestoreBuilder(),  # FUTUREDO: LMS-11227
+)
+MODULESTORE_SETUPS = DIRECT_MODULESTORE_SETUPS + MIXED_MODULESTORE_SETUPS
+
 CONTENTSTORE_SETUPS = (MongoContentstoreBuilder(),)
 COURSE_DATA_NAMES = (
     'toy',