Merge pull request #6180 from edx/jeskew/import_export_asset_xml_pt_2

Import/export modulestore-stored asset metadata as XML from/to exported course.

cms/djangoapps/contentstore/management/commands/import.py

@@ -44,7 +44,7 @@ class Command(BaseCommand):
             mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, course_dirs, load_error_modules=False,
             static_content_store=contentstore(), verbose=True,
             do_import_static=do_import_static,
-            create_new_course_if_not_present=True,
+            create_course_if_not_present=True,
         )
 
         for course in course_items:

cms/djangoapps/contentstore/tests/test_import.py

@@ -56,7 +56,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
             do_import_static=False,
             verbose=True,
             target_course_id=target_course_id,
-            create_new_course_if_not_present=create_new_course_if_not_present,
+            create_course_if_not_present=create_new_course_if_not_present,
         )
         course_id = module_store.make_course_key('edX', 'test_import_course', '2012_Fall')
         course = module_store.get_course(course_id)

cms/djangoapps/contentstore/views/tests/test_assets.py

@@ -204,7 +204,7 @@ class DownloadTestCase(AssetsTestCase):
 
     def test_metadata_found_in_modulestore(self):
         # Insert asset metadata into the modulestore (with no accompanying asset).
-        asset_key = self.course.id.make_asset_key(AssetMetadata.ASSET_TYPE, 'pic1.jpg')
+        asset_key = self.course.id.make_asset_key(AssetMetadata.GENERAL_ASSET_TYPE, 'pic1.jpg')
         asset_md = AssetMetadata(asset_key, {
             'internal_name': 'EKMND332DDBK',
             'basename': 'pix/archive',

common/lib/xmodule/xmodule/assetstore/__init__.py

@@ -7,11 +7,12 @@ import dateutil.parser
 import pytz
 import json
 from contracts import contract, new_contract
-from opaque_keys.edx.keys import AssetKey
+from opaque_keys.edx.keys import CourseKey, AssetKey
 from lxml import etree
 
 
 new_contract('AssetKey', AssetKey)
+new_contract('CourseKey', CourseKey)
 new_contract('datetime', datetime)
 new_contract('basestring', basestring)
 new_contract('AssetElement', lambda x: isinstance(x, etree._Element) and x.tag == "asset")  # pylint: disable=protected-access, no-member
@@ -28,10 +29,25 @@ class AssetMetadata(object):
     EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_by_email', 'edited_on']
     CREATE_INFO_ATTRS = ['created_by', 'created_by_email', 'created_on']
     ATTRS_ALLOWED_TO_UPDATE = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS
-    ALL_ATTRS = ['asset_id'] + ATTRS_ALLOWED_TO_UPDATE + CREATE_INFO_ATTRS
+    ASSET_TYPE_ATTR = 'type'
+    ASSET_BASENAME_ATTR = 'filename'
+    XML_ONLY_ATTRS = [ASSET_TYPE_ATTR, ASSET_BASENAME_ATTR]
+    XML_ATTRS = XML_ONLY_ATTRS + ATTRS_ALLOWED_TO_UPDATE + CREATE_INFO_ATTRS
 
-    # Default type for AssetMetadata objects. A constant for convenience.
-    ASSET_TYPE = 'asset'
+    # Type for assets uploaded by a course author in Studio.
+    GENERAL_ASSET_TYPE = 'asset'
+
+    # Asset section XML tag for asset metadata as XML.
+    ALL_ASSETS_XML_TAG = 'assets'
+
+    # Individual asset XML tag for asset metadata as XML.
+    ASSET_XML_TAG = 'asset'
+
+    # Top-level directory name in exported course XML which holds asset metadata.
+    EXPORTED_ASSET_DIR = 'assets'
+
+    # Filename of all asset metadata exported as XML.
+    EXPORTED_ASSET_FILENAME = 'assets.xml'
 
     @contract(asset_id='AssetKey',
               pathname='basestring|None', internal_name='basestring|None',
@@ -118,6 +134,7 @@ class AssetMetadata(object):
         """
         return {
             'filename': self.asset_id.path,
+            'asset_type': self.asset_id.asset_type,
             'pathname': self.pathname,
             'internal_name': self.internal_name,
             'locked': self.locked,
@@ -169,11 +186,11 @@ class AssetMetadata(object):
         for child in node:
             qname = etree.QName(child)
             tag = qname.localname
-            if tag in self.ALL_ATTRS:
+            if tag in self.XML_ATTRS:
                 value = child.text
-                if tag == 'asset_id':
-                    # Locator.
-                    value = AssetKey.from_string(value)
+                if tag in self.XML_ONLY_ATTRS:
+                    # An AssetLocator is constructed separately from these parts.
+                    continue
                 elif tag == 'locked':
                     # Boolean.
                     value = True if value == "true" else False
@@ -197,13 +214,23 @@ class AssetMetadata(object):
         Add the asset data as XML to the passed-in node.
         The node should already be created as a top-level "asset" element.
         """
-        for attr in self.ALL_ATTRS:
+        for attr in self.XML_ATTRS:
             child = etree.SubElement(node, attr)
-            value = getattr(self, attr)
+            # Get the value.
+            if attr == self.ASSET_TYPE_ATTR:
+                value = self.asset_id.asset_type
+            elif attr == self.ASSET_BASENAME_ATTR:
+                value = self.asset_id.path
+            else:
+                value = getattr(self, attr)
+
+            # Format the value.
             if isinstance(value, bool):
                 value = "true" if value else "false"
             elif isinstance(value, datetime):
                 value = value.isoformat()
+            elif isinstance(value, dict):
+                value = json.dumps(value)
             else:
                 value = unicode(value)
             child.text = value
@@ -218,3 +245,57 @@ class AssetMetadata(object):
         for asset in assets:
             asset_node = etree.SubElement(node, "asset")
             asset.to_xml(asset_node)
+
+
+class CourseAssetsFromStorage(object):
+    """
+    Wrapper class for asset metadata lists returned from modulestore storage.
+    """
+    @contract(course_id='CourseKey', asset_md=dict)
+    def __init__(self, course_id, doc_id, asset_md):
+        """
+        Params:
+            course_id: Course ID for which the asset metadata is stored.
+            doc_id: ObjectId of MongoDB document
+            asset_md: Dict with asset types as keys and lists of storable asset metadata as values.
+        """
+        self.course_id = course_id
+        self._doc_id = doc_id
+        self.asset_md = asset_md
+
+    @property
+    def doc_id(self):
+        """
+        Returns the ID associated with the MongoDB document which stores these course assets.
+        """
+        return self._doc_id
+
+    def setdefault(self, item, default=None):
+        """
+        Provides dict-equivalent setdefault functionality.
+        """
+        return self.asset_md.setdefault(item, default)
+
+    def __getitem__(self, item):
+        return self.asset_md[item]
+
+    def __delitem__(self, item):
+        del self.asset_md[item]
+
+    def __len__(self):
+        return len(self.asset_md)
+
+    def __setitem__(self, key, value):
+        self.asset_md[key] = value
+
+    def get(self, item, default=None):
+        """
+        Provides dict-equivalent get functionality.
+        """
+        return self.asset_md.get(item, default)
+
+    def iteritems(self):
+        """
+        Iterates over the items of the asset dict.
+        """
+        return self.asset_md.iteritems()

common/lib/xmodule/xmodule/assetstore/tests/assets.xsd

@@ -27,7 +27,8 @@
 
 <xs:complexType name="assetType">
   <xs:all>
-    <xs:element name="asset_id" type="stringType"/>
+    <xs:element name="type" type="stringType"/>
+    <xs:element name="filename" type="stringType"/>
     <xs:element name="contenttype" type="stringType"/>
     <xs:element name="pathname" type="stringType"/>
     <xs:element name="internal_name" type="stringType"/>

common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py

@@ -30,7 +30,6 @@ class TestAssetXml(unittest.TestCase):
             self.course_assets.append(asset_md)
 
         # Read in the XML schema definition and make a validator.
-        #xsd_path = path(__file__).abspath().dirname() / xsd_filename
         xsd_path = path(__file__).realpath().parent / xsd_filename
         with open(xsd_path, 'r') as f:
             schema_root = etree.XML(f.read())
@@ -51,7 +50,9 @@ class TestAssetXml(unittest.TestCase):
         new_asset_md = AssetMetadata(new_asset_key)
         new_asset_md.from_xml(asset)
         # Compare asset_md to new_asset_md.
-        for attr in AssetMetadata.ALL_ATTRS:
+        for attr in AssetMetadata.XML_ATTRS:
+            if attr in AssetMetadata.XML_ONLY_ATTRS:
+                continue
             orig_value = getattr(asset_md, attr)
             new_value = getattr(new_asset_md, attr)
             self.assertEqual(orig_value, new_value)

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -36,6 +36,7 @@ log = logging.getLogger('edx.modulestore')
 new_contract('CourseKey', CourseKey)
 new_contract('AssetKey', AssetKey)
 new_contract('AssetMetadata', AssetMetadata)
+new_contract('SortedListWithKey', SortedListWithKey)
 
 
 class ModuleStoreEnum(object):
@@ -279,13 +280,22 @@ class ModuleStoreAssetInterface(object):
     """
     The methods for accessing assets and their metadata
     """
-    def _find_course_assets(self, course_key):
+    @contract(asset_list='SortedListWithKey', asset_id='AssetKey')
+    def _find_asset_in_list(self, asset_list, asset_id):
         """
-        Finds the persisted repr of the asset metadata not converted to AssetMetadata yet.
-        Returns the container holding a dict indexed by asset block_type whose values are a list
-        of raw metadata documents
+        Given a asset list that's a SortedListWithKey, find the index of a particular asset.
+        Returns: Index of asset, if found. None if not found.
         """
-        raise NotImplementedError()
+        # See if this asset already exists by checking the external_filename.
+        # Studio doesn't currently support using multiple course assets with the same filename.
+        # So use the filename as the unique identifier.
+        idx = None
+        idx_left = asset_list.bisect_left({'filename': asset_id.path})
+        idx_right = asset_list.bisect_right({'filename': asset_id.path})
+        if idx_left != idx_right:
+            # Asset was found in the list.
+            idx = idx_left
+        return idx
 
     def _find_course_asset(self, asset_key):
         """
@@ -296,26 +306,16 @@ class ModuleStoreAssetInterface(object):
             asset_key (AssetKey): what to look for
 
         Returns:
-            AssetMetadata[] for all assets of the given asset_key's type, & the index of asset in list
-            (None if asset does not exist)
+            Tuple of:
+            - AssetMetadata[] for all assets of the given asset_key's type
+            - the index of asset in list (None if asset does not exist)
         """
         course_assets = self._find_course_assets(asset_key.course_key)
-        if course_assets is None:
-            return None, None
-
         all_assets = SortedListWithKey([], key=itemgetter('filename'))
         # Assets should be pre-sorted, so add them efficiently without sorting.
         # extend() will raise a ValueError if the passed-in list is not sorted.
         all_assets.extend(course_assets.setdefault(asset_key.block_type, []))
-        # See if this asset already exists by checking the external_filename.
-        # Studio doesn't currently support using multiple course assets with the same filename.
-        # So use the filename as the unique identifier.
-        idx = None
-        idx_left = all_assets.bisect_left({'filename': asset_key.block_id})
-        idx_right = all_assets.bisect_right({'filename': asset_key.block_id})
-        if idx_left != idx_right:
-            # Asset was found in the list.
-            idx = idx_left
+        idx = self._find_asset_in_list(all_assets, asset_key)
 
         return course_assets, idx
 
@@ -340,14 +340,17 @@ class ModuleStoreAssetInterface(object):
         mdata.from_storable(all_assets[asset_idx])
         return mdata
 
-    @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None',)
+    @contract(
+        course_key='CourseKey', asset_type='None | basestring',
+        start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None'
+    )
     def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
         """
         Returns a list of asset metadata for all assets of the given asset_type in the course.
 
         Args:
             course_key (CourseKey): course identifier
-            asset_type (str): the block_type of the assets to return
+            asset_type (str): the block_type of the assets to return. If None, return assets of all types.
             start (int): optional - start at this asset number. Zero-based!
             maxresults (int): optional - return at most this many, -1 means no limit
             sort (array): optional - None means no sort
@@ -359,10 +362,6 @@ class ModuleStoreAssetInterface(object):
             List of AssetMetadata objects.
         """
         course_assets = self._find_course_assets(course_key)
-        if course_assets is None:
-            # If no course assets are found, return None instead of empty list
-            # to distinguish zero assets from "not able to retrieve assets".
-            return None
 
         # Determine the proper sort - with defaults of ('displayname', SortOrder.ascending).
         key_func = itemgetter('filename')
@@ -373,7 +372,14 @@ class ModuleStoreAssetInterface(object):
             if sort[1] == ModuleStoreEnum.SortOrder.descending:
                 sort_order = ModuleStoreEnum.SortOrder.descending
 
-        all_assets = SortedListWithKey(course_assets.get(asset_type, []), key=key_func)
+        if asset_type is None:
+            # Add assets of all types to the sorted list.
+            all_assets = SortedListWithKey([], key=key_func)
+            for asset_type, val in course_assets.iteritems():
+                all_assets.update(val)
+        else:
+            # Add assets of a single type to the sorted list.
+            all_assets = SortedListWithKey(course_assets.get(asset_type, []), key=key_func)
         num_assets = len(all_assets)
 
         start_idx = start
@@ -392,7 +398,8 @@ class ModuleStoreAssetInterface(object):
         ret_assets = []
         for idx in xrange(start_idx, end_idx, step_incr):
             raw_asset = all_assets[idx]
-            new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename']))
+            asset_key = course_key.make_asset_key(raw_asset['asset_type'], raw_asset['filename'])
+            new_asset = AssetMetadata(asset_key)
             new_asset.from_storable(raw_asset)
             ret_assets.append(new_asset)
         return ret_assets
@@ -403,13 +410,29 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
     The write operations for assets and asset metadata
     """
     @contract(asset_metadata='AssetMetadata')
-    def save_asset_metadata(self, asset_metadata, user_id):
+    def save_asset_metadata(self, asset_metadata, user_id, import_only):
         """
         Saves the asset metadata for a particular course's asset.
 
         Arguments:
-            asset_metadata (AssetMetadata): data about the course asset data (must have asset_id
-            set)
+            asset_metadata (AssetMetadata): data about the course asset data
+            user_id (int): user ID saving the asset metadata
+            import_only (bool): True if importing without editing, False if editing
+
+        Returns:
+            True if metadata save was successful, else False
+        """
+        raise NotImplementedError()
+
+    @contract(asset_metadata_list='list(AssetMetadata)')
+    def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only):
+        """
+        Saves a list of asset metadata for a particular course's asset.
+
+        Arguments:
+            asset_metadata (AssetMetadata): data about the course asset data
+            user_id (int): user ID saving the asset metadata
+            import_only (bool): True if importing without editing, False if editing
 
         Returns:
             True if metadata save was successful, else False
@@ -437,6 +460,7 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
             asset_key (AssetKey): asset identifier
             attr (str): which attribute to set
             value: the value to set it to (any type pymongo accepts such as datetime, number, string)
+            user_id (int): user ID saving the asset metadata
 
         Raises:
             ItemNotFoundError if no such item exists
@@ -455,6 +479,7 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
         Arguments:
             source_course_key (CourseKey): identifier of course to copy from
             dest_course_key (CourseKey): identifier of course to copy to
+            user_id (int): user ID copying the asset metadata
         """
         pass
 

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -351,17 +351,40 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         store = self._get_modulestore_for_courseid(course_key)
         return store.delete_course(course_key, user_id)
 
-    @contract(asset_metadata='AssetMetadata')
-    def save_asset_metadata(self, asset_metadata, user_id):
+    @contract(asset_metadata='AssetMetadata', user_id=int, import_only=bool)
+    def save_asset_metadata(self, asset_metadata, user_id, import_only=False):
         """
         Saves the asset metadata for a particular course's asset.
 
         Args:
-        course_key (CourseKey): course identifier
         asset_metadata (AssetMetadata): data about the course asset data
+        user_id (int): user ID saving the asset metadata
+        import_only (bool): True if importing without editing, False if editing
+
+        Returns:
+            True if info save was successful, else False
         """
         store = self._get_modulestore_for_courseid(asset_metadata.asset_id.course_key)
-        return store.save_asset_metadata(asset_metadata, user_id)
+        return store.save_asset_metadata(asset_metadata, user_id, import_only)
+
+    @contract(asset_metadata_list='list(AssetMetadata)', user_id=int, import_only=bool)
+    def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=False):
+        """
+        Saves the asset metadata for each asset in a list of asset metadata.
+        Optimizes the saving of many assets.
+
+        Args:
+        asset_metadata_list (list(AssetMetadata)): list of data about several course assets
+        user_id (int): user ID saving the asset metadata
+        import_only (bool): True if importing without editing, False if editing
+
+        Returns:
+            True if info save was successful, else False
+        """
+        if len(asset_metadata_list) == 0:
+            return True
+        store = self._get_modulestore_for_courseid(asset_metadata_list[0].asset_id.course_key)
+        return store.save_asset_metadata_list(asset_metadata_list, user_id, import_only)
 
     @strip_key
     @contract(asset_key='AssetKey')
@@ -379,7 +402,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         return store.find_asset_metadata(asset_key, **kwargs)
 
     @strip_key
-    @contract(course_key='CourseKey', start=int, maxresults=int, sort='tuple|None')
+    @contract(course_key='CourseKey', asset_type='None | basestring', start=int, maxresults=int, sort='tuple|None')
     def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
         """
         Returns a list of static assets for a course.
@@ -387,6 +410,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
 
         Args:
             course_key (CourseKey): course identifier
+            asset_type (str): type of asset, such as 'asset', 'video', etc. If None, return assets of all types.
             start (int): optional - start at this asset number
             maxresults (int): optional - return at most this many, -1 means no limit
             sort (array): optional - None means no sort
@@ -395,12 +419,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
                 sort_order - one of 'ascending' or 'descending'
 
         Returns:
-            List of asset data dictionaries, which have the following keys:
-                asset_key (AssetKey): asset identifier
-                displayname: The human-readable name of the asset
-                uploadDate (datetime.datetime): The date and time that the file was uploaded
-                contentType: The mimetype string of the asset
-                md5: An md5 hash of the asset content
+            List of AssetMetadata objects.
         """
         store = self._get_modulestore_for_courseid(course_key)
         return store.get_all_asset_metadata(course_key, asset_type, start, maxresults, sort, **kwargs)

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -40,7 +40,7 @@ from xblock.exceptions import InvalidScopeError
 from xblock.fields import Scope, ScopeIds, Reference, ReferenceList, ReferenceValueDict
 from xblock.runtime import KvsFieldData
 
-from xmodule.assetstore import AssetMetadata
+from xmodule.assetstore import AssetMetadata, CourseAssetsFromStorage
 from xmodule.error_module import ErrorDescriptor
 from xmodule.errortracker import null_error_tracker, exc_info_to_str
 from xmodule.exceptions import HeartbeatFailure
@@ -1473,7 +1473,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
             course_key (CourseKey): course identifier
 
         Returns:
-            Asset info for the course
+            CourseAssetsFromStorage object, wrapping the relevant Mongo doc. If asset metadata
+            exists, other keys will be the other asset types with values as lists of asset metadata.
         """
         # Using the course_key, find or insert the course asset metadata document.
         # A single document exists per course to store the course asset metadata.
@@ -1482,25 +1483,27 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
             {'course_id': unicode(course_key)},
         )
 
-        # Pass back 'assets' dict but add the '_id' key to it for document update purposes.
+        doc_id = None if course_assets is None else course_assets['_id']
         if course_assets is None:
-            # Not found, so create.
-            course_assets = {'course_id': unicode(course_key), 'assets': {}}
-            course_assets['assets']['_id'] = self.asset_collection.insert(course_assets)
+            # Check to see if the course is created in the course collection.
+            if self.get_course(course_key) is None:
+                raise ItemNotFoundError(course_key)
+            else:
+                # Course exists, so create matching assets document.
+                course_assets = {'course_id': unicode(course_key), 'assets': {}}
+                doc_id = self.asset_collection.insert(course_assets)
         elif isinstance(course_assets['assets'], list):
             # This record is in the old course assets format.
             # Ensure that no data exists before updating the format.
             assert(len(course_assets['assets']) == 0)
             # Update the format to a dict.
             self.asset_collection.update(
-                {'_id': course_assets['_id']},
+                {'_id': doc_id},
                 {'$set': {'assets': {}}}
             )
-            course_assets['assets'] = {'_id': course_assets['_id']}
-        else:
-            course_assets['assets']['_id'] = course_assets['_id']
 
-        return course_assets['assets']
+        # Pass back wrapped 'assets' dict with the '_id' key added to it for document update purposes.
+        return CourseAssetsFromStorage(course_key, doc_id, course_assets['assets'])
 
     def _make_mongo_asset_key(self, asset_type):
         """
@@ -1508,39 +1511,82 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         """
         return 'assets.{}'.format(asset_type)
 
-    @contract(asset_metadata='AssetMetadata')
-    def save_asset_metadata(self, asset_metadata, user_id):
+    @contract(asset_metadata_list='list(AssetMetadata)', user_id=int)
+    def _save_asset_metadata_list(self, asset_metadata_list, user_id, import_only):
+        """
+        Internal; saves the info for a particular course's asset.
+
+        Arguments:
+            asset_metadata_list (list(AssetMetadata)): list of data about several course assets
+            user_id (int): user ID saving the asset metadata
+            import_only (bool): True if edited_on/by data should remain unchanged.
+        """
+        course_assets = self._find_course_assets(asset_metadata_list[0].asset_id.course_key)
+
+        changed_asset_types = set()
+        assets_by_type = {}
+        for asset_md in asset_metadata_list:
+            asset_type = asset_md.asset_id.asset_type
+            changed_asset_types.add(asset_type)
+            # Lazily create a sorted list if not already created.
+            if asset_type not in assets_by_type:
+                assets_by_type[asset_type] = SortedListWithKey(course_assets.get(asset_type, []), key=itemgetter('filename'))
+            all_assets = assets_by_type[asset_type]
+            asset_idx = self._find_asset_in_list(assets_by_type[asset_type], asset_md.asset_id)
+            if not import_only:
+                asset_md.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
+
+            # Translate metadata to Mongo format.
+            metadata_to_insert = asset_md.to_storable()
+            if asset_idx is None:
+                # Add new metadata sorted into the list.
+                all_assets.add(metadata_to_insert)
+            else:
+                # Replace existing metadata.
+                all_assets[asset_idx] = metadata_to_insert
+
+        # Build an update set with potentially multiple embedded fields.
+        updates_by_type = {}
+        for asset_type in changed_asset_types:
+            updates_by_type[self._make_mongo_asset_key(asset_type)] = assets_by_type[asset_type].as_list()
+
+        # Update the document.
+        self.asset_collection.update(
+            {'_id': course_assets.doc_id},
+            {'$set': updates_by_type}
+        )
+        return True
+
+    @contract(asset_metadata='AssetMetadata', user_id=int)
+    def save_asset_metadata(self, asset_metadata, user_id, import_only=False):
         """
         Saves the info for a particular course's asset.
 
         Arguments:
-            asset_metadata (AssetMetadata): data about the course asset
+            asset_metadata (AssetMetadata): data about the course asset data
+            user_id (int): user ID saving the asset metadata
+            import_only (bool): True if importing without editing, False if editing
 
         Returns:
             True if info save was successful, else False
         """
-        course_assets, asset_idx = self._find_course_asset(asset_metadata.asset_id)
-        all_assets = SortedListWithKey([], key=itemgetter('filename'))
-        # Assets should be pre-sorted, so add them efficiently without sorting.
-        # extend() will raise a ValueError if the passed-in list is not sorted.
-        all_assets.extend(course_assets[asset_metadata.asset_id.block_type])
-        asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
+        return self._save_asset_metadata_list([asset_metadata, ], user_id, import_only)
 
-        # Translate metadata to Mongo format.
-        metadata_to_insert = asset_metadata.to_storable()
-        if asset_idx is None:
-            # Add new metadata sorted into the list.
-            all_assets.add(metadata_to_insert)
-        else:
-            # Replace existing metadata.
-            all_assets[asset_idx] = metadata_to_insert
+    @contract(asset_metadata_list='list(AssetMetadata)', user_id=int)
+    def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=False):
+        """
+        Saves the asset metadata for each asset in a list of asset metadata.
+        Optimizes the saving of many assets.
 
-        # Update the document.
-        self.asset_collection.update(
-            {'_id': course_assets['_id']},
-            {'$set': {self._make_mongo_asset_key(asset_metadata.asset_id.block_type): all_assets.as_list()}}
-        )
-        return True
+        Args:
+            asset_metadata (AssetMetadata): data about the course asset data
+            user_id (int): user ID saving the asset metadata
+            import_only (bool): True if importing without editing, False if editing
+
+        Returns:
+            True if info save was successful, else False
+        """
+        return self._save_asset_metadata_list(asset_metadata_list, user_id, import_only)
 
     @contract(source_course_key='CourseKey', dest_course_key='CourseKey')
     def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
@@ -1554,9 +1600,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
             dest_course_key (CourseKey): identifier of course to copy to
         """
         source_assets = self._find_course_assets(source_course_key)
-        dest_assets = source_assets.copy()
-        del dest_assets['_id']
-        dest_assets = {'assets': dest_assets}
+        dest_assets = {'assets': source_assets.asset_md.copy()}
         dest_assets['course_id'] = unicode(dest_course_key)
 
         self.asset_collection.remove({'course_id': unicode(dest_course_key)})
@@ -1581,7 +1625,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
             raise ItemNotFoundError(asset_key)
 
         # Form an AssetMetadata.
-        all_assets = course_assets[asset_key.block_type]
+        all_assets = course_assets[asset_key.asset_type]
         md = AssetMetadata(asset_key, asset_key.path)
         md.from_storable(all_assets[asset_idx])
         md.update(attr_dict)
@@ -1590,8 +1634,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         all_assets[asset_idx] = md.to_storable()
 
         self.asset_collection.update(
-            {'_id': course_assets['_id']},
-            {"$set": {self._make_mongo_asset_key(asset_key.block_type): all_assets}}
+            {'_id': course_assets.doc_id},
+            {"$set": {self._make_mongo_asset_key(asset_key.asset_type): all_assets}}
         )
 
     @contract(asset_key='AssetKey')
@@ -1609,13 +1653,13 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         if asset_idx is None:
             return 0
 
-        all_asset_info = course_assets[asset_key.block_type]
+        all_asset_info = course_assets[asset_key.asset_type]
         all_asset_info.pop(asset_idx)
 
         # Update the document.
         self.asset_collection.update(
-            {'_id': course_assets['_id']},
-            {'$set': {self._make_mongo_asset_key(asset_key.block_type): all_asset_info}}
+            {'_id': course_assets.doc_id},
+            {'$set': {self._make_mongo_asset_key(asset_key.asset_type): all_asset_info}}
         )
         return 1
 
@@ -1630,8 +1674,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         """
         # Using the course_id, find the course asset metadata document.
         # A single document exists per course to store the course asset metadata.
-        course_assets = self._find_course_assets(course_key)
-        self.asset_collection.remove(course_assets['_id'])
+        try:
+            course_assets = self._find_course_assets(course_key)
+            self.asset_collection.remove(course_assets.doc_id)
+        except ItemNotFoundError:
+            # When deleting asset metadata, if a course's asset metadata is not present, no big deal.
+            pass
 
     def heartbeat(self):
         """

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -2225,7 +2225,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         # So use the filename as the unique identifier.
         accessor = asset_key.block_type
         for idx, asset in enumerate(structure.setdefault(accessor, [])):
-            if asset['filename'] == asset_key.block_id:
+            if asset['filename'] == asset_key.path:
                 return idx
         return None
 
@@ -2256,7 +2256,45 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
                 # update the index entry if appropriate
                 self._update_head(asset_key.course_key, index_entry, asset_key.branch, new_structure['_id'])
 
-    def save_asset_metadata(self, asset_metadata, user_id):
+    def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=False):
+        """
+        A wrapper for functions wanting to manipulate assets. Gets and versions the structure,
+        passes the mutable array for all asset types as well as the idx to the function for it to
+        update, then persists the changed data back into the course.
+
+        The update function can raise an exception if it doesn't want to actually do the commit. The
+        surrounding method probably should catch that exception.
+        """
+        asset_key = asset_metadata_list[0].asset_id
+        course_key = asset_key.course_key
+
+        with self.bulk_operations(course_key):
+            original_structure = self._lookup_course(course_key).structure
+            index_entry = self._get_index_if_valid(course_key)
+            new_structure = self.version_structure(course_key, original_structure, user_id)
+
+            # Add all asset metadata to the structure at once.
+            for asset_metadata in asset_metadata_list:
+                metadata_to_insert = asset_metadata.to_storable()
+                asset_md_key = asset_metadata.asset_id
+
+                asset_idx = self._lookup_course_asset(new_structure.setdefault('assets', {}), asset_md_key)
+
+                all_assets = new_structure['assets'][asset_md_key.asset_type]
+                if asset_idx is None:
+                    all_assets.append(metadata_to_insert)
+                else:
+                    all_assets[asset_idx] = metadata_to_insert
+                new_structure['assets'][asset_md_key.asset_type] = all_assets
+
+            # update index if appropriate and structures
+            self.update_structure(course_key, new_structure)
+
+            if index_entry is not None:
+                # update the index entry if appropriate
+                self._update_head(course_key, index_entry, asset_key.branch, new_structure['_id'])
+
+    def save_asset_metadata(self, asset_metadata, user_id, import_only=False):
         """
         The guts of saving a new or updated asset
         """
@@ -2347,7 +2385,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             index_entry = self._get_index_if_valid(dest_course_key)
             new_structure = self.version_structure(dest_course_key, original_structure, user_id)
 
-            new_structure['assets'] = source_structure.get('assets', [])
+            new_structure['assets'] = source_structure.get('assets', {})
             new_structure['thumbnails'] = source_structure.get('thumbnails', [])
 
             # update index if appropriate and structures

common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py

@@ -477,6 +477,17 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
             update_function
         )
 
+    def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=False):
+        """
+        Updates both the published and draft branches
+        """
+        asset_key = asset_metadata_list[0].asset_id
+        asset_metadata_list[0].asset_id = self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only)
+        # if one call gets an exception, don't do the other call but pass on the exception
+        super(DraftVersioningModuleStore, self).save_asset_metadata_list(asset_metadata_list, user_id, import_only)
+        asset_metadata_list[0].asset_id = self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only)
+        super(DraftVersioningModuleStore, self).save_asset_metadata_list(asset_metadata_list, user_id, import_only)
+
     def _find_course_asset(self, asset_key):
         return super(DraftVersioningModuleStore, self)._find_course_asset(
             self._map_revision_to_branch(asset_key)

common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py

@@ -8,8 +8,10 @@ from nose.plugins.attrib import attr
 import pytz
 import unittest
 
+from opaque_keys.edx.keys import CourseKey
 from xmodule.assetstore import AssetMetadata
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
     MIXED_MODULESTORE_BOTH_SETUP, MODULESTORE_SETUPS, MongoContentstoreBuilder,
@@ -26,7 +28,7 @@ class AssetStoreTestData(object):
     user_email = "me@example.com"
 
     asset_fields = (
-        'filename', 'internal_name', 'pathname', 'locked',
+        AssetMetadata.ASSET_BASENAME_ATTR, 'internal_name', 'pathname', 'locked',
         'edited_by', 'edited_by_email', 'edited_on', 'created_by', 'created_by_email', 'created_on',
         'curr_version', 'prev_version'
     )
@@ -117,11 +119,8 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
         with MongoContentstoreBuilder().build() as contentstore:
             with storebuilder.build(contentstore) as store:
                 course = CourseFactory.create(modulestore=store)
-
                 asset_filename = 'burnside.jpg'
                 new_asset_loc = course.id.make_asset_key('asset', asset_filename)
-                # Confirm that the asset's metadata is not present.
-                self.assertIsNone(store.find_asset_metadata(new_asset_loc))
                 # Save the asset's metadata.
                 new_asset_md = self._make_asset_metadata(new_asset_loc)
                 store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
@@ -134,7 +133,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
     @ddt.data(*MODULESTORE_SETUPS)
     def test_delete(self, storebuilder):
         """
-        Delete non_existent and existent metadata
+        Delete non-existent and existent metadata
         """
         with MongoContentstoreBuilder().build() as contentstore:
             with storebuilder.build(contentstore) as store:
@@ -152,7 +151,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
     @ddt.data(*MODULESTORE_SETUPS)
     def test_find_non_existing_assets(self, storebuilder):
         """
-        Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
+        Find a non-existent asset in an existing course.
         """
         with MongoContentstoreBuilder().build() as contentstore:
             with storebuilder.build(contentstore) as store:
@@ -163,9 +162,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 self.assertIsNone(asset_md)
 
     @ddt.data(*MODULESTORE_SETUPS)
+    def test_get_all_non_existing_assets(self, storebuilder):
+        """
+        Get all assets in an existing course when no assets exist.
+        """
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course = CourseFactory.create(modulestore=store)
+                # Find existing asset metadata.
+                asset_md = store.get_all_asset_metadata(course.id, 'asset')
+                self.assertEquals(asset_md, [])
+
+    @ddt.data(*MODULESTORE_SETUPS)
+    def test_find_assets_in_non_existent_course(self, storebuilder):
+        """
+        Find asset metadata from a non-existent course.
+        """
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course = CourseFactory.create(modulestore=store)
+                fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format(
+                    course.id.org, course.id.course, course.id.run
+                ))
+                new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg')
+                # Find asset metadata from non-existent course.
+                with self.assertRaises(ItemNotFoundError):
+                    store.find_asset_metadata(new_asset_loc)
+                with self.assertRaises(ItemNotFoundError):
+                    store.get_all_asset_metadata(fake_course_id, 'asset')
+
+    @ddt.data(*MODULESTORE_SETUPS)
     def test_add_same_asset_twice(self, storebuilder):
         """
-        Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
+        Add an asset's metadata, then add it again.
         """
         with MongoContentstoreBuilder().build() as contentstore:
             with storebuilder.build(contentstore) as store:
@@ -360,6 +389,58 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 self.assertIsNone(store.find_asset_metadata(unknown_asset_key))
 
     @ddt.data(*MODULESTORE_SETUPS)
+    def test_get_multiple_types(self, storebuilder):
+        """
+        getting all things which are of type other than 'asset'
+        """
+        def check_asset_values(assets, orig):
+            """
+            Check asset values.
+            """
+            for idx, asset in enumerate(orig):
+                self.assertEquals(assets[idx].asset_id.asset_type, asset[0])
+                self.assertEquals(assets[idx].asset_id.path, asset[1])
+
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course = CourseFactory.create(modulestore=store)
+                differents = (('different', 'burn.jpg'),)
+                vrmls = (
+                    ('vrml', 'olympus_mons.vrml'),
+                    ('vrml', 'ponte_vecchio.vrml'),
+                )
+                regular_assets = (('asset', 'zippy.png'),)
+                alls = differents + vrmls + regular_assets
+
+                # Save 'em.
+                for asset_type, filename in alls:
+                    asset_key = course.id.make_asset_key(asset_type, filename)
+                    new_asset = self._make_asset_thumbnail_metadata(
+                        self._make_asset_metadata(asset_key)
+                    )
+                    store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test)
+
+                # Check 'em.
+                for asset_type, asset_list in (
+                    ('different', differents),
+                    ('vrml', vrmls),
+                    ('asset', regular_assets),
+                ):
+                    assets = store.get_all_asset_metadata(course.id, asset_type)
+                    self.assertEquals(len(assets), len(asset_list))
+                    check_asset_values(assets, asset_list)
+
+                self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
+                self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
+
+                assets = store.get_all_asset_metadata(
+                    course.id, None, start=0, maxresults=-1,
+                    sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
+                )
+                self.assertEquals(len(assets), len(alls))
+                check_asset_values(assets, alls)
+
+    @ddt.data(*MODULESTORE_SETUPS)
     def test_delete_all_different_type(self, storebuilder):
         """
         deleting all assets of a given but not 'asset' type
@@ -455,14 +536,8 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
         with storebuilder.build(None) as store:
             course_key = store.make_course_key("org", "course", "run")
             asset_key = course_key.make_asset_key('asset', 'foo.jpg')
-            for method in ['find_asset_metadata']:
-                with self.assertRaises(NotImplementedError):
-                    getattr(store, method)(asset_key)
-            with self.assertRaises(NotImplementedError):
-                # pylint: disable=protected-access
-                store._find_course_asset(asset_key)
-            with self.assertRaises(NotImplementedError):
-                store.get_all_asset_metadata(course_key, 'asset')
+            self.assertEquals(store.find_asset_metadata(asset_key), None)
+            self.assertEquals(store.get_all_asset_metadata(course_key, 'asset'), [])
 
     @ddt.data(*MODULESTORE_SETUPS)
     def test_copy_all_assets_same_modulestore(self, storebuilder):
@@ -485,6 +560,23 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
                 self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
 
+    @ddt.data(*MODULESTORE_SETUPS)
+    def test_copy_all_assets_from_course_with_no_assets(self, storebuilder):
+        """
+        Create a course with *no* assets, and try copy them all to another course in the same modulestore.
+        """
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course1 = CourseFactory.create(modulestore=store)
+                course2 = CourseFactory.create(modulestore=store)
+                store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
+                self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
+                self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
+                all_assets = store.get_all_asset_metadata(
+                    course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
+                )
+                self.assertEquals(len(all_assets), 0)
+
     @ddt.data(
         ('mongo', 'split'),
         ('split', 'mongo'),

common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py

@@ -331,7 +331,8 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
                             course_dirs=[course_data_name],
                             static_content_store=source_content,
                             target_course_id=source_course_key,
-                            create_new_course_if_not_present=True,
+                            create_course_if_not_present=True,
+                            raise_on_failure=True,
                         )
 
                         export_to_xml(
@@ -349,7 +350,8 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
                             course_dirs=['exported_source_course'],
                             static_content_store=dest_content,
                             target_course_id=dest_course_key,
-                            create_new_course_if_not_present=True,
+                            create_course_if_not_present=True,
+                            raise_on_failure=True,
                         )
 
 # NOT CURRENTLY USED
@@ -382,3 +384,10 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
                             dest_content,
                             dest_course_key,
                         )
+
+                        self.assertAssetsMetadataEqual(
+                            source_store,
+                            source_course_key,
+                            dest_store,
+                            dest_course_key,
+                        )

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -1932,7 +1932,7 @@ class TestMixedModuleStore(CourseComparisonTest):
                     self.store, self.user_id, DATA_DIR, ['toy'], load_error_modules=False,
                     static_content_store=contentstore,
                     target_course_id=dest_course_key,
-                    create_new_course_if_not_present=True,
+                    create_course_if_not_present=True,
                 )
                 course_id = courses[0].id
                 # no need to verify course content here as test_cross_modulestore_import_export does that
@@ -1980,7 +1980,7 @@ class TestMixedModuleStore(CourseComparisonTest):
                     self.store, self.user_id, DATA_DIR, ['toy'], load_error_modules=False,
                     static_content_store=contentstore,
                     target_course_id=dest_course_key,
-                    create_new_course_if_not_present=True,
+                    create_course_if_not_present=True,
                 )
                 course_id = courses[0].id
                 # no need to verify course content here as test_cross_modulestore_import_export does that

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -867,18 +867,21 @@ class XMLModuleStore(ModuleStoreReadBase):
         For now this is not implemented, but others should feel free to implement using the asset.json
         which export produces.
         """
-        raise NotImplementedError()
+        log.warning("_find_course_asset request of XML modulestore - not implemented.")
+        return (None, None)
 
     def find_asset_metadata(self, asset_key, **kwargs):
         """
         For now this is not implemented, but others should feel free to implement using the asset.json
         which export produces.
         """
-        raise NotImplementedError()
+        log.warning("find_asset_metadata request of XML modulestore - not implemented.")
+        return None
 
     def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
         """
         For now this is not implemented, but others should feel free to implement using the asset.json
         which export produces.
         """
-        raise NotImplementedError()
+        log.warning("get_all_asset_metadata request of XML modulestore - not implemented.")
+        return []

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

@@ -7,6 +7,7 @@ import lxml.etree
 from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
 from xmodule.contentstore.content import StaticContent
 from xmodule.exceptions import NotFoundError
+from xmodule.assetstore import AssetMetadata
 from xmodule.modulestore import EdxJSONEncoder, ModuleStoreEnum
 from xmodule.modulestore.inheritance import own_metadata
 from xmodule.modulestore.store_utilities import draft_node_constructor, get_draft_subtree_roots
@@ -43,6 +44,7 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
         course = modulestore.get_course(course_key, depth=None)  # None means infinite
         fsm = OSFS(root_dir)
         export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
+        root_course_dir = root_dir + '/' + course_dir
 
         root = lxml.etree.Element('unknown')
 
@@ -57,13 +59,26 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
         with export_fs.open('course.xml', 'w') as course_xml:
             lxml.etree.ElementTree(root).write(course_xml)
 
+        # Export the modulestore's asset metadata.
+        asset_dir = root_course_dir + '/' + AssetMetadata.EXPORTED_ASSET_DIR + '/'
+        if not os.path.isdir(asset_dir):
+            os.makedirs(asset_dir)
+        asset_root = lxml.etree.Element(AssetMetadata.ALL_ASSETS_XML_TAG)
+        course_assets = modulestore.get_all_asset_metadata(course_key, None)
+        for asset_md in course_assets:
+            # All asset types are exported using the "asset" tag - but their asset type is specified in each asset key.
+            asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG)
+            asset_md.to_xml(asset)
+        with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'w') as asset_xml_file:
+            lxml.etree.ElementTree(asset_root).write(asset_xml_file)
+
         # export the static assets
         policies_dir = export_fs.makeopendir('policies')
         if contentstore:
             contentstore.export_all_for_course(
                 course_key,
-                root_dir + '/' + course_dir + '/static/',
-                root_dir + '/' + course_dir + '/policies/assets.json',
+                root_course_dir + '/static/',
+                root_course_dir + '/policies/assets.json',
             )
 
             # If we are using the default course image, export it to the
@@ -79,7 +94,7 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
                 except NotFoundError:
                     pass
                 else:
-                    output_dir = root_dir + '/' + course_dir + '/static/images/'
+                    output_dir = root_course_dir + '/static/images/'
                     if not os.path.isdir(output_dir):
                         os.makedirs(output_dir)
                     with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file:

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -26,6 +26,7 @@ import mimetypes
 from path import path
 import json
 import re
+from lxml import etree
 
 from .xml import XMLModuleStore, ImportSystem, ParentTracker
 from xblock.runtime import KvsFieldData, DictKeyValueStore
@@ -38,6 +39,7 @@ from xmodule.errortracker import make_error_tracker
 from .store_utilities import rewrite_nonportable_content_links
 import xblock
 from xmodule.tabs import CourseTabList
+from xmodule.assetstore import AssetMetadata
 from xmodule.modulestore.django import ASSET_IGNORE_REGEX
 from xmodule.modulestore.exceptions import DuplicateCourseError
 from xmodule.modulestore.mongo.base import MongoRevisionKey
@@ -139,7 +141,8 @@ def import_from_xml(
         default_class='xmodule.raw_module.RawDescriptor',
         load_error_modules=True, static_content_store=None,
         target_course_id=None, verbose=False,
-        do_import_static=True, create_new_course_if_not_present=False):
+        do_import_static=True, create_course_if_not_present=False,
+        raise_on_failure=False):
     """
     Import xml-based courses from data_dir into modulestore.
 
@@ -167,7 +170,7 @@ def import_from_xml(
             time the course is loaded. Static content for some courses may also be
             served directly by nginx, instead of going through django.
 
-        create_new_course_if_not_present: If True, then a new course is created if it doesn't already exist.
+        create_course_if_not_present: If True, then a new course is created if it doesn't already exist.
             Otherwise, it throws an InvalidLocationError if the course does not exist.
 
         default_class, load_error_modules: are arguments for constructing the XMLModuleStore (see its doc)
@@ -196,7 +199,7 @@ def import_from_xml(
 
         runtime = None
         # Creates a new course if it doesn't already exist
-        if create_new_course_if_not_present and not store.has_course(dest_course_id, ignore_case=True):
+        if create_course_if_not_present and not store.has_course(dest_course_id, ignore_case=True):
             try:
                 new_course = store.create_course(dest_course_id.org, dest_course_id.course, dest_course_id.run, user_id)
                 runtime = new_course.runtime
@@ -223,6 +226,9 @@ def import_from_xml(
                 static_content_store, do_import_static, course_data_path, dest_course_id, verbose
             )
 
+            # Import asset metadata stored in XML.
+            _import_course_asset_metadata(store, course_data_path, dest_course_id, raise_on_failure)
+
             # STEP 3: import PUBLISHED items
             # now loop through all the modules depth first and then orphans
             with store.branch_setting(ModuleStoreEnum.Branch.published_only, dest_course_id):
@@ -285,10 +291,60 @@ def import_from_xml(
     return new_courses
 
 
+def _import_course_asset_metadata(store, data_dir, course_id, raise_on_failure):
+    """
+    Read in assets XML file, parse it, and add all asset metadata to the modulestore.
+    """
+    asset_dir = path(data_dir) / AssetMetadata.EXPORTED_ASSET_DIR
+    assets_filename = AssetMetadata.EXPORTED_ASSET_FILENAME
+    asset_xml_file = asset_dir / assets_filename
+
+    def make_asset_id(course_id, asset_xml):
+        """
+        Construct an asset ID out of a complete asset XML section.
+        """
+        asset_type = None
+        asset_name = None
+        for child in asset_xml.iterchildren():
+            if child.tag == AssetMetadata.ASSET_TYPE_ATTR:
+                asset_type = child.text
+            elif child.tag == AssetMetadata.ASSET_BASENAME_ATTR:
+                asset_name = child.text
+        return course_id.make_asset_key(asset_type, asset_name)
+
+    all_assets = []
+    try:
+        xml_data = etree.parse(asset_xml_file).getroot()
+        assert(xml_data.tag == AssetMetadata.ALL_ASSETS_XML_TAG)
+        for asset in xml_data.iterchildren():
+            if asset.tag == AssetMetadata.ASSET_XML_TAG:
+                # Construct the asset key.
+                asset_key = make_asset_id(course_id, asset)
+                asset_md = AssetMetadata(asset_key)
+                asset_md.from_xml(asset)
+                all_assets.append(asset_md)
+    except IOError:
+        logging.info('No {} file is present with asset metadata.'.format(assets_filename))
+        return
+    except Exception:  # pylint: disable=W0703
+        logging.exception('Error while parsing asset xml.')
+        if raise_on_failure:
+            raise
+        else:
+            return
+
+    # Now add all asset metadata to the modulestore.
+    if len(all_assets) > 0:
+        store.save_asset_metadata_list(all_assets, all_assets[0].edited_by, import_only=True)
+
+
 def _import_course_module(
         store, runtime, user_id, data_dir, course_key, dest_course_id, source_course, do_import_static,
         verbose,
 ):
+    """
+    Import a course module.
+    """
     if verbose:
         log.debug("Scanning {0} for course module...".format(course_key))
 
@@ -534,11 +590,11 @@ def _import_course_draft(
         for child in module.get_children():
             _import_module(child)
 
-    # now walk the /vertical directory where each file in there
-    # will be a draft copy of the Vertical
+    # Now walk the /vertical directory.
+    # Each file in the directory will be a draft copy of the vertical.
 
-    # First it is necessary to order the draft items by their desired index in the child list
-    # (order os.walk returns them in is not guaranteed).
+    # First it is necessary to order the draft items by their desired index in the child list,
+    # since the order in which os.walk() returns the files is not guaranteed.
     drafts = []
     for dirname, _dirnames, filenames in os.walk(draft_dir):
         for filename in filenames:

common/lib/xmodule/xmodule/tests/__init__.py

@@ -26,6 +26,7 @@ from xmodule.modulestore.inheritance import InheritanceMixin, own_metadata
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.mako_module import MakoDescriptorSystem
 from xmodule.error_module import ErrorDescriptor
+from xmodule.assetstore import AssetMetadata
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.mongo.draft import DraftModuleStore
 from xmodule.modulestore.xml import CourseLocationManager
@@ -498,3 +499,22 @@ class CourseComparisonTest(BulkAssertionTest):
             actual_thumbs = actual_store.get_all_content_thumbnails_for_course(actual_course_key)
 
             self._assertAssetsEqual(expected_course_key, expected_thumbs, actual_course_key, actual_thumbs)
+
+    def assertAssetsMetadataEqual(self, expected_modulestore, expected_course_key, actual_modulestore, actual_course_key):
+        """
+        Assert that the modulestore asset metdata for the ``expected_course_key`` and the ``actual_course_key``
+        are equivalent.
+        """
+        expected_course_assets = expected_modulestore.get_all_asset_metadata(
+            expected_course_key, None, sort=('displayname', ModuleStoreEnum.SortOrder.descending)
+        )
+        actual_course_assets = actual_modulestore.get_all_asset_metadata(
+            actual_course_key, None, sort=('displayname', ModuleStoreEnum.SortOrder.descending)
+        )
+        self.assertEquals(len(expected_course_assets), len(actual_course_assets))
+        for idx, __ in enumerate(expected_course_assets):
+            for attr in AssetMetadata.ATTRS_ALLOWED_TO_UPDATE:
+                if attr in ('edited_on',):
+                    # edited_on is updated upon import.
+                    continue
+                self.assertEquals(getattr(expected_course_assets[idx], attr), getattr(actual_course_assets[idx], attr))

common/test/data/manual-testing-complete/assets/assets.xml

+<assets>
+    <asset>
+        <type>asset</type>
+        <filename>pic1.jpg</filename>
+        <pathname>pix/archive</pathname>
+        <internal_name>EKMND332DDBK</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{"copyrighted": true}</fields>
+        <curr_version>14</curr_version>
+        <prev_version>13</prev_version>
+        <edited_by>144</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>144</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>shout.ogg</filename>
+        <pathname>sounds</pathname>
+        <internal_name>KFMDONSKF39K</internal_name>
+        <locked>true</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{}</fields>
+        <curr_version>1</curr_version>
+        <prev_version>None</prev_version>
+        <edited_by>144</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>144</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>code.tgz</filename>
+        <pathname>exercises/14</pathname>
+        <internal_name>ZZB2333YBDMW</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{"filesize": 123456}</fields>
+        <curr_version>AB</curr_version>
+        <prev_version>AA</prev_version>
+        <edited_by>288</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>288</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>dog.png</filename>
+        <pathname>pictures/animals</pathname>
+        <internal_name>PUPY4242X</internal_name>
+        <locked>true</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{}</fields>
+        <curr_version>5</curr_version>
+        <prev_version>4</prev_version>
+        <edited_by>432</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>432</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>not_here.txt</filename>
+        <pathname>/dev/null</pathname>
+        <internal_name>JJJCCC747</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{}</fields>
+        <curr_version>50</curr_version>
+        <prev_version>49</prev_version>
+        <edited_by>576</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>576</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>asset.txt</filename>
+        <pathname>/dev/null</pathname>
+        <internal_name>JJJCCC747858</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{}</fields>
+        <curr_version>50</curr_version>
+        <prev_version>49</prev_version>
+        <edited_by>576</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>576</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>roman_history.pdf</filename>
+        <pathname>texts/italy</pathname>
+        <internal_name>JASDUNSADK</internal_name>
+        <locked>true</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{"complicated": true, "thing_list": [14, true, "blue", {"nest": "but no eggs"}]}</fields>
+        <curr_version>1.1</curr_version>
+        <prev_version>1.01</prev_version>
+        <edited_by>1008</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>1008</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>asset</type>
+        <filename>weather_patterns.bmp</filename>
+        <pathname>science</pathname>
+        <internal_name>928SJXX2EB</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{"forecast": "horrible - wear many layers"}</fields>
+        <curr_version>52</curr_version>
+        <prev_version>51</prev_version>
+        <edited_by>1152</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>1152</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+    <asset>
+        <type>video</type>
+        <filename>demo.swf</filename>
+        <pathname>demos/easy</pathname>
+        <internal_name>DFDFGGGG14</internal_name>
+        <locked>false</locked>
+        <contenttype>None</contenttype>
+        <thumbnail>None</thumbnail>
+        <fields>{}</fields>
+        <curr_version>5</curr_version>
+        <prev_version>4</prev_version>
+        <edited_by>1296</edited_by>
+        <edited_by_email>me@example.com</edited_by_email>
+        <edited_on>2014-12-02T23:05:05.196505+00:00</edited_on>
+        <created_by>1296</created_by>
+        <created_by_email>me@example.com</created_by_email>
+        <created_on>2014-12-02T23:05:05.196505+00:00</created_on>
+    </asset>
+</assets>