Commit 22cbe0c4 by Don Mitchell

Merge pull request #5723 from edx/dhm/split_assetstore

Implement asset metadata storage in Split - per-asset functionality
parents ed2ca508 f48289cc
......@@ -3,11 +3,13 @@ Classes representing asset & asset thumbnail metadata.
"""
from datetime import datetime
import pytz
from contracts import contract, new_contract
from opaque_keys.edx.keys import CourseKey, AssetKey
new_contract('AssetKey', AssetKey)
new_contract('datetime', datetime)
new_contract('basestring', basestring)
class IncorrectAssetIdType(Exception):
......@@ -31,12 +33,13 @@ class AssetMetadata(object):
# All AssetMetadata objects should have AssetLocators with this type.
ASSET_TYPE = 'asset'
@contract(asset_id='AssetKey', basename='str | unicode | None', internal_name='str | None', locked='bool | None',
contenttype='str | unicode | None', md5='str | None', curr_version='str | None', prev_version='str | None')
@contract(asset_id='AssetKey', basename='basestring | None', internal_name='str | None', locked='bool | None', contenttype='basestring | None',
md5='str | None', curr_version='str | None', prev_version='str | None', edited_by='int | None', edited_on='datetime | None')
def __init__(self, asset_id,
basename=None, internal_name=None,
locked=None, contenttype=None, md5=None,
curr_version=None, prev_version=None):
curr_version=None, prev_version=None,
edited_by=None, edited_on=None, field_decorator=None):
"""
Construct a AssetMetadata object.
......@@ -48,10 +51,13 @@ class AssetMetadata(object):
contenttype (str): MIME type of the asset.
curr_version (str): Current version of the asset.
prev_version (str): Previous version of the asset.
edited_by (str): Username of last user to upload this asset.
edited_on (datetime): Datetime of last upload of this asset.
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.basename = basename # Path w/o filename.
self.internal_name = internal_name
self.locked = locked
......@@ -59,8 +65,8 @@ class AssetMetadata(object):
self.md5 = md5
self.curr_version = curr_version
self.prev_version = prev_version
self.edited_by = None
self.edited_on = None
self.edited_by = edited_by
self.edited_on = edited_on or datetime.now(pytz.utc)
def __repr__(self):
return """AssetMetadata{!r}""".format((
......@@ -131,7 +137,7 @@ class AssetThumbnailMetadata(object):
ASSET_TYPE = 'thumbnail'
@contract(asset_id='AssetKey', internal_name='str | unicode | None')
def __init__(self, asset_id, internal_name=None):
def __init__(self, asset_id, internal_name=None, field_decorator=None):
"""
Construct a AssetThumbnailMetadata object.
......@@ -141,7 +147,7 @@ class AssetThumbnailMetadata(object):
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.internal_name = internal_name
def __repr__(self):
......
......@@ -913,7 +913,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError()
@contract(course_key='CourseKey', asset_metadata='AssetMetadata', user_id='str | unicode')
@contract(course_key='CourseKey', asset_metadata='AssetMetadata')
def save_asset_metadata(self, course_key, asset_metadata, user_id):
"""
Saves the asset metadata for a particular course's asset.
......@@ -928,7 +928,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
return self._save_asset_info(course_key, asset_metadata, user_id, thumbnail=False)
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata):
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
"""
Saves the asset thumbnail metadata for a particular course asset's thumbnail.
......@@ -939,10 +939,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
True if thumbnail metadata save was successful, else False
"""
return self._save_asset_info(course_key, asset_thumbnail_metadata, '', thumbnail=True)
return self._save_asset_info(course_key, asset_thumbnail_metadata, user_id, thumbnail=True)
@contract(asset_key='AssetKey')
def _find_asset_info(self, asset_key, thumbnail=False):
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
"""
Find the info for a particular course asset/thumbnail.
......@@ -959,16 +959,16 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
if thumbnail:
info = 'thumbnails'
mdata = AssetThumbnailMetadata(asset_key, asset_key.path)
mdata = AssetThumbnailMetadata(asset_key, asset_key.path, **kwargs)
else:
info = 'assets'
mdata = AssetMetadata(asset_key, asset_key.path)
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key):
def find_asset_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
......@@ -978,10 +978,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
return self._find_asset_info(asset_key, thumbnail=False)
return self._find_asset_info(asset_key, thumbnail=False, **kwargs)
@contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key):
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
......@@ -991,10 +991,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
return self._find_asset_info(asset_key, thumbnail=True)
return self._find_asset_info(asset_key, thumbnail=True, **kwargs)
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool')
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False):
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
Returns a list of static asset (or thumbnail) metadata for a course.
......@@ -1018,9 +1018,9 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
return None
if get_thumbnails:
all_assets = course_assets['thumbnails']
all_assets = course_assets.get('thumbnails', [])
else:
all_assets = course_assets['assets']
all_assets = course_assets.get('assets', [])
# DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74
if start and maxresults and sort:
......@@ -1029,17 +1029,24 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
ret_assets = []
for asset in all_assets:
if get_thumbnails:
thumb = AssetThumbnailMetadata(course_key.make_asset_key('thumbnail', asset['filename']),
internal_name=asset['filename'])
thumb = AssetThumbnailMetadata(
course_key.make_asset_key('thumbnail', asset['filename']),
internal_name=asset['filename'], **kwargs
)
ret_assets.append(thumb)
else:
one_asset = AssetMetadata(course_key.make_asset_key('asset', asset['filename']))
one_asset.from_mongo(asset)
ret_assets.append(one_asset)
asset = AssetMetadata(
course_key.make_asset_key('asset', asset['filename']),
basename=asset['filename'],
edited_on=asset['edit_info']['edited_on'],
contenttype=asset['contenttype'],
md5=str(asset['md5']), **kwargs
)
ret_assets.append(asset)
return ret_assets
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None):
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
......@@ -1056,10 +1063,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
List of AssetMetadata objects.
"""
return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False)
return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False, **kwargs)
@contract(course_key='CourseKey')
def get_all_asset_thumbnail_metadata(self, course_key):
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
Returns a list of thumbnails for all course assets.
......@@ -1069,7 +1076,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
List of AssetThumbnailMetadata objects.
"""
return self._get_all_asset_metadata(course_key, get_thumbnails=True)
return self._get_all_asset_metadata(course_key, get_thumbnails=True, **kwargs)
def set_asset_metadata_attrs(self, asset_key, attrs, user_id):
"""
......@@ -1077,7 +1084,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError()
def _delete_asset_data(self, asset_key, thumbnail=False):
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
"""
Base method to over-ride in modulestore.
"""
......@@ -1100,7 +1107,7 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key):
def delete_asset_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
......@@ -1110,10 +1117,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
return self._delete_asset_data(asset_key, thumbnail=False)
return self._delete_asset_data(asset_key, user_id, thumbnail=False)
@contract(asset_key='AssetKey')
def delete_asset_thumbnail_metadata(self, asset_key):
def delete_asset_thumbnail_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
......@@ -1123,10 +1130,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
return self._delete_asset_data(asset_key, thumbnail=True)
return self._delete_asset_data(asset_key, user_id, thumbnail=True)
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key):
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
......
......@@ -315,24 +315,6 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(course_key)
return store.delete_course(course_key, user_id)
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
"""
Base method to over-ride in modulestore.
"""
raise NotImplementedError()
def _delete_asset_data(self, asset_key, thumbnail=False):
"""
Base method to over-ride in modulestore.
"""
raise NotImplementedError()
def _find_course_assets(self, course_key):
"""
Base method to override.
"""
raise NotImplementedError()
@contract(course_key='CourseKey', asset_metadata='AssetMetadata')
def save_asset_metadata(self, course_key, asset_metadata, user_id):
"""
......@@ -341,30 +323,25 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
Args:
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata): data about the course asset data
Returns:
bool: True if metadata save was successful, else False
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_metadata(course_key, asset_metadata, user_id)
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata):
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
"""
Saves the asset thumbnail metadata for a particular course asset's thumbnail.
Arguments:
course_key (CourseKey): course identifier
asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
Returns:
True if thumbnail metadata save was successful, else False
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_metadata(course_key, asset_thumbnail_metadata)
return store.save_asset_thumbnail_metadata(course_key, asset_thumbnail_metadata, user_id)
@strip_key
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key):
def find_asset_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
......@@ -375,10 +352,11 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
asset metadata (AssetMetadata) -or- None if not found
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.find_asset_metadata(asset_key)
return store.find_asset_metadata(asset_key, **kwargs)
@strip_key
@contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key):
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
......@@ -389,10 +367,11 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
asset metadata (AssetMetadata) -or- None if not found
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.find_asset_thumbnail_metadata(asset_key)
return store.find_asset_thumbnail_metadata(asset_key, **kwargs)
@strip_key
@contract(course_key='CourseKey', start=int, maxresults=int, sort='list | None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None):
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
......@@ -415,10 +394,11 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
md5: An md5 hash of the asset content
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_metadata(course_key, start, maxresults, sort)
return store.get_all_asset_metadata(course_key, start, maxresults, sort, **kwargs)
@strip_key
@contract(course_key='CourseKey')
def get_all_asset_thumbnail_metadata(self, course_key):
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
Returns a list of thumbnails for all course assets.
......@@ -429,10 +409,10 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
List of AssetThumbnailMetadata objects.
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_thumbnail_metadata(course_key)
return store.get_all_asset_thumbnail_metadata(course_key, **kwargs)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key):
def delete_asset_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
......@@ -443,10 +423,10 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
Number of asset metadata entries deleted (0 or 1)
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_metadata(asset_key)
return store.delete_asset_metadata(asset_key, user_id)
@contract(asset_key='AssetKey')
def delete_asset_thumbnail_metadata(self, asset_key):
def delete_asset_thumbnail_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
......@@ -457,10 +437,10 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
Number of asset metadata entries deleted (0 or 1)
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_metadata(asset_key)
return store.delete_asset_thumbnail_metadata(asset_key, user_id)
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key):
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
......@@ -468,10 +448,10 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
course_key (CourseKey): course_identifier
"""
store = self._get_modulestore_for_courseid(course_key)
return store.delete_all_asset_metadata(course_key)
return store.delete_all_asset_metadata(course_key, user_id)
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key):
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
......@@ -483,7 +463,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
# Check the modulestores of both the source and dest course_keys. If in different modulestores,
# export all asset data from one modulestore and import it into the dest one.
store = self._get_modulestore_for_courseid(source_course_key)
return store.copy_all_asset_metadata(source_course_key, dest_course_key)
return store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id)
@contract(asset_key='AssetKey', attr=str)
def set_asset_metadata_attr(self, asset_key, attr, value, user_id):
......@@ -500,7 +480,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
AttributeError is attr is one of the build in attrs.
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.set_asset_metadata_attrs(asset_key, attr, value, user_id)
return store.set_asset_metadata_attrs(asset_key, {attr: value}, user_id)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
......
......@@ -1474,7 +1474,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
return course_assets
@contract(course_key='CourseKey', asset_metadata='AssetMetadata | AssetThumbnailMetadata', user_id='str | unicode')
@contract(course_key='CourseKey', asset_metadata='AssetMetadata | AssetThumbnailMetadata')
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
"""
Saves the info for a particular course's asset/thumbnail.
......@@ -1537,7 +1537,6 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
md = AssetMetadata(asset_key, asset_key.path)
md.from_mongo(all_assets[asset_idx])
md.update(attr_dict)
md.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = md.to_mongo()
......@@ -1545,7 +1544,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {'assets': all_assets}})
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, thumbnail=False):
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
......@@ -1572,8 +1571,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_asset_info}})
return 1
# pylint: disable=unused-argument
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key):
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
......
......@@ -80,6 +80,7 @@ from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope
from xmodule.error_module import ErrorDescriptor
from collections import defaultdict
from types import NoneType
from xmodule.assetstore import AssetMetadata
log = logging.getLogger(__name__)
......@@ -1174,7 +1175,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
Find the version_history_depth next versions of this definition. Return as a VersionTree
'''
# TODO implement
raise NotImplementedError()
pass
def create_definition_from_data(self, course_key, new_def_data, category, user_id):
"""
......@@ -2120,6 +2121,180 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
return ModuleStoreEnum.Type.split
def _find_course_assets(self, course_key):
"""
Split specific lookup
"""
return self._lookup_course(course_key).structure
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
structure = self._lookup_course(course_key).structure
return structure, self._lookup_course_asset(structure, filename, get_thumbnail)
def _lookup_course_asset(self, structure, filename, get_thumbnail=False):
"""
Find the course asset in the structure or return None if it does not exist
"""
# See if this asset already exists by checking the external_filename.
# Studio doesn't currently support using multiple course assets with the same filename.
# So use the filename as the unique identifier.
accessor = 'thumbnails' if get_thumbnail else 'assets'
for idx, asset in enumerate(structure.get(accessor, [])):
if asset['filename'] == filename:
return idx
return None
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
"""
A wrapper for functions wanting to manipulate assets. Gets and versions the structure,
passes the mutable array for either 'assets' or 'thumbnails' as well as the idx to the function for it to
update, then persists the changed data back into the course.
The update function can raise an exception if it doesn't want to actually do the commit. The
surrounding method probably should catch that exception.
"""
with self.bulk_operations(asset_key.course_key):
original_structure = self._lookup_course(asset_key.course_key).structure
index_entry = self._get_index_if_valid(asset_key.course_key)
new_structure = self.version_structure(asset_key.course_key, original_structure, user_id)
accessor = 'thumbnails' if get_thumbnail else 'assets'
asset_idx = self._lookup_course_asset(new_structure, asset_key.path, get_thumbnail)
new_structure[accessor] = update_function(new_structure.get(accessor, []), asset_idx)
# update index if appropriate and structures
self.update_structure(asset_key.course_key, new_structure)
if index_entry is not None:
# update the index entry if appropriate
self._update_head(asset_key.course_key, index_entry, asset_key.branch, new_structure['_id'])
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
"""
The guts of saving a new or updated asset
"""
metadata_to_insert = asset_metadata.to_mongo()
def _internal_method(all_assets, asset_idx):
"""
Either replace the existing entry or add a new one
"""
if asset_idx is None:
all_assets.append(metadata_to_insert)
else:
all_assets[asset_idx] = metadata_to_insert
return all_assets
return self._update_course_assets(user_id, asset_metadata.asset_id, _internal_method, thumbnail)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
"""
Add/set the given dict of attrs on the asset at the given location. Value can be any type which pymongo accepts.
Arguments:
asset_key (AssetKey): asset identifier
attr_dict (dict): attribute: value pairs to set
Raises:
ItemNotFoundError if no such item exists
AttributeError is attr is one of the build in attrs.
"""
def _internal_method(all_assets, asset_idx):
"""
Update the found item
"""
if asset_idx is None:
raise ItemNotFoundError(asset_key)
# Form an AssetMetadata.
mdata = AssetMetadata(asset_key, asset_key.path)
mdata.from_mongo(all_assets[asset_idx])
mdata.update(attr_dict)
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = mdata.to_mongo()
return all_assets
self._update_course_assets(user_id, asset_key, _internal_method, False)
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
Arguments:
asset_key (AssetKey): key containing original asset/thumbnail filename
thumbnail: True if thumbnail deletion, False if asset metadata deletion
Returns:
Number of asset metadata/thumbnail entries deleted (0 or 1)
"""
def _internal_method(all_asset_info, asset_idx):
"""
Remove the item if it was found
"""
if asset_idx is None:
raise ItemNotFoundError(asset_key)
all_asset_info.pop(asset_idx)
return all_asset_info
try:
self._update_course_assets(user_id, asset_key, _internal_method, thumbnail)
return 1
except ItemNotFoundError:
return 0
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
Arguments:
course_key (CourseKey): course_identifier
"""
with self.bulk_operations(course_key):
original_structure = self._lookup_course(course_key).structure
index_entry = self._get_index_if_valid(course_key)
new_structure = self.version_structure(course_key, original_structure, user_id)
new_structure['assets'] = []
new_structure['thumbnails'] = []
# update index if appropriate and structures
self.update_structure(course_key, new_structure)
if index_entry is not None:
# update the index entry if appropriate
self._update_head(course_key, index_entry, course_key.branch, new_structure['_id'])
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
Arguments:
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
"""
source_structure = self._lookup_course(source_course_key).structure
with self.bulk_operations(dest_course_key):
original_structure = self._lookup_course(dest_course_key).structure
index_entry = self._get_index_if_valid(dest_course_key)
new_structure = self.version_structure(dest_course_key, original_structure, user_id)
new_structure['assets'] = source_structure.get('assets', [])
new_structure['thumbnails'] = source_structure.get('thumbnails', [])
# update index if appropriate and structures
self.update_structure(dest_course_key, new_structure)
if index_entry is not None:
# update the index entry if appropriate
self._update_head(dest_course_key, index_entry, dest_course_key.branch, new_structure['_id'])
def internal_clean_children(self, course_locator):
"""
Only intended for rather low level methods to use. Goes through the children attrs of
......
......@@ -2,7 +2,7 @@
Module for the dual-branch fall-back Draft->Published Versioning ModuleStore
"""
from split import SplitMongoModuleStore, EXCLUDE_ALL
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.exceptions import InsufficientSpecificationError
......@@ -13,7 +13,7 @@ from opaque_keys.edx.locator import CourseLocator
from xmodule.modulestore.split_mongo import BlockKey
class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleStore):
class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPublished):
"""
A subclass of Split that supports a dual-branch fall-back versioning framework
with a Draft branch that falls back to a Published branch.
......@@ -43,9 +43,9 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
# create any other necessary things as a side effect: ensure they populate the draft branch
# and rely on auto publish to populate the published branch: split's create course doesn't
# call super b/c it needs the auto publish above to have happened before any of the create_items
# in this. The explicit use of SplitMongoModuleStore is intentional
# in this; so, this manually calls the grandparent and above methods.
with self.branch_setting(ModuleStoreEnum.Branch.draft_preferred, item.id):
# pylint: disable=bad-super-call
# NOTE: DO NOT CHANGE THE SUPER. See comment above
super(SplitMongoModuleStore, self).create_course(
org, course, run, user_id, runtime=item.runtime, **kwargs
)
......@@ -229,7 +229,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
if revision == ModuleStoreEnum.RevisionOption.draft_preferred:
revision = ModuleStoreEnum.RevisionOption.draft_only
location = self._map_revision_to_branch(location, revision=revision)
return SplitMongoModuleStore.get_parent_location(self, location, **kwargs)
return super(DraftVersioningModuleStore, self).get_parent_location(location, **kwargs)
def get_orphans(self, course_key, **kwargs):
course_key = self._map_revision_to_branch(course_key)
......@@ -275,8 +275,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
Publishes the subtree under location from the draft branch to the published branch
Returns the newly published item.
"""
SplitMongoModuleStore.copy(
self,
super(DraftVersioningModuleStore, self).copy(
user_id,
# Directly using the replace function rather than the for_branch function
# because for_branch obliterates the version_guid and will lead to missed version conflicts.
......@@ -446,3 +445,62 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
if published_block is not None:
setattr(xblock, '_published_by', published_block['edit_info']['edited_by'])
setattr(xblock, '_published_on', published_block['edit_info']['edited_on'])
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
return super(DraftVersioningModuleStore, self)._find_asset_info(
self._map_revision_to_branch(asset_key), thumbnail, **kwargs
)
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
return super(DraftVersioningModuleStore, self)._get_all_asset_metadata(
self._map_revision_to_branch(course_key), start, maxresults, sort, get_thumbnails, **kwargs
)
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
"""
Updates both the published and draft branches
"""
# if one call gets an exception, don't do the other call but pass on the exception
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only),
update_function, get_thumbnail
)
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only),
update_function, get_thumbnail
)
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
return super(DraftVersioningModuleStore, self)._find_course_asset(
self._map_revision_to_branch(course_key), filename, get_thumbnail=get_thumbnail
)
def _find_course_assets(self, course_key):
"""
Split specific lookup
"""
return super(DraftVersioningModuleStore, self)._find_course_assets(
self._map_revision_to_branch(course_key)
)
def delete_all_asset_metadata(self, course_key, user_id):
"""
Deletes from both branches
"""
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.published_only), user_id
)
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.draft_only), user_id
)
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copies to and from both branches
"""
for revision in [ModuleStoreEnum.RevisionOption.published_only, ModuleStoreEnum.RevisionOption.draft_only]:
super(DraftVersioningModuleStore, self).copy_all_asset_metadata(
self._map_revision_to_branch(source_course_key, revision),
self._map_revision_to_branch(dest_course_key, revision),
user_id
)
"""
Tests for assetstore using any of the modulestores for metadata. May extend to testing the storage options
too.
"""
from datetime import datetime, timedelta
import pytz
import unittest
import ddt
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MODULESTORE_SETUPS, MongoContentstoreBuilder,
)
@ddt.ddt
class TestMongoAssetMetadataStorage(unittest.TestCase):
"""
Tests for storing/querying course asset metadata.
"""
def setUp(self):
super(TestMongoAssetMetadataStorage, self).setUp()
self.addTypeEqualityFunc(datetime, self._compare_datetimes)
self.addTypeEqualityFunc(AssetMetadata, self._compare_metadata)
def _compare_metadata(self, mdata1, mdata2, msg=None):
"""
So we can use the below date comparison
"""
if type(mdata1) != type(mdata2):
self.fail(self._formatMessage(msg, u"{} is not same type as {}".format(mdata1, mdata2)))
for attr in mdata1.ALLOWED_ATTRS:
self.assertEqual(getattr(mdata1, attr), getattr(mdata2, attr), msg)
def _compare_datetimes(self, datetime1, datetime2, msg=None):
"""
Don't compare microseconds as mongo doesn't encode below milliseconds
"""
if not timedelta(seconds=-1) < datetime1 - datetime2 < timedelta(seconds=1):
self.fail(self._formatMessage(msg, u"{} != {}".format(datetime1, datetime2)))
def _make_asset_metadata(self, asset_loc):
"""
Make a single test asset metadata.
"""
return AssetMetadata(asset_loc, internal_name='EKMND332DDBK',
basename='pictures/historical', contenttype='image/jpeg',
locked=False, md5='77631ca4f0e08419b70726a447333ab6',
edited_by=ModuleStoreEnum.UserID.test, edited_on=datetime.now(pytz.utc),
curr_version='v1.0', prev_version='v0.95')
def _make_asset_thumbnail_metadata(self, asset_key):
"""
Make a single test asset thumbnail metadata.
"""
return AssetThumbnailMetadata(asset_key, internal_name='ABC39XJUDN2')
def setup_assets(self, course1_key, course2_key, store=None):
"""
Setup assets. Save in store if given
"""
asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'edited_by', 'edited_on', 'curr_version', 'prev_version')
asset1_vals = ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13')
asset2_vals = ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None)
asset3_vals = ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA')
asset4_vals = ('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4')
asset5_vals = ('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49')
asset1 = dict(zip(asset_fields[1:], asset1_vals[1:]))
asset2 = dict(zip(asset_fields[1:], asset2_vals[1:]))
asset3 = dict(zip(asset_fields[1:], asset3_vals[1:]))
asset4 = dict(zip(asset_fields[1:], asset4_vals[1:]))
non_existent_asset = dict(zip(asset_fields[1:], asset5_vals[1:]))
# Asset6 and thumbnail6 have equivalent information on purpose.
asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49')
asset6 = dict(zip(asset_fields[1:], asset6_vals[1:]))
asset1_key = course1_key.make_asset_key('asset', asset1_vals[0])
asset2_key = course1_key.make_asset_key('asset', asset2_vals[0])
asset3_key = course2_key.make_asset_key('asset', asset3_vals[0])
asset4_key = course2_key.make_asset_key('asset', asset4_vals[0])
asset5_key = course2_key.make_asset_key('asset', asset5_vals[0])
asset6_key = course2_key.make_asset_key('asset', asset6_vals[0])
asset1_md = AssetMetadata(asset1_key, **asset1)
asset2_md = AssetMetadata(asset2_key, **asset2)
asset3_md = AssetMetadata(asset3_key, **asset3)
asset4_md = AssetMetadata(asset4_key, **asset4)
asset5_md = AssetMetadata(asset5_key, **non_existent_asset)
asset6_md = AssetMetadata(asset6_key, **asset6)
if store is not None:
store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test)
# 5 & 6 are not saved on purpose!
return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md)
def setup_thumbnails(self, course1_key, course2_key, store=None):
"""
Setup thumbs. Save in store if given
"""
thumbnail_fields = ('filename', 'internal_name')
thumbnail1_vals = ('cat_thumb.jpg', 'XYXYXYXYXYXY')
thumbnail2_vals = ('kitten_thumb.jpg', '123ABC123ABC')
thumbnail3_vals = ('puppy_thumb.jpg', 'ADAM12ADAM12')
thumbnail4_vals = ('meerkat_thumb.jpg', 'CHIPSPONCH14')
thumbnail5_vals = ('corgi_thumb.jpg', 'RON8LDXFFFF10')
thumbnail1 = dict(zip(thumbnail_fields[1:], thumbnail1_vals[1:]))
thumbnail2 = dict(zip(thumbnail_fields[1:], thumbnail2_vals[1:]))
thumbnail3 = dict(zip(thumbnail_fields[1:], thumbnail3_vals[1:]))
thumbnail4 = dict(zip(thumbnail_fields[1:], thumbnail4_vals[1:]))
non_existent_thumbnail = dict(zip(thumbnail_fields[1:], thumbnail5_vals[1:]))
# Asset6 and thumbnail6 have equivalent information on purpose.
thumbnail6_vals = ('asset.txt', 'JJJCCC747858')
thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:]))
thumb1_key = course1_key.make_asset_key('thumbnail', thumbnail1_vals[0])
thumb2_key = course1_key.make_asset_key('thumbnail', thumbnail2_vals[0])
thumb3_key = course2_key.make_asset_key('thumbnail', thumbnail3_vals[0])
thumb4_key = course2_key.make_asset_key('thumbnail', thumbnail4_vals[0])
thumb5_key = course2_key.make_asset_key('thumbnail', thumbnail5_vals[0])
thumb6_key = course2_key.make_asset_key('thumbnail', thumbnail6_vals[0])
thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1)
thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2)
thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3)
thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4)
thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail)
thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6)
if store is not None:
store.save_asset_thumbnail_metadata(course1_key, thumb1_md, ModuleStoreEnum.UserID.test)
store.save_asset_thumbnail_metadata(course1_key, thumb2_md, ModuleStoreEnum.UserID.test)
store.save_asset_thumbnail_metadata(course2_key, thumb3_md, ModuleStoreEnum.UserID.test)
store.save_asset_thumbnail_metadata(course2_key, thumb4_md, ModuleStoreEnum.UserID.test)
# thumb5 and thumb6 are not saved on purpose!
return (thumb1_md, thumb2_md, thumb3_md, thumb4_md, thumb5_md, thumb6_md)
@ddt.data(*MODULESTORE_SETUPS)
def test_save_one_and_confirm(self, storebuilder):
"""
Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all.
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
asset_filename = 'burnside.jpg'
new_asset_loc = course.id.make_asset_key('asset', asset_filename)
# Confirm that the asset's metadata is not present.
self.assertIsNone(store.find_asset_metadata(new_asset_loc))
# Save the asset's metadata.
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
# Find the asset's metadata and confirm it's the same.
found_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(found_asset_md)
self.assertEquals(new_asset_md, found_asset_md)
# Confirm that only two setup plus one asset's metadata exists.
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
# Delete all metadata and confirm it's gone.
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
# Now delete the non-existent metadata and ensure it doesn't choke
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_delete(self, storebuilder):
"""
Delete non_existent and existent metadata
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
# Attempt to delete an asset that doesn't exist.
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_non_existing_assets(self, storebuilder):
"""
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
# Find existing asset metadata.
asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNone(asset_md)
@ddt.data(*MODULESTORE_SETUPS)
def test_add_same_asset_twice(self, storebuilder):
"""
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Add asset metadata.
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
# Add *the same* asset metadata.
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
# Still one here?
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_lock_unlock_assets(self, storebuilder):
"""
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
locked_state = new_asset_md.locked
# Flip the course asset's locked status.
store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test)
# Find the same course and check its locked status.
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
self.assertEquals(updated_asset_md.locked, not locked_state)
# Now flip it back.
store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test)
reupdated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(reupdated_asset_md)
self.assertEquals(reupdated_asset_md.locked, locked_state)
ALLOWED_ATTRS = (
('basename', '/new/path'),
('internal_name', 'new_filename.txt'),
('locked', True),
('contenttype', 'image/png'),
('md5', '5346682d948cc3f683635b6918f9b3d0'),
('curr_version', 'v1.01'),
('prev_version', 'v1.0'),
('edited_by', 'Mork'),
('edited_on', datetime(1969, 1, 1, tzinfo=pytz.utc)),
)
DISALLOWED_ATTRS = (
('asset_id', 'IAmBogus'),
)
UNKNOWN_ATTRS = (
('lunch_order', 'burger_and_fries'),
('villain', 'Khan')
)
@ddt.data(*MODULESTORE_SETUPS)
def test_set_all_attrs(self, storebuilder):
"""
Save setting each attr one at a time
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.ALLOWED_ATTRS:
# Set the course asset's attr.
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test)
# Find the same course asset and check its changed attr.
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attr, None))
self.assertEquals(getattr(updated_asset_md, attr, None), value)
@ddt.data(*MODULESTORE_SETUPS)
def test_set_disallowed_attrs(self, storebuilder):
"""
setting disallowed attrs should fail
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.DISALLOWED_ATTRS:
original_attr_val = getattr(new_asset_md, attr)
# Set the course asset's attr.
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test)
# Find the same course and check its changed attr.
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attr, None))
# Make sure that the attr is unchanged from its original value.
self.assertEquals(getattr(updated_asset_md, attr, None), original_attr_val)
@ddt.data(*MODULESTORE_SETUPS)
def test_set_unknown_attrs(self, storebuilder):
"""
setting unknown attrs should fail
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.UNKNOWN_ATTRS:
# Set the course asset's attr.
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test)
# Find the same course and check its changed attr.
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
# Make sure the unknown field was *not* added.
with self.assertRaises(AttributeError):
self.assertEquals(getattr(updated_asset_md, attr), value)
@ddt.data(*MODULESTORE_SETUPS)
def test_save_one_thumbnail_and_delete_one_thumbnail(self, storebuilder):
"""
saving and deleting thumbnails
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 1)
self.assertEquals(store.delete_asset_thumbnail_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_thumbnail(self, storebuilder):
"""
finding thumbnails
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertIsNotNone(store.find_asset_thumbnail_metadata(asset_key))
unknown_asset_key = course.id.make_asset_key('thumbnail', 'nosuchfile.jpg')
self.assertIsNone(store.find_asset_thumbnail_metadata(unknown_asset_key))
@ddt.data(*MODULESTORE_SETUPS)
def test_delete_all_thumbnails(self, storebuilder):
"""
deleting all thumbnails
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(
course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test
)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 1)
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0)
def test_get_all_assets_with_paging(self):
pass
def test_copy_all_assets(self):
pass
......@@ -87,6 +87,7 @@ class MongoModulestoreBuilder(object):
doc_store_config = dict(
db='modulestore{}'.format(random.randint(0, 10000)),
collection='xmodule',
asset_collection='asset_metadata',
**COMMON_DOCSTORE_CONFIG
)
......
......@@ -14,7 +14,6 @@ from datetime import datetime
from pytz import UTC
import unittest
from xblock.core import XBlock
from ddt import ddt, data
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xblock.runtime import KeyValueStore
......@@ -31,7 +30,6 @@ from opaque_keys.edx.keys import UsageKey
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
from xmodule.contentstore.mongo import MongoContentStore
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from nose.tools import assert_in
from xmodule.exceptions import NotFoundError
......@@ -665,332 +663,6 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
shutil.rmtree(root_dir)
@ddt
class TestMongoAssetMetadataStorage(TestMongoModuleStore):
"""
Tests for storing/querying course asset metadata from Mongo storage.
"""
def _make_asset_metadata(self, asset_loc):
"""
Make a single test asset metadata.
"""
return AssetMetadata(asset_loc, internal_name='EKMND332DDBK',
basename='pictures/historical', contenttype='image/jpeg',
locked=False, md5='77631ca4f0e08419b70726a447333ab6',
curr_version='v1.0', prev_version='v0.95')
def _make_asset_thumbnail_metadata(self, asset_key):
"""
Make a single test asset thumbnail metadata.
"""
return AssetThumbnailMetadata(asset_key, internal_name='ABC39XJUDN2')
@classmethod
def add_asset_collection(cls, doc_store_config):
"""
Valid asset collection.
"""
doc_store_config['asset_collection'] = ASSET_COLLECTION
@classmethod
def setupClass(cls):
super(TestMongoAssetMetadataStorage, cls).setupClass()
@classmethod
def teardownClass(cls):
super(TestMongoAssetMetadataStorage, cls).teardownClass()
def setup_assets(self):
"""
Setup assets.
"""
asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'curr_version', 'prev_version')
asset1_vals = ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, '14', '13')
asset2_vals = ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, '1', None)
asset3_vals = ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, 'AB', 'AA')
asset4_vals = ('dog.png', 'PUPY4242X', 'pictures/animals', True, '5', '4')
asset5_vals = ('not_here.txt', 'JJJCCC747', '/dev/null', False, '50', '49')
asset1 = dict(zip(asset_fields[1:], asset1_vals[1:]))
asset2 = dict(zip(asset_fields[1:], asset2_vals[1:]))
asset3 = dict(zip(asset_fields[1:], asset3_vals[1:]))
asset4 = dict(zip(asset_fields[1:], asset4_vals[1:]))
non_existent_asset = dict(zip(asset_fields[1:], asset5_vals[1:]))
# Asset6 and thumbnail6 have equivalent information on purpose.
asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, '50', '49')
asset6 = dict(zip(asset_fields[1:], asset6_vals[1:]))
asset1_key = self.course1.id.make_asset_key('asset', asset1_vals[0])
asset2_key = self.course1.id.make_asset_key('asset', asset2_vals[0])
asset3_key = self.course2.id.make_asset_key('asset', asset3_vals[0])
asset4_key = self.course2.id.make_asset_key('asset', asset4_vals[0])
asset5_key = self.course2.id.make_asset_key('asset', asset5_vals[0])
asset6_key = self.course2.id.make_asset_key('asset', asset6_vals[0])
asset1_md = AssetMetadata(asset1_key, **asset1)
asset2_md = AssetMetadata(asset2_key, **asset2)
asset3_md = AssetMetadata(asset3_key, **asset3)
asset4_md = AssetMetadata(asset4_key, **asset4)
asset5_md = AssetMetadata(asset5_key, **non_existent_asset)
asset6_md = AssetMetadata(asset6_key, **asset6)
editing_user = 'Oliver Twist'
self.assertTrue(self.draft_store.save_asset_metadata(self.course1.id, asset1_md, editing_user))
self.assertTrue(self.draft_store.save_asset_metadata(self.course1.id, asset2_md, editing_user))
self.assertTrue(self.draft_store.save_asset_metadata(self.course2.id, asset3_md, editing_user))
self.assertTrue(self.draft_store.save_asset_metadata(self.course2.id, asset4_md, editing_user))
# asset5 and asset6 are not saved on purpose!
return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md)
def setup_thumbnails(self):
"""
Setup thumbs.
"""
thumbnail_fields = ('filename', 'internal_name')
thumbnail1_vals = ('cat_thumb.jpg', 'XYXYXYXYXYXY')
thumbnail2_vals = ('kitten_thumb.jpg', '123ABC123ABC')
thumbnail3_vals = ('puppy_thumb.jpg', 'ADAM12ADAM12')
thumbnail4_vals = ('meerkat_thumb.jpg', 'CHIPSPONCH14')
thumbnail5_vals = ('corgi_thumb.jpg', 'RON8LDXFFFF10')
thumbnail1 = dict(zip(thumbnail_fields[1:], thumbnail1_vals[1:]))
thumbnail2 = dict(zip(thumbnail_fields[1:], thumbnail2_vals[1:]))
thumbnail3 = dict(zip(thumbnail_fields[1:], thumbnail3_vals[1:]))
thumbnail4 = dict(zip(thumbnail_fields[1:], thumbnail4_vals[1:]))
non_existent_thumbnail = dict(zip(thumbnail_fields[1:], thumbnail5_vals[1:]))
# Asset6 and thumbnail6 have equivalent information on purpose.
thumbnail6_vals = ('asset.txt', 'JJJCCC747858')
thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:]))
thumb1_key = self.course1.id.make_asset_key('thumbnail', thumbnail1_vals[0])
thumb2_key = self.course1.id.make_asset_key('thumbnail', thumbnail2_vals[0])
thumb3_key = self.course2.id.make_asset_key('thumbnail', thumbnail3_vals[0])
thumb4_key = self.course2.id.make_asset_key('thumbnail', thumbnail4_vals[0])
thumb5_key = self.course2.id.make_asset_key('thumbnail', thumbnail5_vals[0])
thumb6_key = self.course2.id.make_asset_key('thumbnail', thumbnail6_vals[0])
thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1)
thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2)
thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3)
thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4)
thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail)
thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6)
self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, thumb1_md))
self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, thumb2_md))
self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course2.id, thumb3_md))
self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course2.id, thumb4_md))
# thumb5 and thumb6 are not saved on purpose!
return (thumb1_md, thumb2_md, thumb3_md, thumb4_md, thumb5_md, thumb6_md)
def setUp(self):
"""
Set up a quantity of test asset metadata for testing purposes.
"""
super(TestMongoAssetMetadataStorage, self).setUp()
courses = self.draft_store.get_courses()
self.course1 = courses[0]
self.course2 = courses[1]
(self.asset1_md, self.asset2_md, self.asset3_md, self.asset4_md, self.asset5_md, self.asset6_md) = self.setup_assets()
(self.thumb1_md, self.thumb2_md, self.thumb3_md, self.thumb4_md, self.thumb5_md, self.thumb6_md) = self.setup_thumbnails()
def tearDown(self):
self.draft_store.delete_all_asset_metadata(self.course1.id)
self.draft_store.delete_all_asset_metadata(self.course2.id)
def test_save_one_and_confirm(self):
courses = self.draft_store.get_courses()
course = courses[0]
asset_filename = 'burnside.jpg'
new_asset_loc = course.id.make_asset_key('asset', asset_filename)
# Confirm that the asset's metadata is not present.
self.assertIsNone(self.draft_store.find_asset_metadata(new_asset_loc))
# Save the asset's metadata.
new_asset_md = self._make_asset_metadata(new_asset_loc)
self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, 'John Doe'))
# Find the asset's metadata and confirm it's the same.
found_asset_md = self.draft_store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(found_asset_md)
self.assertEquals(new_asset_md.asset_id, found_asset_md.asset_id)
# Confirm that only two setup plus one asset's metadata exists.
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3)
# Delete all metadata and confirm it's gone.
self.draft_store.delete_all_asset_metadata(course.id)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0)
def test_delete_all_without_creation(self):
courses = self.draft_store.get_courses()
course = courses[0]
# Confirm that only setup asset metadata exists.
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 2)
# Now delete the metadata.
self.draft_store.delete_all_asset_metadata(course.id)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0)
# Now delete the non-existent metadata.
self.draft_store.delete_all_asset_metadata(course.id)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0)
def test_save_many_and_delete_one(self):
# Make sure there's two assets.
self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 2)
# Delete one of the assets.
self.assertEquals(self.draft_store.delete_asset_metadata(self.asset1_md.asset_id), 1)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 1)
# Attempt to delete an asset that doesn't exist.
self.assertEquals(self.draft_store.delete_asset_metadata(self.asset5_md.asset_id), 0)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 1)
def test_find_existing_and_non_existing_assets(self):
# Find existing asset metadata.
asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(asset_md)
# Find non-existent asset metadata.
asset_md = self.draft_store.find_asset_metadata(self.asset5_md.asset_id)
self.assertIsNone(asset_md)
def test_add_same_asset_twice(self):
courses = self.draft_store.get_courses()
course = courses[0]
asset_filename = 'burnside.jpg'
new_asset_loc = course.id.make_asset_key('asset', asset_filename)
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Only the setup stuff here?
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 2)
# Add asset metadata.
self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, "John Do"))
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3)
# Add *the same* asset metadata.
self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, "John Dont"))
# Still one here?
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3)
self.draft_store.delete_all_asset_metadata(course.id)
self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0)
def test_lock_unlock_assets(self):
# Find a course asset and check its locked status.
asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(asset_md)
locked_state = asset_md.locked
# Flip the course asset's locked status.
self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, "locked", not locked_state, 'John Doe')
# Find the same course and check its locked status.
updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(updated_asset_md)
self.assertEquals(updated_asset_md.locked, not locked_state)
# Now flip it back.
self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, "locked", locked_state, 'John Doe')
reupdated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(reupdated_asset_md)
self.assertEquals(reupdated_asset_md.locked, locked_state)
ALLOWED_ATTRS = (
('basename', '/new/path'),
('internal_name', 'new_filename.txt'),
('locked', True),
('contenttype', 'image/png'),
('md5', '5346682d948cc3f683635b6918f9b3d0'),
('curr_version', 'v1.01'),
('prev_version', 'v1.0'),
('edited_by', 'Mork'),
('edited_on', datetime(1969, 1, 1, tzinfo=UTC)),
)
DISALLOWED_ATTRS = (
('asset_id', 'IAmBogus'),
)
UNKNOWN_ATTRS = (
('lunch_order', 'burger_and_fries'),
('villain', 'Khan')
)
@data(*ALLOWED_ATTRS)
def test_set_all_attrs(self, attr_pair):
# Find a course asset.
asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(asset_md)
# Set the course asset's attr.
editing_user = 'user_who_edited'
self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id=editing_user)
# Find the same course asset and check its changed attr.
updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attr_pair[0], None))
if attr_pair[0] == 'edited_by':
# No matter what the edited_by attr_pair is, it gets over-ridden by the passed-in user_id.
self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), editing_user)
elif attr_pair[0] == 'edited_on':
# edited_on is also over-ridden to be the time of update.
pass
else:
self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), attr_pair[1])
@data(*DISALLOWED_ATTRS)
def test_set_disallowed_attrs(self, attr_pair):
# Find a course asset.
asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(asset_md)
original_attr_val = getattr(asset_md, attr_pair[0])
# Set the course asset's attr.
self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id='John Doe')
# Find the same course and check its changed attr.
updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attr_pair[0], None))
# Make sure that the attr is unchanged from its original value.
self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), original_attr_val)
@data(*UNKNOWN_ATTRS)
def test_set_unknown_attrs(self, attr_pair):
# Find a course asset.
asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(asset_md)
# Set the course asset's attr.
self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id='John Smith')
# Find the same course and check its changed attr.
updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id)
self.assertIsNotNone(updated_asset_md)
# Make sure the unknown field was *not* added.
with self.assertRaises(AttributeError):
self.assertEquals(getattr(updated_asset_md, attr_pair[0]), attr_pair[1])
def test_save_one_thumbnail_and_delete_one_thumbnail(self):
thumbnail_filename = 'burn_thumb.jpg'
asset_key = self.course1.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2)
self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, new_asset_thumbnail))
self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 3)
self.assertEquals(self.draft_store.delete_asset_thumbnail_metadata(asset_key), 1)
self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2)
def test_find_thumbnail(self):
self.assertIsNotNone(self.draft_store.find_asset_thumbnail_metadata(self.thumb1_md.asset_id))
self.assertIsNone(self.draft_store.find_asset_thumbnail_metadata(self.thumb5_md.asset_id))
def test_delete_all_thumbnails(self):
self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2)
self.draft_store.delete_all_asset_metadata(self.course1.id)
self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 0)
def test_asset_object_equivalence(self):
# Assets are only equivalent to themselves.
self.assertTrue(self.asset6_md != self.thumb6_md)
self.assertEquals(self.asset1_md, self.asset1_md)
def test_get_all_assets_with_paging(self):
pass
def test_copy_all_assets(self):
pass
class TestMongoModuleStoreWithNoAssetCollection(TestMongoModuleStore):
'''
Tests a situation where no asset_collection is specified.
......@@ -1017,7 +689,7 @@ class TestMongoModuleStoreWithNoAssetCollection(TestMongoModuleStore):
# Confirm that no asset collection means no asset metadata.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
# Now delete the non-existent asset metadata.
self.draft_store.delete_all_asset_metadata(course.id)
self.draft_store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
# Should still be nothing.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment