Skip to content

E
edx-platform
Commit fc83c299 by Don Mitchell
Options

Generalize assets to any block_type but using 

AssetKey and the metadata store
PLAT-204
parent 303b4e66
Showing with 134 additions and 322 deletions
cms/djangoapps/contentstore/views/tests/test_assets.py
"""
Unit tests for the asset upload endpoint.
"""
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0212
from datetime import datetime
from io import BytesIO
from pytz import UTC
......@@ -13,7 +8,7 @@ import json
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
from xmodule.assetstore.assetmgr import UnknownAssetType, AssetMetadataFoundTemporary
from xmodule.assetstore.assetmgr import AssetMetadataFoundTemporary
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
......@@ -35,12 +30,18 @@ class AssetsTestCase(CourseTestCase):
self.url = reverse_course_url('assets_handler', self.course.id)
def upload_asset(self, name="asset-1"):
"""
Post to the asset upload url
"""
f = BytesIO(name)
f.name = name + ".txt"
return self.client.post(self.url, {"name": name, "file": f})
class BasicAssetsTestCase(AssetsTestCase):
"""
Test getting assets via html w/o additional args
"""
def test_basic(self):
resp = self.client.get(self.url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 200)
......@@ -81,6 +82,9 @@ class PaginationTestCase(AssetsTestCase):
Tests the pagination of assets returned from the REST API.
"""
def test_json_responses(self):
"""
Test the ajax asset interfaces
"""
self.upload_asset("asset-1")
self.upload_asset("asset-2")
self.upload_asset("asset-3")
......@@ -100,20 +104,26 @@ class PaginationTestCase(AssetsTestCase):
self.assert_correct_asset_response(self.url + "?page_size=3&page=1", 0, 3, 3)
def assert_correct_asset_response(self, url, expected_start, expected_length, expected_total):
"""
Get from the url and ensure it contains the expected number of responses
"""
resp = self.client.get(url, HTTP_ACCEPT='application/json')
json_response = json.loads(resp.content)
assets = json_response['assets']
assets_response = json_response['assets']
self.assertEquals(json_response['start'], expected_start)
self.assertEquals(len(assets), expected_length)
self.assertEquals(len(assets_response), expected_length)
self.assertEquals(json_response['totalCount'], expected_total)
def assert_correct_sort_response(self, url, sort, direction):
"""
Get from the url w/ a sort option and ensure items honor that sort
"""
resp = self.client.get(url + '?sort=' + sort + '&direction=' + direction, HTTP_ACCEPT='application/json')
json_response = json.loads(resp.content)
assets = json_response['assets']
name1 = assets[0][sort]
name2 = assets[1][sort]
name3 = assets[2][sort]
assets_response = json_response['assets']
name1 = assets_response[0][sort]
name2 = assets_response[1][sort]
name3 = assets_response[2][sort]
if direction == 'asc':
self.assertLessEqual(name1, name2)
self.assertLessEqual(name2, name3)
......@@ -163,12 +173,6 @@ class DownloadTestCase(AssetsTestCase):
resp = self.client.get(url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 404)
def test_download_unknown_asset_type(self):
# Change the asset type to something unknown.
url = self.uploaded_url.replace('/asset/', '/unknown_type/')
with self.assertRaises((UnknownAssetType, NameError)):
self.client.get(url, HTTP_ACCEPT='text/html')
def test_metadata_found_in_modulestore(self):
# Insert asset metadata into the modulestore (with no accompanying asset).
asset_key = self.course.id.make_asset_key(AssetMetadata.ASSET_TYPE, 'pic1.jpg')
......@@ -179,7 +183,7 @@ class DownloadTestCase(AssetsTestCase):
'curr_version': '14',
'prev_version': '13'
})
modulestore().save_asset_metadata(self.course.id, asset_md, 15)
modulestore().save_asset_metadata(asset_md, 15)
# Get the asset metadata and have it be found in the modulestore.
# Currently, no asset metadata should be found in the modulestore. The code is not yet storing it there.
# If asset metadata *is* found there, an exception is raised. This test ensures the exception is indeed raised.
......@@ -201,6 +205,7 @@ class AssetToJsonTestCase(AssetsTestCase):
location = course_key.make_asset_key('asset', 'my_file_name.jpg')
thumbnail_location = course_key.make_asset_key('thumbnail', 'my_file_name_thumb.jpg')
# pylint: disable=protected-access
output = assets._get_asset_json("my_file", upload_date, location, thumbnail_location, True)
self.assertEquals(output["display_name"], "my_file")
......@@ -239,6 +244,7 @@ class LockAssetTestCase(AssetsTestCase):
resp = self.client.post(
url,
# pylint: disable=protected-access
json.dumps(assets._get_asset_json("sample_static.txt", upload_date, asset_location, None, lock)),
"application/json"
)
......
common/lib/xmodule/xmodule/assetstore/__init__.py
"""
Classes representing asset & asset thumbnail metadata.
Classes representing asset metadata.
"""
from datetime import datetime
......@@ -13,74 +13,70 @@ new_contract('datetime', datetime)
new_contract('basestring', basestring)
class IncorrectAssetIdType(Exception):
"""
Raised when the asset ID passed-in to create an AssetMetadata or
AssetThumbnailMetadata is of the wrong type.
"""
pass
class AssetMetadata(object):
"""
Stores the metadata associated with a particular course asset. The asset metadata gets stored
in the modulestore.
"""
TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'md5']
TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'thumbnail', 'fields']
EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_on']
ALLOWED_ATTRS = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS
# All AssetMetadata objects should have AssetLocators with this type.
# Default type for AssetMetadata objects. A constant for convenience.
ASSET_TYPE = 'asset'
@contract(asset_id='AssetKey', basename='basestring|None', internal_name='basestring|None', locked='bool|None', contenttype='basestring|None',
md5='basestring|None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None')
fields='dict | None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None')
def __init__(self, asset_id,
basename=None, internal_name=None,
locked=None, contenttype=None, md5=None,
locked=None, contenttype=None, thumbnail=None, fields=None,
curr_version=None, prev_version=None,
edited_by=None, edited_on=None, field_decorator=None):
edited_by=None, edited_on=None,
field_decorator=None,):
"""
Construct a AssetMetadata object.
Arguments:
asset_id (AssetKey): Key identifying this particular asset.
basename (str): Original path to file at asset upload time.
internal_name (str): Name under which the file is stored internally.
internal_name (str): Name, url, or handle for the storage system to access the file.
locked (bool): If True, only course participants can access the asset.
contenttype (str): MIME type of the asset.
thumbnail (str): the internal_name for the thumbnail if one exists
fields (dict): fields to save w/ the metadata
curr_version (str): Current version of the asset.
prev_version (str): Previous version of the asset.
edited_by (str): Username of last user to upload this asset.
edited_on (datetime): Datetime of last upload of this asset.
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding.
Not saved.
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.basename = basename # Path w/o filename.
self.internal_name = internal_name
self.locked = locked
self.contenttype = contenttype
self.md5 = md5
self.thumbnail = thumbnail
self.curr_version = curr_version
self.prev_version = prev_version
self.edited_by = edited_by
self.edited_on = edited_on or datetime.now(pytz.utc)
self.fields = fields or {}
def __repr__(self):
return """AssetMetadata{!r}""".format((
self.asset_id,
self.basename, self.internal_name,
self.locked, self.contenttype, self.md5,
self.locked, self.contenttype, self.fields,
self.curr_version, self.prev_version,
self.edited_by, self.edited_on
))
def update(self, attr_dict):
"""
Set the attributes on the metadata. Ignore all those outside the known fields.
Set the attributes on the metadata. Any which are not in ALLOWED_ATTRS get put into
fields.
Arguments:
attr_dict: Prop, val dictionary of all attributes to set.
......@@ -88,6 +84,8 @@ class AssetMetadata(object):
for attr, val in attr_dict.iteritems():
if attr in self.ALLOWED_ATTRS:
setattr(self, attr, val)
else:
self.fields[attr] = val
def to_mongo(self):
"""
......@@ -99,7 +97,8 @@ class AssetMetadata(object):
'internal_name': self.internal_name,
'locked': self.locked,
'contenttype': self.contenttype,
'md5': self.md5,
'thumbnail': self.thumbnail,
'fields': self.fields,
'curr_version': self.curr_version,
'prev_version': self.prev_version,
'edited_by': self.edited_by,
......@@ -119,54 +118,9 @@ class AssetMetadata(object):
self.internal_name = asset_doc['internal_name']
self.locked = asset_doc['locked']
self.contenttype = asset_doc['contenttype']
self.md5 = asset_doc['md5']
self.thumbnail = asset_doc['thumbnail']
self.fields = asset_doc['fields']
self.curr_version = asset_doc['curr_version']
self.prev_version = asset_doc['prev_version']
self.edited_by = asset_doc['edited_by']
self.edited_on = asset_doc['edited_on']
class AssetThumbnailMetadata(object):
"""
Stores the metadata associated with the thumbnail of a course asset.
"""
# All AssetThumbnailMetadata objects should have AssetLocators with this type.
ASSET_TYPE = 'thumbnail'
@contract(asset_id='AssetKey', internal_name='basestring|None')
def __init__(self, asset_id, internal_name=None, field_decorator=None):
"""
Construct a AssetThumbnailMetadata object.
Arguments:
asset_id (AssetKey): Key identifying this particular asset.
internal_name (str): Name under which the file is stored internally.
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.internal_name = internal_name
def __repr__(self):
return """AssetMetadata{!r}""".format((self.asset_id, self.internal_name))
def to_mongo(self):
"""
Converts metadata properties into a MongoDB-storable dict.
"""
return {
'filename': self.asset_id.path,
'internal_name': self.internal_name
}
@contract(thumbnail_doc='dict|None')
def from_mongo(self, thumbnail_doc):
"""
Fill in all metadata fields from a MongoDB document.
The asset_id prop is initialized upon construction only.
"""
if thumbnail_doc is None:
return
self.internal_name = thumbnail_doc['internal_name']
common/lib/xmodule/xmodule/assetstore/assetmgr.py
......@@ -15,7 +15,6 @@ from contracts import contract, new_contract
from opaque_keys.edx.keys import AssetKey
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
new_contract('AssetKey', AssetKey)
......@@ -35,13 +34,6 @@ class AssetMetadataNotFound(AssetException):
pass
class UnknownAssetType(AssetException):
"""
Thrown when the asset type is not recognized.
"""
pass
class AssetMetadataFoundTemporary(AssetException):
"""
TEMPORARY: Thrown if asset metadata is actually found in the course modulestore.
......@@ -59,15 +51,7 @@ class AssetManager(object):
"""
Finds a course asset either in the assetstore -or- in the deprecated contentstore.
"""
store = modulestore()
content_md = None
asset_type = asset_key.asset_type
if asset_type == AssetThumbnailMetadata.ASSET_TYPE:
content_md = store.find_asset_thumbnail_metadata(asset_key)
elif asset_type == AssetMetadata.ASSET_TYPE:
content_md = store.find_asset_metadata(asset_key)
else:
raise UnknownAssetType()
content_md = modulestore().find_asset_metadata(asset_key)
# If found, raise an exception.
if content_md:
......
common/lib/xmodule/xmodule/modulestore/mixed.py
......@@ -14,7 +14,7 @@ from contracts import contract, new_contract
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, AssetKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
from . import ModuleStoreWriteBase
from . import ModuleStoreEnum
......@@ -25,7 +25,6 @@ from .split_migrator import SplitMigrator
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata)
log = logging.getLogger(__name__)
......@@ -315,8 +314,8 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(course_key)
return store.delete_course(course_key, user_id)
@contract(course_key='CourseKey', asset_metadata='AssetMetadata')
def save_asset_metadata(self, course_key, asset_metadata, user_id):
@contract(asset_metadata='AssetMetadata')
def save_asset_metadata(self, asset_metadata, user_id):
"""
Saves the asset metadata for a particular course's asset.
......@@ -324,20 +323,8 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata): data about the course asset data
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_metadata(course_key, asset_metadata, user_id)
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
"""
Saves the asset thumbnail metadata for a particular course asset's thumbnail.
Arguments:
course_key (CourseKey): course identifier
asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_thumbnail_metadata(course_key, asset_thumbnail_metadata, user_id)
store = self._get_modulestore_for_courseid(asset_metadata.asset_id.course_key)
return store.save_asset_metadata(asset_metadata, user_id)
@strip_key
@contract(asset_key='AssetKey')
......@@ -355,23 +342,8 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
return store.find_asset_metadata(asset_key, **kwargs)
@strip_key
@contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.find_asset_thumbnail_metadata(asset_key, **kwargs)
@strip_key
@contract(course_key='CourseKey', start=int, maxresults=int, sort='tuple|None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
......@@ -394,22 +366,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
md5: An md5 hash of the asset content
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_metadata(course_key, start, maxresults, sort, **kwargs)
@strip_key
@contract(course_key='CourseKey')
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
Returns a list of thumbnails for all course assets.
Args:
course_key (CourseKey): course identifier
Returns:
List of AssetThumbnailMetadata objects.
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_thumbnail_metadata(course_key, **kwargs)
return store.get_all_asset_metadata(course_key, asset_type, start, maxresults, sort, **kwargs)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key, user_id):
......@@ -425,31 +382,6 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_metadata(asset_key, user_id)
@contract(asset_key='AssetKey')
def delete_asset_thumbnail_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): locator containing original asset filename
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_thumbnail_metadata(asset_key, user_id)
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
Arguments:
course_key (CourseKey): course_identifier
"""
store = self._get_modulestore_for_courseid(course_key)
return store.delete_all_asset_metadata(course_key, user_id)
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
......
common/lib/xmodule/xmodule/modulestore/mongo/base.py
......@@ -47,14 +47,13 @@ from opaque_keys.edx.locator import CourseLocator
from opaque_keys.edx.keys import UsageKey, CourseKey, AssetKey
from xmodule.exceptions import HeartbeatFailure
from xmodule.modulestore.edit_info import EditInfoRuntimeMixin
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
log = logging.getLogger(__name__)
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata)
# sort order that returns DRAFT items first
SORT_REVISION_FAVOR_DRAFT = ('_id.revision', pymongo.DESCENDING)
......@@ -1467,25 +1466,22 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
course_key = self.fill_in_run(course_key)
course_assets = self.asset_collection.find_one(
{'course_id': unicode(course_key)},
fields=('course_id', 'storage', 'assets', 'thumbnails')
)
if course_assets is None:
# Not found, so create.
course_assets = {'course_id': unicode(course_key), 'storage': 'FILLMEIN-TMP', 'assets': [], 'thumbnails': []}
course_assets = {'course_id': unicode(course_key), 'storage': 'FILLMEIN-TMP', 'assets': []}
course_assets['_id'] = self.asset_collection.insert(course_assets)
return course_assets
@contract(course_key='CourseKey', asset_metadata='AssetMetadata | AssetThumbnailMetadata')
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
@contract(asset_metadata='AssetMetadata')
def save_asset_metadata(self, asset_metadata, user_id):
"""
Saves the info for a particular course's asset/thumbnail.
Saves the info for a particular course's asset.
Arguments:
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata/AssetThumbnailMetadata): data about the course asset/thumbnail
thumbnail (bool): True if saving thumbnail metadata, False if saving asset metadata
asset_metadata (AssetMetadata): data about the course asset
Returns:
True if info save was successful, else False
......@@ -1493,16 +1489,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if self.asset_collection is None:
return False
course_assets, asset_idx = self._find_course_asset(course_key, asset_metadata.asset_id.path, thumbnail)
info = 'thumbnails' if thumbnail else 'assets'
course_assets, asset_idx = self._find_course_asset(asset_metadata.asset_id)
all_assets = SortedListWithKey([], key=itemgetter('filename'))
# Assets should be pre-sorted, so add them efficiently without sorting.
# extend() will raise a ValueError if the passed-in list is not sorted.
all_assets.extend(course_assets[info])
# Set the edited information for assets only - not thumbnails.
if not thumbnail:
asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
all_assets.extend(course_assets[asset_metadata.asset_id.block_type])
asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
# Translate metadata to Mongo format.
metadata_to_insert = asset_metadata.to_mongo()
......@@ -1514,30 +1506,31 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
all_assets[asset_idx] = metadata_to_insert
# Update the document.
self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}})
self.asset_collection.update(
{'_id': course_assets['_id']},
{'$set': {asset_metadata.asset_id.block_type: all_assets.as_list()}}
)
return True
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
If dest_course already has assets, this removes the previous value.
It doesn't combine the assets in dest.
Arguments:
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
"""
source_assets = self._find_course_assets(source_course_key)
dest_assets = self._find_course_assets(dest_course_key)
dest_assets['assets'] = source_assets.get('assets', [])
dest_assets['thumbnails'] = source_assets.get('thumbnails', [])
dest_assets = source_assets.copy()
dest_assets['course_id'] = unicode(dest_course_key)
del dest_assets['_id']
self.asset_collection.remove({'course_id': unicode(dest_course_key)})
# Update the document.
self.asset_collection.update(
{'_id': dest_assets['_id']},
{'$set': {'assets': dest_assets['assets'],
'thumbnails': dest_assets['thumbnails']}
}
)
self.asset_collection.insert(dest_assets)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
......@@ -1555,12 +1548,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if self.asset_collection is None:
return
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path)
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
raise ItemNotFoundError(asset_key)
# Form an AssetMetadata.
all_assets = course_assets['assets']
all_assets = course_assets[asset_key.block_type]
md = AssetMetadata(asset_key, asset_key.path)
md.from_mongo(all_assets[asset_idx])
md.update(attr_dict)
......@@ -1568,34 +1561,34 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = md.to_mongo()
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {'assets': all_assets}})
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {asset_key.block_type: all_assets}})
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
def delete_asset_metadata(self, asset_key, user_id):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
Internal; deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): key containing original asset/thumbnail filename
thumbnail: True if thumbnail deletion, False if asset metadata deletion
asset_key (AssetKey): key containing original asset filename
Returns:
Number of asset metadata/thumbnail entries deleted (0 or 1)
Number of asset metadata entries deleted (0 or 1)
"""
if self.asset_collection is None:
return 0
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, get_thumbnail=thumbnail)
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
return 0
info = 'thumbnails' if thumbnail else 'assets'
all_asset_info = course_assets[info]
all_asset_info = course_assets[asset_key.block_type]
all_asset_info.pop(asset_idx)
# Update the document.
self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_asset_info}})
self.asset_collection.update(
{'_id': course_assets['_id']},
{'$set': {asset_key.block_type: all_asset_info}}
)
return 1
# pylint: disable=unused-argument
......
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
......@@ -155,6 +155,7 @@ class DraftModuleStore(MongoModuleStore):
# delete all of the db records for the course
course_query = self._course_key_to_son(course_key)
self.collection.remove(course_query, multi=True)
self.delete_all_asset_metadata(course_key, user_id)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None, **kwargs):
"""
......
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
......@@ -2125,26 +2125,30 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
Split specific lookup
"""
return self._lookup_course(course_key).structure
return self._lookup_course(course_key).structure.get('assets', {})
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
structure = self._lookup_course(course_key).structure
return structure, self._lookup_course_asset(structure, filename, get_thumbnail)
def _find_course_asset(self, asset_key):
"""
Return the raw dict of assets type as well as the index to the one being sought from w/in
it's subvalue (or None)
"""
assets = self._lookup_course(asset_key.course_key).structure.get('assets', {})
return assets, self._lookup_course_asset(assets, asset_key)
def _lookup_course_asset(self, structure, filename, get_thumbnail=False):
def _lookup_course_asset(self, structure, asset_key):
"""
Find the course asset in the structure or return None if it does not exist
"""
# See if this asset already exists by checking the external_filename.
# Studio doesn't currently support using multiple course assets with the same filename.
# So use the filename as the unique identifier.
accessor = 'thumbnails' if get_thumbnail else 'assets'
for idx, asset in enumerate(structure.get(accessor, [])):
if asset['filename'] == filename:
accessor = asset_key.block_type
for idx, asset in enumerate(structure.setdefault(accessor, [])):
if asset['filename'] == asset_key.block_id:
return idx
return None
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
def _update_course_assets(self, user_id, asset_key, update_function):
"""
A wrapper for functions wanting to manipulate assets. Gets and versions the structure,
passes the mutable array for either 'assets' or 'thumbnails' as well as the idx to the function for it to
......@@ -2158,10 +2162,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
index_entry = self._get_index_if_valid(asset_key.course_key)
new_structure = self.version_structure(asset_key.course_key, original_structure, user_id)
accessor = 'thumbnails' if get_thumbnail else 'assets'
asset_idx = self._lookup_course_asset(new_structure, asset_key.path, get_thumbnail)
asset_idx = self._lookup_course_asset(new_structure.setdefault('assets', {}), asset_key)
new_structure[accessor] = update_function(new_structure.get(accessor, []), asset_idx)
new_structure['assets'][asset_key.block_type] = update_function(
new_structure['assets'][asset_key.block_type], asset_idx
)
# update index if appropriate and structures
self.update_structure(asset_key.course_key, new_structure)
......@@ -2170,7 +2175,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# update the index entry if appropriate
self._update_head(asset_key.course_key, index_entry, asset_key.branch, new_structure['_id'])
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
def save_asset_metadata(self, asset_metadata, user_id):
"""
The guts of saving a new or updated asset
"""
......@@ -2186,7 +2191,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
all_assets[asset_idx] = metadata_to_insert
return all_assets
return self._update_course_assets(user_id, asset_metadata.asset_id, _internal_method, thumbnail)
return self._update_course_assets(user_id, asset_metadata.asset_id, _internal_method)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
......@@ -2217,19 +2222,18 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
all_assets[asset_idx] = mdata.to_mongo()
return all_assets
self._update_course_assets(user_id, asset_key, _internal_method, False)
self._update_course_assets(user_id, asset_key, _internal_method)
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
def delete_asset_metadata(self, asset_key, user_id):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
Internal; deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): key containing original asset/thumbnail filename
thumbnail: True if thumbnail deletion, False if asset metadata deletion
asset_key (AssetKey): key containing original asset filename
Returns:
Number of asset metadata/thumbnail entries deleted (0 or 1)
Number of asset metadata entries deleted (0 or 1)
"""
def _internal_method(all_asset_info, asset_idx):
"""
......@@ -2242,34 +2246,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
return all_asset_info
try:
self._update_course_assets(user_id, asset_key, _internal_method, thumbnail)
self._update_course_assets(user_id, asset_key, _internal_method)
return 1
except ItemNotFoundError:
return 0
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
Arguments:
course_key (CourseKey): course_identifier
"""
with self.bulk_operations(course_key):
original_structure = self._lookup_course(course_key).structure
index_entry = self._get_index_if_valid(course_key)
new_structure = self.version_structure(course_key, original_structure, user_id)
new_structure['assets'] = []
new_structure['thumbnails'] = []
# update index if appropriate and structures
self.update_structure(course_key, new_structure)
if index_entry is not None:
# update the index entry if appropriate
self._update_head(course_key, index_entry, course_key.branch, new_structure['_id'])
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
......
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
......@@ -11,6 +11,7 @@ from xmodule.modulestore.draft_and_published import (
)
from opaque_keys.edx.locator import CourseLocator
from xmodule.modulestore.split_mongo import BlockKey
from contracts import contract
class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPublished):
......@@ -446,33 +447,34 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
setattr(xblock, '_published_by', published_block['edit_info']['edited_by'])
setattr(xblock, '_published_on', published_block['edit_info']['edited_on'])
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
return super(DraftVersioningModuleStore, self)._find_asset_info(
self._map_revision_to_branch(asset_key), thumbnail, **kwargs
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key, **kwargs):
return super(DraftVersioningModuleStore, self).find_asset_metadata(
self._map_revision_to_branch(asset_key), **kwargs
)
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
return super(DraftVersioningModuleStore, self)._get_all_asset_metadata(
self._map_revision_to_branch(course_key), start, maxresults, sort, get_thumbnails, **kwargs
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
return super(DraftVersioningModuleStore, self).get_all_asset_metadata(
self._map_revision_to_branch(course_key), asset_type, start, maxresults, sort, **kwargs
)
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
def _update_course_assets(self, user_id, asset_key, update_function):
"""
Updates both the published and draft branches
"""
# if one call gets an exception, don't do the other call but pass on the exception
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only),
update_function, get_thumbnail
update_function
)
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only),
update_function, get_thumbnail
update_function
)
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
def _find_course_asset(self, asset_key):
return super(DraftVersioningModuleStore, self)._find_course_asset(
self._map_revision_to_branch(course_key), filename, get_thumbnail=get_thumbnail
self._map_revision_to_branch(asset_key)
)
def _find_course_assets(self, course_key):
......@@ -483,17 +485,6 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
self._map_revision_to_branch(course_key)
)
def delete_all_asset_metadata(self, course_key, user_id):
"""
Deletes from both branches
"""
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.published_only), user_id
)
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.draft_only), user_id
)
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copies to and from both branches
......
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
......@@ -40,7 +40,7 @@ COMMON_DOCSTORE_CONFIG = {
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
}
DATA_DIR = path(__file__).dirname().parent.parent.parent.parent.parent / "test" / "data"
DATA_DIR = path(__file__).dirname().parent.parent / "tests" / "data" / "xml-course-root"
XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)
......
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
......@@ -686,11 +686,7 @@ class TestMongoModuleStoreWithNoAssetCollection(TestMongoModuleStore):
courses = self.draft_store.get_courses()
course = courses[0]
# Confirm that no asset collection means no asset metadata.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
# Now delete the non-existent asset metadata.
self.draft_store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
# Should still be nothing.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id, 'asset'), None)
class TestMongoKeyValueStore(object):
......
common/lib/xmodule/xmodule/modulestore/xml.py
......@@ -847,14 +847,7 @@ class XMLModuleStore(ModuleStoreReadBase):
raise ValueError(u"Cannot set branch setting to {} on a ReadOnly store".format(branch_setting))
yield
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
def _find_course_asset(self, asset_key):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
......@@ -868,28 +861,7 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
raise NotImplementedError()
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
......
common/lib/xmodule/xmodule/tests/data/xml-course-root 0 → 120000
../../../../../test/data/
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment