Skip to content

E
edx-platform
Commit 436b32a6 by Don Mitchell
Options

Merge pull request #5800 from edx/dhm/xml_assetstore 

Abstract asset methods into own interface class
parents bc20ccb3 aa07355e
Showing with 775 additions and 675 deletions
common/lib/xmodule/xmodule/modulestore/__init__.py
...@@ -266,891 +266,898 @@ class BulkOperationsMixin(object): ...@@ -266,891 +266,898 @@ class BulkOperationsMixin(object):
return self._get_bulk_ops_record(course_key, ignore_case).active return self._get_bulk_ops_record(course_key, ignore_case).active
class ModuleStoreRead(object): class ModuleStoreAssetInterface(object):
""" """
An abstract interface for a database backend that stores XModuleDescriptor The methods for accessing assets and their metadata
instances and extends read-only functionality
""" """
def _find_course_assets(self, course_key):
__metaclass__ = ABCMeta
@abstractmethod
def has_item(self, usage_key):
""" """
Returns True if usage_key exists in this ModuleStore. Base method to override.
""" """
pass raise NotImplementedError()
@abstractmethod def _find_course_asset(self, course_key, filename, get_thumbnail=False):
def get_item(self, usage_key, depth=0, **kwargs):
""" """
Returns an XModuleDescriptor instance for the item at location. Internal; finds or creates course asset info -and- finds existing asset (or thumbnail) metadata.
If any segment of the location is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
usage_key: A :class:`.UsageKey` subclass instance Arguments:
course_key (CourseKey): course identifier
filename (str): filename of the asset or thumbnail
get_thumbnail (bool): True gets thumbnail data, False gets asset data
depth (int): An argument that some module stores may use to prefetch Returns:
descendents of the queried modules for more efficient results later Asset info for the course, index of asset/thumbnail in list (None if asset/thumbnail does not exist)
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
""" """
pass course_assets = self._find_course_assets(course_key)
if course_assets is None:
return None, None
@abstractmethod if get_thumbnail:
def get_course_errors(self, course_key): all_assets = course_assets['thumbnails']
""" else:
Return a list of (msg, exception-or-None) errors that the modulestore all_assets = course_assets['assets']
encountered when loading the course at course_id.
Raises the same exceptions as get_item if the location isn't found or # See if this asset already exists by checking the external_filename.
isn't fully specified. # Studio doesn't currently support using multiple course assets with the same filename.
# So use the filename as the unique identifier.
for idx, asset in enumerate(all_assets):
if asset['filename'] == filename:
return course_assets, idx
Args: return course_assets, None
course_key (:class:`.CourseKey`): The course to check for errors
"""
pass
@abstractmethod @contract(asset_key='AssetKey')
def get_items(self, location, course_id=None, depth=0, qualifiers=None, **kwargs): def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
""" """
Returns a list of XModuleDescriptor instances for the items Find the info for a particular course asset/thumbnail.
that match location. Any element of location that is None is treated
as a wildcard that matches any value
location: Something that can be passed to Location Arguments:
asset_key (AssetKey): key containing original asset filename
thumbnail (bool): True if finding thumbnail, False if finding asset metadata
depth: An argument that some module stores may use to prefetch Returns:
descendents of the queried modules for more efficient results later asset/thumbnail metadata (AssetMetadata/AssetThumbnailMetadata) -or- None if not found
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
""" """
pass course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, thumbnail)
if asset_idx is None:
def _block_matches(self, fields_or_xblock, qualifiers): return None
'''
Return True or False depending on whether the field value (block contents)
matches the qualifiers as per get_items. Note, only finds directly set not
inherited nor default value matches.
For substring matching pass a regex object.
for arbitrary function comparison such as date time comparison, pass
the function as in start=lambda x: x < datetime.datetime(2014, 1, 1, 0, tzinfo=pytz.UTC)
Args: if thumbnail:
fields_or_xblock (dict or XBlock): either the json blob (from the db or get_explicitly_set_fields) info = 'thumbnails'
or the xblock.fields() value or the XBlock from which to get those values mdata = AssetThumbnailMetadata(asset_key, asset_key.path, **kwargs)
qualifiers (dict): field: searchvalue pairs.
'''
if isinstance(fields_or_xblock, XBlock):
fields = fields_or_xblock.fields
xblock = fields_or_xblock
is_xblock = True
else: else:
fields = fields_or_xblock info = 'assets'
is_xblock = False mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
def _is_set_on(key): @contract(asset_key='AssetKey')
""" def find_asset_metadata(self, asset_key, **kwargs):
Is this key set in fields? (return tuple of boolean and value). A helper which can
handle fields either being the json doc or xblock fields. Is inner function to restrict
use and to access local vars.
""" """
if key not in fields: Find the metadata for a particular course asset.
return False, None
field = fields[key]
if is_xblock:
return field.is_set_on(fields_or_xblock), getattr(xblock, key)
else:
return True, field
for key, criteria in qualifiers.iteritems():
is_set, value = _is_set_on(key)
if not is_set:
return False
if not self._value_matches(value, criteria):
return False
return True
def _value_matches(self, target, criteria):
'''
helper for _block_matches: does the target (field value) match the criteria?
If target is a list, do any of the list elements meet the criteria Arguments:
If the criteria is a regex, does the target match it? asset_key (AssetKey): key containing original asset filename
If the criteria is a function, does invoking it on the target yield something truthy?
If criteria is a dict {($nin|$in): []}, then do (none|any) of the list elements meet the criteria
Otherwise, is the target == criteria
'''
if isinstance(target, list):
return any(self._value_matches(ele, criteria) for ele in target)
elif isinstance(criteria, re._pattern_type):
return criteria.search(target) is not None
elif callable(criteria):
return criteria(target)
elif isinstance(criteria, dict) and '$in' in criteria:
# note isn't handling any other things in the dict other than in
return any(self._value_matches(target, test_val) for test_val in criteria['$in'])
elif isinstance(criteria, dict) and '$nin' in criteria:
# note isn't handling any other things in the dict other than nin
return not any(self._value_matches(target, test_val) for test_val in criteria['$nin'])
else:
return criteria == target
@abstractmethod Returns:
def make_course_key(self, org, course, run): asset metadata (AssetMetadata) -or- None if not found
""" """
Return a valid :class:`~opaque_keys.edx.keys.CourseKey` for this modulestore return self._find_asset_info(asset_key, thumbnail=False, **kwargs)
that matches the supplied `org`, `course`, and `run`.
This key may represent a course that doesn't exist in this modulestore. @contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
""" """
pass Find the metadata for a particular course asset.
@abstractmethod
def get_courses(self, **kwargs):
'''
Returns a list containing the top level XModuleDescriptors of the courses
in this modulestore.
'''
pass
@abstractmethod
def get_course(self, course_id, depth=0, **kwargs):
'''
Look for a specific course by its id (:class:`CourseKey`).
Returns the course descriptor, or None if not found.
'''
pass
@abstractmethod
def has_course(self, course_id, ignore_case=False, **kwargs):
'''
Look for a specific course id. Returns whether it exists.
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
to search for whether a potentially conflicting course exists in that case.
'''
pass
@abstractmethod Arguments:
def get_parent_location(self, location, **kwargs): asset_key (AssetKey): key containing original asset filename
'''
Find the location that is the parent of this location in this
course. Needed for path_to_location().
'''
pass
@abstractmethod Returns:
def get_orphans(self, course_key, **kwargs): asset metadata (AssetMetadata) -or- None if not found
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
use children to point to their dependents.
""" """
pass return self._find_asset_info(asset_key, thumbnail=True, **kwargs)
@abstractmethod @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool')
def get_errored_courses(self): def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
Return a dictionary of course_dir -> [(msg, exception_str)], for each
course_dir where course loading failed.
""" """
pass Returns a list of static asset (or thumbnail) metadata for a course.
@abstractmethod Args:
def get_modulestore_type(self, course_id): course_key (CourseKey): course identifier
""" start (int): optional - start at this asset number
Returns a type which identifies which modulestore is servicing the given maxresults (int): optional - return at most this many, -1 means no limit
course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses sort (array): optional - None means no sort
""" (sort_by (str), sort_order (str))
pass sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of 'ascending' or 'descending'
get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata
@abstractmethod Returns:
def get_courses_for_wiki(self, wiki_slug, **kwargs): List of AssetMetadata or AssetThumbnailMetadata objects.
"""
Return the list of courses which use this wiki_slug
:param wiki_slug: the course wiki root slug
:return: list of course keys
""" """
pass course_assets = self._find_course_assets(course_key)
if course_assets is None:
# If no course assets are found, return None instead of empty list
# to distinguish zero assets from "not able to retrieve assets".
return None
@abstractmethod if get_thumbnails:
def has_published_version(self, xblock): all_assets = course_assets.get('thumbnails', [])
""" else:
Returns true if this xblock exists in the published course regardless of whether it's up to date all_assets = course_assets.get('assets', [])
"""
pass
@abstractmethod # DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74
def close_connections(self): if start and maxresults and sort:
"""
Closes any open connections to the underlying databases
"""
pass pass
@contextmanager ret_assets = []
def bulk_operations(self, course_id): for asset in all_assets:
""" if get_thumbnails:
A context manager for notifying the store of bulk operations. This affects only the current thread. thumb = AssetThumbnailMetadata(
course_key.make_asset_key('thumbnail', asset['filename']),
internal_name=asset['filename'], **kwargs
)
ret_assets.append(thumb)
else:
asset = AssetMetadata(
course_key.make_asset_key('asset', asset['filename']),
basename=asset['filename'],
edited_on=asset['edit_info']['edited_on'],
contenttype=asset['contenttype'],
md5=str(asset['md5']), **kwargs
)
ret_assets.append(asset)
return ret_assets
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
""" """
yield Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
Args:
course_key (CourseKey): course identifier
start (int): optional - start at this asset number
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of 'ascending' or 'descending'
def ensure_indexes(self): Returns:
List of AssetMetadata objects.
""" """
Ensure that all appropriate indexes are created that are needed by this modulestore, or raise return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False, **kwargs)
an exception if unable to.
This method is intended for use by tests and administrative commands, and not @contract(course_key='CourseKey')
to be run during server startup. def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
""" """
pass Returns a list of thumbnails for all course assets.
Args:
course_key (CourseKey): course identifier
class ModuleStoreWrite(ModuleStoreRead): Returns:
""" List of AssetThumbnailMetadata objects.
An abstract interface for a database backend that stores XModuleDescriptor
instances and extends both read and write functionality
""" """
return self._get_all_asset_metadata(course_key, get_thumbnails=True, **kwargs)
__metaclass__ = ABCMeta
@abstractmethod class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
def update_item(self, xblock, user_id, allow_not_found=False, force=False, **kwargs):
""" """
Update the given xblock's persisted repr. Pass the user's unique id which the persistent store The write operations for assets and asset metadata
should save with the update if it has that ability.
:param allow_not_found: whether this method should raise an exception if the given xblock
has not been persisted before.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
:raises VersionConflictError: if org, course, run, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
""" """
pass def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
@abstractmethod
def delete_item(self, location, user_id, **kwargs):
""" """
Delete an item and its subtree from persistence. Remove the item from any parents (Note, does not Base method to over-ride in modulestore.
affect parents from other branches or logical branches; thus, in old mongo, deleting something """
whose parent cannot be draft, deletes it from both but deleting a component under a draft vertical raise NotImplementedError()
only deletes it from the draft.
Pass the user's unique id which the persistent store @contract(course_key='CourseKey', asset_metadata='AssetMetadata')
should save with the update if it has that ability. def save_asset_metadata(self, course_key, asset_metadata, user_id):
"""
Saves the asset metadata for a particular course's asset.
:param force: fork the structure and don't update the course draftVersion if there's a version Arguments:
conflict (only applicable to version tracking and conflict detecting persistence stores) course_key (CourseKey): course identifier
asset_metadata (AssetMetadata): data about the course asset data
:raises VersionConflictError: if org, course, run, and version_guid given and the current Returns:
version head != version_guid and force is not True. (only applicable to version tracking stores) True if metadata save was successful, else False
""" """
pass return self._save_asset_info(course_key, asset_metadata, user_id, thumbnail=False)
@abstractmethod @contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def create_course(self, org, course, run, user_id, fields=None, **kwargs): def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
""" """
Creates and returns the course. Saves the asset thumbnail metadata for a particular course asset's thumbnail.
Args: Arguments:
org (str): the organization that owns the course course_key (CourseKey): course identifier
course (str): the name of the course asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
run (str): the name of the run
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a CourseDescriptor Returns:
True if thumbnail metadata save was successful, else False
""" """
pass return self._save_asset_info(course_key, asset_thumbnail_metadata, user_id, thumbnail=True)
@abstractmethod def set_asset_metadata_attrs(self, asset_key, attrs, user_id):
def create_item(self, user_id, course_key, block_type, block_id=None, fields=None, **kwargs):
""" """
Creates and saves a new item in a course. Base method to over-ride in modulestore.
"""
Returns the newly created item. raise NotImplementedError()
Args: def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
user_id: ID of the user creating and saving the xmodule
course_key: A :class:`~opaque_keys.edx.CourseKey` identifying which course to create
this item in
block_type: The type of block to create
block_id: a unique identifier for the new item. If not supplied,
a new identifier will be generated
fields (dict): A dictionary specifying initial values for some or all fields
in the newly created block
""" """
pass Base method to over-ride in modulestore.
"""
raise NotImplementedError()
@abstractmethod @contract(asset_key='AssetKey', attr=str)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None): def set_asset_metadata_attr(self, asset_key, attr, value, user_id):
""" """
Sets up source_course_id to point a course with the same content as the desct_course_id. This Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts.
operation may be cheap or expensive. It may have to copy all assets and all xblock content or
merely setup new pointers.
Backward compatibility: this method used to require in some modulestores that dest_course_id Arguments:
pointed to an empty but already created course. Implementers should support this or should asset_key (AssetKey): asset identifier
enable creating the course from scratch. attr (str): which attribute to set
value: the value to set it to (any type pymongo accepts such as datetime, number, string)
Raises: Raises:
ItemNotFoundError: if the source course doesn't exist (or any of its xblocks aren't found) ItemNotFoundError if no such item exists
DuplicateItemError: if the destination course already exists (with content in some cases) AttributeError is attr is one of the build in attrs.
""" """
pass return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id)
@abstractmethod @contract(asset_key='AssetKey')
def delete_course(self, course_key, user_id, **kwargs): def delete_asset_metadata(self, asset_key, user_id):
""" """
Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions Deletes a single asset's metadata.
depending on the persistence layer and how tightly bound the xblocks are to the course.
Args: Arguments:
course_key (CourseKey): which course to delete asset_key (AssetKey): locator containing original asset filename
user_id: id of the user deleting the course
"""
pass
@abstractmethod Returns:
def _drop_database(self): Number of asset metadata entries deleted (0 or 1)
"""
A destructive operation to drop the underlying database and close all connections.
Intended to be used by test code for cleanup.
""" """
pass return self._delete_asset_data(asset_key, user_id, thumbnail=False)
class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
'''
Implement interface functionality that can be shared.
'''
# pylint: disable=W0613
def __init__(
self,
contentstore=None,
doc_store_config=None, # ignore if passed up
metadata_inheritance_cache_subsystem=None, request_cache=None,
xblock_mixins=(), xblock_select=None,
# temporary parms to enable backward compatibility. remove once all envs migrated
db=None, collection=None, host=None, port=None, tz_aware=True, user=None, password=None,
# allow lower level init args to pass harmlessly
** kwargs
):
'''
Set up the error-tracking logic.
'''
super(ModuleStoreReadBase, self).__init__(**kwargs)
self._course_errors = defaultdict(make_error_tracker) # location -> ErrorLog
# TODO move the inheritance_cache_subsystem to classes which use it
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
self.request_cache = request_cache
self.xblock_mixins = xblock_mixins
self.xblock_select = xblock_select
self.contentstore = contentstore
def get_course_errors(self, course_key): @contract(asset_key='AssetKey')
""" def delete_asset_thumbnail_metadata(self, asset_key, user_id):
Return list of errors for this :class:`.CourseKey`, if any. Raise the same
errors as get_item if course_key isn't present.
""" """
# check that item is present and raise the promised exceptions if needed Deletes a single asset's metadata.
# TODO (vshnayder): post-launch, make errors properties of items
# self.get_item(location)
assert(isinstance(course_key, CourseKey))
return self._course_errors[course_key].errors
def get_errored_courses(self): Arguments:
""" asset_key (AssetKey): locator containing original asset filename
Returns an empty dict.
It is up to subclasses to extend this method if the concept Returns:
of errored courses makes sense for their implementation. Number of asset metadata entries deleted (0 or 1)
""" """
return {} return self._delete_asset_data(asset_key, user_id, thumbnail=True)
def get_course(self, course_id, depth=0, **kwargs): @contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
""" """
See ModuleStoreRead.get_course Copy all the course assets from source_course_key to dest_course_key.
Default impl--linear search through course list Arguments:
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
""" """
assert(isinstance(course_id, CourseKey)) pass
for course in self.get_courses(**kwargs):
if course.id == course_id:
return course
return None
def has_course(self, course_id, ignore_case=False, **kwargs):
"""
Returns the course_id of the course if it was found, else None
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
to search for whether a potentially conflicting course exists in that case.
"""
# linear search through list
assert(isinstance(course_id, CourseKey))
if ignore_case:
return next(
(
c.id for c in self.get_courses()
if c.id.org.lower() == course_id.org.lower() and
c.id.course.lower() == course_id.course.lower() and
c.id.run.lower() == course_id.run.lower()
),
None
)
else:
return next(
(c.id for c in self.get_courses() if c.id == course_id),
None
)
def has_published_version(self, xblock): # pylint: disable=abstract-method
class ModuleStoreRead(ModuleStoreAssetInterface):
""" """
Returns True since this is a read-only store. An abstract interface for a database backend that stores XModuleDescriptor
instances and extends read-only functionality
""" """
return True
def heartbeat(self): __metaclass__ = ABCMeta
"""
Is this modulestore ready?
"""
# default is to say yes by not raising an exception
return {'default_impl': True}
def close_connections(self): @abstractmethod
def has_item(self, usage_key):
""" """
Closes any open connections to the underlying databases Returns True if usage_key exists in this ModuleStore.
""" """
if self.contentstore: pass
self.contentstore.close_connections()
super(ModuleStoreReadBase, self).close_connections()
@contextmanager @abstractmethod
def default_store(self, store_type): def get_item(self, usage_key, depth=0, **kwargs):
""" """
A context manager for temporarily changing the default store Returns an XModuleDescriptor instance for the item at location.
If any segment of the location is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
usage_key: A :class:`.UsageKey` subclass instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
""" """
if self.get_modulestore_type(None) != store_type: pass
raise ValueError(u"Cannot set default store to type {}".format(store_type))
yield
@staticmethod @abstractmethod
def memoize_request_cache(func): def get_course_errors(self, course_key):
""" """
Memoize a function call results on the request_cache if there's one. Creates the cache key by Return a list of (msg, exception-or-None) errors that the modulestore
joining the unicode of all the args with &; so, if your arg may use the default &, it may encountered when loading the course at course_id.
have false hits
Raises the same exceptions as get_item if the location isn't found or
isn't fully specified.
Args:
course_key (:class:`.CourseKey`): The course to check for errors
""" """
@functools.wraps(func) pass
def wrapper(self, *args, **kwargs):
@abstractmethod
def get_items(self, course_id, qualifiers=None, **kwargs):
""" """
Wraps a method to memoize results. Returns a list of XModuleDescriptor instances for the items
that match location. Any element of location that is None is treated
as a wildcard that matches any value
location: Something that can be passed to Location
""" """
if self.request_cache: pass
cache_key = '&'.join([hashvalue(arg) for arg in args])
if cache_key in self.request_cache.data.setdefault(func.__name__, {}):
return self.request_cache.data[func.__name__][cache_key]
result = func(self, *args, **kwargs) def _block_matches(self, fields_or_xblock, qualifiers):
'''
Return True or False depending on whether the field value (block contents)
matches the qualifiers as per get_items. Note, only finds directly set not
inherited nor default value matches.
For substring matching pass a regex object.
for arbitrary function comparison such as date time comparison, pass
the function as in start=lambda x: x < datetime.datetime(2014, 1, 1, 0, tzinfo=pytz.UTC)
self.request_cache.data[func.__name__][cache_key] = result Args:
return result fields_or_xblock (dict or XBlock): either the json blob (from the db or get_explicitly_set_fields)
or the xblock.fields() value or the XBlock from which to get those values
qualifiers (dict): field: searchvalue pairs.
'''
if isinstance(fields_or_xblock, XBlock):
fields = fields_or_xblock.fields
xblock = fields_or_xblock
is_xblock = True
else: else:
return func(self, *args, **kwargs) fields = fields_or_xblock
return wrapper is_xblock = False
def hashvalue(arg): def _is_set_on(key):
""" """
If arg is an xblock, use its location. otherwise just turn it into a string Is this key set in fields? (return tuple of boolean and value). A helper which can
handle fields either being the json doc or xblock fields. Is inner function to restrict
use and to access local vars.
""" """
if isinstance(arg, XBlock): if key not in fields:
return unicode(arg.location) return False, None
field = fields[key]
if is_xblock:
return field.is_set_on(fields_or_xblock), getattr(xblock, key)
else: else:
return unicode(arg) return True, field
for key, criteria in qualifiers.iteritems():
is_set, value = _is_set_on(key)
if not is_set:
return False
if not self._value_matches(value, criteria):
return False
return True
class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite): def _value_matches(self, target, criteria):
''' '''
Implement interface functionality that can be shared. helper for _block_matches: does the target (field value) match the criteria?
If target is a list, do any of the list elements meet the criteria
If the criteria is a regex, does the target match it?
If the criteria is a function, does invoking it on the target yield something truthy?
If criteria is a dict {($nin|$in): []}, then do (none|any) of the list elements meet the criteria
Otherwise, is the target == criteria
''' '''
def __init__(self, contentstore, **kwargs): if isinstance(target, list):
super(ModuleStoreWriteBase, self).__init__(contentstore=contentstore, **kwargs) return any(self._value_matches(ele, criteria) for ele in target)
elif isinstance(criteria, re._pattern_type): # pylint: disable=protected-access
return criteria.search(target) is not None
elif callable(criteria):
return criteria(target)
elif isinstance(criteria, dict) and '$in' in criteria:
# note isn't handling any other things in the dict other than in
return any(self._value_matches(target, test_val) for test_val in criteria['$in'])
elif isinstance(criteria, dict) and '$nin' in criteria:
# note isn't handling any other things in the dict other than nin
return not any(self._value_matches(target, test_val) for test_val in criteria['$nin'])
else:
return criteria == target
# TODO: Don't have a runtime just to generate the appropriate mixin classes (cpennington) @abstractmethod
# This is only used by partition_fields_by_scope, which is only needed because def make_course_key(self, org, course, run):
# the split mongo store is used for item creation as well as item persistence """
self.mixologist = Mixologist(self.xblock_mixins) Return a valid :class:`~opaque_keys.edx.keys.CourseKey` for this modulestore
that matches the supplied `org`, `course`, and `run`.
def partition_fields_by_scope(self, category, fields): This key may represent a course that doesn't exist in this modulestore.
""" """
Return dictionary of {scope: {field1: val, ..}..} for the fields of this potential xblock pass
:param category: the xblock category @abstractmethod
:param fields: the dictionary of {fieldname: value} def get_courses(self, **kwargs):
'''
Returns a list containing the top level XModuleDescriptors of the courses
in this modulestore.
'''
pass
@abstractmethod
def get_course(self, course_id, depth=0, **kwargs):
'''
Look for a specific course by its id (:class:`CourseKey`).
Returns the course descriptor, or None if not found.
'''
pass
@abstractmethod
def has_course(self, course_id, ignore_case=False, **kwargs):
'''
Look for a specific course id. Returns whether it exists.
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
to search for whether a potentially conflicting course exists in that case.
'''
pass
@abstractmethod
def get_parent_location(self, location, **kwargs):
'''
Find the location that is the parent of this location in this
course. Needed for path_to_location().
'''
pass
@abstractmethod
def get_orphans(self, course_key, **kwargs):
""" """
result = collections.defaultdict(dict) Get all of the xblocks in the given course which have no parents and are not of types which are
if fields is None: usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
return result use children to point to their dependents.
cls = self.mixologist.mix(XBlock.load_class(category, select=prefer_xmodules)) """
for field_name, value in fields.iteritems(): pass
field = getattr(cls, field_name)
result[field.scope][field_name] = value
return result
def create_course(self, org, course, run, user_id, fields=None, runtime=None, **kwargs): @abstractmethod
def get_errored_courses(self):
""" """
Creates any necessary other things for the course as a side effect and doesn't return Return a dictionary of course_dir -> [(msg, exception_str)], for each
anything useful. The real subclass should call this before it returns the course. course_dir where course loading failed.
""" """
# clone a default 'about' overview module as well pass
about_location = self.make_course_key(org, course, run).make_usage_key('about', 'overview')
about_descriptor = XBlock.load_class('about') @abstractmethod
overview_template = about_descriptor.get_template('overview.yaml') def get_modulestore_type(self, course_id):
self.create_item( """
user_id, Returns a type which identifies which modulestore is servicing the given
about_location.course_key, course_id. The return can be either "xml" (for XML based courses) or "mongo" for MongoDB backed courses
about_location.block_type, """
block_id=about_location.block_id, pass
definition_data={'data': overview_template.get('data')},
metadata=overview_template.get('metadata'),
runtime=runtime,
continue_version=True,
)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None, **kwargs): @abstractmethod
def get_courses_for_wiki(self, wiki_slug, **kwargs):
""" """
This base method just copies the assets. The lower level impls must do the actual cloning of Return the list of courses which use this wiki_slug
content. :param wiki_slug: the course wiki root slug
:return: list of course keys
""" """
# copy the assets pass
if self.contentstore:
self.contentstore.copy_all_course_assets(source_course_id, dest_course_id)
return dest_course_id
def delete_course(self, course_key, user_id, **kwargs): @abstractmethod
def has_published_version(self, xblock):
""" """
This base method just deletes the assets. The lower level impls must do the actual deleting of Returns true if this xblock exists in the published course regardless of whether it's up to date
content.
""" """
# delete the assets pass
if self.contentstore:
self.contentstore.delete_all_course_assets(course_key)
super(ModuleStoreWriteBase, self).delete_course(course_key, user_id)
def _drop_database(self): @abstractmethod
def close_connections(self):
""" """
A destructive operation to drop the underlying database and close all connections. Closes any open connections to the underlying databases
Intended to be used by test code for cleanup.
""" """
if self.contentstore: pass
self.contentstore._drop_database() # pylint: disable=protected-access
super(ModuleStoreWriteBase, self)._drop_database() # pylint: disable=protected-access
def create_child(self, user_id, parent_usage_key, block_type, block_id=None, fields=None, **kwargs): @contextmanager
def bulk_operations(self, course_id):
""" """
Creates and saves a new xblock that as a child of the specified block A context manager for notifying the store of bulk operations. This affects only the current thread.
"""
yield
Returns the newly created item. def ensure_indexes(self):
"""
Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
an exception if unable to.
Args: This method is intended for use by tests and administrative commands, and not
user_id: ID of the user creating and saving the xmodule to be run during server startup.
parent_usage_key: a :class:`~opaque_key.edx.UsageKey` identifing the
block that this item should be parented under
block_type: The type of block to create
block_id: a unique identifier for the new item. If not supplied,
a new identifier will be generated
fields (dict): A dictionary specifying initial values for some or all fields
in the newly created block
""" """
item = self.create_item(user_id, parent_usage_key.course_key, block_type, block_id=block_id, fields=fields, **kwargs) pass
parent = self.get_item(parent_usage_key)
parent.children.append(item.location)
self.update_item(parent, user_id)
def _find_course_assets(self, course_key):
# pylint: disable=abstract-method
class ModuleStoreWrite(ModuleStoreRead, ModuleStoreAssetWriteInterface):
""" """
Base method to override. An abstract interface for a database backend that stores XModuleDescriptor
instances and extends both read and write functionality
""" """
raise NotImplementedError()
def _find_course_asset(self, course_key, filename, get_thumbnail=False): __metaclass__ = ABCMeta
@abstractmethod
def update_item(self, xblock, user_id, allow_not_found=False, force=False, **kwargs):
""" """
Internal; finds or creates course asset info -and- finds existing asset (or thumbnail) metadata. Update the given xblock's persisted repr. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
Arguments: :param allow_not_found: whether this method should raise an exception if the given xblock
course_key (CourseKey): course identifier has not been persisted before.
filename (str): filename of the asset or thumbnail :param force: fork the structure and don't update the course draftVersion if there's a version
get_thumbnail (bool): True gets thumbnail data, False gets asset data conflict (only applicable to version tracking and conflict detecting persistence stores)
Returns: :raises VersionConflictError: if org, course, run, and version_guid given and the current
Asset info for the course, index of asset/thumbnail in list (None if asset/thumbnail does not exist) version head != version_guid and force is not True. (only applicable to version tracking stores)
""" """
course_assets = self._find_course_assets(course_key) pass
if course_assets is None:
return None, None
if get_thumbnail: @abstractmethod
all_assets = course_assets['thumbnails'] def delete_item(self, location, user_id, **kwargs):
else: """
all_assets = course_assets['assets'] Delete an item and its subtree from persistence. Remove the item from any parents (Note, does not
affect parents from other branches or logical branches; thus, in old mongo, deleting something
whose parent cannot be draft, deletes it from both but deleting a component under a draft vertical
only deletes it from the draft.
# See if this asset already exists by checking the external_filename. Pass the user's unique id which the persistent store
# Studio doesn't currently support using multiple course assets with the same filename. should save with the update if it has that ability.
# So use the filename as the unique identifier.
for idx, asset in enumerate(all_assets):
if asset['filename'] == filename:
return course_assets, idx
return course_assets, None :param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False): :raises VersionConflictError: if org, course, run, and version_guid given and the current
""" version head != version_guid and force is not True. (only applicable to version tracking stores)
Base method to over-ride in modulestore.
""" """
raise NotImplementedError() pass
@contract(course_key='CourseKey', asset_metadata='AssetMetadata') @abstractmethod
def save_asset_metadata(self, course_key, asset_metadata, user_id): def create_course(self, org, course, run, user_id, fields=None, **kwargs):
""" """
Saves the asset metadata for a particular course's asset. Creates and returns the course.
Arguments: Args:
course_key (CourseKey): course identifier org (str): the organization that owns the course
asset_metadata (AssetMetadata): data about the course asset data course (str): the name of the course
run (str): the name of the run
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: Returns: a CourseDescriptor
True if metadata save was successful, else False
""" """
return self._save_asset_info(course_key, asset_metadata, user_id, thumbnail=False) pass
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata') @abstractmethod
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id): def create_item(self, user_id, course_key, block_type, block_id=None, fields=None, **kwargs):
""" """
Saves the asset thumbnail metadata for a particular course asset's thumbnail. Creates and saves a new item in a course.
Arguments: Returns the newly created item.
course_key (CourseKey): course identifier
asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
Returns: Args:
True if thumbnail metadata save was successful, else False user_id: ID of the user creating and saving the xmodule
course_key: A :class:`~opaque_keys.edx.CourseKey` identifying which course to create
this item in
block_type: The type of block to create
block_id: a unique identifier for the new item. If not supplied,
a new identifier will be generated
fields (dict): A dictionary specifying initial values for some or all fields
in the newly created block
""" """
return self._save_asset_info(course_key, asset_thumbnail_metadata, user_id, thumbnail=True) pass
@contract(asset_key='AssetKey') @abstractmethod
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs): def clone_course(self, source_course_id, dest_course_id, user_id, fields=None):
""" """
Find the info for a particular course asset/thumbnail. Sets up source_course_id to point a course with the same content as the desct_course_id. This
operation may be cheap or expensive. It may have to copy all assets and all xblock content or
merely setup new pointers.
Arguments: Backward compatibility: this method used to require in some modulestores that dest_course_id
asset_key (AssetKey): key containing original asset filename pointed to an empty but already created course. Implementers should support this or should
thumbnail (bool): True if finding thumbnail, False if finding asset metadata enable creating the course from scratch.
Returns: Raises:
asset/thumbnail metadata (AssetMetadata/AssetThumbnailMetadata) -or- None if not found ItemNotFoundError: if the source course doesn't exist (or any of its xblocks aren't found)
DuplicateItemError: if the destination course already exists (with content in some cases)
""" """
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, thumbnail) pass
if asset_idx is None:
return None
if thumbnail:
info = 'thumbnails'
mdata = AssetThumbnailMetadata(asset_key, asset_key.path, **kwargs)
else:
info = 'assets'
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
@contract(asset_key='AssetKey') @abstractmethod
def find_asset_metadata(self, asset_key, **kwargs): def delete_course(self, course_key, user_id, **kwargs):
""" """
Find the metadata for a particular course asset. Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions
depending on the persistence layer and how tightly bound the xblocks are to the course.
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns: Args:
asset metadata (AssetMetadata) -or- None if not found course_key (CourseKey): which course to delete
user_id: id of the user deleting the course
""" """
return self._find_asset_info(asset_key, thumbnail=False, **kwargs) pass
@contract(asset_key='AssetKey') @abstractmethod
def find_asset_thumbnail_metadata(self, asset_key, **kwargs): def _drop_database(self):
""" """
Find the metadata for a particular course asset. A destructive operation to drop the underlying database and close all connections.
Intended to be used by test code for cleanup.
"""
pass
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns: # pylint: disable=abstract-method
asset metadata (AssetMetadata) -or- None if not found class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
'''
Implement interface functionality that can be shared.
'''
# pylint: disable=invalid-name
def __init__(
self,
contentstore=None,
doc_store_config=None, # ignore if passed up
metadata_inheritance_cache_subsystem=None, request_cache=None,
xblock_mixins=(), xblock_select=None,
# temporary parms to enable backward compatibility. remove once all envs migrated
db=None, collection=None, host=None, port=None, tz_aware=True, user=None, password=None,
# allow lower level init args to pass harmlessly
** kwargs
):
'''
Set up the error-tracking logic.
'''
super(ModuleStoreReadBase, self).__init__(**kwargs)
self._course_errors = defaultdict(make_error_tracker) # location -> ErrorLog
# pylint: disable=fixme
# TODO move the inheritance_cache_subsystem to classes which use it
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
self.request_cache = request_cache
self.xblock_mixins = xblock_mixins
self.xblock_select = xblock_select
self.contentstore = contentstore
def get_course_errors(self, course_key):
""" """
return self._find_asset_info(asset_key, thumbnail=True, **kwargs) Return list of errors for this :class:`.CourseKey`, if any. Raise the same
errors as get_item if course_key isn't present.
"""
# check that item is present and raise the promised exceptions if needed
# pylint: disable=fixme
# TODO (vshnayder): post-launch, make errors properties of items
# self.get_item(location)
assert(isinstance(course_key, CourseKey))
return self._course_errors[course_key].errors
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool') def get_errored_courses(self):
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
""" """
Returns a list of static asset (or thumbnail) metadata for a course. Returns an empty dict.
Args: It is up to subclasses to extend this method if the concept
course_key (CourseKey): course identifier of errored courses makes sense for their implementation.
start (int): optional - start at this asset number """
maxresults (int): optional - return at most this many, -1 means no limit return {}
sort (array): optional - None means no sort
(sort_by (str), sort_order (str)) def get_course(self, course_id, depth=0, **kwargs):
sort_by - one of 'uploadDate' or 'displayname' """
sort_order - one of 'ascending' or 'descending' See ModuleStoreRead.get_course
get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata
Returns: Default impl--linear search through course list
List of AssetMetadata or AssetThumbnailMetadata objects.
""" """
course_assets = self._find_course_assets(course_key) assert(isinstance(course_id, CourseKey))
if course_assets is None: for course in self.get_courses(**kwargs):
# If no course assets are found, return None instead of empty list if course.id == course_id:
# to distinguish zero assets from "not able to retrieve assets". return course
return None return None
if get_thumbnails: def has_course(self, course_id, ignore_case=False, **kwargs):
all_assets = course_assets.get('thumbnails', []) """
else: Returns the course_id of the course if it was found, else None
all_assets = course_assets.get('assets', []) Args:
course_id (CourseKey):
# DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74 ignore_case (boolean): some modulestores are case-insensitive. Use this flag
if start and maxresults and sort: to search for whether a potentially conflicting course exists in that case.
pass """
# linear search through list
ret_assets = [] assert(isinstance(course_id, CourseKey))
for asset in all_assets: if ignore_case:
if get_thumbnails: return next(
thumb = AssetThumbnailMetadata( (
course_key.make_asset_key('thumbnail', asset['filename']), c.id for c in self.get_courses()
internal_name=asset['filename'], **kwargs if c.id.org.lower() == course_id.org.lower() and
c.id.course.lower() == course_id.course.lower() and
c.id.run.lower() == course_id.run.lower()
),
None
) )
ret_assets.append(thumb)
else: else:
asset = AssetMetadata( return next(
course_key.make_asset_key('asset', asset['filename']), (c.id for c in self.get_courses() if c.id == course_id),
basename=asset['filename'], None
edited_on=asset['edit_info']['edited_on'],
contenttype=asset['contenttype'],
md5=str(asset['md5']), **kwargs
) )
ret_assets.append(asset)
return ret_assets
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None') def has_published_version(self, xblock):
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
""" """
Returns a list of static assets for a course. Returns True since this is a read-only store.
By default all assets are returned, but start and maxresults can be provided to limit the query.
Args:
course_key (CourseKey): course identifier
start (int): optional - start at this asset number
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of 'ascending' or 'descending'
Returns:
List of AssetMetadata objects.
""" """
return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False, **kwargs) return True
@contract(course_key='CourseKey') def heartbeat(self):
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
""" """
Returns a list of thumbnails for all course assets. Is this modulestore ready?
Args:
course_key (CourseKey): course identifier
Returns:
List of AssetThumbnailMetadata objects.
""" """
return self._get_all_asset_metadata(course_key, get_thumbnails=True, **kwargs) # default is to say yes by not raising an exception
return {'default_impl': True}
def set_asset_metadata_attrs(self, asset_key, attrs, user_id): def close_connections(self):
""" """
Base method to over-ride in modulestore. Closes any open connections to the underlying databases
""" """
raise NotImplementedError() if self.contentstore:
self.contentstore.close_connections()
super(ModuleStoreReadBase, self).close_connections()
def _delete_asset_data(self, asset_key, user_id, thumbnail=False): @contextmanager
def default_store(self, store_type):
""" """
Base method to over-ride in modulestore. A context manager for temporarily changing the default store
""" """
raise NotImplementedError() if self.get_modulestore_type(None) != store_type:
raise ValueError(u"Cannot set default store to type {}".format(store_type))
yield
@contract(asset_key='AssetKey', attr=str) @staticmethod
def set_asset_metadata_attr(self, asset_key, attr, value, user_id): def memoize_request_cache(func):
""" """
Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts. Memoize a function call results on the request_cache if there's one. Creates the cache key by
joining the unicode of all the args with &; so, if your arg may use the default &, it may
have false hits
"""
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
"""
Wraps a method to memoize results.
"""
if self.request_cache:
cache_key = '&'.join([hashvalue(arg) for arg in args])
if cache_key in self.request_cache.data.setdefault(func.__name__, {}):
return self.request_cache.data[func.__name__][cache_key]
Arguments: result = func(self, *args, **kwargs)
asset_key (AssetKey): asset identifier
attr (str): which attribute to set
value: the value to set it to (any type pymongo accepts such as datetime, number, string)
Raises: self.request_cache.data[func.__name__][cache_key] = result
ItemNotFoundError if no such item exists return result
AttributeError is attr is one of the build in attrs. else:
return func(self, *args, **kwargs)
return wrapper
def hashvalue(arg):
""" """
return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id) If arg is an xblock, use its location. otherwise just turn it into a string
"""
if isinstance(arg, XBlock):
return unicode(arg.location)
else:
return unicode(arg)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key, user_id): # pylint: disable=abstract-method
class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
'''
Implement interface functionality that can be shared.
'''
def __init__(self, contentstore, **kwargs):
super(ModuleStoreWriteBase, self).__init__(contentstore=contentstore, **kwargs)
self.mixologist = Mixologist(self.xblock_mixins)
def partition_fields_by_scope(self, category, fields):
""" """
Deletes a single asset's metadata. Return dictionary of {scope: {field1: val, ..}..} for the fields of this potential xblock
Arguments: :param category: the xblock category
asset_key (AssetKey): locator containing original asset filename :param fields: the dictionary of {fieldname: value}
"""
result = collections.defaultdict(dict)
if fields is None:
return result
cls = self.mixologist.mix(XBlock.load_class(category, select=prefer_xmodules))
for field_name, value in fields.iteritems():
field = getattr(cls, field_name)
result[field.scope][field_name] = value
return result
Returns: def create_course(self, org, course, run, user_id, fields=None, runtime=None, **kwargs):
Number of asset metadata entries deleted (0 or 1)
""" """
return self._delete_asset_data(asset_key, user_id, thumbnail=False) Creates any necessary other things for the course as a side effect and doesn't return
anything useful. The real subclass should call this before it returns the course.
"""
# clone a default 'about' overview module as well
about_location = self.make_course_key(org, course, run).make_usage_key('about', 'overview')
@contract(asset_key='AssetKey') about_descriptor = XBlock.load_class('about')
def delete_asset_thumbnail_metadata(self, asset_key, user_id): overview_template = about_descriptor.get_template('overview.yaml')
self.create_item(
user_id,
about_location.course_key,
about_location.block_type,
block_id=about_location.block_id,
definition_data={'data': overview_template.get('data')},
metadata=overview_template.get('metadata'),
runtime=runtime,
continue_version=True,
)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None, **kwargs):
""" """
Deletes a single asset's metadata. This base method just copies the assets. The lower level impls must do the actual cloning of
content.
"""
# copy the assets
if self.contentstore:
self.contentstore.copy_all_course_assets(source_course_id, dest_course_id)
return dest_course_id
Arguments: def delete_course(self, course_key, user_id, **kwargs):
asset_key (AssetKey): locator containing original asset filename """
This base method just deletes the assets. The lower level impls must do the actual deleting of
content.
"""
# delete the assets
if self.contentstore:
self.contentstore.delete_all_course_assets(course_key)
super(ModuleStoreWriteBase, self).delete_course(course_key, user_id)
Returns: def _drop_database(self):
Number of asset metadata entries deleted (0 or 1)
""" """
return self._delete_asset_data(asset_key, user_id, thumbnail=True) A destructive operation to drop the underlying database and close all connections.
Intended to be used by test code for cleanup.
"""
if self.contentstore:
self.contentstore._drop_database() # pylint: disable=protected-access
super(ModuleStoreWriteBase, self)._drop_database() # pylint: disable=protected-access
@contract(source_course_key='CourseKey', dest_course_key='CourseKey') def create_child(self, user_id, parent_usage_key, block_type, block_id=None, fields=None, **kwargs):
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
""" """
Copy all the course assets from source_course_key to dest_course_key. Creates and saves a new xblock that as a child of the specified block
Arguments: Returns the newly created item.
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to Args:
user_id: ID of the user creating and saving the xmodule
parent_usage_key: a :class:`~opaque_key.edx.UsageKey` identifing the
block that this item should be parented under
block_type: The type of block to create
block_id: a unique identifier for the new item. If not supplied,
a new identifier will be generated
fields (dict): A dictionary specifying initial values for some or all fields
in the newly created block
""" """
pass item = self.create_item(user_id, parent_usage_key.course_key, block_type, block_id=block_id, fields=fields, **kwargs)
parent = self.get_item(parent_usage_key)
parent.children.append(item.location)
self.update_item(parent, user_id)
def only_xmodules(identifier, entry_points): def only_xmodules(identifier, entry_points):
......
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
...@@ -12,7 +12,7 @@ from xmodule.modulestore import ModuleStoreEnum ...@@ -12,7 +12,7 @@ from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MODULESTORE_SETUPS, MongoContentstoreBuilder, MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder
) )
...@@ -392,3 +392,21 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -392,3 +392,21 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
def test_copy_all_assets(self): def test_copy_all_assets(self):
pass pass
@ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())]))
def test_xml_not_yet_implemented(self, storebuilder):
"""
Test coverage which shows that for now xml read operations are not implemented
"""
with storebuilder.build(None) as store:
course_key = store.make_course_key("org", "course", "run")
asset_key = course_key.make_asset_key('asset', 'foo.jpg')
for method in ['_find_asset_info', 'find_asset_metadata', 'find_asset_thumbnail_metadata']:
with self.assertRaises(NotImplementedError):
getattr(store, method)(asset_key)
with self.assertRaises(NotImplementedError):
# pylint: disable=protected-access
store._find_course_asset(course_key, asset_key.block_id)
for method in ['_get_all_asset_metadata', 'get_all_asset_metadata', 'get_all_asset_thumbnail_metadata']:
with self.assertRaises(NotImplementedError):
getattr(store, method)(course_key)
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
...@@ -17,6 +17,7 @@ import random ...@@ -17,6 +17,7 @@ import random
from contextlib import contextmanager, nested from contextlib import contextmanager, nested
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
from path import path
from xmodule.tests import CourseComparisonTest from xmodule.tests import CourseComparisonTest
...@@ -30,13 +31,14 @@ from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleSto ...@@ -30,13 +31,14 @@ from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleSto
from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
from xmodule.modulestore.inheritance import InheritanceMixin from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.x_module import XModuleMixin from xmodule.x_module import XModuleMixin
from xmodule.modulestore.xml import XMLModuleStore
COMMON_DOCSTORE_CONFIG = { COMMON_DOCSTORE_CONFIG = {
'host': MONGO_HOST, 'host': MONGO_HOST,
'port': MONGO_PORT_NUM, 'port': MONGO_PORT_NUM,
} }
DATA_DIR = path(__file__).dirname().parent.parent.parent.parent.parent / "test" / "data"
XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin) XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)
...@@ -163,6 +165,30 @@ class VersioningModulestoreBuilder(object): ...@@ -163,6 +165,30 @@ class VersioningModulestoreBuilder(object):
return 'SplitModulestoreBuilder()' return 'SplitModulestoreBuilder()'
class XmlModulestoreBuilder(object):
"""
A builder class for a XMLModuleStore.
"""
# pylint: disable=unused-argument
@contextmanager
def build(self, contentstore=None, course_ids=None):
"""
A contextmanager that returns an isolated xml modulestore
Args:
contentstore: The contentstore that this modulestore should use to store
all of its assets.
"""
modulestore = XMLModuleStore(
DATA_DIR,
course_ids=course_ids,
default_class='xmodule.hidden_module.HiddenDescriptor',
xblock_mixins=XBLOCK_MIXINS,
)
yield modulestore
class MixedModulestoreBuilder(object): class MixedModulestoreBuilder(object):
""" """
A builder class for a MixedModuleStore. A builder class for a MixedModuleStore.
......
common/lib/xmodule/xmodule/modulestore/xml.py
...@@ -846,3 +846,52 @@ class XMLModuleStore(ModuleStoreReadBase): ...@@ -846,3 +846,52 @@ class XMLModuleStore(ModuleStoreReadBase):
if branch_setting != ModuleStoreEnum.Branch.published_only: if branch_setting != ModuleStoreEnum.Branch.published_only:
raise ValueError(u"Cannot set branch setting to {} on a ReadOnly store".format(branch_setting)) raise ValueError(u"Cannot set branch setting to {} on a ReadOnly store".format(branch_setting))
yield yield
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def find_asset_metadata(self, asset_key, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment