Commit 7f126f13 by Don Mitchell

Merge pull request #624 from edx/dhm/flatten_kvs

xblock fields persist w/o breaking by scope
parents 5b1b73cb 4c286840
'''
Created on May 7, 2013
@author: dmitchell
'''
import unittest
from xmodule import templates
from xmodule.modulestore.tests import persistent_factories
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.seq_module import SequenceDescriptor
from xmodule.x_module import XModuleDescriptor
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.html_module import HtmlDescriptor
from xmodule.modulestore import inheritance
from xmodule.x_module import XModuleDescriptor
class TemplateTests(unittest.TestCase):
......@@ -74,8 +70,8 @@ class TemplateTests(unittest.TestCase):
test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
display_name='fun test course', user_id='testbot')
test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
'metadata': {'display_name': 'chapter n'}},
test_chapter = self.load_from_json({'category': 'chapter',
'fields': {'display_name': 'chapter n'}},
test_course.system, parent_xblock=test_course)
self.assertIsInstance(test_chapter, SequenceDescriptor)
self.assertEqual(test_chapter.display_name, 'chapter n')
......@@ -83,8 +79,8 @@ class TemplateTests(unittest.TestCase):
# test w/ a definition (e.g., a problem)
test_def_content = '<problem>boo</problem>'
test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
'definition': {'data': test_def_content}},
test_problem = self.load_from_json({'category': 'problem',
'fields': {'data': test_def_content}},
test_course.system, parent_xblock=test_chapter)
self.assertIsInstance(test_problem, CapaDescriptor)
self.assertEqual(test_problem.data, test_def_content)
......@@ -98,12 +94,13 @@ class TemplateTests(unittest.TestCase):
"""
test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
display_name='fun test course', user_id='testbot')
test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
'metadata': {'display_name': 'chapter n'}},
test_chapter = self.load_from_json({'category': 'chapter',
'fields': {'display_name': 'chapter n'}},
test_course.system, parent_xblock=test_course)
test_def_content = '<problem>boo</problem>'
test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
'definition': {'data': test_def_content}},
# create child
_ = self.load_from_json({'category': 'problem',
'fields': {'data': test_def_content}},
test_course.system, parent_xblock=test_chapter)
# better to pass in persisted parent over the subdag so
# subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
......@@ -152,15 +149,24 @@ class TemplateTests(unittest.TestCase):
parent_location=test_course.location, user_id='testbot')
sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
parent_location=chapter.location, user_id='testbot', category='vertical')
first_problem = persistent_factories.ItemFactory.create(display_name='problem 1',
parent_location=sub.location, user_id='testbot', category='problem', data="<problem></problem>")
first_problem = persistent_factories.ItemFactory.create(
display_name='problem 1', parent_location=sub.location, user_id='testbot', category='problem',
data="<problem></problem>"
)
first_problem.max_attempts = 3
first_problem.save() # decache the above into the kvs
updated_problem = modulestore('split').update_item(first_problem, 'testbot')
updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot')
self.assertIsNotNone(updated_problem.previous_version)
self.assertEqual(updated_problem.previous_version, first_problem.update_version)
self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)
second_problem = persistent_factories.ItemFactory.create(display_name='problem 2',
second_problem = persistent_factories.ItemFactory.create(
display_name='problem 2',
parent_location=BlockUsageLocator(updated_loc, usage_id=sub.location.usage_id),
user_id='testbot', category='problem', data="<problem></problem>")
user_id='testbot', category='problem',
data="<problem></problem>"
)
# course root only updated 2x
version_history = modulestore('split').get_block_generations(test_course.location)
......@@ -184,3 +190,48 @@ class TemplateTests(unittest.TestCase):
version_history = modulestore('split').get_block_generations(second_problem.location)
self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)
# ================================= JSON PARSING ===========================
# These are example methods for creating xmodules in memory w/o persisting them.
# They were in x_module but since xblock is not planning to support them but will
# allow apps to use this type of thing, I put it here.
@staticmethod
def load_from_json(json_data, system, default_class=None, parent_xblock=None):
"""
This method instantiates the correct subclass of XModuleDescriptor based
on the contents of json_data. It does not persist it and can create one which
has no usage id.
parent_xblock is used to compute inherited metadata as well as to append the new xblock.
json_data:
- 'location' : must have this field
- 'category': the xmodule category (required or location must be a Location)
- 'metadata': a dict of locally set metadata (not inherited)
- 'children': a list of children's usage_ids w/in this course
- 'definition':
- '_id' (optional): the usage_id of this. Will generate one if not given one.
"""
class_ = XModuleDescriptor.load_class(
json_data.get('category', json_data.get('location', {}).get('category')),
default_class
)
usage_id = json_data.get('_id', None)
if not '_inherited_settings' in json_data and parent_xblock is not None:
json_data['_inherited_settings'] = parent_xblock.xblock_kvs.get_inherited_settings().copy()
json_fields = json_data.get('fields', {})
for field in inheritance.INHERITABLE_METADATA:
if field in json_fields:
json_data['_inherited_settings'][field] = json_fields[field]
new_block = system.xblock_from_json(class_, usage_id, json_data)
if parent_xblock is not None:
children = parent_xblock.children
children.append(new_block)
# trigger setter method by using top level field access
parent_xblock.children = children
# decache pending children field settings (Note, truly persisting at this point would break b/c
# persistence assumes children is a list of ids not actual xblocks)
parent_xblock.save()
return new_block
......@@ -58,14 +58,12 @@ def save_item(request):
# 'apply' the submitted metadata, so we don't end up deleting system metadata
existing_item = modulestore().get_item(item_location)
for metadata_key in request.POST.get('nullout', []):
# [dhm] see comment on _get_xblock_field
_get_xblock_field(existing_item, metadata_key).write_to(existing_item, None)
# update existing metadata with submitted metadata (which can be partial)
# IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
# the intent is to make it None, use the nullout field
for metadata_key, value in request.POST.get('metadata', {}).items():
# [dhm] see comment on _get_xblock_field
field = _get_xblock_field(existing_item, metadata_key)
if value is None:
......@@ -82,32 +80,15 @@ def save_item(request):
return JsonResponse()
# [DHM] A hack until we implement a permanent soln. Proposed perm solution is to make namespace fields also top level
# fields in xblocks rather than requiring dereference through namespace but we'll need to consider whether there are
# plausible use cases for distinct fields w/ same name in different namespaces on the same blocks.
# The idea is that consumers of the xblock, and particularly the web client, shouldn't know about our internal
# representation (namespaces as means of decorating all modules).
# Given top-level access, the calls can simply be setattr(existing_item, field, value) ...
# Really, this method should be elsewhere (e.g., xblock). We also need methods for has_value (v is_default)...
def _get_xblock_field(xblock, field_name):
"""
A temporary function to get the xblock field either from the xblock or one of its namespaces by name.
:param xblock:
:param field_name:
"""
def find_field(fields):
for field in fields:
if field.name == field_name:
return field
found = find_field(xblock.fields)
if found:
return found
for namespace in xblock.namespaces:
found = find_field(getattr(xblock, namespace).fields)
if found:
return found
for field in xblock.iterfields():
if field.name == field_name:
return field
@login_required
@expect_json
......
......@@ -11,18 +11,17 @@ from .split_mongo_kvs import SplitMongoKVS, SplitMongoKVSid
log = logging.getLogger(__name__)
# TODO should this be here or w/ x_module or ???
class CachingDescriptorSystem(MakoDescriptorSystem):
"""
A system that has a cache of a course version's json that it will use to load modules
from, with a backup of calling to the underlying modulestore for more data.
Computes the metadata inheritance upon creation.
Computes the settings (nee 'metadata') inheritance upon creation.
"""
def __init__(self, modulestore, course_entry, module_data, lazy,
default_class, error_tracker, render_template):
"""
Computes the metadata inheritance and sets up the cache.
Computes the settings inheritance and sets up the cache.
modulestore: the module store that can be used to retrieve additional
modules
......@@ -50,9 +49,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.default_class = default_class
# TODO see if self.course_id is needed: is already in course_entry but could be > 1 value
# Compute inheritance
modulestore.inherit_metadata(course_entry.get('blocks', {}),
course_entry.get('blocks', {})
.get(course_entry.get('root')))
modulestore.inherit_settings(
course_entry.get('blocks', {}),
course_entry.get('blocks', {}).get(course_entry.get('root'))
)
def _load_item(self, usage_id, course_entry_override=None):
# TODO ensure all callers of system.load_item pass just the id
......@@ -73,9 +73,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
def xblock_from_json(self, class_, usage_id, json_data, course_entry_override=None):
if course_entry_override is None:
course_entry_override = self.course_entry
# most likely a lazy loader but not the id directly
# most likely a lazy loader or the id directly
definition = json_data.get('definition', {})
metadata = json_data.get('metadata', {})
block_locator = BlockUsageLocator(
version_guid=course_entry_override['_id'],
......@@ -86,9 +85,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
kvs = SplitMongoKVS(
definition,
json_data.get('children', []),
metadata,
json_data.get('_inherited_metadata'),
json_data.get('fields', {}),
json_data.get('_inherited_settings'),
block_locator,
json_data.get('category'))
model_data = DbModel(kvs, class_, None,
......@@ -111,10 +109,11 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
error_msg=exc_info_to_str(sys.exc_info())
)
module.edited_by = json_data.get('edited_by')
module.edited_on = json_data.get('edited_on')
module.previous_version = json_data.get('previous_version')
module.update_version = json_data.get('update_version')
edit_info = json_data.get('edit_info', {})
module.edited_by = edit_info.get('edited_by')
module.edited_on = edit_info.get('edited_on')
module.previous_version = edit_info.get('previous_version')
module.update_version = edit_info.get('update_version')
module.definition_locator = self.modulestore.definition_locator(definition)
# decache any pending field settings
module.save()
......
......@@ -11,44 +11,24 @@ class PersistentCourseFactory(factory.Factory):
"""
Create a new course (not a new version of a course, but a whole new index entry).
keywords:
keywords: any xblock field plus (note, the below are filtered out; so, if they
become legitimate xblock fields, they won't be settable via this factory)
* org: defaults to textX
* prettyid: defaults to 999
* display_name
* user_id
* data (optional) the data payload to save in the course item
* metadata (optional) the metadata payload. If display_name is in the metadata, that takes
precedence over any display_name provided directly.
* master_branch: (optional) defaults to 'draft'
* user_id: (optional) defaults to 'test_user'
* display_name (xblock field): will default to 'Robot Super Course' unless provided
"""
FACTORY_FOR = CourseDescriptor
org = 'testX'
prettyid = '999'
display_name = 'Robot Super Course'
user_id = "test_user"
data = None
metadata = None
master_version = 'draft'
# pylint: disable=W0613
@classmethod
def _create(cls, target_class, *args, **kwargs):
org = kwargs.get('org')
prettyid = kwargs.get('prettyid')
display_name = kwargs.get('display_name')
user_id = kwargs.get('user_id')
data = kwargs.get('data')
metadata = kwargs.get('metadata', {})
if metadata is None:
metadata = {}
if 'display_name' not in metadata:
metadata['display_name'] = display_name
def _create(cls, target_class, org='testX', prettyid='999', user_id='test_user', master_branch='draft', **kwargs):
# Write the data to the mongo datastore
new_course = modulestore('split').create_course(
org, prettyid, user_id, metadata=metadata, course_data=data, id_root=prettyid,
master_version=kwargs.get('master_version'))
org, prettyid, user_id, fields=kwargs, id_root=prettyid,
master_branch=master_branch)
return new_course
......@@ -60,36 +40,24 @@ class PersistentCourseFactory(factory.Factory):
class ItemFactory(factory.Factory):
FACTORY_FOR = XModuleDescriptor
category = 'chapter'
user_id = 'test_user'
display_name = factory.LazyAttributeSequence(lambda o, n: "{} {}".format(o.category, n))
# pylint: disable=W0613
@classmethod
def _create(cls, target_class, *args, **kwargs):
def _create(cls, target_class, parent_location, category='chapter',
user_id='test_user', definition_locator=None, **kwargs):
"""
Uses *kwargs*:
*parent_location* (required): the location of the course & possibly parent
passes *kwargs* as the new item's field values:
*category* (defaults to 'chapter')
:param parent_location: (required) the location of the course & possibly parent
*data* (optional): the data for the item
:param category: (defaults to 'chapter')
definition_locator (optional): the DescriptorLocator for the definition this uses or branches
*display_name* (optional): the display name of the item
*metadata* (optional): dictionary of metadata attributes (display_name here takes
precedence over the above attr)
:param definition_locator (optional): the DescriptorLocator for the definition this uses or branches
"""
metadata = kwargs.get('metadata', {})
if 'display_name' not in metadata and 'display_name' in kwargs:
metadata['display_name'] = kwargs['display_name']
return modulestore('split').create_item(kwargs['parent_location'], kwargs['category'],
kwargs['user_id'], definition_locator=kwargs.get('definition_locator'),
new_def_data=kwargs.get('data'), metadata=metadata)
return modulestore('split').create_item(
parent_location, category, user_id, definition_locator, fields=kwargs
)
@classmethod
def _build(cls, target_class, *args, **kwargs):
......
......@@ -7,7 +7,7 @@ from lxml import etree
from collections import namedtuple
from pkg_resources import resource_listdir, resource_string, resource_isdir
from xmodule.modulestore import inheritance, Location
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError, InvalidLocationError
from xblock.core import XBlock, Scope, String, Integer, Float, List, ModelType
......@@ -557,75 +557,6 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
"""
return False
# ================================= JSON PARSING ===========================
@staticmethod
def load_from_json(json_data, system, default_class=None, parent_xblock=None):
"""
This method instantiates the correct subclass of XModuleDescriptor based
on the contents of json_data. It does not persist it and can create one which
has no usage id.
parent_xblock is used to compute inherited metadata as well as to append the new xblock.
json_data:
- 'location' : must have this field
- 'category': the xmodule category (required or location must be a Location)
- 'metadata': a dict of locally set metadata (not inherited)
- 'children': a list of children's usage_ids w/in this course
- 'definition':
- '_id' (optional): the usage_id of this. Will generate one if not given one.
"""
class_ = XModuleDescriptor.load_class(
json_data.get('category', json_data.get('location', {}).get('category')),
default_class
)
return class_.from_json(json_data, system, parent_xblock)
@classmethod
def from_json(cls, json_data, system, parent_xblock=None):
"""
Creates an instance of this descriptor from the supplied json_data.
This may be overridden by subclasses
json_data: A json object with the keys 'definition' and 'metadata',
definition: A json object with the keys 'data' and 'children'
data: A json value
children: A list of edX Location urls
metadata: A json object with any keys
This json_data is transformed to model_data using the following rules:
1) The model data contains all of the fields from metadata
2) The model data contains the 'children' array
3) If 'definition.data' is a json object, model data contains all of its fields
Otherwise, it contains the single field 'data'
4) Any value later in this list overrides a value earlier in this list
json_data:
- 'category': the xmodule category (required)
- 'metadata': a dict of locally set metadata (not inherited)
- 'children': a list of children's usage_ids w/in this course
- 'definition':
- '_id' (optional): the usage_id of this. Will generate one if not given one.
"""
usage_id = json_data.get('_id', None)
if not '_inherited_metadata' in json_data and parent_xblock is not None:
json_data['_inherited_metadata'] = parent_xblock.xblock_kvs.get_inherited_metadata().copy()
json_metadata = json_data.get('metadata', {})
for field in inheritance.INHERITABLE_METADATA:
if field in json_metadata:
json_data['_inherited_metadata'][field] = json_metadata[field]
new_block = system.xblock_from_json(cls, usage_id, json_data)
if parent_xblock is not None:
children = parent_xblock.children
children.append(new_block)
# trigger setter method by using top level field access
parent_xblock.children = children
# decache pending children field settings (Note, truly persisting at this point would break b/c
# persistence assumes children is a list of ids not actual xblocks)
parent_xblock.save()
return new_block
@classmethod
def _translate(cls, key):
'VS[compat]'
......@@ -726,6 +657,17 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
)
)
def iterfields(self):
"""
A generator for iterate over the fields of this xblock (including the ones in namespaces).
Example usage: [field.name for field in module.iterfields()]
"""
for field in self.fields:
yield field
for namespace in self.namespaces:
for field in getattr(self, namespace).fields:
yield field
@property
def non_editable_metadata_fields(self):
"""
......@@ -736,6 +678,27 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
# We are not allowing editing of xblock tag and name fields at this time (for any component).
return [XBlock.tags, XBlock.name]
def get_explicitly_set_fields_by_scope(self, scope=Scope.content):
"""
Get a dictionary of the fields for the given scope which are set explicitly on this xblock. (Including
any set to None.)
"""
if scope == Scope.settings and hasattr(self, '_inherited_metadata'):
inherited_metadata = getattr(self, '_inherited_metadata')
result = {}
for field in self.iterfields():
if (field.scope == scope and
field.name in self._model_data and
field.name not in inherited_metadata):
result[field.name] = self._model_data[field.name]
return result
else:
result = {}
for field in self.iterfields():
if (field.scope == scope and field.name in self._model_data):
result[field.name] = self._model_data[field.name]
return result
@property
def editable_metadata_fields(self):
"""
......
......@@ -2,7 +2,7 @@
{
"_id":"head12345_12",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -43,15 +43,17 @@
},
"wiki_slug":null
},
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364481713238},
"previous_version":"head12345_11",
"original_version":"head12345_10"
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364481713238},
"previous_version":"head12345_11",
"original_version":"head12345_10"
}
},
{
"_id":"head12345_11",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -92,15 +94,17 @@
},
"wiki_slug":null
},
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364481713238},
"previous_version":"head12345_10",
"original_version":"head12345_10"
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364481713238},
"previous_version":"head12345_10",
"original_version":"head12345_10"
}
},
{
"_id":"head12345_10",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -141,15 +145,17 @@
},
"wiki_slug":null
},
"edited_by":"test@edx.org",
"edited_on":{"$date": 1364473713238},
"previous_version":null,
"original_version":"head12345_10"
"edit_info": {
"edited_by":"test@edx.org",
"edited_on":{"$date": 1364473713238},
"previous_version":null,
"original_version":"head12345_10"
}
},
{
"_id":"head23456_1",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -190,15 +196,17 @@
},
"wiki_slug":null
},
"edited_by":"test@edx.org",
"edited_on":{"$date": 1364481313238},
"previous_version":"head23456_0",
"original_version":"head23456_0"
"edit_info": {
"edited_by":"test@edx.org",
"edited_on":{"$date": 1364481313238},
"previous_version":"head23456_0",
"original_version":"head23456_0"
}
},
{
"_id":"head23456_0",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -239,15 +247,17 @@
},
"wiki_slug":null
},
"edited_by":"test@edx.org",
"edited_on":{"$date" : 1364481313238},
"previous_version":null,
"original_version":"head23456_0"
"edit_info": {
"edited_by":"test@edx.org",
"edited_on":{"$date" : 1364481313238},
"previous_version":null,
"original_version":"head23456_0"
}
},
{
"_id":"head345679_1",
"category":"course",
"data":{
"fields":{
"textbooks":[
],
......@@ -281,54 +291,66 @@
},
"wiki_slug":null
},
"edited_by":"test@edx.org",
"edited_on":{"$date" : 1364481313238},
"previous_version":null,
"original_version":"head23456_0"
"edit_info": {
"edited_by":"test@edx.org",
"edited_on":{"$date" : 1364481313238},
"previous_version":null,
"original_version":"head23456_0"
}
},
{
"_id":"chapter12345_1",
"category":"chapter",
"data":null,
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_1"
"fields":{},
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_1"
}
},
{
"_id":"chapter12345_2",
"category":"chapter",
"data":null,
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_2"
"fields":{},
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_2"
}
},
{
"_id":"chapter12345_3",
"category":"chapter",
"data":null,
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_3"
"fields":{},
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"chapter12345_3"
}
},
{
"_id":"problem12345_3_1",
"category":"problem",
"data":"",
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"problem12345_3_1"
"fields": {"data": ""},
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"problem12345_3_1"
}
},
{
"_id":"problem12345_3_2",
"category":"problem",
"data":"",
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"problem12345_3_2"
"fields": {"data": ""},
"edit_info": {
"edited_by":"testassist@edx.org",
"edited_on":{"$date" : 1364483713238},
"previous_version":null,
"original_version":"problem12345_3_2"
}
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment