Merge remote-tracking branch 'edx/opaque-keys' into opaque-keys-merge-master

cms/djangoapps/contentstore/tests/test_crud.py

@@ -12,7 +12,6 @@ from xmodule.html_module import HtmlDescriptor
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 
 
-@unittest.skip("Not fixing split until we land opaque-keys 0.9")
 class TemplateTests(unittest.TestCase):
     """
     Test finding and using the templates (boilerplates) for xblocks.
@@ -55,25 +54,25 @@ class TemplateTests(unittest.TestCase):
 
     def test_factories(self):
         test_course = persistent_factories.PersistentCourseFactory.create(
-            course_id='testx.tempcourse', org='testx',
+            offering='tempcourse', org='testx',
             display_name='fun test course', user_id='testbot'
         )
         self.assertIsInstance(test_course, CourseDescriptor)
         self.assertEqual(test_course.display_name, 'fun test course')
-        index_info = modulestore('split').get_course_index_info(test_course.location)
+        index_info = modulestore('split').get_course_index_info(test_course.id)
         self.assertEqual(index_info['org'], 'testx')
-        self.assertEqual(index_info['_id'], 'testx.tempcourse')
+        self.assertEqual(index_info['offering'], 'tempcourse')
 
         test_chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
             parent_location=test_course.location)
         self.assertIsInstance(test_chapter, SequenceDescriptor)
         # refetch parent which should now point to child
-        test_course = modulestore('split').get_course(test_course.id)
+        test_course = modulestore('split').get_course(test_course.id.version_agnostic())
         self.assertIn(test_chapter.location.block_id, test_course.children)
 
         with self.assertRaises(DuplicateCourseError):
             persistent_factories.PersistentCourseFactory.create(
-                course_id='testx.tempcourse', org='testx', 
+                offering='tempcourse', org='testx',
                 display_name='fun test course', user_id='testbot'
             )
 
@@ -82,7 +81,7 @@ class TemplateTests(unittest.TestCase):
         Test create_xblock to create non persisted xblocks
         """
         test_course = persistent_factories.PersistentCourseFactory.create(
-            course_id='testx.tempcourse', org='testx',
+            offering='tempcourse', org='testx',
             display_name='fun test course', user_id='testbot'
         )
 
@@ -109,7 +108,7 @@ class TemplateTests(unittest.TestCase):
         try saving temporary xblocks
         """
         test_course = persistent_factories.PersistentCourseFactory.create(
-            course_id='testx.tempcourse', org='testx', 
+            offering='tempcourse', org='testx',
             display_name='fun test course', user_id='testbot'
         )
         test_chapter = modulestore('split').create_xblock(
@@ -148,7 +147,7 @@ class TemplateTests(unittest.TestCase):
 
     def test_delete_course(self):
         test_course = persistent_factories.PersistentCourseFactory.create(
-            course_id='edu.harvard.history.doomed', org='testx',
+            offering='history.doomed', org='edu.harvard',
             display_name='doomed test course',
             user_id='testbot')
         persistent_factories.ItemFactory.create(display_name='chapter 1',
@@ -171,7 +170,7 @@ class TemplateTests(unittest.TestCase):
         Test get_block_generations
         """
         test_course = persistent_factories.PersistentCourseFactory.create(
-            course_id='edu.harvard.history.hist101', org='testx',
+            offering='history.hist101', org='edu.harvard',
             display_name='history test course',
             user_id='testbot'
         )
@@ -193,7 +192,9 @@ class TemplateTests(unittest.TestCase):
 
         second_problem = persistent_factories.ItemFactory.create(
             display_name='problem 2',
-            parent_location=BlockUsageLocator.make_relative(updated_loc, block_id=sub.location.block_id),
+            parent_location=BlockUsageLocator.make_relative(
+                updated_loc, block_type='problem', block_id=sub.location.block_id
+            ),
             user_id='testbot', category='problem',
             data="<problem></problem>"
         )

cms/djangoapps/contentstore/views/tests/test_course_index.py

@@ -7,7 +7,7 @@ import lxml
 from contentstore.tests.utils import CourseTestCase
 from contentstore.utils import reverse_course_url
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
-from xmodule.modulestore import parsers
+from xmodule.modulestore.locator import Locator
 
 
 class TestCourseIndex(CourseTestCase):
@@ -38,7 +38,7 @@ class TestCourseIndex(CourseTestCase):
         for link in course_link_eles:
             self.assertRegexpMatches(
                 link.get("href"),
-                'course/slashes:{0}'.format(parsers.ALLOWED_ID_CHARS)
+                'course/slashes:{0}'.format(Locator.ALLOWED_ID_CHARS)
             )
             # now test that url
             outline_response = authed_client.get(link.get("href"), {}, HTTP_ACCEPT='text/html')

common/djangoapps/cache_toolbox/core.py

@@ -118,4 +118,9 @@ def get_cached_content(location):
 
 
 def del_cached_content(location):
-    cache.delete(unicode(location).encode("utf-8"))
+    # delete content for the given location, as well as for content with run=None.
+    # it's possible that the content could have been cached without knowing the
+    # course_key - and so without having the run.
+    cache.delete_many(
+        [unicode(loc).encode("utf-8") for loc in [location, location.replace(run=None)]]
+    )

common/djangoapps/student/migrations/0035_access_roles.py

@@ -9,6 +9,7 @@ from opaque_keys import InvalidKeyError
 import bson.son
 import logging
 from django.db.models.query_utils import Q
+from django.db.utils import IntegrityError
 
 log = logging.getLogger(__name__)
 
@@ -28,69 +29,72 @@ class Migration(DataMigration):
         loc_map_collection = loc_mapper().location_map
         # b/c the Groups table had several entries for each course, we need to ensure we process each unique
         # course only once. The below datastructures help ensure that.
-        hold = {}
-        done = set()
-        orgs = {}
-        query = Q(name__startswith='course_creator_group')
+        hold = {}  # key of course_id_strings with array of group objects. Should only be org scoped entries
+        # or deleted courses
+        orgs = {}  # downcased org to last recorded normal case of the org
+        query = Q(name='course_creator_group')
         for role in ['staff', 'instructor', 'beta_testers', ]:
             query = query | Q(name__startswith=role)
         for group in orm['auth.Group'].objects.filter(query).all():
-            def _migrate_users(correct_course_key, lower_org):
+            def _migrate_users(correct_course_key, role, lower_org):
                 """
                 Get all the users from the old group and migrate to this course key in the new table
                 """
                 for user in orm['auth.user'].objects.filter(groups=group).all():
                     entry = orm['student.courseaccessrole'](
-                        role=parsed_entry.group('role_id'), user=user,
-                        org=correct_course_key.org, course_id=correct_course_key.to_deprecated_string()
+                        role=role, user=user,
+                        org=correct_course_key.org, course_id=correct_course_key
                     )
-                    entry.save()
+                    try:
+                        entry.save()
+                    except IntegrityError:
+                        # already stored
+                        pass
                 orgs[lower_org] = correct_course_key.org
-                done.add(correct_course_key)
 
-            # should this actually loop through all groups and log any which are not compliant? That is,
-            # remove the above filter
             parsed_entry = self.GROUP_ENTRY_RE.match(group.name)
-            if parsed_entry.group('role_id') == 'course_creator_group':
+            role = parsed_entry.group('role_id')
+            if role == 'course_creator_group':
                 for user in orm['auth.user'].objects.filter(groups=group).all():
-                    entry = orm['student.courseaccessrole'](role=parsed_entry.group('role_id'), user=user)
+                    entry = orm['student.courseaccessrole'](role=role, user=user)
                     entry.save()
             else:
                 course_id_string = parsed_entry.group('course_id_string')
                 try:
                     course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id_string)
-                    if course_key not in done:
-                        # is the downcased version, get the normal cased one. loc_mapper() has no
-                        # methods taking downcased SSCK; so, need to do it manually here
-                        correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection, done)
-                        _migrate_users(correct_course_key, course_key.org)
-                        done.add(course_key)
+                    # course_key is the downcased version, get the normal cased one. loc_mapper() has no
+                    # methods taking downcased SSCK; so, need to do it manually here
+                    correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection)
+                    if correct_course_key is not None:
+                        _migrate_users(correct_course_key, role, course_key.org)
                 except InvalidKeyError:
                     entry = loc_map_collection.find_one({
                         'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
                     })
                     if entry is None:
-                        # not a course_id as far as we can tell
-                        if course_id_string not in done:
-                            hold[course_id_string] = group
+                        hold.setdefault(course_id_string, []).append(group)
                     else:
-                        correct_course_key = self._cache_done_return_ssck(entry, done)
-                        _migrate_users(correct_course_key, entry['lower_id']['org'])
+                        correct_course_key = SlashSeparatedCourseKey(*entry['_id'].values())
+                        _migrate_users(correct_course_key, role, entry['lower_id']['org'])
 
-        # see if any in hold ere missed above
-        for not_ssck, group in hold.iteritems():
-            if not_ssck not in done:
-                if not_ssck in orgs:
+        # see if any in hold were missed above
+        for held_auth_scope, groups in hold.iteritems():
+            # orgs indexed by downcased org
+            held_auth_scope = held_auth_scope.lower()
+            if held_auth_scope in orgs:
+                for group in groups:
+                    role = self.GROUP_ENTRY_RE.match(group.name).group('role_id')
                     # they have org permission
                     for user in orm['auth.user'].objects.filter(groups=group).all():
                         entry = orm['student.courseaccessrole'](
-                            role=parsed_entry.group('role_id'), user=user,
-                            org=orgs[not_ssck],
+                            role=role,
+                            user=user,
+                            org=orgs[held_auth_scope],
                         )
                         entry.save()
-                else:
-                    # should this just log or really make an effort to do the conversion?
-                    log.warn("Didn't convert role %s", group.name)
+            else:
+                # don't silently skip unexpected roles
+                log.warn("Didn't convert roles %s", [group.name for group in groups])
 
     def backwards(self, orm):
         "Write your backwards methods here."
@@ -98,10 +102,9 @@ class Migration(DataMigration):
         # the semantic of backwards should be other than perhaps clearing the table.
         orm['student.courseaccessrole'].objects.all().delete()
 
-    def _map_downcased_ssck(self, downcased_ssck, loc_map_collection, done):
+    def _map_downcased_ssck(self, downcased_ssck, loc_map_collection):
         """
-        Get the normal cased version of this downcased slash sep course key and add
-        the lowercased locator form to done map
+        Get the normal cased version of this downcased slash sep course key
         """
         # given the regex, the son may be an overkill
         course_son = bson.son.SON([
@@ -111,25 +114,11 @@ class Migration(DataMigration):
         ])
         entry = loc_map_collection.find_one(course_son)
         if entry:
-            return self._cache_done_return_ssck(entry, done)
+            idpart = entry['_id']
+            return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
         else:
             return None
 
-    def _cache_done_return_ssck(self, entry, done):
-        """
-        Add all the various formats which auth may use to the done set and return the ssck for the entry
-        """
-        # cache that the dotted form is done too
-        if 'lower_course_id' in entry:
-            done.add(entry['lower_course_id'])
-        elif 'course_id' in entry:
-            done.add(entry['course_id'].lower())
-        elif 'lower_org' in entry:
-            done.add('{}.{}'.format(entry['lower_org'], entry['lower_offering']))
-        else:
-            done.add('{}.{}'.format(entry['org'].lower(), entry['offering'].lower()))
-        return SlashSeparatedCourseKey(*entry['_id'].values())
-
 
     models = {
         'auth.group': {

common/lib/xmodule/setup.py

@@ -79,5 +79,8 @@ setup(
             'asset-location = xmodule.modulestore.locations:AssetLocation',
             'edx = xmodule.modulestore.locator:BlockUsageLocator',
         ],
+        'definition_key': [
+            'defx = xmodule.modulestore.locator:DefinitionLocator',
+        ],
     },
 )

common/lib/xmodule/xmodule/modulestore/keys.py

@@ -58,7 +58,7 @@ class DefinitionKey(OpaqueKey):
     KEY_TYPE = 'definition_key'
     __slots__ = ()
 
-    @abstractmethod
+    @abstractproperty
     def block_type(self):
         """
         The XBlock type of this definition.
@@ -125,6 +125,10 @@ class UsageKey(CourseObjectMixin, OpaqueKey):
         """
         raise NotImplementedError()
 
+    @property
+    def block_type(self):
+        return self.category
+
 
 class OpaqueKeyReader(IdReader):
     """

common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py

@@ -150,6 +150,7 @@ class LocMapperStore(object):
             entry = self._migrate_if_necessary([entry])[0]
 
         block_id = entry['block_map'].get(self.encode_key_for_mongo(location.name))
+        category = location.category
         if block_id is None:
             if add_entry_if_missing:
                 block_id = self._add_to_block_map(
@@ -159,14 +160,15 @@ class LocMapperStore(object):
                 raise ItemNotFoundError(location)
         else:
             # jump_to_id uses a None category.
-            if location.category is None:
+            if category is None:
                 if len(block_id) == 1:
                     # unique match (most common case)
+                    category = block_id.keys()[0]
                     block_id = block_id.values()[0]
                 else:
                     raise InvalidLocationError()
-            elif location.category in block_id:
-                block_id = block_id[location.category]
+            elif category in block_id:
+                block_id = block_id[category]
             elif add_entry_if_missing:
                 block_id = self._add_to_block_map(location, course_son, entry['block_map'])
             else:
@@ -179,10 +181,12 @@ class LocMapperStore(object):
         )
         published_usage = BlockUsageLocator(
             prod_course_locator,
+            block_type=category,
             block_id=block_id
         )
         draft_usage = BlockUsageLocator(
             prod_course_locator.for_branch(entry['draft_branch']),
+            block_type=category,
             block_id=block_id
         )
         if published:
@@ -285,6 +289,7 @@ class LocMapperStore(object):
                         org=entry[entry_org], offering=entry[entry_offering],
                         branch=entry['prod_branch']
                     ),
+                    block_type=category,
                     block_id=block_id
                 )
                 draft_locator = BlockUsageLocator(
@@ -292,6 +297,7 @@ class LocMapperStore(object):
                         org=entry[entry_org], offering=entry[entry_offering],
                         branch=entry['draft_branch']
                     ),
+                    block_type=category,
                     block_id=block_id
                 )
                 self._cache_location_map_entry(location, published_locator, draft_locator)

common/lib/xmodule/xmodule/modulestore/parsers.py

-import re
-
-# Prefix for the branch portion of a locator URL
-BRANCH_PREFIX = r"branch/"
-# Prefix for the block portion of a locator URL
-BLOCK_PREFIX = r"block/"
-# Prefix for the version portion of a locator URL, when it is preceded by a course ID
-VERSION_PREFIX = r"version/"
-
-ALLOWED_ID_CHARS = r'[\w\-~.:+]'
-ALLOWED_ID_RE = re.compile(r'^{}+$'.format(ALLOWED_ID_CHARS), re.UNICODE)
-
-# NOTE: if we need to support period in place of +, make it aggressive (take the first period in the string)
-URL_RE_SOURCE = r"""
-    ((?P<org>{ALLOWED_ID_CHARS}+)\+(?P<offering>{ALLOWED_ID_CHARS}+)/?)?
-    ({BRANCH_PREFIX}(?P<branch>{ALLOWED_ID_CHARS}+)/?)?
-    ({VERSION_PREFIX}(?P<version_guid>[A-F0-9]+)/?)?
-    ({BLOCK_PREFIX}(?P<block_id>{ALLOWED_ID_CHARS}+))?
-    """.format(
-        ALLOWED_ID_CHARS=ALLOWED_ID_CHARS, BRANCH_PREFIX=BRANCH_PREFIX,
-        VERSION_PREFIX=VERSION_PREFIX, BLOCK_PREFIX=BLOCK_PREFIX
-    )
-
-URL_RE = re.compile('^' + URL_RE_SOURCE + '$', re.IGNORECASE | re.VERBOSE | re.UNICODE)
-
-
-def parse_url(string):
-    """
-    followed by either a version_guid or a org + offering pair. If tag_optional, then
-    the url does not have to start with the tag and edx will be assumed.
-
-    Examples:
-        'edx:version/0123FFFF'
-        'edx:mit.eecs.6002x'
-        'edx:mit.eecs.6002x/branch/published'
-        'edx:mit.eecs.6002x/branch/published/block/HW3'
-        'edx:mit.eecs.6002x/branch/published/version/000eee12345/block/HW3'
-
-    This returns None if string cannot be parsed.
-
-    If it can be parsed as a version_guid with no preceding org + offering, returns a dict
-    with key 'version_guid' and the value,
-
-    If it can be parsed as a org + offering, returns a dict
-    with key 'id' and optional keys 'branch' and 'version_guid'.
-    """
-    match = URL_RE.match(string)
-    if not match:
-        return None
-    matched_dict = match.groupdict()
-    return matched_dict
-
-
-def parse_block_ref(string):
-    r"""
-    A block_ref is a string of url safe characters (see ALLOWED_ID_CHARS)
-
-    If string is a block_ref, returns a dict with key 'block_ref' and the value,
-    otherwise returns None.
-    """
-    if ALLOWED_ID_RE.match(string):
-        return {'block_id': string}
-    return None

common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py

@@ -108,6 +108,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 offering=course_entry_override.get('offering'),
                 branch=course_entry_override.get('branch'),
             ),
+            block_type=json_data.get('category'),
             block_id=block_id,
         )
 
@@ -131,6 +132,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 self,
                 BlockUsageLocator(
                     CourseLocator(version_guid=course_entry_override['structure']['_id']),
+                    block_type='error',
                     block_id=block_id
                 ),
                 error_msg=exc_info_to_str(sys.exc_info())

common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py

@@ -8,14 +8,14 @@ class DefinitionLazyLoader(object):
     object doesn't force access during init but waits until client wants the
     definition. Only works if the modulestore is a split mongo store.
     """
-    def __init__(self, modulestore, definition_id):
+    def __init__(self, modulestore, block_type, definition_id):
         """
         Simple placeholder for yet-to-be-fetched data
         :param modulestore: the pymongo db connection with the definitions
         :param definition_locator: the id of the record in the above to fetch
         """
         self.modulestore = modulestore
-        self.definition_locator = DefinitionLocator(definition_id)
+        self.definition_locator = DefinitionLocator(block_type, definition_id)
 
     def fetch(self):
         """

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -152,7 +152,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
 
         if lazy:
             for block in new_module_data.itervalues():
-                block['definition'] = DefinitionLazyLoader(self, block['definition'])
+                block['definition'] = DefinitionLazyLoader(self, block['category'], block['definition'])
         else:
             # Load all descendants by id
             descendent_definitions = self.db_connection.find_matching_definitions({
@@ -242,11 +242,6 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
 
         :param course_locator: any subclass of CourseLocator
         '''
-        # NOTE: if and when this uses cache, the update if changed logic will break if the cache
-        # holds the same objects as the descriptors!
-        if not course_locator.is_fully_specified():
-            raise InsufficientSpecificationError('Not fully specified: %s' % course_locator)
-
         if course_locator.org and course_locator.offering and course_locator.branch:
             # use the course id
             index = self.db_connection.get_course_index(course_locator)
@@ -258,6 +253,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
             if course_locator.version_guid is not None and version_guid != course_locator.version_guid:
                 # This may be a bit too touchy but it's hard to infer intent
                 raise VersionConflictError(course_locator, version_guid)
+        elif course_locator.version_guid is None:
+            raise InsufficientSpecificationError(course_locator)
         else:
             # TODO should this raise an exception if branch was provided?
             version_guid = course_locator.version_guid
@@ -322,9 +319,6 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
     def get_course(self, course_id, depth=None):
         '''
         Gets the course descriptor for the course identified by the locator
-        which may or may not be a blockLocator.
-
-        raises InsufficientSpecificationError
         '''
         assert(isinstance(course_id, CourseLocator))
         course_entry = self._lookup_course(course_id)
@@ -458,6 +452,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         return [
             BlockUsageLocator.make_relative(
                 locator,
+                block_type=course['structure']['blocks'][parent_id].get('category'),
                 block_id=LocMapperStore.decode_key_from_mongo(parent_id),
             )
             for parent_id in items
@@ -471,12 +466,13 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         course = self._lookup_course(course_key)
         items = {LocMapperStore.decode_key_from_mongo(block_id) for block_id in course['structure']['blocks'].keys()}
         items.remove(course['structure']['root'])
-        for block_id, block_data in course['structure']['blocks'].iteritems():
+        blocks = course['structure']['blocks']
+        for block_id, block_data in blocks.iteritems():
             items.difference_update(block_data.get('fields', {}).get('children', []))
             if block_data['category'] in detached_categories:
                 items.discard(LocMapperStore.decode_key_from_mongo(block_id))
         return [
-            BlockUsageLocator(course_key=course_key, block_id=block_id)
+            BlockUsageLocator(course_key=course_key, block_type=blocks[block_id]['category'], block_id=block_id)
             for block_id in items
         ]
 
@@ -613,11 +609,11 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         # convert the results value sets to locators
         for k, versions in result.iteritems():
             result[k] = [
-                BlockUsageLocator(CourseLocator(version_guid=version), block_id=block_id)
+                block_locator.for_version(version)
                 for version in versions
             ]
         return VersionTree(
-            BlockUsageLocator(CourseLocator(version_guid=possible_roots[0]), block_id=block_id),
+            block_locator.for_version(possible_roots[0]),
             result
         )
 
@@ -650,7 +646,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
             'schema_version': self.SCHEMA_VERSION,
         }
         self.db_connection.insert_definition(document)
-        definition_locator = DefinitionLocator(new_id)
+        definition_locator = DefinitionLocator(category, new_id)
         return definition_locator
 
     def update_definition_from_data(self, definition_locator, new_def_data, user_id):
@@ -685,7 +681,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
             old_definition['edit_info']['previous_version'] = definition_locator.definition_id
             old_definition['schema_version'] = self.SCHEMA_VERSION
             self.db_connection.insert_definition(old_definition)
-            return DefinitionLocator(old_definition['_id']), True
+            return DefinitionLocator(old_definition['category'], old_definition['_id']), True
         else:
             return definition_locator, False
 
@@ -829,11 +825,13 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
                 self._update_head(index_entry, course_or_parent_locator.branch, new_id)
             item_loc = BlockUsageLocator(
                 course_or_parent_locator.version_agnostic(),
+                block_type=category,
                 block_id=new_block_id,
             )
         else:
             item_loc = BlockUsageLocator(
                 CourseLocator(version_guid=new_id),
+                block_type=category,
                 block_id=new_block_id,
             )
 
@@ -1029,7 +1027,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
                 course_key = CourseLocator(version_guid=new_id)
 
             # fetch and return the new item--fetching is unnecessary but a good qc step
-            new_locator = BlockUsageLocator(course_key, descriptor.location.block_id)
+            new_locator = descriptor.location.map_into_course(course_key)
             return self.get_item(new_locator)
         else:
             # nothing changed, just return the one sent in
@@ -1101,18 +1099,14 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
                 self._update_head(index_entry, xblock.location.branch, new_id)
 
             # fetch and return the new item--fetching is unnecessary but a good qc step
-            return self.get_item(
-                BlockUsageLocator(
-                    xblock.location.course_key.for_version(new_id),
-                    block_id=xblock.location.block_id,
-                )
-            )
+            return self.get_item(xblock.location.for_version(new_id))
         else:
             return xblock
 
     def _persist_subdag(self, xblock, user_id, structure_blocks, new_id):
         # persist the definition if persisted != passed
         new_def_data = self._filter_special_fields(xblock.get_explicitly_set_fields_by_scope(Scope.content))
+        is_updated = False
         if xblock.definition_locator is None or isinstance(xblock.definition_locator.definition_id, LocalId):
             xblock.definition_locator = self.create_definition_from_data(
                 new_def_data, xblock.category, user_id)
@@ -1134,9 +1128,6 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         else:
             is_new = False
             encoded_block_id = LocMapperStore.encode_key_for_mongo(xblock.location.block_id)
-            is_updated = is_updated or (
-                xblock.has_children and structure_blocks[encoded_block_id]['fields']['children'] != xblock.children
-            )
 
         children = []
         if xblock.has_children:
@@ -1147,6 +1138,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
                     children.append(child_block.location.block_id)
                 else:
                     children.append(child)
+            is_updated = is_updated or structure_blocks[encoded_block_id]['fields']['children'] != children
 
         block_fields = xblock.get_explicitly_set_fields_by_scope(Scope.settings)
         if not is_new and not is_updated:
@@ -1419,9 +1411,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         if isinstance(definition, DefinitionLazyLoader):
             return definition.definition_locator
         elif '_id' not in definition:
-            return DefinitionLocator(LocalId())
+            return DefinitionLocator(definition.get('category'), LocalId())
         else:
-            return DefinitionLocator(definition['_id'])
+            return DefinitionLocator(definition['category'], definition['_id'])
 
     def get_modulestore_type(self, course_id):
         """

common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py

@@ -32,14 +32,14 @@ class PersistentCourseFactory(SplitFactory):
 
     # pylint: disable=W0613
     @classmethod
-    def _create(cls, target_class, course_id='testX.999', org='testX', user_id='test_user',
+    def _create(cls, target_class, offering='999', org='testX', user_id='test_user',
                 master_branch='draft', **kwargs):
 
         modulestore = kwargs.pop('modulestore')
         root_block_id = kwargs.pop('root_block_id', 'course')
         # Write the data to the mongo datastore
         new_course = modulestore.create_course(
-            course_id, org, user_id, fields=kwargs,
+            org, offering, user_id, fields=kwargs,
             master_branch=master_branch, root_block_id=root_block_id
         )
 

common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py

@@ -267,6 +267,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         )
         prob_locator = BlockUsageLocator(
             prob_course_key,
+            block_type='problem',
             block_id='problem2',
         )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
@@ -289,20 +290,21 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
         self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
         # explicit branch
-        prob_locator = BlockUsageLocator(
-            prob_course_key.for_branch('draft'), block_id=prob_locator.block_id
-        )
+        prob_locator = prob_locator.for_branch('draft')
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
         # Even though the problem was set as draft, we always return revision=None to work
         # with old mongo/draft modulestores.
         self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
-        prob_locator = BlockUsageLocator(prob_course_key.for_branch('production'), block_id='problem2')
+        prob_locator = BlockUsageLocator(
+            prob_course_key.for_branch('production'),
+            block_type='problem', block_id='problem2'
+        )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
         self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
         # same for chapter except chapter cannot be draft in old system
         chap_locator = BlockUsageLocator(
             prob_course_key.for_branch('production'),
-            block_id='chapter48f',
+            block_type='chapter', block_id='chapter48f',
         )
         chap_location = loc_mapper().translate_locator_to_location(chap_locator)
         self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
@@ -311,7 +313,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         chap_location = loc_mapper().translate_locator_to_location(chap_locator)
         self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
         chap_locator = BlockUsageLocator(
-            prob_course_key.for_branch('production'), block_id='chapter48f'
+            prob_course_key.for_branch('production'), block_type='chapter', block_id='chapter48f'
         )
         chap_location = loc_mapper().translate_locator_to_location(chap_locator)
         self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
@@ -319,7 +321,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         # look for non-existent problem
         prob_locator2 = BlockUsageLocator(
             prob_course_key.for_branch('draft'),
-            block_id='problem3'
+            block_type='problem', block_id='problem3'
         )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
         self.assertIsNone(prob_location, 'Found non-existent problem')

common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py

@@ -105,6 +105,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
             # create pointer for split
             course_or_parent_locator = BlockUsageLocator(
                 course_key=self.split_course_key,
+                block_type=parent_category,
                 block_id=parent_name
             )
         else:

common/lib/xmodule/xmodule/open_ended_grading_classes/controller_query_service.py

@@ -37,7 +37,7 @@ class ControllerQueryService(GradingService):
     def check_combined_notifications(self, course_id, student_id, user_is_staff, last_time_viewed):
         params = {
             'student_id': student_id,
-            'course_id': course_id,
+            'course_id': course_id.to_deprecated_string(),
             'user_is_staff': user_is_staff,
             'last_time_viewed': last_time_viewed,
         }

common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py

@@ -85,7 +85,7 @@ class PeerGradingService(GradingService):
         return result
 
     def get_problem_list(self, course_id, grader_id):
-        params = {'course_id': course_id, 'student_id': grader_id}
+        params = {'course_id': course_id.to_deprecated_string(), 'student_id': grader_id}
         result = self.get(self.get_problem_list_url, params)
 
         if 'problem_list' in result:
@@ -100,7 +100,7 @@ class PeerGradingService(GradingService):
         return result
 
     def get_notifications(self, course_id, grader_id):
-        params = {'course_id': course_id, 'student_id': grader_id}
+        params = {'course_id': course_id.to_deprecated_string(), 'student_id': grader_id}
         result = self.get(self.get_notifications_url, params)
         self._record_result(
             'get_notifications',

lms/djangoapps/django_comment_client/base/views.py

@@ -182,7 +182,7 @@ def _create_comment(request, course_key, thread_id=None, parent_id=None):
         'anonymous': anonymous,
         'anonymous_to_peers': anonymous_to_peers,
         'user_id': request.user.id,
-        'course_id': course_key,
+        'course_id': course_key.to_deprecated_string(),
         'thread_id': thread_id,
         'parent_id': parent_id,
     })

lms/djangoapps/django_comment_client/forum/tests.py

@@ -285,7 +285,7 @@ class UserProfileTestCase(ModuleStoreTestCase):
             StringEndsWithMatcher('/users/{}/active_threads'.format(self.profiled_user.id)),
             data=None,
             params=PartialDictMatcher({
-                "course_id": self.course.id,
+                "course_id": self.course.id.to_deprecated_string(),
                 "page": params.get("page", 1),
                 "per_page": views.THREADS_PER_PAGE
                 }),

lms/djangoapps/open_ended_grading/staff_grading_service.py

@@ -9,7 +9,6 @@ from django.conf import settings
 from django.http import HttpResponse, Http404
 from django.utils.translation import ugettext as _
 
-from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.locations import SlashSeparatedCourseKey
 from xmodule.open_ended_grading_classes.grading_service_module import GradingService, GradingServiceError
 from xmodule.modulestore.django import ModuleI18nService
@@ -116,7 +115,7 @@ class StaffGradingService(GradingService):
         Raises:
             GradingServiceError: something went wrong with the connection.
         """
-        params = {'course_id': course_id, 'grader_id': grader_id}
+        params = {'course_id': course_id.to_deprecated_string(), 'grader_id': grader_id}
         result = self.get(self.get_problem_list_url, params)
         tags = [u'course_id:{}'.format(course_id)]
         self._record_result('get_problem_list', result, tags)
@@ -148,7 +147,7 @@ class StaffGradingService(GradingService):
             self.get(
                 self.get_next_url,
                 params={
-                    'location': location,
+                    'location': location.to_deprecated_string(),
                     'grader_id': grader_id
                 }
             )
@@ -170,7 +169,7 @@ class StaffGradingService(GradingService):
         Raises:
             GradingServiceError if there's a problem connecting.
         """
-        data = {'course_id': course_id,
+        data = {'course_id': course_id.to_deprecated_string(),
                 'submission_id': submission_id,
                 'score': score,
                 'feedback': feedback,
@@ -186,7 +185,7 @@ class StaffGradingService(GradingService):
         return result
 
     def get_notifications(self, course_id):
-        params = {'course_id': course_id}
+        params = {'course_id': course_id.to_deprecated_string()}
         result = self.get(self.get_notifications_url, params)
         tags = [
             u'course_id:{}'.format(course_id),
@@ -274,7 +273,7 @@ def get_next(request, course_id):
             ', '.join(missing)))
     grader_id = unique_id_for_user(request.user)
     p = request.POST
-    location = p['location']
+    location = course_key.make_usage_key_from_deprecated_string(p['location'])
 
     return HttpResponse(json.dumps(_get_next(course_key, grader_id, location)),
                         mimetype="application/json")
@@ -400,7 +399,7 @@ def save_grade(request, course_id):
 
     grader_id = unique_id_for_user(request.user)
 
-    location = p['location']
+    location = course_key.make_usage_key_from_deprecated_string(p['location'])
 
     try:
         result = staff_grading_service().save_grade(course_key,

lms/djangoapps/open_ended_grading/tests.py

@@ -109,13 +109,13 @@ class TestStaffGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.student = 'view@test.com'
         self.instructor = 'view2@test.com'
         self.password = 'foo'
-        self.location = 'TestLocation'
         self.create_account('u1', self.student, self.password)
         self.create_account('u2', self.instructor, self.password)
         self.activate_user(self.student)
         self.activate_user(self.instructor)
 
         self.course_id = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
+        self.location_string = self.course_id.make_usage_key('html', 'TestLocation').to_deprecated_string()
         self.toy = modulestore().get_course(self.course_id)
 
         make_instructor(self.toy, self.instructor)
@@ -140,7 +140,7 @@ class TestStaffGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.login(self.instructor, self.password)
 
         url = reverse('staff_grading_get_next', kwargs={'course_id': self.course_id.to_deprecated_string()})
-        data = {'location': self.location}
+        data = {'location': self.location_string}
 
         response = check_for_post_code(self, 200, url, data)
 
@@ -165,7 +165,7 @@ class TestStaffGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         data = {'score': '12',
                 'feedback': 'great!',
                 'submission_id': '123',
-                'location': self.location,
+                'location': self.location_string,
                 'submission_flagged': "true",
                 'rubric_scores[]': ['1', '2']}
         if skip:
@@ -227,7 +227,7 @@ class TestStaffGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
             'score': '12',
             'feedback': '',
             'submission_id': '123',
-            'location': self.location,
+            'location': self.location_string,
             'submission_flagged': "false",
             'rubric_scores[]': ['1', '2']
         }
@@ -262,13 +262,13 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.student = 'view@test.com'
         self.instructor = 'view2@test.com'
         self.password = 'foo'
-        self.location = 'TestLocation'
         self.create_account('u1', self.student, self.password)
         self.create_account('u2', self.instructor, self.password)
         self.activate_user(self.student)
         self.activate_user(self.instructor)
 
         self.course_id = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
+        self.location_string = self.course_id.make_usage_key('html', 'TestLocation').to_deprecated_string()
         self.toy = modulestore().get_course(self.course_id)
         location = "i4x://edX/toy/peergrading/init"
         field_data = DictFieldData({'data': "<peergrading/>", 'location': location, 'category':'peergrading'})
@@ -292,7 +292,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.logout()
 
     def test_get_next_submission_success(self):
-        data = {'location': self.location}
+        data = {'location': self.location_string}
 
         response = self.peer_module.get_next_submission(data)
         content = response
@@ -312,7 +312,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
     def test_save_grade_success(self):
         data = {
             'rubric_scores[]': [0, 0],
-            'location': self.location,
+            'location': self.location_string,
             'submission_id': 1,
             'submission_key': 'fake key',
             'score': 2,
@@ -342,7 +342,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.assertTrue(d['error'].find('Missing required keys:') > -1)
 
     def test_is_calibrated_success(self):
-        data = {'location': self.location}
+        data = {'location': self.location_string}
         response = self.peer_module.is_student_calibrated(data)
 
         self.assertTrue(response['success'])
@@ -355,7 +355,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         self.assertFalse('calibrated' in response)
 
     def test_show_calibration_essay_success(self):
-        data = {'location': self.location}
+        data = {'location': self.location_string}
 
         response = self.peer_module.show_calibration_essay(data)
 
@@ -376,7 +376,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
     def test_save_calibration_essay_success(self):
         data = {
             'rubric_scores[]': [0, 0],
-            'location': self.location,
+            'location': self.location_string,
             'submission_id': 1,
             'submission_key': 'fake key',
             'score': 2,
@@ -410,7 +410,7 @@ class TestPeerGradingService(ModuleStoreTestCase, LoginEnrollmentTestCase):
         """
         data = {
             'rubric_scores[]': [0, 0],
-            'location': self.location,
+            'location': self.location_string,
             'submission_id': 1,
             'submission_key': 'fake key',
             'score': 2,

lms/djangoapps/open_ended_grading/views.py

 import logging
 
-from django.conf import settings
 from django.views.decorators.cache import cache_control
 from edxmako.shortcuts import render_to_response
 from django.core.urlresolvers import reverse
 
-from student.models import unique_id_for_user
 from courseware.courses import get_course_with_access
 
 from xmodule.open_ended_grading_classes.grading_service_module import GradingServiceError
@@ -20,11 +18,11 @@ from xmodule.modulestore import SlashSeparatedCourseKey
 from xmodule.modulestore.exceptions import NoPathToItem
 
 from django.http import HttpResponse, Http404, HttpResponseRedirect
-from edxmako.shortcuts import render_to_string
 from django.utils.translation import ugettext as _
 
-from open_ended_grading.utils import (STAFF_ERROR_MESSAGE, STUDENT_ERROR_MESSAGE,
-                                      StudentProblemList, generate_problem_url, create_controller_query_service)
+from open_ended_grading.utils import (
+    STAFF_ERROR_MESSAGE, StudentProblemList, generate_problem_url, create_controller_query_service
+)
 
 log = logging.getLogger(__name__)
 
@@ -68,9 +66,10 @@ def staff_grading(request, course_id):
     """
     Show the instructor grading interface.
     """
-    course = get_course_with_access(request.user, 'staff', course_id)
+    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
+    course = get_course_with_access(request.user, 'staff', course_key)
 
-    ajax_url = _reverse_with_slash('staff_grading', course_id)
+    ajax_url = _reverse_with_slash('staff_grading', course_key)
 
     return render_to_response('instructor/staff_grading.html', {
         'course': course,
@@ -118,9 +117,9 @@ def peer_grading(request, course_id):
     When a student clicks on the "peer grading" button in the open ended interface, link them to a peer grading
     xmodule in the course.
     '''
-
+    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
     #Get the current course
-    course = get_course_with_access(request.user, 'load', course_id)
+    course = get_course_with_access(request.user, 'load', course_key)
 
     found_module, problem_url = find_peer_grading_module(course)
     if not found_module:
@@ -187,13 +186,11 @@ def flagged_problem_list(request, course_id):
     '''
     course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
     course = get_course_with_access(request.user, 'staff', course_key)
-    student_id = unique_id_for_user(request.user)
 
     # call problem list service
     success = False
     error_text = ""
     problem_list = []
-    base_course_url = reverse('courses')
 
     # Make a service that can query edX ORA.
     controller_qs = create_controller_query_service()

lms/lib/comment_client/user.py

@@ -86,7 +86,7 @@ class User(models.Model):
         if not self.course_id:
             raise CommentClientRequestError("Must provide course_id when retrieving active threads for the user")
         url = _url_for_user_active_threads(self.id)
-        params = {'course_id': self.course_id}
+        params = {'course_id': self.course_id.to_deprecated_string()}
         params = merge_dict(params, query_params)
         response = perform_request(
             'get',
@@ -102,7 +102,7 @@ class User(models.Model):
         if not self.course_id:
             raise CommentClientRequestError("Must provide course_id when retrieving subscribed threads for the user")
         url = _url_for_user_subscribed_threads(self.id)
-        params = {'course_id': self.course_id}
+        params = {'course_id': self.course_id.to_deprecated_string()}
         params = merge_dict(params, query_params)
         response = perform_request(
             'get',
@@ -118,7 +118,7 @@ class User(models.Model):
         url = self.url(action='get', params=self.attributes)
         retrieve_params = self.default_retrieve_params
         if self.attributes.get('course_id'):
-            retrieve_params['course_id'] = self.course_id
+            retrieve_params['course_id'] = self.course_id.to_deprecated_string()
         try:
             response = perform_request(
                 'get',