Merge pull request #661 from edx/dhm/location_mapper

Location to/from locator mapping facilities

common/lib/xmodule/xmodule/modulestore/django.py

@@ -8,6 +8,7 @@ from __future__ import absolute_import
 from importlib import import_module
 
 from django.conf import settings
+from xmodule.modulestore.loc_mapper_store import LocMapperStore
 
 _MODULESTORES = {}
 
@@ -53,3 +54,18 @@ def modulestore(name='default'):
                                                           settings.MODULESTORE[name]['OPTIONS'])
 
     return _MODULESTORES[name]
+
+_loc_singleton = None
+def loc_mapper():
+    """
+    Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
+    a singleton accessor.
+    """
+    # pylint: disable=W0603
+    global _loc_singleton
+    # pylint: disable=W0212
+    if _loc_singleton is None:
+        # instantiate
+        _loc_singleton = LocMapperStore(settings.modulestore_options)
+    return _loc_singleton
+

common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py

+'''
+Method for converting among our differing Location/Locator whatever reprs
+'''
+from random import randint
+import re
+import pymongo
+
+from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, DuplicateItemError
+from xmodule.modulestore.locator import BlockUsageLocator
+from xmodule.modulestore.mongo import draft
+from xmodule.modulestore import Location
+
+class LocMapperStore(object):
+    '''
+    This store persists mappings among the addressing schemes. At this time, it's between the old i4x Location
+    tuples and the split mongo Course and Block Locator schemes.
+
+    edX has used several different addressing schemes. The original ones were organically created based on
+    immediate needs and were overly restrictive esp wrt course ids. These were slightly extended to support
+    some types of blocks may need to have draft states during editing to keep live courses from seeing the wip.
+    A later refactoring generalized course ids to enable governance and more complex naming, branch naming with
+    anything able to be in any branch.
+
+    The expectation is that the configuration will have this use the same store as whatever is the default
+    or dominant store, but that's not a requirement. This store creates its own connection.
+    '''
+
+    def __init__(self, host, db, collection, port=27017, user=None, password=None, **kwargs):
+        '''
+        Constructor
+        '''
+        self.db = pymongo.database.Database(pymongo.MongoClient(
+            host=host,
+            port=port,
+            tz_aware=True,
+            **kwargs
+        ), db)
+        if user is not None and password is not None:
+            self.db.authenticate(user, password)
+
+        self.location_map = self.db[collection + '.location_map']
+        self.location_map.write_concern = {'w': 1}
+
+
+    # location_map functions
+    def create_map_entry(self, course_location, course_id=None, draft_branch='draft', prod_branch='published',
+        block_map=None):
+        """
+        Add a new entry to map this course_location to the new style CourseLocator.course_id. If course_id is not
+        provided, it creates the default map of using org.course.name from the location (just like course_id) if
+        the location.cateogry = 'course'; otherwise, it uses org.course.
+
+        You can create more than one mapping to the
+        same course_id target. In that case, the reverse translate will be arbitrary (no guarantee of which wins).
+        The use
+        case for more than one mapping is to map both org/course/run and org/course to the same new course_id thus
+        making a default for org/course. When querying for just org/course, the translator will prefer any entry
+        which does not have a name in the _id; otherwise, it will return an arbitrary match.
+
+        Note: the opposite is not true. That is, it never makes sense to use 2 different CourseLocator.course_id
+        keys to index the same old Locator org/course/.. pattern. There's no checking to ensure you don't do this.
+
+        NOTE: if there's already an entry w the given course_location, this may either overwrite that entry or
+        throw an error depending on how mongo is configured.
+
+        :param course_location: a Location preferably whose category is 'course'. Unlike the other
+        map methods, this one doesn't take the old-style course_id.  It should be called with
+        a course location not a block location; however, if called w/ a non-course Location, it creates
+        a "default" map for the org/course pair to a new course_id.
+        :param course_id: the CourseLocator style course_id
+        :param draft_branch: the branch name to assign for drafts. This is hardcoded because old mongo had
+        a fixed notion that there was 2 and only 2 versions for modules: draft and production. The old mongo
+        did not, however, require that a draft version exist. The new one, however, does require a draft to
+        exist.
+        :param prod_branch: the branch name to assign for the production (live) copy. In old mongo, every course
+        had to have a production version (whereas new split mongo does not require that until the author's ready
+        to publish).
+        :param block_map: an optional map to specify preferred names for blocks where the keys are the
+        Location block names and the values are the BlockUsageLocator.block_id.
+        """
+        if course_id is None:
+            if course_location.category == 'course':
+                course_id = "{0.org}.{0.course}.{0.name}".format(course_location)
+            else:
+                course_id = "{0.org}.{0.course}".format(course_location)
+        # very like _interpret_location_id but w/o the _id
+        location_id = {'org': course_location.org, 'course': course_location.course}
+        if course_location.category == 'course':
+            location_id['name'] = course_location.name
+
+        self.location_map.insert({
+                '_id': location_id,
+                'course_id': course_id,
+                'draft_branch': draft_branch,
+                'prod_branch': prod_branch,
+                'block_map': block_map or {},
+            })
+
+
+    def translate_location(self, old_style_course_id, location, published=True, add_entry_if_missing=True):
+        """
+        Translate the given module location to a Locator. If the mapping has the run id in it, then you
+        should provide old_style_course_id with that run id in it to disambiguate the mapping if there exists more
+        than one entry in the mapping table for the org.course.
+
+        The rationale for auto adding entries was that there should be a reasonable default translation
+        if the code just trips into this w/o creating translations. The downfall is that ambiguous course
+        locations may generate conflicting block_ids.
+
+        Will raise ItemNotFoundError if there's no mapping and add_entry_if_missing is False.
+
+        :param old_style_course_id: the course_id used in old mongo not the new one (optional, will use location)
+        :param location:  a Location pointing to a module
+        :param published: a boolean to indicate whether the caller wants the draft or published branch.
+        :param add_entry_if_missing: a boolean as to whether to raise ItemNotFoundError or to create an entry if
+        the course
+        or block is not found in the map.
+
+        NOTE: unlike old mongo, draft branches contain the whole course; so, it applies to all category
+        of locations including course.
+        """
+        location_id = self._interpret_location_course_id(old_style_course_id, location)
+
+        maps = self.location_map.find(location_id).sort('_id.name', pymongo.ASCENDING)
+        if maps.count() == 0:
+            if add_entry_if_missing:
+                # create a new map
+                course_location = location.replace(category='course', name=location_id['_id.name'])
+                self.create_map_entry(course_location)
+                entry = self.location_map.find_one(location_id)
+            else:
+                raise ItemNotFoundError()
+        elif maps.count() > 1:
+            # if more than one, prefer the one w/o a name if that exists. Otherwise, choose the first (alphabetically)
+            entry = maps[0]
+        else:
+            entry = maps[0]
+
+        if published:
+            branch = entry['prod_branch']
+        else:
+            branch = entry['draft_branch']
+
+        usage_id = entry['block_map'].get(location.name)
+        if usage_id is None:
+            if add_entry_if_missing:
+                usage_id = self._add_to_block_map(location, location_id, entry['block_map'])
+            else:
+                raise ItemNotFoundError()
+        elif isinstance(usage_id, dict):
+            # name is not unique, look through for the right category
+            if location.category in usage_id:
+                usage_id = usage_id[location.category]
+            elif add_entry_if_missing:
+                usage_id = self._add_to_block_map(location, location_id, entry['block_map'])
+            else:
+                raise ItemNotFoundError()
+        else:
+            raise InvalidLocationError()
+
+        return BlockUsageLocator(course_id=entry['course_id'], branch=branch, usage_id=usage_id)
+
+    def translate_locator_to_location(self, locator):
+        """
+        Returns an old style Location for the given Locator if there's an appropriate entry in the
+        mapping collection. Note, it requires that the course was previously mapped (a side effect of
+        translate_location or explicitly via create_map_entry) and
+        the block's usage_id was previously stored in the
+        map (a side effect of translate_location or via add|update_block_location).
+
+        If there are no matches, it returns None.
+
+        If there's more than one location to locator mapping to the same course_id, it looks for the first
+        one with a mapping for the block usage_id and picks that arbitrary course location.
+
+        :param locator: a BlockUsageLocator
+        """
+        # This does not require that the course exist in any modulestore
+        # only that it has a mapping entry.
+        maps = self.location_map.find({'course_id': locator.course_id})
+        # look for one which maps to this block usage_id
+        if maps.count() == 0:
+            return None
+        for candidate in maps:
+            for old_name, cat_to_usage in candidate['block_map'].iteritems():
+                for category, usage_id in cat_to_usage.iteritems():
+                    if usage_id == locator.usage_id:
+                        # figure out revision
+                        # enforce the draft only if category in [..] logic
+                        if category in draft.DIRECT_ONLY_CATEGORIES:
+                            revision = None
+                        elif locator.branch == candidate['draft_branch']:
+                            revision = draft.DRAFT
+                        else:
+                            revision = None
+                        return Location(
+                            'i4x',
+                            candidate['_id']['org'],
+                            candidate['_id']['course'],
+                            category,
+                            old_name,
+                            revision)
+        return None
+
+
+    def add_block_location_translator(self, location, old_course_id=None, usage_id=None):
+        """
+        Similar to translate_location which adds an entry if none is found, but this cannot create a new
+        course mapping entry, only a block within such a mapping entry. If it finds no existing
+        course maps, it raises ItemNotFoundError.
+
+        In the case that there are more than one mapping record for the course identified by location, this
+        method adds the mapping to all matching records! (translate_location only adds to one)
+
+        It allows the caller to specify
+        the new-style usage_id for the target rather than having the translate concoct its own.
+        If the provided usage_id already exists in one of the found maps for the org/course, this function
+        raises DuplicateItemError unless the old item id == the new one.
+
+        If the caller does not provide a usage_id and there exists an entry in one of the course variants,
+        it will use that entry. If more than one variant uses conflicting entries, it will raise DuplicateItemError.
+
+        Returns the usage_id used in the mapping
+
+        :param location: a fully specified Location
+        :param old_course_id: the old-style org/course or org/course/run string (optional)
+        :param usage_id: the desired new block_id. If left as None, this will generate one as per translate_location
+        """
+        location_id = self._interpret_location_course_id(old_course_id, location)
+
+        maps = self.location_map.find(location_id)
+        if maps.count() == 0:
+            raise ItemNotFoundError()
+
+        # turn maps from cursor to list
+        map_list = list(maps)
+        # check whether there's already a usage_id for this location (and it agrees w/ any passed in or found)
+        for map_entry in map_list:
+            if (location.name in map_entry['block_map'] and
+                    location.category in map_entry['block_map'][location.name]):
+                if usage_id is None:
+                    usage_id = map_entry['block_map'][location.name][location.category]
+                elif usage_id != map_entry['block_map'][location.name][location.category]:
+                    raise DuplicateItemError()
+
+        computed_usage_id = usage_id
+
+        # update the maps (and generate a usage_id if it's not been set yet)
+        for map_entry in map_list:
+            if computed_usage_id is None:
+                computed_usage_id = self._add_to_block_map(location, location_id, map_entry['block_map'])
+            elif (location.name not in map_entry['block_map'] or
+                     location.category not in map_entry['block_map'][location.name]):
+                alt_usage_id = self._verify_uniqueness(computed_usage_id, map_entry['block_map'])
+                if alt_usage_id != computed_usage_id:
+                    if usage_id is not None:
+                        raise DuplicateItemError()
+                    else:
+                        # revise already set ones and add to remaining ones
+                        computed_usage_id = self.update_block_location_translator(
+                            location,
+                            alt_usage_id,
+                            old_course_id,
+                            True
+                        )
+
+                map_entry['block_map'].setdefault(location.name, {})[location.category] = computed_usage_id
+                self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
+
+        return computed_usage_id
+
+
+    def update_block_location_translator(self, location, usage_id, old_course_id=None, autogenerated_usage_id=False):
+        """
+        Update all existing maps from location's block to the new usage_id. Used for changing the usage_id,
+        thus the usage_id is required.
+
+        Returns the usage_id. (which is primarily useful in the case of autogenerated_usage_id)
+
+        :param location: a fully specified Location
+        :param usage_id: the desired new block_id.
+        :param old_course_id: the old-style org/course or org/course/run string (optional)
+        :param autogenerated_usage_id: a flag used mostly for internal calls to indicate that this usage_id
+        was autogenerated and thus can be overridden if it's not unique. If you set this flag, the stored
+        usage_id may not be the one you submitted.
+        """
+        location_id = self._interpret_location_course_id(old_course_id, location)
+
+        maps = self.location_map.find(location_id)
+        for map_entry in maps:
+            # handle noop of renaming to same name
+            if (location.name in map_entry['block_map'] and
+                    map_entry['block_map'][location.name].get(location.category) == usage_id):
+                continue
+            alt_usage_id = self._verify_uniqueness(usage_id, map_entry['block_map'])
+            if alt_usage_id != usage_id:
+                if autogenerated_usage_id:
+                    # revise already set ones and add to remaining ones
+                    usage_id = self.update_block_location_translator(location, alt_usage_id, old_course_id, True)
+                    return usage_id
+                else:
+                    raise DuplicateItemError()
+
+            if location.category in map_entry['block_map'].setdefault(location.name, {}):
+                map_entry['block_map'][location.name][location.category] = usage_id
+                self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
+
+        return usage_id
+
+
+    def delete_block_location_translator(self, location, old_course_id=None):
+        """
+        Remove all existing maps from location's block.
+
+        :param location: a fully specified Location
+        :param old_course_id: the old-style org/course or org/course/run string (optional)
+        """
+        location_id = self._interpret_location_course_id(old_course_id, location)
+
+        maps = self.location_map.find(location_id)
+        for map_entry in maps:
+            if location.category in map_entry['block_map'].setdefault(location.name, {}):
+                if len(map_entry['block_map'][location.name]) == 1:
+                    del map_entry['block_map'][location.name]
+                else:
+                    del map_entry['block_map'][location.name][location.category]
+                self.location_map.update({'_id': map_entry['_id']}, {'$set': {'block_map': map_entry['block_map']}})
+
+    def _add_to_block_map(self, location, location_id, block_map):
+        '''add the given location to the block_map and persist it'''
+        if self._block_id_is_guid(location.name):
+            # This makes the ids more meaningful with a small probability of name collision.
+            # The downside is that if there's more than one course mapped to from the same org/course root
+            # the block ids will likely be out of sync and collide from an id perspective. HOWEVER,
+            # if there are few == org/course roots or their content is unrelated, this will work well.
+            usage_id = self._verify_uniqueness(location.category + location.name[:3], block_map)
+        block_map.setdefault(location.name, {})[location.category] = usage_id
+        self.location_map.update(location_id, {'$set': {'block_map': block_map}})
+        return usage_id
+
+    def _interpret_location_course_id(self, course_id, location):
+        """
+        Take the old style course id (org/course/run) and return a dict for querying the mapping table.
+        If the course_id is empty, it uses location, but this may result in an inadequate id.
+
+        :param course_id: old style 'org/course/run' id from Location.course_id where Location.category = 'course'
+        :param location: a Location object which may be to a module or a course. Provides partial info
+        if course_id is omitted.
+        """
+        if course_id:
+            # re doesn't allow ?P<_id.org> and ilk
+            m = re.match(r'([^/]+)/([^/]+)/([^/]+)', course_id)
+            return dict(zip(['_id.org', '_id.course', '_id.name'], m.groups()))
+
+        location_id = {'_id.org': location.org, '_id.course': location.course}
+        if location.category == 'course':
+            location_id['_id.name'] = location.name
+        return location_id
+
+    def _block_id_is_guid(self, name):
+        return len(name) == 32 and re.search(r'[^0-9A-Fa-f]', name) is None
+
+    def _verify_uniqueness(self, name, block_map):
+        '''
+        Verify that the name doesn't occur elsewhere in block_map. If it does, keep adding to it until
+        it's unique.
+        '''
+        for targets in block_map.itervalues():
+            if isinstance(targets, dict):
+                for values in targets.itervalues():
+                    if values == name:
+                        name += str(randint(0, 9))
+                        return self._verify_uniqueness(name, block_map)
+
+            elif targets == name:
+                name += str(randint(0, 9))
+                return self._verify_uniqueness(name, block_map)
+        return name

common/lib/xmodule/xmodule/modulestore/parsers.py

@@ -19,10 +19,10 @@ def parse_url(string):
 
     Examples:
         'edx://version/0123FFFF'
-        'edx://edu.mit.eecs.6002x'
-        'edx://edu.mit.eecs.6002x/branch/published'
-        'edx://edu.mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
-        'edx://edu.mit.eecs.6002x/branch/published/block/HW3'
+        'edx://mit.eecs.6002x'
+        'edx://mit.eecs.6002x;published'
+        'edx://mit.eecs.6002x;published/block/HW3'
+        'edx://mit.eecs.6002x;published/version/000eee12345/block/HW3'
 
     This returns None if string cannot be parsed.
 
@@ -97,11 +97,11 @@ def parse_course_id(string):
 
     Examples of valid course_ids:
 
-      'edu.mit.eecs.6002x'
-      'edu.mit.eecs.6002x/branch/published'
-      'edu.mit.eecs.6002x/block/HW3'
-      'edu.mit.eecs.6002x/branch/published/block/HW3'
-      'edu.mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
+      'mit.eecs.6002x'
+      'mit.eecs.6002x/branch/published'
+      'mit.eecs.6002x/block/HW3'
+      'mit.eecs.6002x/branch/published/block/HW3'
+      'mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
 
 
     Syntax:

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -10,9 +10,8 @@ from xmodule.errortracker import null_error_tracker
 from xmodule.x_module import XModuleDescriptor
 from xmodule.modulestore.locator import BlockUsageLocator, DescriptionLocator, CourseLocator, VersionTree
 from xmodule.modulestore.exceptions import InsufficientSpecificationError, VersionConflictError
-from xmodule.modulestore import inheritance
+from xmodule.modulestore import inheritance, ModuleStoreBase
 
-from .. import ModuleStoreBase
 from ..exceptions import ItemNotFoundError
 from .definition_lazy_loader import DefinitionLazyLoader
 from .caching_descriptor_system import CachingDescriptorSystem
@@ -62,14 +61,11 @@ class SplitMongoModuleStore(ModuleStoreBase):
             **kwargs
         ), db)
 
-        # TODO add caching of structures to thread_cache to prevent repeated fetches (but not index b/c
-        # it changes w/o having a change in id)
         self.course_index = self.db[collection + '.active_versions']
         self.structures = self.db[collection + '.structures']
         self.definitions = self.db[collection + '.definitions']
 
-        # ??? Code review question: those familiar w/ python threading. Should I instead
-        # use django cache? How should I expire entries?
+        # Code review question: How should I expire entries?
         # _add_cache could use a lru mechanism to control the cache size?
         self.thread_cache = threading.local()
 
@@ -1178,6 +1174,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         else:
             return DescriptionLocator(definition['_id'])
 
+
     def _block_matches(self, value, qualifiers):
         '''
         Return True or False depending on whether the value (block contents)

common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py

+'''
+Created on Aug 5, 2013
+
+@author: dmitchell
+'''
+import unittest
+import uuid
+from xmodule.modulestore import Location
+from xmodule.modulestore.locator import BlockUsageLocator
+from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
+from xmodule.modulestore.loc_mapper_store import LocMapperStore
+
+
+class TestLocationMapper(unittest.TestCase):
+
+    def setUp(self):
+        modulestore_options = {
+            'host': 'localhost',
+            'db': 'test_xmodule',
+            'collection': 'modulestore{0}'.format(uuid.uuid4().hex),
+        }
+
+        # pylint: disable=W0142
+        TestLocationMapper.loc_store = LocMapperStore(**modulestore_options)
+
+
+    def tearDown(self):
+        db = TestLocationMapper.loc_store.db
+        db.drop_collection(TestLocationMapper.loc_store.location_map)
+        db.connection.close()
+        TestLocationMapper.loc_store = None
+
+    def test_create_map(self):
+        org = 'foo_org'
+        course = 'bar_course'
+        loc_mapper().create_map_entry(Location('i4x', org, course, 'course', 'baz_run'))
+        entry = loc_mapper().location_map.find_one({
+            '_id': {'org': org, 'course': course, 'name': 'baz_run'}
+        })
+        self.assertIsNotNone(entry, "Didn't find entry")
+        self.assertEqual(entry['course_id'], '{}.{}.baz_run'.format(org, course))
+        self.assertEqual(entry['draft_branch'], 'draft')
+        self.assertEqual(entry['prod_branch'], 'published')
+        self.assertEqual(entry['block_map'], {})
+
+        # ensure create_entry does the right thing when not given a course (creates org/course
+        # rather than org/course/run course_id)
+        loc_mapper().create_map_entry(Location('i4x', org, course, 'vertical', 'baz_vert'))
+        entry = loc_mapper().location_map.find_one({
+            '_id': {'org': org, 'course': course}
+        })
+        self.assertIsNotNone(entry, "Didn't find entry")
+        self.assertEqual(entry['course_id'], '{}.{}'.format(org, course))
+
+        course = 'quux_course'
+        # oldname: {category: newname}
+        block_map = {'abc123': {'problem': 'problem2'}}
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'problem', 'abc123', 'draft'),
+            'foo_org.geek_dept.quux_course.baz_run',
+            'wip',
+            'live',
+            block_map)
+        entry = loc_mapper().location_map.find_one({
+            '_id': {'org': org, 'course': course}
+        })
+        self.assertIsNotNone(entry, "Didn't find entry")
+        self.assertEqual(entry['course_id'], 'foo_org.geek_dept.quux_course.baz_run')
+        self.assertEqual(entry['draft_branch'], 'wip')
+        self.assertEqual(entry['prod_branch'], 'live')
+        self.assertEqual(entry['block_map'], block_map)
+
+    def test_translate_location_read_only(self):
+        """
+        Test the variants of translate_location which don't create entries, just decode
+        """
+        # lookup before there are any maps
+        org = 'foo_org'
+        course = 'bar_course'
+        old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
+        with self.assertRaises(ItemNotFoundError):
+            _ = loc_mapper().translate_location(
+                old_style_course_id,
+                Location('i4x', org, course, 'problem', 'abc123'),
+                add_entry_if_missing=False
+            )
+
+        new_style_course_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+        block_map = {'abc123': {'problem': 'problem2'}}
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'baz_run'),
+            new_style_course_id,
+            block_map=block_map
+        )
+        # only one course matches
+        prob_locator = loc_mapper().translate_location(
+            old_style_course_id,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+        self.assertEqual(prob_locator.branch, 'published')
+        self.assertEqual(prob_locator.usage_id, 'problem2')
+        # look for w/ only the Location (works b/c there's only one possible course match)
+        prob_locator = loc_mapper().translate_location(
+            None,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+        # look for non-existent problem
+        with self.assertRaises(ItemNotFoundError):
+            prob_locator = loc_mapper().translate_location(
+                None,
+                Location('i4x', org, course, 'problem', '1def23'),
+                add_entry_if_missing=False
+            )
+
+        # add a distractor course
+        block_map = {'abc123': {'problem': 'problem3'}}
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'),
+            block_map=block_map
+        )
+        prob_locator = loc_mapper().translate_location(
+            old_style_course_id,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+        self.assertEqual(prob_locator.usage_id, 'problem2')
+        # look for w/ only the Location (not unique; so, just verify it returns something)
+        prob_locator = loc_mapper().translate_location(
+            None,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            add_entry_if_missing=False
+        )
+        self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
+
+        # add a default course pointing to the delta_run
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'problem', '789abc123efg456'),
+            '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'),
+            block_map=block_map
+        )
+        # now the ambiguous query should return delta
+        prob_locator = loc_mapper().translate_location(
+            None,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'))
+        self.assertEqual(prob_locator.usage_id, 'problem3')
+
+        # get the draft one (I'm sorry this is getting long)
+        prob_locator = loc_mapper().translate_location(
+            None,
+            Location('i4x', org, course, 'problem', 'abc123'),
+            published=False,
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'))
+        self.assertEqual(prob_locator.usage_id, 'problem3')
+        self.assertEqual(prob_locator.branch, 'draft')
+
+    def test_translate_location_dwim(self):
+        """
+        Test the location translation mechanisms which try to do-what-i-mean by creating new
+        entries for never seen queries.
+        """
+        org = 'foo_org'
+        course = 'bar_course'
+        old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
+        problem_name = 'abc123abc123abc123abc123abc123f9'
+        location = Location('i4x', org, course, 'problem', problem_name)
+        prob_locator = loc_mapper().translate_location(
+            old_style_course_id,
+            location,
+            add_entry_if_missing=True
+        )
+        new_style_course_id = '{}.{}.{}'.format(org, course, 'baz_run')
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+        self.assertEqual(prob_locator.branch, 'published')
+        self.assertEqual(prob_locator.usage_id, 'problemabc')
+        # look for w/ only the Location (works b/c there's only one possible course match)
+        prob_locator = loc_mapper().translate_location(
+            None,
+            location,
+            add_entry_if_missing=True
+        )
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+
+        # add a distractor course
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'),
+            block_map={problem_name: {'problem': 'problem3'}}
+        )
+        prob_locator = loc_mapper().translate_location(
+            old_style_course_id,
+            location,
+            add_entry_if_missing=True
+        )
+        self.assertEqual(prob_locator.course_id, new_style_course_id)
+        self.assertEqual(prob_locator.usage_id, 'problemabc')
+        # look for w/ only the Location (not unique; so, just verify it returns something)
+        prob_locator = loc_mapper().translate_location(
+            None,
+            location,
+            add_entry_if_missing=True
+        )
+        self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
+
+        # add a default course pointing to the delta_run
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'problem', '789abc123efg456'),
+            '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'),
+            block_map={problem_name: {'problem': 'problem3'}}
+        )
+        # now the ambiguous query should return delta
+        prob_locator = loc_mapper().translate_location(
+            None,
+            location,
+            add_entry_if_missing=False
+        )
+        self.assertEqual(prob_locator.course_id, '{}.geek_dept.{}.{}'.format(org, course, 'delta_run'))
+        self.assertEqual(prob_locator.usage_id, 'problem3')
+
+    def test_translate_locator(self):
+        """
+        tests translate_locator_to_location(BlockUsageLocator)
+        """
+        # lookup for non-existent course
+        org = 'foo_org'
+        course = 'bar_course'
+        new_style_course_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+        prob_locator = BlockUsageLocator(
+            course_id=new_style_course_id,
+            usage_id='problem2'
+        )
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        self.assertIsNone(prob_location, 'found entry in empty map table')
+
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'baz_run'),
+            new_style_course_id,
+            block_map={
+                'abc123': {'problem': 'problem2'},
+                '48f23a10395384929234': {'chapter': 'chapter48f'}
+            }
+        )
+        # only one course matches
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        # default branch
+        self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
+        # explicit branch
+        prob_locator = BlockUsageLocator(prob_locator, branch='draft')
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', 'draft'))
+        prob_locator = BlockUsageLocator(
+            course_id=new_style_course_id, usage_id='problem2', branch='production'
+        )
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
+        # same for chapter except chapter cannot be draft in old system
+        chap_locator = BlockUsageLocator(
+            course_id=new_style_course_id,
+            usage_id='chapter48f'
+        )
+        chap_location = loc_mapper().translate_locator_to_location(chap_locator)
+        self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+        # explicit branch
+        chap_locator = BlockUsageLocator(chap_locator, branch='draft')
+        chap_location = loc_mapper().translate_locator_to_location(chap_locator)
+        self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+        chap_locator = BlockUsageLocator(
+            course_id=new_style_course_id, usage_id='chapter48f', branch='production'
+        )
+        chap_location = loc_mapper().translate_locator_to_location(chap_locator)
+        self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+
+        # look for non-existent problem
+        prob_locator2 = BlockUsageLocator(
+            course_id=new_style_course_id,
+            branch='draft',
+            usage_id='problem3'
+        )
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
+        self.assertIsNone(prob_location, 'Found non-existent problem')
+
+        # add a distractor course
+        new_style_course_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            new_style_course_id,
+            block_map={'abc123': {'problem': 'problem3'}}
+        )
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
+
+        # add a default course pointing to the delta_run
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'problem', '789abc123efg456'),
+            new_style_course_id,
+            block_map={'abc123': {'problem': 'problem3'}}
+        )
+        # now query delta (2 entries point to it)
+        prob_locator = BlockUsageLocator(
+            course_id=new_style_course_id,
+            branch='production',
+            usage_id='problem3'
+        )
+        prob_location = loc_mapper().translate_locator_to_location(prob_locator)
+        self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123'))
+
+    def test_add_block(self):
+        """
+        Test add_block_location_translator(location, old_course_id=None, usage_id=None)
+        """
+        # call w/ no matching courses
+        org = 'foo_org'
+        course = 'bar_course'
+        old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
+        problem_name = 'abc123abc123abc123abc123abc123f9'
+        location = Location('i4x', org, course, 'problem', problem_name)
+        with self.assertRaises(ItemNotFoundError):
+            loc_mapper().add_block_location_translator(location)
+        with self.assertRaises(ItemNotFoundError):
+            loc_mapper().add_block_location_translator(location, old_style_course_id)
+
+        # w/ one matching course
+        new_style_course_id = '{}.{}.{}'.format(org, course, 'baz_run')
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'baz_run'),
+            new_style_course_id,
+        )
+        new_usage_id = loc_mapper().add_block_location_translator(location)
+        self.assertEqual(new_usage_id, 'problemabc')
+        # look it up
+        translated_loc = loc_mapper().translate_location(old_style_course_id, location, add_entry_if_missing=False)
+        self.assertEqual(translated_loc.course_id, new_style_course_id)
+        self.assertEqual(translated_loc.usage_id, new_usage_id)
+
+        # w/ one distractor which has one entry already
+        new_style_course_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            new_style_course_id,
+            block_map={'48f23a10395384929234': {'chapter': 'chapter48f'}}
+        )
+        # try adding the one added before
+        new_usage_id2 = loc_mapper().add_block_location_translator(location)
+        self.assertEqual(new_usage_id, new_usage_id2)
+        # it should be in the distractor now
+        new_location = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id, usage_id=new_usage_id2)
+        )
+        self.assertEqual(new_location, location)
+        # add one close to the existing chapter (cause name collision)
+        location = Location('i4x', org, course, 'chapter', '48f23a103953849292341234567890ab')
+        new_usage_id = loc_mapper().add_block_location_translator(location)
+        self.assertRegexpMatches(new_usage_id, r'^chapter48f\d')
+        # retrievable from both courses
+        new_location = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id, usage_id=new_usage_id)
+        )
+        self.assertEqual(new_location, location)
+        new_location = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id='{}.{}.{}'.format(org, course, 'baz_run'), usage_id=new_usage_id)
+        )
+        self.assertEqual(new_location, location)
+
+        # provoke duplicate item errors
+        location = location.replace(name='44f23a103953849292341234567890ab')
+        with self.assertRaises(DuplicateItemError):
+            loc_mapper().add_block_location_translator(location, usage_id=new_usage_id)
+        new_usage_id = loc_mapper().add_block_location_translator(location, old_course_id=old_style_course_id)
+        other_course_old_style = '{}/{}/{}'.format(org, course, 'delta_run')
+        new_usage_id2 = loc_mapper().add_block_location_translator(
+            location,
+            old_course_id=other_course_old_style,
+            usage_id='{}b'.format(new_usage_id)
+        )
+        with self.assertRaises(DuplicateItemError):
+            loc_mapper().add_block_location_translator(location)
+
+    def test_update_block(self):
+        """
+        test update_block_location_translator(location, usage_id, old_course_id=None)
+        """
+        org = 'foo_org'
+        course = 'bar_course'
+        new_style_course_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'baz_run'),
+            new_style_course_id,
+            block_map={
+                'abc123': {'problem': 'problem2'},
+                '48f23a10395384929234': {'chapter': 'chapter48f'},
+                '1': {'chapter': 'chapter1', 'problem': 'problem1'},
+            }
+        )
+        new_style_course_id2 = '{}.geek_dept.{}.delta_run'.format(org, course)
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            new_style_course_id2,
+            block_map={
+                'abc123': {'problem': 'problem3'},
+                '48f23a10395384929234': {'chapter': 'chapter48b'},
+                '1': {'chapter': 'chapter2', 'problem': 'problem2'},
+            }
+        )
+        location = Location('i4x', org, course, 'problem', '1')
+        # change in all courses to same value
+        loc_mapper().update_block_location_translator(location, 'problem1')
+        trans_loc = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id, usage_id='problem1')
+        )
+        self.assertEqual(trans_loc, location)
+        trans_loc = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id2, usage_id='problem1')
+        )
+        self.assertEqual(trans_loc, location)
+        # try to change to overwrite used usage_id
+        location = Location('i4x', org, course, 'chapter', '48f23a10395384929234')
+        with self.assertRaises(DuplicateItemError):
+            loc_mapper().update_block_location_translator(location, 'chapter2')
+        # just change the one course
+        loc_mapper().update_block_location_translator(location, 'chapter2', '{}/{}/{}'.format(org, course, 'baz_run'))
+        trans_loc = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id, usage_id='chapter2')
+        )
+        self.assertEqual(trans_loc.name, '48f23a10395384929234')
+        # but this still points to the old
+        trans_loc = loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id2, usage_id='chapter2')
+        )
+        self.assertEqual(trans_loc.name, '1')
+
+
+    def test_delete_block(self):
+        """
+        test delete_block_location_translator(location, old_course_id=None)
+        """
+        org = 'foo_org'
+        course = 'bar_course'
+        new_style_course_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'baz_run'),
+            new_style_course_id,
+            block_map={
+                'abc123': {'problem': 'problem2'},
+                '48f23a10395384929234': {'chapter': 'chapter48f'},
+                '1': {'chapter': 'chapter1', 'problem': 'problem1'},
+            }
+        )
+        new_style_course_id2 = '{}.geek_dept.{}.delta_run'.format(org, course)
+        loc_mapper().create_map_entry(
+            Location('i4x', org, course, 'course', 'delta_run'),
+            new_style_course_id2,
+            block_map={
+                'abc123': {'problem': 'problem3'},
+                '48f23a10395384929234': {'chapter': 'chapter48b'},
+                '1': {'chapter': 'chapter2', 'problem': 'problem2'},
+            }
+        )
+        location = Location('i4x', org, course, 'problem', '1')
+        # delete from all courses
+        loc_mapper().delete_block_location_translator(location)
+        self.assertIsNone(loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id, usage_id='problem1')
+        ))
+        self.assertIsNone(loc_mapper().translate_locator_to_location(
+            BlockUsageLocator(course_id=new_style_course_id2, usage_id='problem2')
+        ))
+        # delete from one course
+        location = location.replace(name='abc123')
+        loc_mapper().delete_block_location_translator(location, '{}/{}/{}'.format(org, course, 'baz_run'))
+        with self.assertRaises(ItemNotFoundError):
+            loc_mapper().translate_location(
+                '{}/{}/{}'.format(org, course, 'baz_run'),
+                location,
+                add_entry_if_missing=False
+            )
+        locator = loc_mapper().translate_location(
+            '{}/{}/{}'.format(org, course, 'delta_run'),
+            location,
+            add_entry_if_missing=False
+        )
+        self.assertEqual(locator.usage_id, 'problem3')
+
+#==================================
+# functions to mock existing services
+def loc_mapper():
+    return TestLocationMapper.loc_store
+
+def render_to_template_mock(*_args):
+    pass

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -992,8 +992,8 @@ class TestInheritance(SplitModuleTest):
 # This mocks the django.modulestore() function and is intended purely to disentangle
 # the tests from django
 def modulestore():
-    def load_function(path):
-        module_path, _, name = path.rpartition('.')
+    def load_function(engine_path):
+        module_path, _, name = engine_path.rpartition('.')
         return getattr(import_module(module_path), name)
 
     if SplitModuleTest.modulestore is None: