Merge pull request #4413 from edx/remove-loc-mapper

Remove LocMapperStore

common/djangoapps/student/migrations/0035_access_roles.py

 # -*- coding: utf-8 -*-
-from south.db import db
 from south.v2 import DataMigration
-from django.db import models
-from xmodule.modulestore.django import loc_mapper
 import re
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from opaque_keys import InvalidKeyError
@@ -10,6 +7,10 @@ import bson.son
 import logging
 from django.db.models.query_utils import Q
 from django.db.utils import IntegrityError
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.mixed import MixedModuleStore
+import itertools
 
 log = logging.getLogger(__name__)
 
@@ -25,8 +26,20 @@ class Migration(DataMigration):
         """
         Converts group table entries for write access and beta_test roles to course access roles table.
         """
+        store = modulestore()
+        if isinstance(store, MixedModuleStore):
+            self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
+            self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
+        elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
+            self.mongostore = store
+            self.xmlstore = None
+        elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
+            self.mongostore = None
+            self.xmlstore = store
+        else:
+            return
+
         # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
-        loc_map_collection = loc_mapper().location_map
         # b/c the Groups table had several entries for each course, we need to ensure we process each unique
         # course only once. The below datastructures help ensure that.
         hold = {}  # key of course_id_strings with array of group objects. Should only be org scoped entries
@@ -64,21 +77,27 @@ class Migration(DataMigration):
                     course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id_string)
                     # course_key is the downcased version, get the normal cased one. loc_mapper() has no
                     # methods taking downcased SSCK; so, need to do it manually here
-                    correct_course_key = self._map_downcased_ssck(course_key, loc_map_collection)
+                    correct_course_key = self._map_downcased_ssck(course_key)
                     if correct_course_key is not None:
                         _migrate_users(correct_course_key, role, course_key.org)
                 except InvalidKeyError:
-                    entry = loc_map_collection.find_one({
-                        'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
-                    })
-                    if entry is None:
+                    # old dotted format, try permutations
+                    parts = course_id_string.split('.')
+                    if len(parts) < 3:
                         hold.setdefault(course_id_string, []).append(group)
+                    elif len(parts) == 3:
+                        course_key = SlashSeparatedCourseKey(*parts)
+                        correct_course_key = self._map_downcased_ssck(course_key)
+                        if correct_course_key is None:
+                            hold.setdefault(course_id_string, []).append(group)
+                        else:
+                            _migrate_users(correct_course_key, role, course_key.org)
                     else:
-                        correct_course_key = SlashSeparatedCourseKey(*entry['_id'].values())
-                        if 'lower_id' in entry:
-                            _migrate_users(correct_course_key, role, entry['lower_id']['org'])
+                        correct_course_key = self.divide_parts_find_key(parts)
+                        if correct_course_key is None:
+                            hold.setdefault(course_id_string, []).append(group)
                         else:
-                            _migrate_users(correct_course_key, role, entry['_id']['org'].lower())
+                            _migrate_users(correct_course_key, role, course_key.org)
 
         # see if any in hold were missed above
         for held_auth_scope, groups in hold.iteritems():
@@ -99,28 +118,50 @@ class Migration(DataMigration):
                 # don't silently skip unexpected roles
                 log.warn("Didn't convert roles %s", [group.name for group in groups])
 
+    def divide_parts_find_key(self, parts):
+        """
+        Look for all possible org/course/run patterns from a possibly dotted source
+        """
+        for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
+            org = '.'.join(parts[:org_stop])
+            course = '.'.join(parts[org_stop:course_stop])
+            run = '.'.join(parts[course_stop:])
+            course_key = SlashSeparatedCourseKey(org, course, run)
+            correct_course_key = self._map_downcased_ssck(course_key)
+            if correct_course_key is not None:
+                return correct_course_key
+        return None
+
     def backwards(self, orm):
-        "Write your backwards methods here."
+        "Removes the new table."
         # Since this migration is non-destructive (monotonically adds information), I'm not sure what
         # the semantic of backwards should be other than perhaps clearing the table.
         orm['student.courseaccessrole'].objects.all().delete()
 
-    def _map_downcased_ssck(self, downcased_ssck, loc_map_collection):
+    def _map_downcased_ssck(self, downcased_ssck):
         """
         Get the normal cased version of this downcased slash sep course key
         """
-        # given the regex, the son may be an overkill
-        course_son = bson.son.SON([
-            ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
-            ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
-            ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
-        ])
-        entry = loc_map_collection.find_one(course_son)
-        if entry:
-            idpart = entry['_id']
-            return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
-        else:
-            return None
+        if self.mongostore is not None:
+            course_son = bson.son.SON([
+                ('_id.tag', 'i4x'),
+                ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
+                ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
+                ('_id.category', 'course'),
+                ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
+            ])
+            entry = self.mongostore.collection.find_one(course_son)
+            if entry:
+                idpart = entry['_id']
+                return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
+        if self.xmlstore is not None:
+            for course in self.xmlstore.get_courses():
+                if (
+                    course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
+                    and course.id.run.lower() == downcased_ssck.run
+                ):
+                    return course.id
+        return None
 
 
     models = {

common/djangoapps/student/migrations/0036_access_roles_orgless.py

 # -*- coding: utf-8 -*-
 from south.v2 import DataMigration
-from xmodule.modulestore.django import loc_mapper, modulestore
+from xmodule.modulestore.django import modulestore
 import re
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
-from opaque_keys import InvalidKeyError
 import logging
 from django.db.models.query_utils import Q
 from django.db.utils import IntegrityError
 from xmodule.modulestore import ModuleStoreEnum
+import bson.son
 from xmodule.modulestore.mixed import MixedModuleStore
+import itertools
 
 log = logging.getLogger(__name__)
 
@@ -24,11 +25,18 @@ class Migration(DataMigration):
         """
         Converts group table entries for write access and beta_test roles to course access roles table.
         """
-        # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
-        loc_map_collection = loc_mapper().location_map
-        mixed_ms = modulestore()
-        xml_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
-        mongo_ms = mixed_ms._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
+        store = modulestore()
+        if isinstance(store, MixedModuleStore):
+            self.mongostore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
+            self.xmlstore = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
+        elif store.get_modulestore_type() == ModuleStoreEnum.Type.mongo:
+            self.mongostore = store
+            self.xmlstore = None
+        elif store.get_modulestore_type() == ModuleStoreEnum.Type.xml:
+            self.mongostore = None
+            self.xmlstore = store
+        else:
+            return
 
         query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
         for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():
@@ -59,10 +67,7 @@ class Migration(DataMigration):
             role = parsed_entry.group('role_id')
             course_id_string = parsed_entry.group('course_id_string')
             # if it's a full course_id w/ dots, ignore it
-            entry = loc_map_collection.find_one({
-                'course_id': re.compile(r'^{}$'.format(course_id_string), re.IGNORECASE)
-            })
-            if entry is None:
+            if u'/' not in course_id_string and not self.dotted_course(course_id_string):
                 # check new table to see if it's been added as org permission
                 if not orm['student.courseaccessrole'].objects.filter(
                     role=role,
@@ -70,14 +75,14 @@ class Migration(DataMigration):
                 ).exists():
                     # old auth was of form role_coursenum. Grant access to all such courses wildcarding org and run
                     # look in xml for matching courses
-                    if xml_ms is not None:
-                        for course in xml_ms.get_courses():
+                    if self.xmlstore is not None:
+                        for course in self.xmlstore.get_courses():
                             if course_id_string == course.id.course.lower():
                                 _migrate_users(course.id, role)
 
-                    if mongo_ms is not None:
+                    if self.mongostore is not None:
                         mongo_query = re.compile(ur'^{}$'.format(course_id_string), re.IGNORECASE)
-                        for mongo_entry in mongo_ms.collection.find(
+                        for mongo_entry in self.mongostore.collection.find(
                             {"_id.category": "course", "_id.course": mongo_query}, fields=["_id"]
                         ):
                             mongo_id_dict = mongo_entry['_id']
@@ -86,6 +91,44 @@ class Migration(DataMigration):
                             )
                             _migrate_users(course_key, role)
 
+    def dotted_course(self, parts):
+        """
+        Look for all possible org/course/run patterns from a possibly dotted source
+        """
+        for org_stop, course_stop in itertools.combinations(range(1, len(parts)), 2):
+            org = '.'.join(parts[:org_stop])
+            course = '.'.join(parts[org_stop:course_stop])
+            run = '.'.join(parts[course_stop:])
+            course_key = SlashSeparatedCourseKey(org, course, run)
+            correct_course_key = self._map_downcased_ssck(course_key)
+            if correct_course_key is not None:
+                return correct_course_key
+        return False
+
+    def _map_downcased_ssck(self, downcased_ssck):
+        """
+        Get the normal cased version of this downcased slash sep course key
+        """
+        if self.mongostore is not None:
+            course_son = bson.son.SON([
+                ('_id.tag', 'i4x'),
+                ('_id.org', re.compile(r'^{}$'.format(downcased_ssck.org), re.IGNORECASE)),
+                ('_id.course', re.compile(r'^{}$'.format(downcased_ssck.course), re.IGNORECASE)),
+                ('_id.category', 'course'),
+                ('_id.name', re.compile(r'^{}$'.format(downcased_ssck.run), re.IGNORECASE)),
+            ])
+            entry = self.mongostore.collection.find_one(course_son)
+            if entry:
+                idpart = entry['_id']
+                return SlashSeparatedCourseKey(idpart['org'], idpart['course'], idpart['name'])
+        if self.xmlstore is not None:
+            for course in self.xmlstore.get_courses():
+                if (
+                    course.id.org.lower() == downcased_ssck.org and course.id.course.lower() == downcased_ssck.course
+                    and course.id.run.lower() == downcased_ssck.run
+                ):
+                    return course.id
+        return None
 
     def backwards(self, orm):
         "No obvious way to reverse just this migration, but reversing 0035 will reverse this."

common/lib/xmodule/xmodule/modulestore/django.py

@@ -14,7 +14,6 @@ import django.utils
 import re
 import threading
 
-from xmodule.modulestore.loc_mapper_store import LocMapperStore
 from xmodule.util.django import get_current_request_hostname
 import xmodule.modulestore  # pylint: disable=unused-import
 from xmodule.contentstore.django import contentstore
@@ -102,36 +101,8 @@ def clear_existing_modulestores():
 
     This is useful for flushing state between unit tests.
     """
-    global _MIXED_MODULESTORE, _loc_singleton  # pylint: disable=global-statement
+    global _MIXED_MODULESTORE  # pylint: disable=global-statement
     _MIXED_MODULESTORE = None
-    # pylint: disable=W0603
-    cache = getattr(_loc_singleton, "cache", None)
-    if cache:
-        cache.clear()
-    _loc_singleton = None
-
-
-# singleton instance of the loc_mapper
-_loc_singleton = None
-
-
-def loc_mapper():
-    """
-    Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
-    a singleton accessor.
-    """
-    # pylint: disable=W0603
-    global _loc_singleton
-    # pylint: disable=W0212
-    if _loc_singleton is None:
-        try:
-            loc_cache = get_cache('loc_cache')
-        except InvalidCacheBackendError:
-            loc_cache = get_cache('default')
-        # instantiate
-        _loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
-
-    return _loc_singleton
 
 
 class ModuleI18nService(object):

common/lib/xmodule/xmodule/modulestore/split_migrator.py

@@ -173,7 +173,7 @@ class SplitMigrator(object):
         """
         def get_translation(location):
             """
-            Convert the location and add to loc mapper
+            Convert the location
             """
             return new_course_key.make_usage_key(
                 location.category,
@@ -207,7 +207,7 @@ class SplitMigrator(object):
         """
         def get_translation(location):
             """
-            Convert the location and add to loc mapper
+            Convert the location
             """
             return new_course_key.make_usage_key(
                 location.category,

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -5,10 +5,8 @@ Modulestore configuration for test cases.
 from uuid import uuid4
 from django.test import TestCase
 from django.contrib.auth.models import User
-from xmodule.modulestore.django import (
-    modulestore, clear_existing_modulestores, loc_mapper
-)
 from xmodule.contentstore.django import _CONTENTSTORE
+from xmodule.modulestore.django import modulestore, clear_existing_modulestores
 from xmodule.modulestore import ModuleStoreEnum
 
 
@@ -197,11 +195,6 @@ class ModuleStoreTestCase(TestCase):
             module_store._drop_database()  # pylint: disable=protected-access
         _CONTENTSTORE.clear()
 
-        location_mapper = loc_mapper()
-        if location_mapper.db:
-            location_mapper.location_map.drop()
-            location_mapper.db.connection.close()
-
     @classmethod
     def setUpClass(cls):
         """

docs/en_us/developers/source/modulestore.rst

@@ -33,10 +33,6 @@ Modulestore Helpers
 These packages provide utilities for easier use of modulestores,
 and migrating data between modulestores.
 
-.. automodule:: xmodule.modulestore.loc_mapper_store
-    :members:
-    :show-inheritance:
-
 .. automodule:: xmodule.modulestore.search
     :members:
     :show-inheritance:

mongo_indexes.md

@@ -12,14 +12,6 @@ db.collection_name
 ```
 as in ```db.location_map.ensureIndex({'course_id': 1}{background: true})```
 
-location_map:
-=============
-
-```
-ensureIndex({'org': 1, 'offering': 1})
-ensureIndex({'schema': 1})
-```
-
 fs.files:
 =========