Merge pull request #1954 from edx/dhm/loc_mapper_memoize

loc mapper memoize

CHANGELOG.rst

@@ -17,6 +17,11 @@ Common: Switch from mitx.db to edx.db for sqlite databases. This will effectivel
   reset state for local instances of the code, unless you manually rename your
   mitx.db file to edx.db.
 
+Common: significant performance improvement for authorization checks and location translations.
+  Ensure all auth checks, check all possible permutations of the auth key (Instructor dashboard
+  now shows when it should for all courses in lms).
+  Made queries for Studio dashboard 2 orders of magnitude faster (and fewer).
+
 Blades: Video Transcripts: Fix clear and download buttons. BLD-438.
 
 Common: Switch over from MITX_FEATURES to just FEATURES. To override items in

cms/djangoapps/auth/authz.py

@@ -63,7 +63,7 @@ def get_all_course_role_groupnames(location, role, use_filter=True):
     # filter to the ones which exist
     default = groupnames[0]
     if use_filter:
-        groupnames = [group for group in groupnames if Group.objects.filter(name=group).exists()]
+        groupnames = [group.name for group in Group.objects.filter(name__in=groupnames)]
     return groupnames, default
 
 
@@ -203,12 +203,8 @@ def remove_user_from_course_group(caller, user, location, role):
 
     # see if the user is actually in that role, if not then we don't have to do anything
     groupnames, _ = get_all_course_role_groupnames(location, role)
-    for groupname in groupnames:
-        groups = user.groups.filter(name=groupname)
-        if groups:
-            # will only be one with that name
-            user.groups.remove(groups[0])
-            user.save()
+    user.groups.remove(*user.groups.filter(name__in=groupnames))
+    user.save()
 
 
 def remove_user_from_creator_group(caller, user):
@@ -243,7 +239,7 @@ def is_user_in_course_group_role(user, location, role, check_staff=True):
         if check_staff and user.is_staff:
             return True
         groupnames, _ = get_all_course_role_groupnames(location, role)
-        return any(user.groups.filter(name=groupname).exists() for groupname in groupnames)
+        return user.groups.filter(name__in=groupnames).exists()
 
     return False
 
@@ -266,7 +262,7 @@ def is_user_in_creator_group(user):
 
     # Feature flag for using the creator group setting. Will be removed once the feature is complete.
     if settings.FEATURES.get('ENABLE_CREATOR_GROUP', False):
-        return user.groups.filter(name=COURSE_CREATOR_GROUP_NAME).count() > 0
+        return user.groups.filter(name=COURSE_CREATOR_GROUP_NAME).exists()
 
     return True
 

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -7,7 +7,6 @@ from textwrap import dedent
 
 from django.test.utils import override_settings
 from django.conf import settings
-from django.core.urlresolvers import reverse
 from path import path
 from tempdir import mkdtemp_clean
 from fs.osfs import OSFS
@@ -427,7 +426,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         course = module_store.get_item(course_location)
         num_tabs = len(course.tabs)
-        last_tab = course.tabs[num_tabs - 1]
+        last_tab = course.tabs[-1]
         url_slug = last_tab['url_slug']
         delete_url = self._get_tab_locator(course, last_tab).url_reverse('xblock')
 
@@ -446,7 +445,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
     def _get_tab_locator(self, course, tab):
         """ Returns the locator for a given tab. """
-        tab_location = 'i4x://MITx/999/static_tab/{0}'.format(tab['url_slug'])
+        tab_location = 'i4x://edX/999/static_tab/{0}'.format(tab['url_slug'])
         return loc_mapper().translate_location(
             course.location.course_id, Location(tab_location), False, True
         )

cms/djangoapps/contentstore/views/course.py

@@ -128,7 +128,8 @@ def course_listing(request):
     """
     List all courses available to the logged in user
     """
-    courses = modulestore('direct').get_items(Location('i4x', None, None, 'course', None))
+    # there's an index on category which will be used if none of its antecedents are set
+    courses = modulestore('direct').get_items(Location(None, None, None, 'course', None))
 
     # filter out courses that we don't have access too
     def course_filter(course):

cms/envs/dev.py

@@ -129,7 +129,12 @@ CACHES = {
         'LOCATION': '/var/tmp/mongo_metadata_inheritance',
         'TIMEOUT': 300,
         'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
+    },
+    'loc_cache': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'edx_location_mem_cache',
+    },
+
 }
 
 # Make the keyedcache startup warnings go away

cms/envs/test.py

@@ -138,7 +138,12 @@ CACHES = {
         'LOCATION': '/var/tmp/mongo_metadata_inheritance',
         'TIMEOUT': 300,
         'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
+    },
+    'loc_cache': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'edx_location_mem_cache',
+    },
+
 }
 
 # hide ratelimit warnings while running tests

common/lib/xmodule/xmodule/modulestore/django.py

@@ -129,8 +129,12 @@ def loc_mapper():
     global _loc_singleton
     # pylint: disable=W0212
     if _loc_singleton is None:
+        try:
+            loc_cache = get_cache('loc_cache')
+        except InvalidCacheBackendError:
+            loc_cache = get_cache('default')
         # instantiate
-        _loc_singleton = LocMapperStore(**settings.DOC_STORE_CONFIG)
+        _loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
     # inject into split mongo modulestore
     if 'split' in _MODULESTORES:
         _MODULESTORES['split'].loc_mapper = _loc_singleton

common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py

@@ -43,7 +43,10 @@ class TestMigration(unittest.TestCase):
 
     def setUp(self):
         super(TestMigration, self).setUp()
-        self.loc_mapper = LocMapperStore(**self.db_config)
+        noop_cache = mock.Mock(spec=['get', 'set_many'])
+        noop_cache.configure_mock(**{'get.return_value': None})
+        # pylint: disable=W0142
+        self.loc_mapper = LocMapperStore(noop_cache, **self.db_config)
         self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
         self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
         self.split_mongo = SplitMongoModuleStore(

lms/djangoapps/courseware/roles.py

@@ -4,7 +4,6 @@ adding users, removing users, and listing members
 """
 
 from abc import ABCMeta, abstractmethod
-from functools import partial
 
 from django.contrib.auth.models import User, Group
 

lms/envs/dev.py

@@ -85,7 +85,11 @@ CACHES = {
         'LOCATION': '/var/tmp/mongo_metadata_inheritance',
         'TIMEOUT': 300,
         'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
+    },
+    'loc_cache': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'edx_location_mem_cache',
+    },
 }
 
 

lms/envs/test.py

@@ -165,7 +165,12 @@ CACHES = {
         'LOCATION': '/var/tmp/mongo_metadata_inheritance',
         'TIMEOUT': 300,
         'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
+    },
+    'loc_cache': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'edx_location_mem_cache',
+    },
+
 }
 
 # Dummy secret key for dev

mongo_indexes.md

+These are the indexes each mongo db should have in order to perform well.
+Each section states the collection name and then the indexes. To create an index,
+you'll typically either use the mongohq type web interface or a standard terminal console.
+If a terminal, this assumes you've logged in and gotten to the mongo prompt 
+```
+mongo mydatabasename
+```
+
+If using the terminal, to add an index to a collection, you'll need to prefix ```ensureIndex``` with
+```
+db.collection_name
+```
+as in ```db.location_map.ensureIndex({'course_id': 1}{background: true})```
+
+location_map:
+=============
+
+```
+ensureIndex({'_id.org': 1, '_id.course': 1})
+ensureIndex({'course_id': 1})
+```