Merge pull request #9295 from edx/bbeggs/ensure-indexes-PLAT-785

Adding django admin command to create mongodb indexes

AUTHORS

@@ -232,3 +232,4 @@ William Ono <william.ono@ubc.ca>
 Dongwook Yoon <dy252@cornell.edu>
 Awais Qureshi <awais.qureshi@arbisoft.com>
 Eric Fischer <efischer@edx.org>
+Brian Beggs <macdiesel@gmail.com>

common/djangoapps/track/backends/mongodb.py

@@ -83,8 +83,8 @@ class MongoBackend(BaseBackend):
         # TODO: The creation of indexes can be moved to a Django
         # management command or equivalent. There is also an option to
         # run the indexing on the background, without locking.
-        self.collection.ensure_index([('time', pymongo.DESCENDING)])
-        self.collection.ensure_index('event_type')
+        self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True)
+        self.collection.ensure_index('event_type', background=True)
 
     def send(self, event):
         """Insert the event in to the Mongo collection"""

common/djangoapps/xblock_django/management/commands/ensure_indexes.py

+"""
+Creates Indexes on contentstore and modulestore databases.
+"""
+from django.core.management.base import BaseCommand
+
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.django import modulestore
+
+
+class Command(BaseCommand):
+    """
+    This command will create indexes on the stores used for both contentstore and modulestore.
+    """
+    args = ''
+    help = 'Creates the indexes for ContentStore and ModuleStore databases'
+
+    def handle(self, *args, **options):
+        contentstore().ensure_indexes()
+        modulestore().ensure_indexes()
+        print 'contentstore and modulestore indexes created!'

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -393,36 +393,80 @@ class MongoContentStore(ContentStore):
         return dbkey
 
     def ensure_indexes(self):
-
         # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort
         # which can be `uploadDate`, `display_name`,
-
         self.fs_files.create_index(
-            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('_id.name', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('_id.tag', pymongo.ASCENDING),
+                ('_id.org', pymongo.ASCENDING),
+                ('_id.course', pymongo.ASCENDING),
+                ('_id.category', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
+        )
+        self.fs_files.create_index(
+            [
+                ('content_son.org', pymongo.ASCENDING),
+                ('content_son.course', pymongo.ASCENDING),
+                ('uploadDate', pymongo.DESCENDING)
+            ],
+            sparse=True,
+            background=True
+        )
+        self.fs_files.create_index(
+            [
+                ('_id.org', pymongo.ASCENDING),
+                ('_id.course', pymongo.ASCENDING),
+                ('_id.name', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
         self.fs_files.create_index(
-            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
-             ('content_son.name', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('content_son.org', pymongo.ASCENDING),
+                ('content_son.course', pymongo.ASCENDING),
+                ('content_son.name', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
         self.fs_files.create_index(
-            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('_id.org', pymongo.ASCENDING),
+                ('_id.course', pymongo.ASCENDING),
+                ('uploadDate', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
         self.fs_files.create_index(
-            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('_id.org', pymongo.ASCENDING),
+                ('_id.course', pymongo.ASCENDING),
+                ('display_name', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
         self.fs_files.create_index(
-            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
-             ('uploadDate', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('content_son.org', pymongo.ASCENDING),
+                ('content_son.course', pymongo.ASCENDING),
+                ('uploadDate', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
         self.fs_files.create_index(
-            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
-             ('display_name', pymongo.ASCENDING)],
-            sparse=True
+            [
+                ('content_son.org', pymongo.ASCENDING),
+                ('content_son.course', pymongo.ASCENDING),
+                ('display_name', pymongo.ASCENDING)
+            ],
+            sparse=True,
+            background=True
         )
 
 

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -1914,21 +1914,25 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         """
 
         # Because we often query for some subset of the id, we define this index:
-        self.collection.create_index([
-            ('_id.org', pymongo.ASCENDING),
-            ('_id.course', pymongo.ASCENDING),
-            ('_id.category', pymongo.ASCENDING),
-            ('_id.name', pymongo.ASCENDING),
-        ])
+        self.collection.create_index(
+            [
+                ('_id.tag', pymongo.ASCENDING),
+                ('_id.org', pymongo.ASCENDING),
+                ('_id.course', pymongo.ASCENDING),
+                ('_id.category', pymongo.ASCENDING),
+                ('_id.name', pymongo.ASCENDING),
+                ('_id.revision', pymongo.ASCENDING),
+            ],
+            background=True)
 
         # Because we often scan for all category='course' regardless of the value of the other fields:
-        self.collection.create_index('_id.category')
+        self.collection.create_index('_id.category', background=True)
 
         # Because lms calls get_parent_locations frequently (for path generation):
-        self.collection.create_index('definition.children', sparse=True)
+        self.collection.create_index('definition.children', sparse=True, background=True)
 
         # To allow prioritizing draft vs published material
-        self.collection.create_index('_id.revision')
+        self.collection.create_index('_id.revision', background=True)
 
     # Some overrides that still need to be implemented by subclasses
     def convert_to_draft(self, location, user_id):

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -532,5 +532,6 @@ class MongoConnection(object):
                 ('course', pymongo.ASCENDING),
                 ('run', pymongo.ASCENDING)
             ],
-            unique=True
+            unique=True,
+            background=True
         )