Make MongoDB index creation idempotent by catching errors about existing

indexes with different options. Emit warning instead and continue.

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -15,7 +15,7 @@ from xmodule.contentstore.content import XASSET_LOCATION_TAG
 from xmodule.exceptions import NotFoundError
 from xmodule.modulestore.django import ASSET_IGNORE_REGEX
 from xmodule.util.misc import escape_invalid_characters
-from xmodule.mongo_connection import connect_to_mongodb
+from xmodule.mongo_utils import connect_to_mongodb, create_collection_index
 from .content import StaticContent, ContentStore, StaticContentStream
 
 
@@ -399,7 +399,8 @@ class MongoContentStore(ContentStore):
     def ensure_indexes(self):
         # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort
         # which can be `uploadDate`, `display_name`,
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('_id.tag', pymongo.ASCENDING),
                 ('_id.org', pymongo.ASCENDING),
@@ -409,7 +410,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('content_son.org', pymongo.ASCENDING),
                 ('content_son.course', pymongo.ASCENDING),
@@ -418,7 +420,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('_id.org', pymongo.ASCENDING),
                 ('_id.course', pymongo.ASCENDING),
@@ -427,7 +430,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('content_son.org', pymongo.ASCENDING),
                 ('content_son.course', pymongo.ASCENDING),
@@ -436,7 +440,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('_id.org', pymongo.ASCENDING),
                 ('_id.course', pymongo.ASCENDING),
@@ -445,7 +450,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('_id.org', pymongo.ASCENDING),
                 ('_id.course', pymongo.ASCENDING),
@@ -454,7 +460,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('content_son.org', pymongo.ASCENDING),
                 ('content_son.course', pymongo.ASCENDING),
@@ -463,7 +470,8 @@ class MongoContentStore(ContentStore):
             sparse=True,
             background=True
         )
-        self.fs_files.create_index(
+        create_collection_index(
+            self.fs_files,
             [
                 ('content_son.org', pymongo.ASCENDING),
                 ('content_son.course', pymongo.ASCENDING),

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -44,7 +44,7 @@ from xmodule.error_module import ErrorDescriptor
 from xmodule.errortracker import null_error_tracker, exc_info_to_str
 from xmodule.exceptions import HeartbeatFailure
 from xmodule.mako_module import MakoDescriptorSystem
-from xmodule.mongo_connection import connect_to_mongodb
+from xmodule.mongo_utils import connect_to_mongodb, create_collection_index
 from xmodule.modulestore import ModuleStoreWriteBase, ModuleStoreEnum, BulkOperationsMixin, BulkOpsRecord
 from xmodule.modulestore.draft_and_published import ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES
 from xmodule.modulestore.edit_info import EditInfoRuntimeMixin
@@ -1947,9 +1947,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         This method is intended for use by tests and administrative commands, and not
         to be run during server startup.
         """
-
         # Because we often query for some subset of the id, we define this index:
-        self.collection.create_index(
+        create_collection_index(
+            self.collection,
             [
                 ('_id.tag', pymongo.ASCENDING),
                 ('_id.org', pymongo.ASCENDING),
@@ -1958,16 +1958,17 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
                 ('_id.name', pymongo.ASCENDING),
                 ('_id.revision', pymongo.ASCENDING),
             ],
-            background=True)
+            background=True
+        )
 
         # Because we often scan for all category='course' regardless of the value of the other fields:
-        self.collection.create_index('_id.category', background=True)
+        create_collection_index(self.collection, '_id.category', background=True)
 
         # Because lms calls get_parent_locations frequently (for path generation):
-        self.collection.create_index('definition.children', sparse=True, background=True)
+        create_collection_index(self.collection, 'definition.children', sparse=True, background=True)
 
         # To allow prioritizing draft vs published material
-        self.collection.create_index('_id.revision', background=True)
+        create_collection_index(self.collection, '_id.revision', background=True)
 
     # Some overrides that still need to be implemented by subclasses
     def convert_to_draft(self, location, user_id):

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -27,7 +27,7 @@ from mongodb_proxy import autoretry_read
 from xmodule.exceptions import HeartbeatFailure
 from xmodule.modulestore import BlockData
 from xmodule.modulestore.split_mongo import BlockKey
-from xmodule.mongo_connection import connect_to_mongodb
+from xmodule.mongo_utils import connect_to_mongodb, create_collection_index
 
 
 new_contract('BlockData', BlockData)
@@ -546,7 +546,8 @@ class MongoConnection(object):
         This method is intended for use by tests and administrative commands, and not
         to be run during server startup.
         """
-        self.course_index.create_index(
+        create_collection_index(
+            self.course_index,
             [
                 ('org', pymongo.ASCENDING),
                 ('course', pymongo.ASCENDING),

common/lib/xmodule/xmodule/mongo_connection.py → common/lib/xmodule/xmodule/mongo_utils.py

@@ -3,6 +3,10 @@ Common MongoDB connection functions.
 """
 import pymongo
 from mongodb_proxy import MongoProxy
+import logging
+
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 # pylint: disable=bad-continuation
@@ -51,3 +55,33 @@ def connect_to_mongodb(
         mongo_conn.authenticate(user, password)
 
     return mongo_conn
+
+
+def create_collection_index(
+    collection, keys,
+    ignore_created=True, ignore_created_opts=True, **kwargs
+):
+    """
+    Create a MongoDB index in a collection. Optionally,
+    ignore errors related to the index already existing.
+    """
+    # For an explanation of the error codes:
+    # https://github.com/mongodb/mongo/blob/v3.0/src/mongo/db/catalog/index_catalog.cpp#L542-L583
+    # https://github.com/mongodb/mongo/blob/v3.0/src/mongo/base/error_codes.err#L70-L87
+    # pylint: disable=invalid-name
+    INDEX_ALREADY_EXISTS = 68
+    INDEX_OPTIONS_CONFLICT = 85
+    try:
+        collection.create_index(keys, **kwargs)
+    except pymongo.errors.OperationFailure as exc:
+        errors_to_ignore = []
+        if ignore_created:
+            errors_to_ignore.append(INDEX_ALREADY_EXISTS)
+        if ignore_created_opts:
+            errors_to_ignore.append(INDEX_OPTIONS_CONFLICT)
+        if exc.code in errors_to_ignore:
+            logger.warning("Existing index in collection '{}' remained unchanged!: {}".format(
+                collection.full_name, exc.details['errmsg'])
+            )
+        else:
+            raise exc