Merge pull request #4748 from Stanford-Online/jbau/parallel-unit-tests

make unit tests respect mongo port/host settings (with default)

cms/envs/test.py

@@ -21,6 +21,12 @@ from uuid import uuid4
 # import settings from LMS for consistent behavior with CMS
 from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME)
 
+# mongo connection settings
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
+THIS_UUID = uuid4().hex[:5]
+
 # Nose Test Runner
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
@@ -87,15 +93,18 @@ update_module_store_settings(
     },
     doc_store_settings={
         'db': 'test_xmodule',
-        'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
+        'collection': 'test_modulestore{0}'.format(THIS_UUID),
     },
 )
 
 CONTENTSTORE = {
     'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
     'DOC_STORE_CONFIG': {
-        'host': 'localhost',
+        'host': MONGO_HOST,
         'db': 'test_xcontent',
+        'port': MONGO_PORT_NUM,
         'collection': 'dont_trip',
     },
     # allow for additional options that can be keyed on a name, e.g. 'trashcan'

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -25,7 +25,7 @@ class MongoContentStore(ContentStore):
 
         :param collection: ignores but provided for consistency w/ other doc_store_config patterns
         """
-        logging.debug('Using MongoDB for static content serving at host={0} db={1}'.format(host, db))
+        logging.debug('Using MongoDB for static content serving at host={0} port={1} db={2}'.format(host, port, db))
         _db = pymongo.database.Database(
             pymongo.MongoClient(
                 host=host,

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -2,7 +2,6 @@
 """
 Modulestore configuration for test cases.
 """
-
 from uuid import uuid4
 from django.test import TestCase
 from django.contrib.auth.models import User
@@ -13,6 +12,7 @@ import datetime
 import pytz
 from xmodule.tabs import CoursewareTab, CourseInfoTab, StaticTab, DiscussionTab, ProgressTab, WikiTab
 from xmodule.modulestore.tests.sample_courses import default_block_info_tree, TOY_BLOCK_INFO_TREE
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 
 def mixed_store_config(data_dir, mappings):
@@ -67,7 +67,8 @@ def draft_mongo_store_config(data_dir):
             'NAME': 'draft',
             'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
             'DOC_STORE_CONFIG': {
-                'host': 'localhost',
+                'host': MONGO_HOST,
+                'port': MONGO_PORT_NUM,
                 'db': 'test_xmodule',
                 'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
             },
@@ -93,7 +94,8 @@ def split_mongo_store_config(data_dir):
             'NAME': 'draft',
             'ENGINE': 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore',
             'DOC_STORE_CONFIG': {
-                'host': 'localhost',
+                'host': MONGO_HOST,
+                'port': MONGO_PORT_NUM,
                 'db': 'test_xmodule',
                 'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
             },
@@ -229,6 +231,8 @@ class ModuleStoreTestCase(TestCase):
         if hasattr(module_store, '_drop_database'):
             module_store._drop_database()  # pylint: disable=protected-access
         _CONTENTSTORE.clear()
+        if hasattr(module_store, 'close_connections'):
+            module_store.close_connections()
 
     @classmethod
     def setUpClass(cls):

common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py

+"""
+This file is intended to provide settings for the mongodb connection used for tests.
+The settings can be provided by environment variables in the shell running the tests.  This reads
+in a variety of environment variables but provides sensible defaults in case those env var
+overrides don't exist
+"""
+import os
+
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')

common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py

 """
  Test contentstore.mongo functionality
 """
+import os
 import logging
 from uuid import uuid4
 import unittest
@@ -17,12 +18,12 @@ from xmodule.contentstore.content import StaticContent
 from xmodule.exceptions import NotFoundError
 import ddt
 from __builtin__ import delattr
-
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 log = logging.getLogger(__name__)
 
-HOST = 'localhost'
-PORT = 27017
+HOST = MONGO_HOST
+PORT = MONGO_PORT_NUM
 DB = 'test_mongo_%s' % uuid4().hex[:5]
 
 

common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py

@@ -11,7 +11,6 @@ and then for each combination of modulestores, performing the sequence:
     4) Compare all modules in the source and destination modulestores to make sure that they line up
 
 """
-
 import ddt
 import itertools
 import random
@@ -28,9 +27,12 @@ from xmodule.contentstore.mongo import MongoContentStore
 from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.xml_exporter import export_to_xml
 from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleStore
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
+
 
 COMMON_DOCSTORE_CONFIG = {
-    'host': 'localhost'
+    'host': MONGO_HOST,
+    'port': MONGO_PORT_NUM,
 }
 
 

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -15,6 +15,7 @@ from xmodule.exceptions import InvalidVersionError
 
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
+
 # Mixed modulestore depends on django, so we'll manually configure some django settings
 # before importing the module
 # TODO remove this import and the configuration -- xmodule should not depend on django!
@@ -26,6 +27,7 @@ if not settings.configured:
     settings.configure()
 from xmodule.modulestore.mixed import MixedModuleStore
 from xmodule.modulestore.draft_and_published import UnsupportedRevisionError
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 
 @ddt.ddt
@@ -34,8 +36,8 @@ class TestMixedModuleStore(unittest.TestCase):
     Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
     Location-based dbs)
     """
-    HOST = 'localhost'
-    PORT = 27017
+    HOST = MONGO_HOST
+    PORT = MONGO_PORT_NUM
     DB = 'test_mongo_%s' % uuid4().hex[:5]
     COLLECTION = 'modulestore'
     FS_ROOT = DATA_DIR
@@ -54,6 +56,7 @@ class TestMixedModuleStore(unittest.TestCase):
     }
     DOC_STORE_CONFIG = {
         'host': HOST,
+        'port': PORT,
         'db': DB,
         'collection': COLLECTION,
     }

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -36,12 +36,12 @@ from xmodule.exceptions import NotFoundError
 from git.test.lib.asserts import assert_not_none
 from xmodule.x_module import XModuleMixin
 from xmodule.modulestore.mongo.base import as_draft
-
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 log = logging.getLogger(__name__)
 
-HOST = 'localhost'
-PORT = 27017
+HOST = MONGO_HOST
+PORT = MONGO_PORT_NUM
 DB = 'test_mongo_%s' % uuid4().hex[:5]
 COLLECTION = 'modulestore'
 FS_ROOT = DATA_DIR  # TODO (vshnayder): will need a real fs_root for testing load_item
@@ -91,12 +91,13 @@ class TestMongoModuleStore(unittest.TestCase):
         # connect to the db
         doc_store_config = {
             'host': HOST,
+            'port': PORT,
             'db': DB,
             'collection': COLLECTION,
         }
         # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
         # as well
-        content_store = MongoContentStore(HOST, DB)
+        content_store = MongoContentStore(HOST, DB, port=PORT)
         #
         # Also test draft store imports
         #
@@ -148,7 +149,7 @@ class TestMongoModuleStore(unittest.TestCase):
     def test_mongo_modulestore_type(self):
         store = DraftModuleStore(
             None,
-            {'host': HOST, 'db': DB, 'collection': COLLECTION},
+            {'host': HOST, 'db': DB, 'port': PORT, 'collection': COLLECTION},
             FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS
         )
         assert_equals(store.get_modulestore_type(''), ModuleStoreEnum.Type.mongo)

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -22,6 +22,7 @@ from xmodule.x_module import XModuleMixin
 from xmodule.fields import Date, Timedelta
 from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
 from xmodule.modulestore.tests.test_modulestore import check_has_course_method
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 
 BRANCH_NAME_DRAFT = ModuleStoreEnum.BranchName.draft
@@ -36,8 +37,9 @@ class SplitModuleTest(unittest.TestCase):
     '''
     # Snippets of what would be in the django settings envs file
     DOC_STORE_CONFIG = {
-        'host': 'localhost',
+        'host': MONGO_HOST,
         'db': 'test_xmodule',
+        'port': MONGO_PORT_NUM,
         'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
     }
     modulestore_options = {

common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py

@@ -9,6 +9,7 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
 from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
 from xmodule.modulestore.mongo import DraftMongoModuleStore
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 
 class SplitWMongoCourseBoostrapper(unittest.TestCase):
@@ -27,7 +28,8 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
     """
         # Snippet of what would be in the django settings envs file
     db_config = {
-        'host': 'localhost',
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
         'db': 'test_xmodule',
     }
 

common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py

@@ -10,6 +10,7 @@ from opaque_keys.edx.locations import Location
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.inheritance import InheritanceMixin
 from xmodule.modulestore.xml_importer import _import_module_and_update_references
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.tests import DATA_DIR
 from uuid import uuid4
@@ -21,8 +22,8 @@ class ModuleStoreNoSettings(unittest.TestCase):
     """
     A mixin to create a mongo modulestore that avoids settings
     """
-    HOST = 'localhost'
-    PORT = 27017
+    HOST = MONGO_HOST
+    PORT = MONGO_PORT_NUM
     DB = 'test_mongo_%s' % uuid4().hex[:5]
     COLLECTION = 'modulestore'
     FS_ROOT = DATA_DIR
@@ -36,6 +37,7 @@ class ModuleStoreNoSettings(unittest.TestCase):
     }
     DOC_STORE_CONFIG = {
         'host': HOST,
+        'port': PORT,
         'db': DB,
         'collection': COLLECTION,
     }

lms/djangoapps/dashboard/git_import.py

@@ -128,6 +128,7 @@ def add_repo(repo, rdir_in, branch=None):
     # Set defaults even if it isn't defined in settings
     mongo_db = {
         'host': 'localhost',
+        'port': 27017,
         'user': '',
         'password': '',
         'db': 'xlog',
@@ -135,7 +136,7 @@ def add_repo(repo, rdir_in, branch=None):
 
     # Allow overrides
     if hasattr(settings, 'MONGODB_LOG'):
-        for config_item in ['host', 'user', 'password', 'db', ]:
+        for config_item in ['host', 'user', 'password', 'db', 'port']:
             mongo_db[config_item] = settings.MONGODB_LOG.get(
                 config_item, mongo_db[config_item])
 
@@ -258,13 +259,13 @@ def add_repo(repo, rdir_in, branch=None):
                                               cwd=os.path.abspath(cdir)))
 
     # store import-command-run output in mongo
-    mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
+    mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db)
 
     try:
         if mongo_db['user'] and mongo_db['password']:
             mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
         else:
-            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
+            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port'])
     except mongoengine.connection.ConnectionError:
         log.exception('Unable to connect to mongodb to save log, please '
                       'check MONGODB_LOG settings')

lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py

 """
 Provide tests for git_add_course management command.
 """
-
 import logging
 import os
 import shutil
@@ -21,9 +20,12 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 import dashboard.git_import as git_import
 from dashboard.git_import import GitImportError
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
+
 
 TEST_MONGODB_LOG = {
-    'host': 'localhost',
+    'host': MONGO_HOST,
+    'port': MONGO_PORT_NUM,
     'user': '',
     'password': '',
     'db': 'test_xlog',

lms/djangoapps/dashboard/tests/test_sysadmin.py

 """
 Provide tests for sysadmin dashboard feature in sysadmin.py
 """
-
 import glob
 import os
 import re
@@ -30,10 +29,12 @@ from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.xml import XMLModuleStore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 
 
 TEST_MONGODB_LOG = {
-    'host': 'localhost',
+    'host': MONGO_HOST,
+    'port': MONGO_PORT_NUM,
     'user': '',
     'password': '',
     'db': 'test_xlog',

lms/envs/test.py

@@ -18,8 +18,14 @@ from path import path
 from warnings import filterwarnings, simplefilter
 from uuid import uuid4
 
+# mongo connection settings
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
 os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = 'localhost:8000-9000'
 
+THIS_UUID = uuid4().hex[:5]
+
 # can't test start dates with this True, but on the other hand,
 # can test everything else :)
 FEATURES['DISABLE_START_DATES'] = True
@@ -125,16 +131,19 @@ update_module_store_settings(
         'data_dir': COMMON_TEST_DATA_ROOT,
     },
     doc_store_settings={
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
         'db': 'test_xmodule',
-        'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
+        'collection': 'test_modulestore{0}'.format(THIS_UUID),
     },
 )
 
 CONTENTSTORE = {
     'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
     'DOC_STORE_CONFIG': {
-        'host': 'localhost',
+        'host': MONGO_HOST,
         'db': 'xcontent',
+        'port': MONGO_PORT_NUM,
     }
 }
 
@@ -345,3 +354,12 @@ VERIFY_STUDENT["SOFTWARE_SECURE"] = {
 VIDEO_CDN_URL = {
     'CN': 'http://api.xuetangx.com/edx/video?s3_url='
 }
+
+######### dashboard git log settings #########
+MONGODB_LOG = {
+    'host': MONGO_HOST,
+    'port': MONGO_PORT_NUM,
+    'user': '',
+    'password': '',
+    'db': 'xlog',
+}

pavelib/utils/test/utils.py

@@ -3,6 +3,11 @@ Helper functions for test tasks
 """
 from paver.easy import sh, task
 from pavelib.utils.envs import Env
+import os
+
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
 
 __test__ = False  # do not collect
 
@@ -43,4 +48,8 @@ def clean_mongo():
     """
     Clean mongo test databases
     """
-    sh("mongo {repo_root}/scripts/delete-mongo-test-dbs.js".format(repo_root=Env.REPO_ROOT))
+    sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format(
+        host=MONGO_HOST,
+        port=MONGO_PORT_NUM,
+        repo_root=Env.REPO_ROOT,
+    ))