Merge pull request #3915 from edx/studio/uses-mixed-modulestore

Enable mixed modulestore

cms/djangoapps/contentstore/course_info_model.py

@@ -28,16 +28,15 @@ from xmodule.html_module import CourseInfoModule
 log = logging.getLogger(__name__)
 
 
-def get_course_updates(location, provided_id):
+def get_course_updates(location, provided_id, user_id):
     """
     Retrieve the relevant course_info updates and unpack into the model which the client expects:
     [{id : index, date : string, content : html string}]
     """
     try:
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
     except ItemNotFoundError:
-        modulestore('direct').create_and_save_xmodule(location)
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().create_and_save_xmodule(location, user_id)
 
     course_update_items = get_course_update_items(course_updates, provided_id)
     return _get_visible_update(course_update_items)
@@ -50,10 +49,9 @@ def update_course_updates(location, update, passed_id=None, user=None):
     into the html structure.
     """
     try:
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
     except ItemNotFoundError:
-        modulestore('direct').create_and_save_xmodule(location)
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().create_and_save_xmodule(location, user.id)
 
     course_update_items = list(reversed(get_course_update_items(course_updates)))
 
@@ -135,7 +133,7 @@ def delete_course_update(location, update, passed_id, user):
         return HttpResponseBadRequest()
 
     try:
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
     except ItemNotFoundError:
         return HttpResponseBadRequest()
 
@@ -239,6 +237,6 @@ def save_course_update_items(location, course_updates, course_update_items, user
     course_updates.data = _get_html(course_update_items)
 
     # update db record
-    modulestore('direct').update_item(course_updates, user.id)
+    modulestore().update_item(course_updates, user.id)
 
     return course_updates

cms/djangoapps/contentstore/features/video.py

 # pylint: disable=C0111
 
 from lettuce import world, step
-from contentstore.utils import get_modulestore
 from selenium.webdriver.common.keys import Keys
+from xmodule.modulestore.django import modulestore
 
 VIDEO_BUTTONS = {
     'CC': '.hide-subtitles',
@@ -137,9 +137,9 @@ def xml_only_video(step):
     world.wait(1)
 
     course = world.scenario_dict['COURSE']
-    store = get_modulestore(course.location)
+    store = modulestore()
 
-    parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location
+    parent_location = store.get_items(course.id, category='vertical')[0].location
 
     youtube_id = 'ABCDEFG'
     world.scenario_dict['YOUTUBE_ID'] = youtube_id

cms/djangoapps/contentstore/git_export_utils.py

@@ -128,8 +128,8 @@ def export_to_git(course_id, repo, user='', rdir=None):
     root_dir = os.path.dirname(rdirp)
     course_dir = os.path.splitext(os.path.basename(rdirp))[0]
     try:
-        export_to_xml(modulestore('direct'), contentstore(), course_id,
-                      root_dir, course_dir, modulestore())
+        export_to_xml(modulestore(), contentstore(), course_id,
+                      root_dir, course_dir)
     except (EnvironmentError, AttributeError):
         log.exception('Failed export to xml')
         raise GitExportError(GitExportError.XML_EXPORT_FAIL)

cms/djangoapps/contentstore/management/commands/clone_course.py

@@ -4,6 +4,7 @@ Script for cloning a course
 from django.core.management.base import BaseCommand, CommandError
 from xmodule.modulestore.store_utilities import clone_course
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
 from xmodule.contentstore.django import contentstore
 from student.roles import CourseInstructorRole, CourseStaffRole
 from opaque_keys.edx.keys import CourseKey
@@ -35,19 +36,18 @@ class Command(BaseCommand):
         source_course_id = self.course_key_from_arg(args[0])
         dest_course_id = self.course_key_from_arg(args[1])
 
-        mstore = modulestore('direct')
+        mstore = modulestore()
         cstore = contentstore()
 
-        mstore.ignore_write_events_on_courses.add(dest_course_id)
-
         print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))
 
-        if clone_course(mstore, cstore, source_course_id, dest_course_id):
-            print("copying User permissions...")
-            # purposely avoids auth.add_user b/c it doesn't have a caller to authorize
-            CourseInstructorRole(dest_course_id).add_users(
-                *CourseInstructorRole(source_course_id).users_with_role()
-            )
-            CourseStaffRole(dest_course_id).add_users(
-                *CourseStaffRole(source_course_id).users_with_role()
-            )
+        with store_bulk_write_operations_on_course(mstore, dest_course_id):
+            if clone_course(mstore, cstore, source_course_id, dest_course_id, None):
+                print("copying User permissions...")
+                # purposely avoids auth.add_user b/c it doesn't have a caller to authorize
+                CourseInstructorRole(dest_course_id).add_users(
+                    *CourseInstructorRole(source_course_id).users_with_role()
+                )
+                CourseStaffRole(dest_course_id).add_users(
+                    *CourseStaffRole(source_course_id).users_with_role()
+                )

cms/djangoapps/contentstore/management/commands/course_id_clash.py

@@ -3,6 +3,7 @@ Script for finding all courses whose org/name pairs == other courses when ignori
 """
 from django.core.management.base import BaseCommand
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
 
 
 #
@@ -10,12 +11,12 @@ from xmodule.modulestore.django import modulestore
 #
 class Command(BaseCommand):
     """
-    Script for finding all courses whose org/name pairs == other courses when ignoring case
+    Script for finding all courses in the Mongo Modulestore whose org/name pairs == other courses when ignoring case
     """
-    help = 'List all courses ids which may collide when ignoring case'
+    help = 'List all courses ids in the Mongo Modulestore which may collide when ignoring case'
 
     def handle(self, *args, **options):
-        mstore = modulestore()
+        mstore = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)  # pylint: disable=protected-access
         if hasattr(mstore, 'collection'):
             map_fn = '''
                 function () {

cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py

@@ -22,7 +22,7 @@ class Command(BaseCommand):
 
             course_ids = [course_key]
         else:
-            course_ids = [course.id for course in modulestore('direct').get_courses()]
+            course_ids = [course.id for course in modulestore().get_courses()]
 
         if query_yes_no("Emptying trashcan. Confirm?", default="no"):
             empty_asset_trashcan(course_ids)

cms/djangoapps/contentstore/management/commands/export.py

@@ -35,4 +35,4 @@ class Command(BaseCommand):
         root_dir = os.path.dirname(output_path)
         course_dir = os.path.splitext(os.path.basename(output_path))[0]
 
-        export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
+        export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)

cms/djangoapps/contentstore/management/commands/export_all_courses.py

@@ -19,7 +19,7 @@ class Command(BaseCommand):
         output_path = args[0]
 
         cs = contentstore()
-        ms = modulestore('direct')
+        ms = modulestore()
         root_dir = output_path
         courses = ms.get_courses()
 
@@ -35,7 +35,7 @@ class Command(BaseCommand):
             if 1:
                 try:
                     course_dir = course_id.replace('/', '...')
-                    export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
+                    export_to_xml(ms, cs, course_id, root_dir, course_dir)
                 except Exception as err:
                     print("="*30 + "> Oops, failed to export %s" % course_id)
                     print("Error:")

cms/djangoapps/contentstore/management/commands/import.py

@@ -37,15 +37,10 @@ class Command(BaseCommand):
             data=data_dir,
             courses=course_dirs,
             dis=do_import_static))
-        try:
-            mstore = modulestore('direct')
-        except KeyError:
-            self.stdout.write('Unable to load direct modulestore, trying '
-                              'default\n')
-            mstore = modulestore('default')
+        mstore = modulestore()
 
         _, course_items = import_from_xml(
-            mstore, data_dir, course_dirs, load_error_modules=False,
+            mstore, "**replace_user**", data_dir, course_dirs, load_error_modules=False,
             static_content_store=contentstore(), verbose=True,
             do_import_static=do_import_static,
             create_new_course_if_not_present=True,

cms/djangoapps/contentstore/management/commands/tests/test_import.py

@@ -8,16 +8,13 @@ import shutil
 import tempfile
 
 from django.core.management import call_command
-from django.test.utils import override_settings
 
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 from django_comment_common.utils import are_permissions_roles_seeded
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestImport(ModuleStoreTestCase):
     """
     Unit tests for importing a course from command line

cms/djangoapps/contentstore/management/commands/tests/test_migrate_to_split.py

@@ -5,9 +5,8 @@ import unittest
 
 from django.contrib.auth.models import User
 from django.core.management import CommandError, call_command
-from django.test.utils import override_settings
 from contentstore.management.commands.migrate_to_split import Command
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
+from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, REVISION_OPTION_PUBLISHED_ONLY
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.django import modulestore, clear_existing_modulestores
@@ -45,7 +44,6 @@ class TestArgParsing(unittest.TestCase):
 
 
 @unittest.skip("Not fixing split mongo until we land this long branch")
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestMigrateToSplit(ModuleStoreTestCase):
     """
     Unit tests for migrating a course from old mongo to split mongo
@@ -58,7 +56,7 @@ class TestMigrateToSplit(ModuleStoreTestCase):
         password = 'foo'
         self.user = User.objects.create_user(uname, email, password)
         self.course = CourseFactory()
-        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
+        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, SPLIT_MONGO_MODULESTORE_TYPE)
         self.addCleanup(clear_existing_modulestores)
 
     def test_user_email(self):
@@ -86,6 +84,6 @@ class TestMigrateToSplit(ModuleStoreTestCase):
             str(self.user.id),
             "org.dept+name.run",
         )
-        locator = CourseLocator(org="org.dept", offering="name.run", branch="published")
+        locator = CourseLocator(org="org.dept", offering="name.run", branch=REVISION_OPTION_PUBLISHED_ONLY)
         course_from_split = modulestore('split').get_course(locator)
         self.assertIsNotNone(course_from_split)

cms/djangoapps/contentstore/management/commands/tests/test_rollback_split_course.py

@@ -7,9 +7,7 @@ from mock import patch
 
 from django.contrib.auth.models import User
 from django.core.management import CommandError, call_command
-from django.test.utils import override_settings
 from contentstore.management.commands.rollback_split_course import Command
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.persistent_factories import PersistentCourseFactory
 from xmodule.modulestore.tests.factories import CourseFactory
@@ -39,7 +37,6 @@ class TestArgParsing(unittest.TestCase):
 
 
 @unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
     """
     Unit tests for rolling back a split-mongo course from command line,
@@ -58,7 +55,6 @@ class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
 
 
 @unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
     """
     Unit tests for rolling back a split-mongo course from command line,
@@ -77,7 +73,6 @@ class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
 
 
 @unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestRollbackSplitCourse(ModuleStoreTestCase):
     """
     Unit tests for rolling back a split-mongo course from command line

cms/djangoapps/contentstore/tests/modulestore_config.py

-"""
-Define test configuration for modulestores.
-"""
-
-from xmodule.modulestore.tests.django_utils import studio_store_config
-from django.conf import settings
-
-TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")

cms/djangoapps/contentstore/tests/test_course_listing.py

@@ -16,7 +16,8 @@ from contentstore.tests.utils import AjaxEnabledTestClient
 from student.tests.factories import UserFactory
 from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff, OrgStaffRole, OrgInstructorRole
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.modulestore.django import modulestore
 from xmodule.error_module import ErrorDescriptor
@@ -197,6 +198,14 @@ class TestCourseListing(ModuleStoreTestCase):
         self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed)
         self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed)
 
+        # Now count the db queries
+        store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
+        with check_mongo_calls(store.collection, USER_COURSES_COUNT):
+            courses_list = _accessible_courses_list_from_groups(self.request)
+
+        with check_mongo_calls(store.collection, 1):
+            courses_list = _accessible_courses_list(self.request)
+
     def test_get_course_list_with_same_course_id(self):
         """
         Test getting courses with same id but with different name case. Then try to delete one of them and
@@ -253,18 +262,20 @@ class TestCourseListing(ModuleStoreTestCase):
         Create good courses, courses that won't load, and deleted courses which still have
         roles. Test course listing.
         """
+        store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
+
         course_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
         self._create_course_with_access_groups(course_location, self.user)
 
         course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
         self._create_course_with_access_groups(course_location, self.user)
-        modulestore().delete_course(course_location)
+        store.delete_course(course_location)
 
         course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
         course = self._create_course_with_access_groups(course_location, self.user)
-        course_db_record = modulestore()._find_one(course.location)
+        course_db_record = store._find_one(course.location)
         course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
-        modulestore().collection.update(
+        store.collection.update(
             {'_id': course.location.to_deprecated_son()},
             {'$set': {
                 'metadata.tabs': course_db_record['metadata']['tabs'],

cms/djangoapps/contentstore/tests/test_course_settings.py

@@ -11,7 +11,7 @@ from django.test.utils import override_settings
 
 from models.settings.course_details import (CourseDetails, CourseSettingsEncoder)
 from models.settings.course_grading import CourseGradingModel
-from contentstore.utils import get_modulestore, EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
+from contentstore.utils import EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
 from xmodule.modulestore.tests.factories import CourseFactory
 
 from models.settings.course_metadata import CourseMetadata
@@ -335,7 +335,7 @@ class CourseGradingTest(CourseTestCase):
 
     def test_update_section_grader_type(self):
         # Get the descriptor and the section_grader_type and assert they are the default values
-        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
+        descriptor = modulestore().get_item(self.course.location)
         section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
 
         self.assertEqual('notgraded', section_grader_type['graderType'])
@@ -344,7 +344,7 @@ class CourseGradingTest(CourseTestCase):
 
         # Change the default grader type to Homework, which should also mark the section as graded
         CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user)
-        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
+        descriptor = modulestore().get_item(self.course.location)
         section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
 
         self.assertEqual('Homework', section_grader_type['graderType'])
@@ -353,7 +353,7 @@ class CourseGradingTest(CourseTestCase):
 
         # Change the grader type back to notgraded, which should also unmark the section as graded
         CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user)
-        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
+        descriptor = modulestore().get_item(self.course.location)
         section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
 
         self.assertEqual('notgraded', section_grader_type['graderType'])
@@ -413,8 +413,7 @@ class CourseGradingTest(CourseTestCase):
         Populate the course, grab a section, get the url for the assignment type access
         """
         self.populate_course()
-        sequential_usage_key = self.course.id.make_usage_key("sequential", None)
-        sections = get_modulestore(self.course.id).get_items(sequential_usage_key)
+        sections = modulestore().get_items(self.course.id, category="sequential")
         # see if test makes sense
         self.assertGreater(len(sections), 0, "No sections found")
         section = sections[0]  # just take the first one
@@ -470,7 +469,7 @@ class CourseMetadataEditingTest(CourseTestCase):
          )
         self.update_check(test_model)
         # try fresh fetch to ensure persistence
-        fresh = modulestore('direct').get_course(self.course.id)
+        fresh = modulestore().get_course(self.course.id)
         test_model = CourseMetadata.fetch(fresh)
         self.update_check(test_model)
         # now change some of the existing metadata

cms/djangoapps/contentstore/tests/test_export_git.py

@@ -16,7 +16,7 @@ from .utils import CourseTestCase
 import contentstore.git_export_utils as git_export_utils
 from xmodule.contentstore.django import _CONTENTSTORE
 from xmodule.modulestore.django import modulestore
-from contentstore.utils import get_modulestore, reverse_course_url
+from contentstore.utils import reverse_course_url
 
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@@ -66,7 +66,7 @@ class TestExportGit(CourseTestCase):
         Test failed course export response.
         """
         self.course_module.giturl = 'foobar'
-        get_modulestore(self.course_module.location).update_item(self.course_module)
+        modulestore().update_item(self.course_module, '**replace_user**')
 
         response = self.client.get('{}?action=push'.format(self.test_url))
         self.assertIn('Export Failed:', response.content)
@@ -76,7 +76,7 @@ class TestExportGit(CourseTestCase):
         Regression test for making sure errors are properly stringified
         """
         self.course_module.giturl = 'foobar'
-        get_modulestore(self.course_module.location).update_item(self.course_module)
+        modulestore().update_item(self.course_module, '**replace_user**')
 
         response = self.client.get('{}?action=push'.format(self.test_url))
         self.assertNotIn('django.utils.functional.__proxy__', response.content)
@@ -99,7 +99,7 @@ class TestExportGit(CourseTestCase):
 
         self.populate_course()
         self.course_module.giturl = 'file://{}'.format(bare_repo_dir)
-        get_modulestore(self.course_module.location).update_item(self.course_module)
+        modulestore().update_item(self.course_module, '**replace_user**')
 
         response = self.client.get('{}?action=push'.format(self.test_url))
         self.assertIn('Export Succeeded', response.content)

cms/djangoapps/contentstore/tests/test_i18n.py

 from unittest import skip
 
 from django.contrib.auth.models import User
-from django.test.utils import override_settings
 
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 from contentstore.tests.utils import AjaxEnabledTestClient
 
 
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class InternationalizationTest(ModuleStoreTestCase):
     """
     Tests to validate Internationalization.

cms/djangoapps/contentstore/tests/test_import.py

@@ -13,13 +13,11 @@ import copy
 from django.contrib.auth.models import User
 
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
 from xmodule.modulestore.xml_importer import import_from_xml
-from xmodule.contentstore.content import StaticContent
 from xmodule.contentstore.django import _CONTENTSTORE
 
 from xmodule.exceptions import NotFoundError
@@ -30,7 +28,7 @@ TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
 
 
-@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class ContentStoreImportTest(ModuleStoreTestCase):
     """
     Tests that rely on the toy and test_import_course courses.
@@ -38,8 +36,6 @@ class ContentStoreImportTest(ModuleStoreTestCase):
     """
     def setUp(self):
 
-        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
-        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
         uname = 'testuser'
         email = 'test+courses@edx.org'
         password = 'foo'
@@ -69,9 +65,10 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         (for do_import_static=False behavior).
         '''
         content_store = contentstore()
-        module_store = modulestore('direct')
+        module_store = modulestore()
         import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['test_import_course'],
             static_content_store=content_store,
@@ -91,6 +88,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         module_store, __, course = self.load_test_import_course()
         __, course_items = import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data',
             ['test_import_course_2'],
             target_course_id=course.id,
@@ -102,10 +100,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         """
         # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
         """
-        module_store = modulestore('direct')
+        module_store = modulestore()
         course_id = SlashSeparatedCourseKey(u'Юникода', u'unicode_course', u'échantillon')
         import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['2014_Uni'],
             target_course_id=course_id
@@ -150,8 +149,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         '''
         content_store = contentstore()
 
-        module_store = modulestore('direct')
-        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
+        module_store = modulestore()
+        import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
 
         course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
 
@@ -161,8 +160,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         self.assertEqual(count, 0)
 
     def test_no_static_link_rewrites_on_import(self):
-        module_store = modulestore('direct')
-        _, courses = import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
+        module_store = modulestore()
+        _, courses = import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
         course_key = courses[0].id
 
         handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
@@ -177,10 +176,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         self.assertEqual(course.tabs[2]['name'], 'Syllabus')
 
     def test_rewrite_reference_list(self):
-        module_store = modulestore('direct')
+        module_store = modulestore()
         target_course_id = SlashSeparatedCourseKey('testX', 'conditional_copy', 'copy_run')
         import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['conditional'],
             target_course_id=target_course_id
@@ -206,10 +206,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         )
 
     def test_rewrite_reference(self):
-        module_store = modulestore('direct')
+        module_store = modulestore()
         target_course_id = SlashSeparatedCourseKey('testX', 'peergrading_copy', 'copy_run')
         import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['open_ended'],
             target_course_id=target_course_id
@@ -224,10 +225,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
         )
 
     def test_rewrite_reference_value_dict(self):
-        module_store = modulestore('direct')
+        module_store = modulestore()
         target_course_id = SlashSeparatedCourseKey('testX', 'split_test_copy', 'copy_run')
         import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['split_test_module'],
             target_course_id=target_course_id

cms/djangoapps/contentstore/tests/test_import_draft_order.py

-from django.test.utils import override_settings
-
 from xmodule.modulestore.xml_importer import import_from_xml
 
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.django import modulestore
 
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
-
 
 # This test is in the CMS module because the test configuration to use a draft
 # modulestore is dependent on django.
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class DraftReorderTestCase(ModuleStoreTestCase):
 
     def test_order(self):
-        store = modulestore('direct')
-        draft_store = modulestore('default')
-        _, course_items = import_from_xml(store, 'common/test/data/', ['import_draft_order'], draft_store=draft_store)
+        store = modulestore()
+        _, course_items = import_from_xml(store, '**replace_user**', 'common/test/data/', ['import_draft_order'])
         course_key = course_items[0].id
-        sequential = draft_store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
+        sequential = store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
         verticals = sequential.children
 
         # The order that files are read in from the file system is not guaranteed (cannot rely on
@@ -39,7 +33,7 @@ class DraftReorderTestCase(ModuleStoreTestCase):
         self.assertEqual(course_key.make_usage_key('vertical', 'c'), verticals[6])
 
         # Now also test that the verticals in a second sequential are correct.
-        sequential = draft_store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
+        sequential = store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
         verticals = sequential.children
         # 'asecond' and 'zsecond' are drafts with 'index_in_children_list' 0 and 2, respectively.
         # 'secondsubsection' is a public vertical.

cms/djangoapps/contentstore/tests/test_import_pure_xblock.py

@@ -2,8 +2,6 @@
 Integration tests for importing courses containing pure XBlocks.
 """
 
-from django.test.utils import override_settings
-
 from xblock.core import XBlock
 from xblock.fields import String
 
@@ -11,7 +9,6 @@ from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.mongo.draft import as_draft
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 
 
 class StubXBlock(XBlock):
@@ -29,12 +26,10 @@ class StubXBlock(XBlock):
     test_field = String(default="default")
 
 
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class XBlockImportTest(ModuleStoreTestCase):
 
     def setUp(self):
-        self.store = modulestore('direct')
-        self.draft_store = modulestore('default')
+        self.store = modulestore()
 
     @XBlock.register_temp_plugin(StubXBlock)
     def test_import_public(self):
@@ -67,8 +62,7 @@ class XBlockImportTest(ModuleStoreTestCase):
 
         """
         _, courses = import_from_xml(
-            self.store, 'common/test/data', [course_dir],
-            draft_store=self.draft_store
+            self.store, '**replace_user**', 'common/test/data', [course_dir]
         )
 
         xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test')
@@ -81,6 +75,7 @@ class XBlockImportTest(ModuleStoreTestCase):
         self.assertEqual(xblock.test_field, expected_field_val)
 
         if has_draft:
-            draft_xblock = self.draft_store.get_item(xblock_location)
+            draft_xblock = self.store.get_item(xblock_location)
+            self.assertTrue(getattr(draft_xblock, 'is_draft', False))
             self.assertTrue(isinstance(draft_xblock, StubXBlock))
             self.assertEqual(draft_xblock.test_field, expected_field_val)

cms/djangoapps/contentstore/tests/test_orphan.py

@@ -32,8 +32,10 @@ class TestOrphan(CourseTestCase):
 
     def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
         location = self.course.location.replace(category=category, name=name)
-        store = modulestore('direct')
-        store.create_and_save_xmodule(location, data, metadata, runtime)
+        store = modulestore()
+        store.create_and_save_xmodule(
+            location, self.user.id, definition_data=data, metadata=metadata, runtime=runtime
+        )
         if parent_name:
             # add child to parent in mongo
             parent_location = self.course.location.replace(category=parent_category, name=parent_name)

cms/djangoapps/contentstore/tests/test_permissions.py

@@ -3,11 +3,9 @@ Test CRUD for authorization.
 """
 import copy
 
-from django.test.utils import override_settings
 from django.contrib.auth.models import User
 
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 from contentstore.tests.utils import AjaxEnabledTestClient
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from contentstore.utils import reverse_url, reverse_course_url
@@ -16,7 +14,6 @@ from contentstore.views.access import has_course_access
 from student import auth
 
 
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestCourseAccess(ModuleStoreTestCase):
     """
     Course-based access (as opposed to access of a non-course xblock)

cms/djangoapps/contentstore/tests/test_transcripts_utils.py

@@ -21,7 +21,6 @@ from xmodule.exceptions import NotFoundError
 from xmodule.contentstore.django import contentstore, _CONTENTSTORE
 from xmodule.video_module import transcripts_utils
 
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
 
@@ -76,7 +75,7 @@ class TestGenerateSubs(unittest.TestCase):
         )
 
 
-@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class TestSaveSubsToStore(ModuleStoreTestCase):
     """Tests for `save_subs_to_store` function."""
 
@@ -156,7 +155,7 @@ class TestSaveSubsToStore(ModuleStoreTestCase):
         _CONTENTSTORE.clear()
 
 
-@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class TestDownloadYoutubeSubs(ModuleStoreTestCase):
     """Tests for `download_youtube_subs` function."""
 

cms/djangoapps/contentstore/tests/tests.py

@@ -15,14 +15,12 @@ from contentstore.tests.utils import parse_json, user, registration, AjaxEnabled
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from contentstore.tests.test_course_settings import CourseTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 import datetime
 from pytz import UTC
 
 from freezegun import freeze_time
 
 
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class ContentStoreTestCase(ModuleStoreTestCase):
     def _login(self, email, password):
         """

cms/djangoapps/contentstore/tests/utils.py

@@ -6,12 +6,10 @@ import json
 
 from django.contrib.auth.models import User
 from django.test.client import Client
-from django.test.utils import override_settings
 
+from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
-from contentstore.utils import get_modulestore
 from student.models import Registration
 
 
@@ -58,8 +56,6 @@ class AjaxEnabledTestClient(Client):
         return self.get(path, data or {}, follow, HTTP_ACCEPT="application/json", **extra)
 
 
-
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class CourseTestCase(ModuleStoreTestCase):
     def setUp(self):
         """
@@ -91,7 +87,7 @@ class CourseTestCase(ModuleStoreTestCase):
             number='999',
             display_name='Robot Super Course',
         )
-        self.store = get_modulestore(self.course.location)
+        self.store = modulestore()
 
     def create_non_staff_authed_user_client(self, authenticate=True):
         """

cms/djangoapps/contentstore/utils.py

@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse
 from xmodule.contentstore.content import StaticContent
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
 from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
 from xmodule.modulestore.store_utilities import delete_course
 from student.roles import CourseInstructorRole, CourseStaffRole
@@ -30,31 +30,22 @@ def delete_course_and_groups(course_id, commit=False):
     This deletes the courseware associated with a course_id as well as cleaning update_item
     the various user table stuff (groups, permissions, etc.)
     """
-    module_store = modulestore('direct')
+    module_store = modulestore()
     content_store = contentstore()
 
-    module_store.ignore_write_events_on_courses.add(course_id)
+    with store_bulk_write_operations_on_course(module_store, course_id):
+        if delete_course(module_store, content_store, course_id, commit):
 
-    if delete_course(module_store, content_store, course_id, commit):
-
-        print 'removing User permissions from course....'
-        # in the django layer, we need to remove all the user permissions groups associated with this course
-        if commit:
-            try:
-                staff_role = CourseStaffRole(course_id)
-                staff_role.remove_users(*staff_role.users_with_role())
-                instructor_role = CourseInstructorRole(course_id)
-                instructor_role.remove_users(*instructor_role.users_with_role())
-            except Exception as err:
-                log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
-
-
-def get_modulestore(category_or_location):
-    """
-    This function no longer does anything more than just calling `modulestore()`. It used
-    to select 'direct' v 'draft' based on the category.
-    """
-    return modulestore()
+            print 'removing User permissions from course....'
+            # in the django layer, we need to remove all the user permissions groups associated with this course
+            if commit:
+                try:
+                    staff_role = CourseStaffRole(course_id)
+                    staff_role.remove_users(*staff_role.users_with_role())
+                    instructor_role = CourseInstructorRole(course_id)
+                    instructor_role.remove_users(*instructor_role.users_with_role())
+                except Exception as err:
+                    log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
 
 
 def get_lms_link_for_item(location, preview=False):
@@ -124,35 +115,18 @@ def course_image_url(course):
     return path
 
 
-class PublishState(object):
-    """
-    The publish state for a given xblock-- either 'draft', 'private', or 'public'.
-
-    Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
-    """
-    draft = 'draft'
-    private = 'private'
-    public = 'public'
-
-
 def compute_publish_state(xblock):
     """
-    Returns whether this xblock is 'draft', 'public', or 'private'.
+    Returns whether this xblock is draft, public, or private.
 
-    'draft' content is in the process of being edited, but still has a previous
-        version visible in the LMS
-    'public' content is locked and visible in the LMS
-    'private' content is editable and not visible in the LMS
+    Returns:
+        PublishState.draft - content is in the process of being edited, but still has a previous
+            version deployed to LMS
+        PublishState.public - content is locked and deployed to LMS
+        PublishState.private - content is editable and not deployed to LMS
     """
 
-    if getattr(xblock, 'is_draft', False):
-        try:
-            modulestore('direct').get_item(xblock.location)
-            return PublishState.draft
-        except ItemNotFoundError:
-            return PublishState.private
-    else:
-        return PublishState.public
+    return modulestore().compute_publish_state(xblock)
 
 
 def add_extra_panel_tab(tab_type, course):

cms/djangoapps/contentstore/views/checklist.py

@@ -11,7 +11,7 @@ from django.http import HttpResponseNotFound
 from django.core.exceptions import PermissionDenied
 from opaque_keys.edx.keys import CourseKey
 from xmodule.modulestore.django import modulestore
-from contentstore.utils import get_modulestore, reverse_course_url
+from contentstore.utils import reverse_course_url
 
 from .access import has_course_access
 from xmodule.course_module import CourseDescriptor
@@ -47,7 +47,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
         # from the template.
         if not course_module.checklists:
             course_module.checklists = CourseDescriptor.checklists.default
-            get_modulestore(course_module.location).update_item(course_module, request.user.id)
+            modulestore().update_item(course_module, request.user.id)
 
         expanded_checklists = expand_all_action_urls(course_module)
         if json_request:
@@ -76,7 +76,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
             # not default
             course_module.checklists = course_module.checklists
             course_module.save()
-            get_modulestore(course_module.location).update_item(course_module, request.user.id)
+            modulestore().update_item(course_module, request.user.id)
             expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
             return JsonResponse(localize_checklist_text(expanded_checklist))
         else:

cms/djangoapps/contentstore/views/component.py

@@ -13,6 +13,7 @@ from edxmako.shortcuts import render_to_response
 
 from util.date_utils import get_default_time_display
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import PublishState
 
 from xblock.core import XBlock
 from xblock.django.request import webob_to_django_response, django_to_webob_request
@@ -21,7 +22,7 @@ from xblock.fields import Scope
 from xblock.plugin import PluginMissingError
 from xblock.runtime import Mixologist
 
-from contentstore.utils import get_lms_link_for_item, compute_publish_state, PublishState, get_modulestore
+from contentstore.utils import get_lms_link_for_item, compute_publish_state
 from contentstore.views.helpers import get_parent_xblock
 
 from models.settings.course_grading import CourseGradingModel
@@ -413,7 +414,7 @@ def _get_item_in_course(request, usage_key):
         raise PermissionDenied()
 
     course = modulestore().get_course(course_key)
-    item = get_modulestore(usage_key).get_item(usage_key, depth=1)
+    item = modulestore().get_item(usage_key, depth=1)
     lms_link = get_lms_link_for_item(usage_key)
 
     return course, item, lms_link
@@ -436,7 +437,7 @@ def component_handler(request, usage_key_string, handler, suffix=''):
 
     usage_key = UsageKey.from_string(usage_key_string)
 
-    descriptor = get_modulestore(usage_key).get_item(usage_key)
+    descriptor = modulestore().get_item(usage_key)
     # Let the module handle the AJAX
     req = django_to_webob_request(request)
 
@@ -449,6 +450,6 @@ def component_handler(request, usage_key_string, handler, suffix=''):
 
     # unintentional update to handle any side effects of handle call; so, request user didn't author
     # the change
-    get_modulestore(usage_key).update_item(descriptor, None)
+    modulestore().update_item(descriptor, None)
 
     return webob_to_django_response(resp)

cms/djangoapps/contentstore/views/course.py

@@ -31,7 +31,6 @@ from contentstore.utils import (
     get_lms_link_for_item,
     add_extra_panel_tab,
     remove_extra_panel_tab,
-    get_modulestore,
     reverse_course_url
 )
 from models.settings.course_details import CourseDetails, CourseSettingsEncoder
@@ -165,7 +164,7 @@ def _accessible_courses_list(request):
     """
     List all courses available to the logged in user by iterating through all the courses
     """
-    courses = modulestore('direct').get_courses()
+    courses = modulestore().get_courses()
 
     # filter out courses that we don't have access to
     def course_filter(course):
@@ -202,14 +201,15 @@ def _accessible_courses_list_from_groups(request):
         if course_key is None:
             # If the course_access does not have a course_id, it's an org-based role, so we fall back
             raise AccessListFallback
-        try:
-            course = modulestore('direct').get_course(course_key)
-        except ItemNotFoundError:
-            # If a user has access to a course that doesn't exist, don't do anything with that course
-            pass
-        if course is not None and not isinstance(course, ErrorDescriptor):
-            # ignore deleted or errored courses
-            courses_list[course_key] = course
+        if course_key not in courses_list:
+            try:
+                course = modulestore().get_course(course_key)
+            except ItemNotFoundError:
+                # If a user has access to a course that doesn't exist, don't do anything with that course
+                pass
+            if course is not None and not isinstance(course, ErrorDescriptor):
+                # ignore deleted or errored courses
+                courses_list[course_key] = course
 
     return courses_list.values()
 
@@ -333,7 +333,7 @@ def create_new_course(request):
         fields.update(metadata)
 
         # Creating the course raises InvalidLocationError if an existing course with this org/name is found
-        new_course = modulestore('direct').create_course(
+        new_course = modulestore().create_course(
             course_key.org,
             course_key.offering,
             fields=fields,
@@ -440,7 +440,7 @@ def course_info_update_handler(request, course_key_string, provided_id=None):
         raise PermissionDenied()
 
     if request.method == 'GET':
-        course_updates = get_course_updates(usage_key, provided_id)
+        course_updates = get_course_updates(usage_key, provided_id, request.user.id)
         if isinstance(course_updates, dict) and course_updates.get('error'):
             return JsonResponse(course_updates, course_updates.get('status', 400))
         else:
@@ -739,7 +739,7 @@ def textbooks_list_handler(request, course_key_string):
     """
     course_key = CourseKey.from_string(course_key_string)
     course = _get_course_module(course_key, request.user)
-    store = get_modulestore(course.location)
+    store = modulestore()
 
     if not "application/json" in request.META.get('HTTP_ACCEPT', 'text/html'):
         # return HTML page
@@ -814,7 +814,7 @@ def textbooks_detail_handler(request, course_key_string, textbook_id):
     """
     course_key = CourseKey.from_string(course_key_string)
     course_module = _get_course_module(course_key, request.user)
-    store = get_modulestore(course_module.location)
+    store = modulestore()
     matching_id = [tb for tb in course_module.pdf_textbooks
                    if unicode(tb.get("id")) == unicode(textbook_id)]
     if matching_id:

cms/djangoapps/contentstore/views/helpers.py

@@ -39,34 +39,16 @@ def render_from_lms(template_name, dictionary, context=None, namespace='main'):
     return render_to_string(template_name, dictionary, context, namespace="lms." + namespace)
 
 
-def _xmodule_recurse(item, action, ignore_exception=()):
-    """
-    Recursively apply provided action on item and its children
-
-    ignore_exception (Exception Object): A optional argument; when passed ignores the corresponding
-        exception raised during xmodule recursion,
-    """
-    for child in item.get_children():
-        _xmodule_recurse(child, action, ignore_exception)
-
-    try:
-        return action(item)
-    except ignore_exception:
-        return
-
-
 def get_parent_xblock(xblock):
     """
     Returns the xblock that is the parent of the specified xblock, or None if it has no parent.
     """
     locator = xblock.location
-    parent_locations = modulestore().get_parent_locations(locator,)
+    parent_location = modulestore().get_parent_location(locator)
 
-    if len(parent_locations) == 0:
+    if parent_location is None:
         return None
-    elif len(parent_locations) > 1:
-        logging.error('Multiple parents have been found for %s', unicode(locator))
-    return modulestore().get_item(parent_locations[0])
+    return modulestore().get_item(parent_location)
 
 
 def is_unit(xblock):

cms/djangoapps/contentstore/views/import_export.py

@@ -217,13 +217,13 @@ def import_handler(request, course_key_string):
                             shutil.move(dirpath / fname, course_dir)
 
                     _module_store, course_items = import_from_xml(
-                        modulestore('direct'),
+                        modulestore(),
+                        request.user.id,
                         settings.GITHUB_REPO_ROOT,
                         [course_subdir],
                         load_error_modules=False,
                         static_content_store=contentstore(),
                         target_course_id=course_key,
-                        draft_store=modulestore()
                     )
 
                     new_location = course_items[0].location
@@ -322,7 +322,7 @@ def export_handler(request, course_key_string):
         root_dir = path(mkdtemp())
 
         try:
-            export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore())
+            export_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
 
             logging.debug('tar file being generated at {0}'.format(export_file.name))
             with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
@@ -334,10 +334,10 @@ def export_handler(request, course_key_string):
             parent = None
             try:
                 failed_item = modulestore().get_item(exc.location)
-                parent_locs = modulestore().get_parent_locations(failed_item.location)
+                parent_loc = modulestore().get_parent_location(failed_item.location)
 
-                if len(parent_locs) > 0:
-                    parent = modulestore().get_item(parent_locs[0])
+                if parent_loc is not None:
+                    parent = modulestore().get_item(parent_loc)
                     if parent.location.category == 'vertical':
                         unit = parent
             except:  # pylint: disable=bare-except

cms/djangoapps/contentstore/views/tabs.py

@@ -117,7 +117,7 @@ def reorder_tabs_handler(course_item, request):
 
     # persist the new order of the tabs
     course_item.tabs = new_tab_list
-    modulestore('direct').update_item(course_item, request.user.id)
+    modulestore().update_item(course_item, request.user.id)
 
     return JsonResponse()
 
@@ -140,7 +140,7 @@ def edit_tab_handler(course_item, request):
     if 'is_hidden' in request.json:
         # set the is_hidden attribute on the requested tab
         tab.is_hidden = request.json['is_hidden']
-        modulestore('direct').update_item(course_item, request.user.id)
+        modulestore().update_item(course_item, request.user.id)
     else:
         raise NotImplementedError('Unsupported request to edit tab: {0}'.format(request.json))
 
@@ -163,7 +163,7 @@ def get_tab_by_locator(tab_list, usage_key_string):
     Look for a tab with the specified locator.  Returns the first matching tab.
     """
     tab_location = UsageKey.from_string(usage_key_string)
-    item = modulestore('direct').get_item(tab_location)
+    item = modulestore().get_item(tab_location)
     static_tab = StaticTab(
         name=item.display_name,
         url_slug=item.location.name,
@@ -192,7 +192,7 @@ def primitive_delete(course, num):
     # Note for future implementations: if you delete a static_tab, then Chris Dodge
     # points out that there's other stuff to delete beyond this element.
     # This code happens to not delete static_tab so it doesn't come up.
-    modulestore('direct').update_item(course, '**replace_user**')
+    modulestore().update_item(course, '**replace_user**')
 
 
 def primitive_insert(course, num, tab_type, name):
@@ -201,5 +201,5 @@ def primitive_insert(course, num, tab_type, name):
     new_tab = CourseTab.from_json({u'type': unicode(tab_type), u'name': unicode(name)})
     tabs = course.tabs
     tabs.insert(num, new_tab)
-    modulestore('direct').update_item(course, '**replace_user**')
+    modulestore().update_item(course, '**replace_user**')
 

cms/djangoapps/contentstore/views/tests/test_assets.py

@@ -48,9 +48,10 @@ class BasicAssetsTestCase(AssetsTestCase):
         self.assertEquals(path, '/static/my_file_name.jpg')
 
     def test_pdf_asset(self):
-        module_store = modulestore('direct')
+        module_store = modulestore()
         _, course_items = import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['toy'],
             static_content_store=contentstore(),
@@ -191,9 +192,10 @@ class LockAssetTestCase(AssetsTestCase):
             return json.loads(resp.content)
 
         # Load the toy course.
-        module_store = modulestore('direct')
+        module_store = modulestore()
         _, course_items = import_from_xml(
             module_store,
+            '**replace_user**',
             'common/test/data/',
             ['toy'],
             static_content_store=contentstore(),

cms/djangoapps/contentstore/views/tests/test_checklists.py

 """ Unit tests for checklist methods in views.py. """
-from contentstore.utils import get_modulestore, reverse_course_url
+from contentstore.utils import reverse_course_url
 from contentstore.views.checklist import expand_checklist_action_url
 from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.django import modulestore
 
 import json
 from contentstore.tests.utils import CourseTestCase
@@ -21,8 +22,7 @@ class ChecklistTestCase(CourseTestCase):
 
     def get_persisted_checklists(self):
         """ Returns the checklists as persisted in the modulestore. """
-        modulestore = get_modulestore(self.course.location)
-        return modulestore.get_item(self.course.location).checklists
+        return modulestore().get_item(self.course.location).checklists
 
     def compare_checklists(self, persisted, request):
         """
@@ -54,8 +54,7 @@ class ChecklistTestCase(CourseTestCase):
         self.course.checklists = None
         # Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore
         self.course.save()
-        modulestore = get_modulestore(self.course.location)
-        modulestore.update_item(self.course, self.user.id)
+        modulestore().update_item(self.course, self.user.id)
         self.assertEqual(self.get_persisted_checklists(), None)
         response = self.client.get(self.checklists_url)
         self.assertEqual(payload, response.content)

cms/djangoapps/contentstore/views/tests/test_container_page.py

@@ -3,8 +3,9 @@ Unit tests for the container page.
 """
 
 import re
-from contentstore.utils import compute_publish_state, PublishState
+from contentstore.utils import compute_publish_state
 from contentstore.views.tests.utils import StudioPageTestCase
+from xmodule.modulestore import PublishState
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.factories import ItemFactory
 
@@ -29,6 +30,7 @@ class ContainerPageTestCase(StudioPageTestCase):
                                                  category='vertical', display_name='Child Vertical')
         self.video = ItemFactory.create(parent_location=self.child_vertical.location,
                                         category="video", display_name="My Video")
+        self.store = modulestore()
 
     def test_container_html(self):
         self._test_html_content(
@@ -49,12 +51,12 @@ class ContainerPageTestCase(StudioPageTestCase):
         Create the scenario of an xblock with children (non-vertical) on the container page.
         This should create a container page that is a child of another container page.
         """
-        published_container = ItemFactory.create(
+        draft_container = ItemFactory.create(
             parent_location=self.child_container.location,
             category="wrapper", display_name="Wrapper"
         )
         ItemFactory.create(
-            parent_location=published_container.location,
+            parent_location=draft_container.location,
             category="html", display_name="Child HTML"
         )
 
@@ -63,7 +65,7 @@ class ContainerPageTestCase(StudioPageTestCase):
                 xblock,
                 expected_section_tag=(
                     '<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
-                    'data-locator="{0}" data-course-key="{0.course_key}">'.format(published_container.location)
+                    'data-locator="{0}" data-course-key="{0.course_key}">'.format(draft_container.location)
                 ),
                 expected_breadcrumbs=(
                     r'<a href="/unit/{unit}"\s*'
@@ -77,13 +79,12 @@ class ContainerPageTestCase(StudioPageTestCase):
                 )
             )
 
-        # Test the published version of the container
-        test_container_html(published_container)
+        # Test the draft version of the container
+        test_container_html(draft_container)
 
-        # Now make the unit and its children into a draft and validate the container again
-        modulestore('draft').convert_to_draft(self.vertical.location)
-        modulestore('draft').convert_to_draft(self.child_vertical.location)
-        draft_container = modulestore('draft').convert_to_draft(published_container.location)
+        # Now publish the unit and validate again
+        self.store.publish(self.vertical.location, self.user.id)
+        draft_container = self.store.get_item(draft_container.location)
         test_container_html(draft_container)
 
     def _test_html_content(self, xblock, expected_section_tag, expected_breadcrumbs):
@@ -98,7 +99,6 @@ class ContainerPageTestCase(StudioPageTestCase):
         self.assertRegexpMatches(html, expected_breadcrumbs)
 
         # Verify the link that allows users to change publish status.
-        expected_message = None
         if publish_state == PublishState.public:
             expected_message = 'you need to edit unit <a href="/unit/{}">Unit</a> as a draft.'
         else:
@@ -110,25 +110,25 @@ class ContainerPageTestCase(StudioPageTestCase):
         """
         Verify that a public xblock's container preview returns the expected HTML.
         """
-        self.validate_preview_html(self.vertical, self.container_view,
+        published_unit = self.store.publish(self.vertical.location, self.user.id)
+        published_child_container = self.store.get_item(self.child_container.location)
+        published_child_vertical = self.store.get_item(self.child_vertical.location)
+        self.validate_preview_html(published_unit, self.container_view,
                                    can_edit=False, can_reorder=False, can_add=False)
-        self.validate_preview_html(self.child_container, self.container_view,
+        self.validate_preview_html(published_child_container, self.container_view,
                                    can_edit=False, can_reorder=False, can_add=False)
-        self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
+        self.validate_preview_html(published_child_vertical, self.reorderable_child_view,
                                    can_edit=False, can_reorder=False, can_add=False)
 
     def test_draft_container_preview_html(self):
         """
         Verify that a draft xblock's container preview returns the expected HTML.
         """
-        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
-        draft_child_container = modulestore('draft').convert_to_draft(self.child_container.location)
-        draft_child_vertical = modulestore('draft').convert_to_draft(self.child_vertical.location)
-        self.validate_preview_html(draft_unit, self.container_view,
+        self.validate_preview_html(self.vertical, self.container_view,
                                    can_edit=True, can_reorder=True, can_add=True)
-        self.validate_preview_html(draft_child_container, self.container_view,
+        self.validate_preview_html(self.child_container, self.container_view,
                                    can_edit=True, can_reorder=True, can_add=True)
-        self.validate_preview_html(draft_child_vertical, self.reorderable_child_view,
+        self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
                                    can_edit=True, can_reorder=True, can_add=True)
 
     def test_public_child_container_preview_html(self):
@@ -137,7 +137,8 @@ class ContainerPageTestCase(StudioPageTestCase):
         """
         empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                    category='split_test', display_name='Split Test')
-        self.validate_preview_html(empty_child_container, self.reorderable_child_view,
+        published_empty_child_container = self.store.publish(empty_child_container.location, '**replace_user**')
+        self.validate_preview_html(published_empty_child_container, self.reorderable_child_view,
                                    can_reorder=False, can_edit=False, can_add=False)
 
     def test_draft_child_container_preview_html(self):
@@ -146,7 +147,5 @@ class ContainerPageTestCase(StudioPageTestCase):
         """
         empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                    category='split_test', display_name='Split Test')
-        modulestore('draft').convert_to_draft(self.vertical.location)
-        draft_empty_child_container = modulestore('draft').convert_to_draft(empty_child_container.location)
-        self.validate_preview_html(draft_empty_child_container, self.reorderable_child_view,
+        self.validate_preview_html(empty_child_container, self.reorderable_child_view,
                                    can_reorder=True, can_edit=True, can_add=False)

cms/djangoapps/contentstore/views/tests/test_course_updates.py

@@ -129,13 +129,12 @@ class CourseUpdateTest(CourseTestCase):
         '''
         # get the updates and populate 'data' field with some data.
         location = self.course.id.make_usage_key('course_info', 'updates')
-        modulestore('direct').create_and_save_xmodule(location)
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().create_and_save_xmodule(location, self.user.id)
         update_date = u"January 23, 2014"
         update_content = u"Hello world!"
         update_data = u"<ol><li><h2>" + update_date + "</h2>" + update_content + "</li></ol>"
         course_updates.data = update_data
-        modulestore('direct').update_item(course_updates, self.user.id)
+        modulestore().update_item(course_updates, self.user.id)
 
         # test getting all updates list
         course_update_url = self.create_update_url()
@@ -155,7 +154,7 @@ class CourseUpdateTest(CourseTestCase):
 
         # test that while updating it converts old data (with string format in 'data' field)
         # to new data (with list format in 'items' field) and respectively updates 'data' field.
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
         self.assertEqual(course_updates.items, [])
         # now try to update first update item
         update_content = 'Testing'
@@ -164,20 +163,20 @@ class CourseUpdateTest(CourseTestCase):
             course_update_url + '1', payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
         )
         self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
         self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
         # course_updates 'data' field should update accordingly
         update_data = u"<section><article><h2>{date}</h2>{content}</article></section>".format(date=update_date, content=update_content)
         self.assertEqual(course_updates.data, update_data)
 
         # test delete course update item (soft delete)
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
         self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
         # now try to delete first update item
         resp = self.client.delete(course_update_url + '1')
         self.assertEqual(json.loads(resp.content), [])
         # confirm that course update is soft deleted ('status' flag set to 'deleted') in db
-        course_updates = modulestore('direct').get_item(location)
+        course_updates = modulestore().get_item(location)
         self.assertEqual(course_updates.items,
                          [{u'date': update_date, u'content': update_content, u'id': 1, u'status': 'deleted'}])
 
@@ -204,10 +203,10 @@ class CourseUpdateTest(CourseTestCase):
         '''Test trying to add to a saved course_update which is not an ol.'''
         # get the updates and set to something wrong
         location = self.course.id.make_usage_key('course_info', 'updates')
-        modulestore('direct').create_and_save_xmodule(location)
-        course_updates = modulestore('direct').get_item(location)
+        modulestore().create_and_save_xmodule(location, self.user.id)
+        course_updates = modulestore().get_item(location)
         course_updates.data = 'bad news'
-        modulestore('direct').update_item(course_updates, self.user.id)
+        modulestore().update_item(course_updates, self.user.id)
 
         init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
         content = init_content + '</iframe>'

cms/djangoapps/contentstore/views/tests/test_import_export.py

@@ -294,7 +294,8 @@ class ExportTestCase(CourseTestCase):
         """
         Export failure.
         """
-        ItemFactory.create(parent_location=self.course.location, category='aawefawef')
+        fake_xblock = ItemFactory.create(parent_location=self.course.location, category='aawefawef')
+        self.store.publish(fake_xblock.location, self.user.id)
         self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')
 
     def test_export_failure_subsection_level(self):

cms/djangoapps/contentstore/views/tests/test_transcripts.py

@@ -21,12 +21,11 @@ from xmodule.exceptions import NotFoundError
 from opaque_keys.edx.keys import UsageKey
 from xmodule.video_module import transcripts_utils
 
-from contentstore.tests.modulestore_config import TEST_MODULESTORE
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
 
 
-@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class Basetranscripts(CourseTestCase):
     """Base test class for transcripts tests."""
 

cms/djangoapps/contentstore/views/tests/test_unit_page.py

@@ -19,11 +19,13 @@ class UnitPageTestCase(StudioPageTestCase):
                                            category='vertical', display_name='Unit')
         self.video = ItemFactory.create(parent_location=self.vertical.location,
                                         category="video", display_name="My Video")
+        self.store = modulestore()
 
     def test_public_unit_page_html(self):
         """
         Verify that an xblock returns the expected HTML for a public unit page.
         """
+
         html = self.get_page_html(self.vertical)
         self.validate_html_for_add_buttons(html)
 
@@ -31,14 +33,14 @@ class UnitPageTestCase(StudioPageTestCase):
         """
         Verify that an xblock returns the expected HTML for a draft unit page.
         """
-        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
-        html = self.get_page_html(draft_unit)
+        html = self.get_page_html(self.vertical)
         self.validate_html_for_add_buttons(html)
 
     def test_public_component_preview_html(self):
         """
         Verify that a public xblock's preview returns the expected HTML.
         """
+        published_video = self.store.publish(self.video.location, '**replace_user**')
         self.validate_preview_html(self.video, STUDENT_VIEW,
                                    can_edit=True, can_reorder=True, can_add=False)
 
@@ -46,9 +48,7 @@ class UnitPageTestCase(StudioPageTestCase):
         """
         Verify that a draft xblock's preview returns the expected HTML.
         """
-        modulestore('draft').convert_to_draft(self.vertical.location)
-        draft_video = modulestore('draft').convert_to_draft(self.video.location)
-        self.validate_preview_html(draft_video, STUDENT_VIEW,
+        self.validate_preview_html(self.video, STUDENT_VIEW,
                                    can_edit=True, can_reorder=True, can_add=False)
 
     def test_public_child_container_preview_html(self):
@@ -60,7 +60,8 @@ class UnitPageTestCase(StudioPageTestCase):
                                              category='split_test', display_name='Split Test')
         ItemFactory.create(parent_location=child_container.location,
                            category='html', display_name='grandchild')
-        self.validate_preview_html(child_container, STUDENT_VIEW,
+        published_child_container = self.store.publish(child_container.location, '**replace_user**')
+        self.validate_preview_html(published_child_container, STUDENT_VIEW,
                                    can_reorder=True, can_edit=True, can_add=False)
 
     def test_draft_child_container_preview_html(self):
@@ -72,7 +73,6 @@ class UnitPageTestCase(StudioPageTestCase):
                                              category='split_test', display_name='Split Test')
         ItemFactory.create(parent_location=child_container.location,
                            category='html', display_name='grandchild')
-        modulestore('draft').convert_to_draft(self.vertical.location)
-        draft_child_container = modulestore('draft').get_item(child_container.location)
+        draft_child_container = self.store.get_item(child_container.location)
         self.validate_preview_html(draft_child_container, STUDENT_VIEW,
                                    can_reorder=True, can_edit=True, can_add=False)

cms/djangoapps/models/settings/course_details.py

@@ -6,7 +6,7 @@ from json.encoder import JSONEncoder
 
 from opaque_keys.edx.locations import Location
 from xmodule.modulestore.exceptions import ItemNotFoundError
-from contentstore.utils import get_modulestore, course_image_url
+from contentstore.utils import course_image_url
 from models.settings import course_grading
 from xmodule.fields import Date
 from xmodule.modulestore.django import modulestore
@@ -34,7 +34,7 @@ class CourseDetails(object):
         """
         Fetch the course details for the given course from persistence and return a CourseDetails model.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
         course_details = cls(course_key.org, course_key.course, course_key.run)
 
         course_details.start_date = descriptor.start
@@ -46,31 +46,31 @@ class CourseDetails(object):
 
         temploc = course_key.make_usage_key('about', 'syllabus')
         try:
-            course_details.syllabus = get_modulestore(temploc).get_item(temploc).data
+            course_details.syllabus = modulestore().get_item(temploc).data
         except ItemNotFoundError:
             pass
 
         temploc = course_key.make_usage_key('about', 'short_description')
         try:
-            course_details.short_description = get_modulestore(temploc).get_item(temploc).data
+            course_details.short_description = modulestore().get_item(temploc).data
         except ItemNotFoundError:
             pass
 
         temploc = course_key.make_usage_key('about', 'overview')
         try:
-            course_details.overview = get_modulestore(temploc).get_item(temploc).data
+            course_details.overview = modulestore().get_item(temploc).data
         except ItemNotFoundError:
             pass
 
         temploc = course_key.make_usage_key('about', 'effort')
         try:
-            course_details.effort = get_modulestore(temploc).get_item(temploc).data
+            course_details.effort = modulestore().get_item(temploc).data
         except ItemNotFoundError:
             pass
 
         temploc = course_key.make_usage_key('about', 'video')
         try:
-            raw_video = get_modulestore(temploc).get_item(temploc).data
+            raw_video = modulestore().get_item(temploc).data
             course_details.intro_video = CourseDetails.parse_video_tag(raw_video)
         except ItemNotFoundError:
             pass
@@ -84,14 +84,14 @@ class CourseDetails(object):
         delete the about item.
         """
         temploc = course_key.make_usage_key('about', about_key)
-        store = get_modulestore(temploc)
+        store = modulestore()
         if data is None:
-            store.delete_item(temploc)
+            store.delete_item(temploc, user.id)
         else:
             try:
                 about_item = store.get_item(temploc)
             except ItemNotFoundError:
-                about_item = store.create_xmodule(temploc, system=course.runtime)
+                about_item = store.create_xmodule(temploc, runtime=course.runtime)
             about_item.data = data
             store.update_item(about_item, user.id)
 
@@ -100,7 +100,7 @@ class CourseDetails(object):
         """
         Decode the json into CourseDetails and save any changed attrs to the db
         """
-        module_store = modulestore('direct')
+        module_store = modulestore()
         descriptor = module_store.get_course(course_key)
 
         dirty = False

cms/djangoapps/models/settings/course_grading.py

@@ -21,7 +21,7 @@ class CourseGradingModel(object):
         """
         Fetch the course grading policy for the given course from persistence and return a CourseGradingModel.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
         model = cls(descriptor)
         return model
 
@@ -31,7 +31,7 @@ class CourseGradingModel(object):
         Fetch the course's nth grader
         Returns an empty dict if there's no such grader.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
         index = int(index)
         if len(descriptor.raw_grader) > index:
             return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
@@ -52,14 +52,14 @@ class CourseGradingModel(object):
         Decode the json into CourseGradingModel and save any changes. Returns the modified model.
         Probably not the usual path for updates as it's too coarse grained.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
 
         graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]
 
         descriptor.raw_grader = graders_parsed
         descriptor.grade_cutoffs = jsondict['grade_cutoffs']
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
 
         CourseGradingModel.update_grace_period_from_json(course_key, jsondict['grace_period'], user)
 
@@ -71,7 +71,7 @@ class CourseGradingModel(object):
         Create or update the grader of the given type (string key) for the given course. Returns the modified
         grader which is a full model on the client but not on the server (just a dict)
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
 
         # parse removes the id; so, grab it before parse
         index = int(grader.get('id', len(descriptor.raw_grader)))
@@ -82,7 +82,7 @@ class CourseGradingModel(object):
         else:
             descriptor.raw_grader.append(grader)
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
 
         return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
 
@@ -92,10 +92,10 @@ class CourseGradingModel(object):
         Create or update the grade cutoffs for the given course. Returns sent in cutoffs (ie., no extra
         db fetch).
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
         descriptor.grade_cutoffs = cutoffs
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
 
         return cutoffs
 
@@ -106,7 +106,7 @@ class CourseGradingModel(object):
         grace_period entry in an enclosing dict. It is also safe to call this method with a value of
         None for graceperiodjson.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
 
         # Before a graceperiod has ever been created, it will be None (once it has been
         # created, it cannot be set back to None).
@@ -117,14 +117,14 @@ class CourseGradingModel(object):
             grace_timedelta = timedelta(**graceperiodjson)
             descriptor.graceperiod = grace_timedelta
 
-            modulestore('direct').update_item(descriptor, user.id)
+            modulestore().update_item(descriptor, user.id)
 
     @staticmethod
     def delete_grader(course_key, index, user):
         """
         Delete the grader of the given type from the given course.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
 
         index = int(index)
         if index < len(descriptor.raw_grader):
@@ -132,22 +132,22 @@ class CourseGradingModel(object):
             # force propagation to definition
             descriptor.raw_grader = descriptor.raw_grader
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
 
     @staticmethod
     def delete_grace_period(course_key, user):
         """
         Delete the course's grace period.
         """
-        descriptor = modulestore('direct').get_course(course_key)
+        descriptor = modulestore().get_course(course_key)
 
         del descriptor.graceperiod
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
 
     @staticmethod
     def get_section_grader_type(location):
-        descriptor = modulestore('direct').get_item(location)
+        descriptor = modulestore().get_item(location)
         return {
             "graderType": descriptor.format if descriptor.format is not None else 'notgraded',
             "location": unicode(location),
@@ -162,7 +162,7 @@ class CourseGradingModel(object):
             del descriptor.format
             del descriptor.graded
 
-        modulestore('direct').update_item(descriptor, user.id)
+        modulestore().update_item(descriptor, user.id)
         return {'graderType': grader_type}
 
     @staticmethod

cms/djangoapps/models/settings/course_metadata.py

 from xblock.fields import Scope
-
-from contentstore.utils import get_modulestore
+from xmodule.modulestore.django import modulestore
 from django.utils.translation import ugettext as _
 
-
 class CourseMetadata(object):
     '''
     For CRUD operations on metadata fields which do not have specific editors
@@ -86,6 +84,6 @@ class CourseMetadata(object):
             setattr(descriptor, key, value)
 
         if len(key_values) > 0:
-            get_modulestore(descriptor.location).update_item(descriptor, user.id if user else None)
+            modulestore().update_item(descriptor, user.id if user else None)
 
         return cls.fetch(descriptor)

cms/envs/acceptance.py

@@ -34,35 +34,17 @@ logging.getLogger('track.middleware').setLevel(logging.CRITICAL)
 # use, so these are not set up for TOS and PRIVACY
 logging.getLogger('edxmako.shortcuts').setLevel(logging.ERROR)
 
-DOC_STORE_CONFIG = {
-    'host': 'localhost',
-    'db': 'acceptance_xmodule',
-    'collection': 'acceptance_modulestore_%s' % seed(),
-}
-
-MODULESTORE_OPTIONS = {
-    'default_class': 'xmodule.raw_module.RawDescriptor',
-    'fs_root': TEST_ROOT / "data",
-    'render_template': 'edxmako.shortcuts.render_to_string',
-}
-
-MODULESTORE = {
-    'default': {
-        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
-    },
-    'direct': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
+update_module_store_settings(
+    MODULESTORE,
+    doc_store_settings={
+        'db': 'acceptance_xmodule',
+        'collection': 'acceptance_modulestore_%s' % seed(),
     },
-    'draft': {
-        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
+    module_store_options={
+        'default_class': 'xmodule.raw_module.RawDescriptor',
+        'fs_root': TEST_ROOT / "data",
     }
-}
+)
 
 CONTENTSTORE = {
     'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',

cms/envs/aws.py

@@ -229,7 +229,7 @@ if AWS_SECRET_ACCESS_KEY == "":
     AWS_SECRET_ACCESS_KEY = None
 
 DATABASES = AUTH_TOKENS['DATABASES']
-MODULESTORE = AUTH_TOKENS['MODULESTORE']
+MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS['MODULESTORE'])
 CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
 # Datadog for events!

cms/envs/bok_choy.auth.json

@@ -48,55 +48,47 @@
     },
     "MODULESTORE": {
         "default": {
-            "DOC_STORE_CONFIG": {
-                "collection": "modulestore",
-                "db": "test",
-                "host": [
-                    "localhost"
-                ],
-                "password": "password",
-                "port": 27017,
-                "user": "edxapp"
-            },
-            "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
+            "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
             "OPTIONS": {
-                "collection": "modulestore",
-                "db": "test",
-                "default_class": "xmodule.hidden_module.HiddenDescriptor",
-                "fs_root": "** OVERRIDDEN **",
-                "host": [
-                    "localhost"
-                ],
-                "password": "password",
-                "port": 27017,
-                "render_template": "edxmako.shortcuts.render_to_string",
-                "user": "edxapp"
-            }
-        },
-        "direct": {
-            "DOC_STORE_CONFIG": {
-                "collection": "modulestore",
-                "db": "test",
-                "host": [
-                    "localhost"
-                ],
-                "password": "password",
-                "port": 27017,
-                "user": "edxapp"
-            },
-            "ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
-            "OPTIONS": {
-                "collection": "modulestore",
-                "db": "test",
-                "default_class": "xmodule.hidden_module.HiddenDescriptor",
-                "fs_root": "** OVERRIDDEN **",
-                "host": [
-                    "localhost"
-                ],
-                "password": "password",
-                "port": 27017,
-                "render_template": "edxmako.shortcuts.render_to_string",
-                "user": "edxapp"
+                "mappings": {},
+                "reference_type": "Location",
+                "stores": [
+                    {
+                        "NAME": "draft",
+                        "DOC_STORE_CONFIG": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "host": [
+                                "localhost"
+                            ],
+                            "password": "password",
+                            "port": 27017,
+                            "user": "edxapp"
+                        },
+                        "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
+                        "OPTIONS": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor",
+                            "fs_root": "** OVERRIDDEN **",
+                            "host": [
+                                "localhost"
+                            ],
+                            "password": "password",
+                            "port": 27017,
+                            "render_template": "edxmako.shortcuts.render_to_string",
+                            "user": "edxapp"
+                        }
+                    },
+                    {
+                        "NAME": "xml",
+                        "ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
+                        "OPTIONS": {
+                            "data_dir": "** OVERRIDDEN **",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor"
+                        }
+                    }
+                ]
             }
         }
     },

cms/envs/bok_choy.py

@@ -27,9 +27,16 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=E11
 GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
 LOG_DIR = (TEST_ROOT / "log").abspath()
 
-# Configure Mongo modulestore to use the test folder within the repo
-for store in ["default", "direct"]:
-    MODULESTORE[store]['OPTIONS']['fs_root'] = (TEST_ROOT / "data").abspath()  # pylint: disable=E1120
+# Configure modulestore to use the test folder within the repo
+update_module_store_settings(
+    MODULESTORE,
+    module_store_options={
+        'fs_root': (TEST_ROOT / "data").abspath(),  # pylint: disable=E1120
+    },
+    xml_store_options={
+        'data_dir': (TEST_ROOT / "data").abspath(),
+    },
+)
 
 # Enable django-pipeline and staticfiles
 STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()

cms/envs/common.py

@@ -27,10 +27,12 @@ Longer TODO:
 import imp
 import sys
 import lms.envs.common
+# Although this module itself may not use these imported variables, other dependent modules may.
 from lms.envs.common import (
-    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED
+    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE
 )
 from path import path
+from lms.envs.modulestore_settings import *
 
 from lms.lib.xblock.mixin import LmsBlockMixin
 from dealer.git import git
@@ -244,6 +246,9 @@ XBLOCK_MIXINS = (LmsBlockMixin, InheritanceMixin, XModuleMixin)
 # xblocks can be added via advanced settings
 XBLOCK_SELECT_FUNCTION = prefer_xmodules
 
+############################ Modulestore Configuration ################################
+MODULESTORE_BRANCH = 'draft'
+
 ############################ DJANGO_BUILTINS ################################
 # Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
 DEBUG = False

cms/envs/dev.py

@@ -19,30 +19,13 @@ LOGGING = get_logger_config(ENV_ROOT / "log",
                             dev_env=True,
                             debug=True)
 
-modulestore_options = {
-    'default_class': 'xmodule.raw_module.RawDescriptor',
-    'fs_root': GITHUB_REPO_ROOT,
-    'render_template': 'edxmako.shortcuts.render_to_string',
-}
-
-MODULESTORE = {
-    'default': {
-        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': modulestore_options
-    },
-    'direct': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': modulestore_options
-    },
-    'split': {
-        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': modulestore_options
+update_module_store_settings(
+    MODULESTORE,
+    module_store_options={
+        'default_class': 'xmodule.raw_module.RawDescriptor',
+        'fs_root': GITHUB_REPO_ROOT,
     }
-}
-
+)
 
 # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
 # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)

cms/envs/test.py

@@ -16,6 +16,7 @@ from .common import *
 import os
 from path import path
 from warnings import filterwarnings
+from uuid import uuid4
 
 # import settings from LMS for consistent behavior with CMS
 from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME)
@@ -58,40 +59,29 @@ STATICFILES_DIRS += [
     if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
 ]
 
-DOC_STORE_CONFIG = {
-    'host': 'localhost',
-    'db': 'test_xmodule',
-    'collection': 'test_modulestore',
-}
-
-MODULESTORE_OPTIONS = {
-    'default_class': 'xmodule.raw_module.RawDescriptor',
-    'fs_root': TEST_ROOT / "data",
-    'render_template': 'edxmako.shortcuts.render_to_string',
-}
-
-MODULESTORE = {
-    'default': {
-        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
+# Add split as another store for testing
+MODULESTORE['default']['OPTIONS']['stores'].append(
+    {
+        'NAME': 'split',
+        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
         'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
+        'OPTIONS': {
+            'render_template': 'edxmako.shortcuts.render_to_string',
+        }
     },
-    'direct': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
+)
+# Update module store settings per defaults for tests
+update_module_store_settings(
+    MODULESTORE,
+    module_store_options={
+        'default_class': 'xmodule.raw_module.RawDescriptor',
+        'fs_root': TEST_ROOT / "data",
     },
-    'draft': {
-        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
+    doc_store_settings={
+        'db': 'test_xmodule',
+        'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
     },
-    'split': {
-        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
-        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
-        'OPTIONS': MODULESTORE_OPTIONS
-    }
-}
+)
 
 CONTENTSTORE = {
     'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',

cms/static/coffee/src/views/unit.coffee

@@ -159,7 +159,7 @@ define ["jquery", "jquery.ui", "gettext", "backbone",
             @wait(true)
             $.ajax({
                 type: 'DELETE',
-                url: @model.url() + "?" + $.param({recurse: true})
+                url: @model.url()
             }).success(=>
 
                 analytics.track "Deleted Draft",

cms/static/js/base.js

@@ -284,7 +284,7 @@ function _deleteItem($el, type) {
 
                     $.ajax({
                         type: 'DELETE',
-                        url: ModuleUtils.getUpdateUrl(locator) +'?'+ $.param({recurse: true, all_versions: true}),
+                        url: ModuleUtils.getUpdateUrl(locator),
                         success: function () {
                             $el.remove();
                             deleting.hide();

cms/static/js/views/pages/container.js

@@ -161,8 +161,7 @@ define(["jquery", "underscore", "gettext", "js/views/baseview", "js/views/contai
                                 return $.ajax({
                                     type: 'DELETE',
                                     url: self.getURLRoot() + "/" +
-                                        xblockElement.data('locator') + "?" +
-                                        $.param({recurse: true, all_versions: false})
+                                        xblockElement.data('locator')
                                 }).success(_.bind(self.onDelete, self, xblockElement));
                             });
                     });

cms/templates/container.html

@@ -2,7 +2,7 @@
 <%!
 import json
 
-from contentstore.utils import PublishState
+from xmodule.modulestore import PublishState
 from contentstore.views.helpers import xblock_studio_url, EDITING_TEMPLATES
 from django.utils.translation import ugettext as _
 %>

common/djangoapps/contentserver/tests/test.py

@@ -17,8 +17,7 @@ from student.models import CourseEnrollment
 from xmodule.contentstore.django import contentstore, _CONTENTSTORE
 from xmodule.modulestore.django import modulestore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
-from xmodule.modulestore.tests.django_utils import (studio_store_config,
-    ModuleStoreTestCase)
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.xml_importer import import_from_xml
 
 log = logging.getLogger(__name__)
@@ -26,10 +25,8 @@ log = logging.getLogger(__name__)
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
 
-TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
 
-
-@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class ContentStoreToyCourseTest(ModuleStoreTestCase):
     """
     Tests that use the toy course.
@@ -39,16 +36,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         """
         Create user and login.
         """
-
-        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
-        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
-
         self.client = Client()
         self.contentstore = contentstore()
 
         self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
 
-        import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
+        import_from_xml(modulestore(), '**replace_user**', 'common/test/data/', ['toy'],
                 static_content_store=self.contentstore, verbose=True)
 
         # A locked asset

common/djangoapps/external_auth/tests/test_shib.py

@@ -18,7 +18,7 @@ from django.utils.importlib import import_module
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
 from xmodule.modulestore.inheritance import own_metadata
-from xmodule.modulestore.django import editable_modulestore
+from xmodule.modulestore.django import modulestore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 from external_auth.models import ExternalAuthMap
@@ -80,7 +80,7 @@ class ShibSPTest(ModuleStoreTestCase):
     request_factory = RequestFactory()
 
     def setUp(self):
-        self.store = editable_modulestore()
+        self.store = modulestore()
 
     @unittest.skipUnless(settings.FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
     def test_exception_shib_login(self):

common/djangoapps/heartbeat/tests/test_heartbeat.py

@@ -9,9 +9,9 @@ import mock
 from django.test.utils import override_settings
 from django.conf import settings
 from django.test.testcases import TestCase
-from xmodule.modulestore.tests.django_utils import mongo_store_config
+from xmodule.modulestore.tests.django_utils import draft_mongo_store_config
 
-TEST_MODULESTORE = mongo_store_config(settings.TEST_ROOT / "data")
+TEST_MODULESTORE = draft_mongo_store_config(settings.TEST_ROOT / "data")
 
 @override_settings(MODULESTORE=TEST_MODULESTORE)
 class HeartbeatTestCase(TestCase):

common/djangoapps/student/migrations/0036_access_roles_orgless.py

@@ -24,27 +24,11 @@ class Migration(DataMigration):
         """
         Converts group table entries for write access and beta_test roles to course access roles table.
         """
-        def get_modulestore(ms_type, key):
-            """
-            Find the modulestore of the given type trying the key first
-            """
-            try:
-                store = modulestore(key)
-                if isinstance(store, MixedModuleStore):
-                    store = store.modulestores[key]
-                if store.get_modulestore_type(None) == ms_type:
-                    return store
-                else:
-                    return None
-            except KeyError:
-                return None
-
         # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
         loc_map_collection = loc_mapper().location_map
-        xml_ms = get_modulestore(XML_MODULESTORE_TYPE, 'xml')
-        mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'default')
-        if mongo_ms is None:
-            mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'direct')
+        mixed_ms = modulestore()
+        xml_ms = mixed_ms._get_modulestore_by_type(XML_MODULESTORE_TYPE)
+        mongo_ms = mixed_ms._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
 
         query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
         for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():

common/djangoapps/student/tests/test_course_listing.py

@@ -6,7 +6,8 @@ from mock import patch, Mock
 
 from student.tests.factories import UserFactory
 from student.roles import GlobalStaff
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, studio_store_config
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.modulestore.django import modulestore
@@ -14,13 +15,8 @@ from xmodule.error_module import ErrorDescriptor
 from django.test.client import Client
 from student.models import CourseEnrollment
 from student.views import get_course_enrollment_pairs
-from django.conf import settings
-from django.test.utils import override_settings
 
-TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
 
-
-@override_settings(MODULESTORE=TEST_MODULESTORE)
 class TestCourseListing(ModuleStoreTestCase):
     """
     Unit tests for getting the list of courses for a logged in user
@@ -44,8 +40,7 @@ class TestCourseListing(ModuleStoreTestCase):
         course = CourseFactory.create(
             org=course_location.org,
             number=course_location.course,
-            run=course_location.run,
-            modulestore=modulestore('direct'),
+            run=course_location.run
         )
 
         CourseEnrollment.enroll(self.student, course.id)
@@ -84,7 +79,7 @@ class TestCourseListing(ModuleStoreTestCase):
         self._create_course_with_access_groups(course_key)
 
         with patch('xmodule.modulestore.mongo.base.MongoKeyValueStore', Mock(side_effect=Exception)):
-            self.assertIsInstance(modulestore('direct').get_course(course_key), ErrorDescriptor)
+            self.assertIsInstance(modulestore().get_course(course_key), ErrorDescriptor)
 
             # get courses through iterating all courses
             courses_list = list(get_course_enrollment_pairs(self.student, None, []))
@@ -95,18 +90,20 @@ class TestCourseListing(ModuleStoreTestCase):
         Create good courses, courses that won't load, and deleted courses which still have
         roles. Test course listing.
         """
+        mongo_store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
+
         good_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
         self._create_course_with_access_groups(good_location)
 
         course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
         self._create_course_with_access_groups(course_location)
-        modulestore('direct').delete_course(course_location)
+        mongo_store.delete_course(course_location)
 
         course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
         course = self._create_course_with_access_groups(course_location)
-        course_db_record = modulestore('direct')._find_one(course.location)
+        course_db_record = mongo_store._find_one(course.location)
         course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
-        modulestore('direct').collection.update(
+        mongo_store.collection.update(
             {'_id': course.location.to_deprecated_son()},
             {'$set': {
                 'metadata.tabs': course_db_record['metadata']['tabs'],

common/djangoapps/student/tests/test_login.py

@@ -17,7 +17,7 @@ from student.views import _parse_course_id_from_string, _get_course_enrollment_d
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
 from xmodule.modulestore.inheritance import own_metadata
-from xmodule.modulestore.django import editable_modulestore
+from xmodule.modulestore.django import modulestore
 
 from external_auth.models import ExternalAuthMap
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -289,7 +289,7 @@ class ExternalAuthShibTest(ModuleStoreTestCase):
     Tests how login_user() interacts with ExternalAuth, in particular Shib
     """
     def setUp(self):
-        self.store = editable_modulestore()
+        self.store = modulestore()
         self.course = CourseFactory.create(org='Stanford', number='456', display_name='NO SHIB')
         self.shib_course = CourseFactory.create(org='Stanford', number='123', display_name='Shib Only')
         self.shib_course.enrollment_domain = 'shib:https://idp.stanford.edu/'

common/djangoapps/terrain/browser.py

@@ -19,6 +19,7 @@ from json import dumps
 from pymongo import MongoClient
 import xmodule.modulestore.django
 from xmodule.contentstore.django import _CONTENTSTORE
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
 
 # There is an import issue when using django-staticfiles with lettuce
 # Lettuce assumes that we are using django.contrib.staticfiles,
@@ -189,7 +190,7 @@ def reset_databases(scenario):
     mongo.drop_database(settings.CONTENTSTORE['DOC_STORE_CONFIG']['db'])
     _CONTENTSTORE.clear()
 
-    modulestore = xmodule.modulestore.django.editable_modulestore()
+    modulestore = xmodule.modulestore.django.modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
     modulestore.collection.drop()
     xmodule.modulestore.django.clear_existing_modulestores()
 

common/djangoapps/terrain/course_helpers.py

@@ -5,7 +5,8 @@ import urllib
 from lettuce import world
 from django.contrib.auth.models import User, Group
 from student.models import CourseEnrollment
-from xmodule.modulestore.django import editable_modulestore
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
 from xmodule.contentstore.django import contentstore
 
 
@@ -71,5 +72,6 @@ def clear_courses():
     # (though it shouldn't), do this manually
     # from the bash shell to drop it:
     # $ mongo test_xmodule --eval "db.dropDatabase()"
-    editable_modulestore().collection.drop()
+    store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
+    store.collection.drop()
     contentstore().fs_files.drop()

common/lib/xmodule/xmodule/contentstore/content.py

@@ -48,7 +48,7 @@ class StaticContent(object):
         - course_key: the course that this asset belongs to
         - path: is the name of the static asset
         - revision: is the object's revision information
-        - is_tumbnail: is whether or not we want the thumbnail version of this
+        - is_thumbnail: is whether or not we want the thumbnail version of this
             asset
         """
         path = path.replace('/', '_')

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -12,7 +12,6 @@ from xmodule.exceptions import NotFoundError
 from fs.osfs import OSFS
 import os
 import json
-import bson.son
 from bson.son import SON
 from opaque_keys.edx.locations import AssetLocation
 

common/lib/xmodule/xmodule/error_module.py

@@ -13,7 +13,7 @@ from xmodule.x_module import XModule, XModuleDescriptor
 from xmodule.errortracker import exc_info_to_str
 from xblock.fields import String, Scope, ScopeIds
 from xblock.field_data import DictFieldData
-from xmodule.modulestore.xml_exporter import EdxJSONEncoder
+from xmodule.modulestore import EdxJSONEncoder
 
 
 log = logging.getLogger(__name__)

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -5,6 +5,8 @@ that are stored in a database an accessible using their Location as an identifie
 
 import logging
 import re
+import json
+import datetime
 
 from collections import namedtuple, defaultdict
 import collections
@@ -20,15 +22,60 @@ from opaque_keys import InvalidKeyError
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xblock.runtime import Mixologist
 from xblock.core import XBlock
-import datetime
 
 log = logging.getLogger('edx.modulestore')
 
+# Modulestore Types
+
 SPLIT_MONGO_MODULESTORE_TYPE = 'split'
 MONGO_MODULESTORE_TYPE = 'mongo'
 XML_MODULESTORE_TYPE = 'xml'
 
 
+# Key Revision constants to use for Location and Usage Keys
+# Note: These values are persisted in the database, so should not be changed without migrations
+KEY_REVISION_DRAFT = 'draft'
+KEY_REVISION_PUBLISHED = None
+
+
+# Revision constants to use for Module Store operations
+# Note: These values are passed into store APIs and only used at run time
+
+# both DRAFT and PUBLISHED versions are queried, with preference to DRAFT versions
+REVISION_OPTION_DRAFT_PREFERRED = 'rev-opt-draft-preferred'
+
+# only DRAFT versions are queried and no PUBLISHED versions
+REVISION_OPTION_DRAFT_ONLY = 'rev-opt-draft-only'
+
+# # only PUBLISHED versions are queried and no DRAFT versions
+REVISION_OPTION_PUBLISHED_ONLY = 'rev-opt-published-only'
+
+# all revisions are queried
+REVISION_OPTION_ALL = 'rev-opt-all'
+
+
+# Branch constants to use for stores, such as Mongo, that have only 2 branches: DRAFT and PUBLISHED
+# Note: These values are taken from server configuration settings, so should not be changed without alerting DevOps
+BRANCH_DRAFT_PREFERRED = 'draft'
+BRANCH_PUBLISHED_ONLY = 'published'
+
+
+# Branch constants to use for stores, such as Split, that have named branches
+BRANCH_NAME_DRAFT = 'draft'
+BRANCH_NAME_PUBLISHED = 'published'
+
+
+class PublishState(object):
+    """
+    The publish state for a given xblock-- either 'draft', 'private', or 'public'.
+
+    Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
+    """
+    draft = 'draft'
+    private = 'private'
+    public = 'public'
+
+
 class ModuleStoreRead(object):
     """
     An abstract interface for a database backend that stores XModuleDescriptor
@@ -184,11 +231,9 @@ class ModuleStoreRead(object):
         pass
 
     @abstractmethod
-    def get_parent_locations(self, location):
-        '''Find all locations that are the parents of this location in this
+    def get_parent_location(self, location, **kwargs):
+        '''Find the location that is the parent of this location in this
         course.  Needed for path_to_location().
-
-        returns an iterable of things that can be passed to Location.
         '''
         pass
 
@@ -245,11 +290,14 @@ class ModuleStoreWrite(ModuleStoreRead):
     @abstractmethod
     def delete_item(self, location, user_id=None, **kwargs):
         """
-        Delete an item from persistence. Pass the user's unique id which the persistent store
+        Delete an item and its subtree from persistence. Remove the item from any parents (Note, does not
+        affect parents from other branches or logical branches; thus, in old mongo, deleting something
+        whose parent cannot be draft, deletes it from both but deleting a component under a draft vertical
+        only deletes it from the draft.
+
+        Pass the user's unique id which the persistent store
         should save with the update if it has that ability.
 
-        :param delete_all_versions: removes both the draft and published version of this item from
-        the course if using draft and old mongo. Split may or may not implement this.
         :param force: fork the structure and don't update the course draftVersion if there's a version
         conflict (only applicable to version tracking and conflict detecting persistence stores)
 
@@ -346,7 +394,7 @@ class ModuleStoreReadBase(ModuleStoreRead):
 
     def has_course(self, course_id, ignore_case=False):
         """
-        Look for a specific course id.  Returns whether it exists.
+        Returns the course_id of the course if it was found, else None
         Args:
             course_id (CourseKey):
             ignore_case (boolean): some modulestores are case-insensitive. Use this flag
@@ -355,12 +403,18 @@ class ModuleStoreReadBase(ModuleStoreRead):
         # linear search through list
         assert(isinstance(course_id, CourseKey))
         if ignore_case:
-            return any(
-                (c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower())
-                for c in self.get_courses()
+            return next(
+                (
+                    c.id for c in self.get_courses()
+                    if c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower()
+                ),
+                None
             )
         else:
-            return any(c.id == course_id for c in self.get_courses())
+            return next(
+                (c.id for c in self.get_courses() if c.id == course_id),
+                None
+            )
 
     def heartbeat(self):
         """
@@ -411,13 +465,12 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
         """
         raise NotImplementedError
 
-    def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
+    def delete_item(self, location, user_id=None, force=False):
         """
         Delete an item from persistence. Pass the user's unique id which the persistent store
         should save with the update if it has that ability.
 
-        :param delete_all_versions: removes both the draft and published version of this item from
-        the course if using draft and old mongo. Split may or may not implement this.
+        :param user_id: ID of the user deleting the item
         :param force: fork the structure and don't update the course draftVersion if there's a version
         conflict (only applicable to version tracking and conflict detecting persistence stores)
 
@@ -426,6 +479,21 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
         """
         raise NotImplementedError
 
+    def create_and_save_xmodule(self, location, user_id, definition_data=None, metadata=None, runtime=None, fields={}):
+        """
+        Create the new xmodule and save it.
+
+        :param location: a Location--must have a category
+        :param user_id: ID of the user creating and saving the xmodule
+        :param definition_data: can be empty. The initial definition_data for the kvs
+        :param metadata: can be empty, the initial metadata for the kvs
+        :param runtime: if you already have an xblock from the course, the xblock.runtime value
+        :param fields: a dictionary of field names and values for the new xmodule
+        """
+        new_object = self.create_xmodule(location, definition_data, metadata, runtime, fields)
+        self.update_item(new_object, user_id, allow_not_found=True)
+        return new_object
+
 
 def only_xmodules(identifier, entry_points):
     """Only use entry_points that are supplied by the xmodule package"""
@@ -441,3 +509,25 @@ def prefer_xmodules(identifier, entry_points):
         return default_select(identifier, from_xmodule)
     else:
         return default_select(identifier, entry_points)
+
+
+class EdxJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
+
+    `Location`s are encoded as their url string form, and `datetime`s as
+    ISO date strings
+    """
+    def default(self, obj):
+        if isinstance(obj, Location):
+            return obj.to_deprecated_string()
+        elif isinstance(obj, datetime.datetime):
+            if obj.tzinfo is not None:
+                if obj.utcoffset() is None:
+                    return obj.isoformat() + 'Z'
+                else:
+                    return obj.isoformat()
+            else:
+                return obj.isoformat()
+        else:
+            return super(EdxJSONEncoder, self).default(obj)

common/lib/xmodule/xmodule/modulestore/django.py

@@ -5,16 +5,18 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 """
 
 from __future__ import absolute_import
-from importlib import import_module
-import re
 
+from importlib import import_module
 from django.conf import settings
 from django.core.cache import get_cache, InvalidCacheBackendError
 import django.utils
 
+import re
+import threading
+
 from xmodule.modulestore.loc_mapper_store import LocMapperStore
 from xmodule.util.django import get_current_request_hostname
-
+import xmodule.modulestore  # pylint: disable=unused-import
 
 # We may not always have the request_cache module available
 try:
@@ -23,10 +25,6 @@ try:
 except ImportError:
     HAS_REQUEST_CACHE = False
 
-_MODULESTORES = {}
-
-FUNCTION_KEYS = ['render_template']
-
 
 def load_function(path):
     """
@@ -48,6 +46,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
     _options = {}
     _options.update(options)
 
+    FUNCTION_KEYS = ['render_template']
     for key in FUNCTION_KEYS:
         if key in _options and isinstance(_options[key], basestring):
             _options[key] = load_function(_options[key])
@@ -69,59 +68,50 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
         xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
         doc_store_config=doc_store_config,
         i18n_service=i18n_service or ModuleI18nService(),
+        branch_setting_func=_get_modulestore_branch_setting,
         **_options
     )
 
 
-def get_default_store_name_for_current_request():
-    """
-    This method will return the appropriate default store mapping for the current Django request,
-    else 'default' which is the system default
-    """
-    store_name = 'default'
+# A singleton instance of the Mixed Modulestore
+_MIXED_MODULESTORE = None
 
-    # see what request we are currently processing - if any at all - and get hostname for the request
-    hostname = get_current_request_hostname()
 
-    # get mapping information which is defined in configurations
-    mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
-
-    # compare hostname against the regex expressions set of mappings
-    # which will tell us which store name to use
-    if hostname and mappings:
-        for key in mappings.keys():
-            if re.match(key, hostname):
-                store_name = mappings[key]
-                return store_name
+def modulestore():
+    """
+    Returns the Mixed modulestore
+    """
+    global _MIXED_MODULESTORE  # pylint: disable=global-statement
+    if _MIXED_MODULESTORE is None:
+        _MIXED_MODULESTORE = create_modulestore_instance(
+            settings.MODULESTORE['default']['ENGINE'],
+            settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}),
+            settings.MODULESTORE['default'].get('OPTIONS', {})
+        )
 
-    return store_name
+    return _MIXED_MODULESTORE
 
 
-def modulestore(name=None):
-    """
-    This returns an instance of a modulestore of given name. This will wither return an existing
-    modulestore or create a new one
+def clear_existing_modulestores():
     """
+    Clear the existing modulestore instances, causing
+    them to be re-created when accessed again.
 
-    if not name:
-        # If caller did not specify name then we should
-        # determine what should be the default
-        name = get_default_store_name_for_current_request()
+    This is useful for flushing state between unit tests.
+    """
+    global _MIXED_MODULESTORE, _loc_singleton  # pylint: disable=global-statement
+    _MIXED_MODULESTORE = None
+    # pylint: disable=W0603
+    cache = getattr(_loc_singleton, "cache", None)
+    if cache:
+        cache.clear()
+    _loc_singleton = None
 
-    if name not in _MODULESTORES:
-        _MODULESTORES[name] = create_modulestore_instance(
-            settings.MODULESTORE[name]['ENGINE'],
-            settings.MODULESTORE[name].get('DOC_STORE_CONFIG', {}),
-            settings.MODULESTORE[name].get('OPTIONS', {})
-        )
-        # inject loc_mapper into newly created modulestore if it needs it
-        if name == 'split' and _loc_singleton is not None:
-            _MODULESTORES['split'].loc_mapper = _loc_singleton
 
-    return _MODULESTORES[name]
+# singleton instance of the loc_mapper
+_loc_singleton = None
 
 
-_loc_singleton = None
 def loc_mapper():
     """
     Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
@@ -137,56 +127,8 @@ def loc_mapper():
             loc_cache = get_cache('default')
         # instantiate
         _loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
-    # inject into split mongo modulestore
-    if 'split' in _MODULESTORES:
-        _MODULESTORES['split'].loc_mapper = _loc_singleton
-    return _loc_singleton
-
 
-def clear_existing_modulestores():
-    """
-    Clear the existing modulestore instances, causing
-    them to be re-created when accessed again.
-
-    This is useful for flushing state between unit tests.
-    """
-    _MODULESTORES.clear()
-    # pylint: disable=W0603
-    global _loc_singleton
-    cache = getattr(_loc_singleton, "cache", None)
-    if cache:
-        cache.clear()
-    _loc_singleton = None
-
-
-def editable_modulestore(name='default'):
-    """
-    Retrieve a modulestore that we can modify.
-    This is useful for tests that need to insert test
-    data into the modulestore.
-
-    Currently, only Mongo-backed modulestores can be modified.
-    Returns `None` if no editable modulestore is available.
-    """
-
-    # Try to retrieve the ModuleStore
-    # Depending on the settings, this may or may not
-    # be editable.
-    store = modulestore(name)
-
-    # If this is a `MixedModuleStore`, then we will need
-    # to retrieve the actual Mongo instance.
-    # We assume that the default is Mongo.
-    if hasattr(store, 'modulestores'):
-        store = store.modulestores['default']
-
-    # At this point, we either have the ability to create
-    # items in the store, or we do not.
-    if hasattr(store, 'create_course'):
-        return store
-
-    else:
-        return None
+    return _loc_singleton
 
 
 class ModuleI18nService(object):
@@ -217,3 +159,40 @@ class ModuleI18nService(object):
         # then Cale was a liar.
         from util.date_utils import strftime_localized
         return strftime_localized(*args, **kwargs)
+
+
+# thread local cache
+_THREAD_CACHE = threading.local()
+
+
+def _get_modulestore_branch_setting():
+    """
+    Returns the branch setting for the module store from the current Django request if configured,
+    else returns the branch value from the configuration settings if set,
+    else returns None
+
+    The value of the branch setting is cached in a thread-local variable so it is not repeatedly recomputed
+    """
+    def get_branch_setting():
+        """
+        Finds and returns the branch setting based on the Django request and the configuration settings
+        """
+        branch = None
+        hostname = get_current_request_hostname()
+        if hostname:
+            # get mapping information which is defined in configurations
+            mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
+
+            # compare hostname against the regex expressions set of mappings which will tell us which branch to use
+            if mappings:
+                for key in mappings.iterkeys():
+                    if re.match(key, hostname):
+                        return mappings[key]
+        if branch is None:
+            branch = getattr(settings, 'MODULESTORE_BRANCH', None)
+        return branch
+
+    # cache the branch setting for this thread so we don't have to recompute it each time
+    if not hasattr(_THREAD_CACHE, 'branch_setting'):
+        _THREAD_CACHE.branch_setting = get_branch_setting()
+    return _THREAD_CACHE.branch_setting

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -27,6 +27,15 @@ class NoPathToItem(Exception):
     pass
 
 
+class ReferentialIntegrityError(Exception):
+    """
+    An incorrect pointer to an object exists. For example, 2 parents point to the same child, an
+    xblock points to a nonexistent child (which probably raises ItemNotFoundError instead depending
+    on context).
+    """
+    pass
+
+
 class DuplicateItemError(Exception):
     """
     Attempted to create an item which already exists.
@@ -66,3 +75,13 @@ class DuplicateCourseError(Exception):
         super(DuplicateCourseError, self).__init__()
         self.course_id = course_id
         self.existing_entry = existing_entry
+
+
+class InvalidBranchSetting(Exception):
+    """
+    Raised when the process' branch setting did not match the required setting for the attempted operation on a store.
+    """
+    def __init__(self, expected_setting, actual_setting):
+        super(InvalidBranchSetting, self).__init__()
+        self.expected_setting = expected_setting
+        self.actual_setting = actual_setting

common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py

@@ -7,6 +7,7 @@ import pymongo
 import bson.son
 import urllib
 
+from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT
 from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
 from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -53,8 +54,8 @@ class LocMapperStore(object):
         self.cache = cache
 
     # location_map functions
-    def create_map_entry(self, course_key, org=None, offering=None, draft_branch='draft', prod_branch='published',
-                         block_map=None):
+    def create_map_entry(self, course_key, org=None, offering=None,
+                         draft_branch=BRANCH_NAME_DRAFT, prod_branch=BRANCH_NAME_PUBLISHED, block_map=None):
         """
         Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & offering. If
         org and offering are not provided, it defaults them based on course_key.
@@ -244,7 +245,7 @@ class LocMapperStore(object):
         for old_name, cat_to_usage in entry['block_map'].iteritems():
             for category, block_id in cat_to_usage.iteritems():
                 # cache all entries and then figure out if we have the one we want
-                # Always return revision=None because the
+                # Always return revision=KEY_REVISION_PUBLISHED because the
                 # old draft module store wraps locations as draft before
                 # trying to access things.
                 location = old_course_id.make_usage_key(

common/lib/xmodule/xmodule/modulestore/search.py

-from itertools import repeat
 from .exceptions import (ItemNotFoundError, NoPathToItem)
 
 
@@ -53,11 +52,8 @@ def path_to_location(modulestore, usage_key):
         while len(queue) > 0:
             (next_usage, path) = queue.pop()  # Takes from the end
 
-            # get_parent_locations should raise ItemNotFoundError if location
-            # isn't found so we don't have to do it explicitly.  Call this
-            # first to make sure the location is there (even if it's a course, and
-            # we would otherwise immediately exit).
-            parents = modulestore.get_parent_locations(next_usage)
+            # get_parent_location raises ItemNotFoundError if location isn't found
+            parent = modulestore.get_parent_location(next_usage)
 
             # print 'Processing loc={0}, path={1}'.format(next_usage, path)
             if next_usage.definition_key.block_type == "course":
@@ -67,7 +63,7 @@ def path_to_location(modulestore, usage_key):
 
             # otherwise, add parent locations at the end
             newpath = (next_usage, path)
-            queue.extend(zip(parents, repeat(newpath)))
+            queue.append((parent, newpath))
 
         # If we're here, there is no path
         return None

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -2,6 +2,7 @@ import re
 import logging
 
 from xmodule.contentstore.content import StaticContent
+from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_ONLY
 
 
 def _prefix_only_url_replace_regex(prefix):
@@ -87,10 +88,12 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
     return text
 
 
-def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
+def _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id):
     for module in modules:
         original_loc = module.location
         module.location = module.location.map_into_course(dest_course_id)
+        if module.location.category == 'course':
+            module.location = module.location.replace(name=module.location.run)
 
         print "Cloning module {0} to {1}....".format(original_loc, module.location)
 
@@ -108,10 +111,10 @@ def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
 
             module.children = new_children
 
-        modulestore.update_item(module, '**replace_user**')
+        modulestore.update_item(module, user_id, allow_not_found=True)
 
 
-def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
+def clone_course(modulestore, contentstore, source_course_id, dest_course_id, user_id):
     # check to see if the dest_location exists as an empty course
     # we need an empty course because the app layers manage the permissions and users
     if not modulestore.has_course(dest_course_id):
@@ -120,29 +123,26 @@ def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
     # verify that the dest_location really is an empty course, which means only one with an optional 'overview'
     dest_modules = modulestore.get_items(dest_course_id)
 
-    basically_empty = True
     for module in dest_modules:
-        if module.location.category == 'course' or (module.location.category == 'about'
-                                                    and module.location.name == 'overview'):
+        if module.location.category == 'course' or (
+                module.location.category == 'about' and module.location.name == 'overview'
+        ):
             continue
-
-        basically_empty = False
-        break
-
-    if not basically_empty:
-        raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
+        # only course and about overview allowed
+        raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_course_id))
 
     # check to see if the source course is actually there
     if not modulestore.has_course(source_course_id):
         raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
 
     # Get all modules under this namespace which is (tag, org, course) tuple
+    modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_PUBLISHED_ONLY)
+    _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
+    course_location = dest_course_id.make_usage_key('course', dest_course_id.run)
+    modulestore.publish(course_location, user_id)
 
-    modules = modulestore.get_items(source_course_id, revision=None)
-    _clone_modules(modulestore, modules, source_course_id, dest_course_id)
-
-    modules = modulestore.get_items(source_course_id, revision='draft')
-    _clone_modules(modulestore, modules, source_course_id, dest_course_id)
+    modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_DRAFT_ONLY)
+    _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
 
     # now iterate through all of the assets and clone them
     # first the thumbnails

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -5,7 +5,8 @@ Modulestore configuration for test cases.
 from uuid import uuid4
 from django.test import TestCase
 from xmodule.modulestore.django import (
-    editable_modulestore, clear_existing_modulestores, loc_mapper)
+    modulestore, clear_existing_modulestores, loc_mapper)
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
 from xmodule.contentstore.django import contentstore
 
 
@@ -25,7 +26,7 @@ def mixed_store_config(data_dir, mappings):
     where 'xml' and 'default' are the two options provided by this configuration,
     mapping (respectively) to XML-backed and Mongo-backed modulestores..
     """
-    mongo_config = mongo_store_config(data_dir)
+    draft_mongo_config = draft_mongo_store_config(data_dir)
     xml_config = xml_store_config(data_dir)
 
     store = {
@@ -33,40 +34,13 @@ def mixed_store_config(data_dir, mappings):
             'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
             'OPTIONS': {
                 'mappings': mappings,
-                'stores': {
-                    'default': mongo_config['default'],
-                    'xml': xml_config['default']
-                }
+                'stores': [
+                    draft_mongo_config['default'],
+                    xml_config['default']
+                ]
             }
         }
     }
-    store['direct'] = store['default']
-    return store
-
-
-def mongo_store_config(data_dir):
-    """
-    Defines default module store using MongoModuleStore.
-
-    Use of this config requires mongo to be running.
-    """
-    store = {
-        'default': {
-            'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-            'DOC_STORE_CONFIG': {
-                'host': 'localhost',
-                'db': 'test_xmodule',
-                'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
-            },
-            'OPTIONS': {
-                'default_class': 'xmodule.raw_module.RawDescriptor',
-                'fs_root': data_dir,
-                'render_template': 'edxmako.shortcuts.render_to_string'
-            }
-        }
-    }
-
-    store['direct'] = store['default']
     return store
 
 
@@ -83,6 +57,7 @@ def draft_mongo_store_config(data_dir):
 
     store = {
         'default': {
+            'NAME': 'draft',
             'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
             'DOC_STORE_CONFIG': {
                 'host': 'localhost',
@@ -93,7 +68,6 @@ def draft_mongo_store_config(data_dir):
         }
     }
 
-    store['direct'] = store['default']
     return store
 
 
@@ -103,6 +77,7 @@ def xml_store_config(data_dir):
     """
     store = {
         'default': {
+            'NAME': 'xml',
             'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
             'OPTIONS': {
                 'data_dir': data_dir,
@@ -111,48 +86,6 @@ def xml_store_config(data_dir):
         }
     }
 
-    store['direct'] = store['default']
-    return store
-
-
-def studio_store_config(data_dir):
-    """
-    Defines modulestore structure used by Studio tests.
-    """
-    store_config = {
-        'host': 'localhost',
-        'db': 'test_xmodule',
-        'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
-    }
-    options = {
-        'default_class': 'xmodule.raw_module.RawDescriptor',
-        'fs_root': data_dir,
-        'render_template': 'edxmako.shortcuts.render_to_string',
-    }
-
-    store = {
-        'default': {
-            'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-            'DOC_STORE_CONFIG': store_config,
-            'OPTIONS': options
-        },
-        'direct': {
-            'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-            'DOC_STORE_CONFIG': store_config,
-            'OPTIONS': options
-        },
-        'draft': {
-            'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
-            'DOC_STORE_CONFIG': store_config,
-            'OPTIONS': options
-        },
-        'split': {
-            'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
-            'DOC_STORE_CONFIG': store_config,
-            'OPTIONS': options
-        }
-    }
-
     return store
 
 
@@ -203,20 +136,19 @@ class ModuleStoreTestCase(TestCase):
         'course' is an instance of CourseDescriptor for which we want
         to update metadata.
         """
-        store = editable_modulestore()
+        store = modulestore()
         store.update_item(course, '**replace_user**')
         updated_course = store.get_course(course.id)
         return updated_course
 
     @staticmethod
-    def drop_mongo_collections(store_name='default'):
+    def drop_mongo_collections(modulestore_type=MONGO_MODULESTORE_TYPE):
         """
         If using a Mongo-backed modulestore & contentstore, drop the collections.
         """
-
-        # This will return the mongo-backed modulestore
-        # even if we're using a mixed modulestore
-        store = editable_modulestore(store_name)
+        store = modulestore()
+        if hasattr(store, '_get_modulestore_by_type'):
+            store = store._get_modulestore_by_type(modulestore_type)  # pylint: disable=W0212
         if hasattr(store, 'collection'):
             connection = store.collection.database.connection
             store.collection.drop()
@@ -268,7 +200,7 @@ class ModuleStoreTestCase(TestCase):
 
     def _pre_setup(self):
         """
-        Flush the ModuleStore before each test.
+        Flush the ModuleStore.
         """
 
         # Flush the Mongo modulestore

common/lib/xmodule/xmodule/modulestore/tests/factories.py

@@ -5,6 +5,10 @@ from uuid import uuid4
 from xmodule.modulestore import prefer_xmodules
 from opaque_keys.edx.locations import Location
 from xblock.core import XBlock
+from xmodule.tabs import StaticTab
+from decorator import contextmanager
+from mock import Mock, patch
+from nose.tools import assert_less_equal
 
 
 class Dummy(object):
@@ -23,10 +27,8 @@ class XModuleFactory(Factory):
 
     @lazy_attribute
     def modulestore(self):
-        # Delayed import so that we only depend on django if the caller
-        # hasn't provided their own modulestore
-        from xmodule.modulestore.django import editable_modulestore
-        return editable_modulestore('direct')
+        from xmodule.modulestore.django import modulestore
+        return modulestore()
 
 
 class CourseFactory(XModuleFactory):
@@ -63,7 +65,7 @@ class CourseFactory(XModuleFactory):
         # Save the attributes we just set
         new_course.save()
         # Update the data in the mongo datastore
-        store.update_item(new_course)
+        store.update_item(new_course, '**replace_user**')
         return new_course
 
 
@@ -141,6 +143,7 @@ class ItemFactory(XModuleFactory):
         display_name = kwargs.pop('display_name', None)
         metadata = kwargs.pop('metadata', {})
         location = kwargs.pop('location')
+        user_id = kwargs.pop('user_id', 999)
 
         assert isinstance(location, Location)
         assert location != parent_location
@@ -162,7 +165,8 @@ class ItemFactory(XModuleFactory):
         # replace the display name with an optional parameter passed in from the caller
         if display_name is not None:
             metadata['display_name'] = display_name
-        store.create_and_save_xmodule(location, metadata=metadata, definition_data=data)
+        runtime = parent.runtime if parent else None
+        store.create_and_save_xmodule(location, user_id, metadata=metadata, definition_data=data, runtime=runtime)
 
         module = store.get_item(location)
 
@@ -171,10 +175,53 @@ class ItemFactory(XModuleFactory):
         # Save the attributes we just set
         module.save()
 
-        store.update_item(module)
+        store.update_item(module, '**replace_user**')
 
         if 'detached' not in module._class_tags:
             parent.children.append(location)
             store.update_item(parent, '**replace_user**')
 
+        # VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
+        # if we add one then we need to also add it to the policy information (i.e. metadata)
+        # we should remove this once we can break this reference from the course to static tabs
+        if category == 'static_tab':
+            course = store.get_course(location.course_key)
+            course.tabs.append(
+                StaticTab(
+                    name=display_name,
+                    url_slug=location.name,
+                )
+            )
+            store.update_item(course, '**replace_user**')
+
         return store.get_item(location)
+
+
+@contextmanager
+def check_mongo_calls(mongo_store, max_finds=0, max_sends=None):
+    """
+    Instruments the given store to count the number of calls to find (incl find_one) and the number
+    of calls to send_message which is for insert, update, and remove (if you provide max_sends). At the
+    end of the with statement, it compares the counts to the max_finds and max_sends using a simple
+    assertLessEqual.
+
+    :param mongo_store: the MongoModulestore or subclass to watch
+    :param max_finds: the maximum number of find calls to allow
+    :param max_sends: If none, don't instrument the send calls. If non-none, count and compare to
+        the given int value.
+    """
+    try:
+        find_wrap = Mock(wraps=mongo_store.collection.find)
+        wrap_patch = patch.object(mongo_store.collection, 'find', find_wrap)
+        wrap_patch.start()
+        if max_sends:
+            sends_wrap = Mock(wraps=mongo_store.database.connection._send_message)
+            sends_patch = patch.object(mongo_store.database.connection, '_send_message', sends_wrap)
+            sends_patch.start()
+        yield
+    finally:
+        wrap_patch.stop()
+        if max_sends:
+            sends_patch.stop()
+            assert_less_equal(sends_wrap.call_count, max_sends)
+        assert_less_equal(find_wrap.call_count, max_finds)

common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py

+from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, BRANCH_NAME_DRAFT
 from xmodule.course_module import CourseDescriptor
 from xmodule.x_module import XModuleDescriptor
 import factory
@@ -14,7 +15,7 @@ class SplitFactory(factory.Factory):
         # Delayed import so that we only depend on django if the caller
         # hasn't provided their own modulestore
         from xmodule.modulestore.django import modulestore
-        return modulestore('split')
+        return modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
 
 
 class PersistentCourseFactory(SplitFactory):
@@ -24,7 +25,7 @@ class PersistentCourseFactory(SplitFactory):
     keywords: any xblock field plus (note, the below are filtered out; so, if they
     become legitimate xblock fields, they won't be settable via this factory)
     * org: defaults to textX
-    * master_branch: (optional) defaults to 'draft'
+    * master_branch: (optional) defaults to BRANCH_NAME_DRAFT
     * user_id: (optional) defaults to 'test_user'
     * display_name (xblock field): will default to 'Robot Super Course' unless provided
     """
@@ -33,7 +34,7 @@ class PersistentCourseFactory(SplitFactory):
     # pylint: disable=W0613
     @classmethod
     def _create(cls, target_class, offering='999', org='testX', user_id='test_user',
-                master_branch='draft', **kwargs):
+                master_branch=BRANCH_NAME_DRAFT, **kwargs):
 
         modulestore = kwargs.pop('modulestore')
         root_block_id = kwargs.pop('root_block_id', 'course')

common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py

@@ -5,6 +5,7 @@ import unittest
 import uuid
 from opaque_keys.edx.locations import Location
 from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
+from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT, KEY_REVISION_PUBLISHED
 from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
 from xmodule.modulestore.loc_mapper_store import LocMapperStore
 from mock import Mock
@@ -62,8 +63,8 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         self.assertIsNotNone(entry, "Didn't find entry")
         self.assertEqual(entry['org'], org)
         self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
-        self.assertEqual(entry['draft_branch'], 'draft')
-        self.assertEqual(entry['prod_branch'], 'published')
+        self.assertEqual(entry['draft_branch'], BRANCH_NAME_DRAFT)
+        self.assertEqual(entry['prod_branch'], BRANCH_NAME_PUBLISHED)
         self.assertEqual(entry['block_map'], {})
 
         course2 = 'quux_course'
@@ -123,7 +124,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         """
         prob_locator = loc_mapper().translate_location(
             location,
-            published=(branch == 'published'),
+            published=(branch == BRANCH_NAME_PUBLISHED),
             add_entry_if_missing=add_entry
         )
         self.assertEqual(prob_locator.org, org)
@@ -133,7 +134,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
 
         course_locator = loc_mapper().translate_location_to_course_locator(
             location.course_key,
-            published=(branch == 'published'),
+            published=(branch == BRANCH_NAME_PUBLISHED),
         )
         self.assertEqual(course_locator.org, org)
         self.assertEqual(course_locator.offering, offering)
@@ -168,7 +169,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         )
         test_problem_locn = Location(org, course, run, 'problem', 'abc123')
 
-        self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published')
+        self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED)
         # look for non-existent problem
         with self.assertRaises(ItemNotFoundError):
             loc_mapper().translate_location(
@@ -183,7 +184,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         test_no_cat_locn = test_no_cat_locn.replace(name='def456')
 
         self.translate_n_check(
-            test_no_cat_locn, new_style_org, new_style_offering, 'problem4', 'published'
+            test_no_cat_locn, new_style_org, new_style_offering, 'problem4', BRANCH_NAME_PUBLISHED
         )
 
         # add a distractor course (note that abc123 has a different translation in this one)
@@ -202,12 +203,12 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         )
         # test that old translation still works
         self.translate_n_check(
-            test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published'
+            test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED
         )
         # and new returns new id
         self.translate_n_check(
             test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
-            'problem3', 'published'
+            'problem3', BRANCH_NAME_PUBLISHED
         )
 
     def test_translate_location_dwim(self):
@@ -221,11 +222,11 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         problem_name = 'abc123abc123abc123abc123abc123f9'
         location = Location(org, course, run, 'problem', problem_name)
         new_offering = '{}.{}'.format(course, run)
-        self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
+        self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
 
         # create an entry w/o a guid name
         other_location = Location(org, course, run, 'chapter', 'intro')
-        self.translate_n_check(other_location, org, new_offering, 'intro', 'published', True)
+        self.translate_n_check(other_location, org, new_offering, 'intro', BRANCH_NAME_PUBLISHED, True)
 
         # add a distractor course
         delta_new_org = '{}.geek_dept'.format(org)
@@ -237,7 +238,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
             delta_new_org, delta_new_offering,
             block_map={problem_name: {'problem': 'problem3'}}
         )
-        self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
+        self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
 
         # add a new one to both courses (ensure name doesn't have same beginning)
         new_prob_name = uuid.uuid4().hex
@@ -245,10 +246,10 @@ class TestLocationMapper(LocMapperSetupSansDjango):
             new_prob_name = uuid.uuid4().hex
         new_prob_locn = location.replace(name=new_prob_name)
         new_usage_id = 'problem{}'.format(new_prob_name[:3])
-        self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, 'published', True)
+        self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True)
         new_prob_locn = new_prob_locn.replace(run=run)
         self.translate_n_check(
-            new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, 'published', True
+            new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True
         )
 
     def test_translate_locator(self):
@@ -263,7 +264,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         new_style_offering = '{}.{}'.format(course, run)
         prob_course_key = CourseLocator(
             org=new_style_org, offering=new_style_offering,
-            branch='published',
+            branch=BRANCH_NAME_PUBLISHED,
         )
         prob_locator = BlockUsageLocator(
             prob_course_key,
@@ -285,22 +286,22 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         # only one course matches
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
         # default branch
-        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
+        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
         # test get_course keyword
         prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
         self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
         # explicit branch
-        prob_locator = prob_locator.for_branch('draft')
+        prob_locator = prob_locator.for_branch(BRANCH_NAME_DRAFT)
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
-        # Even though the problem was set as draft, we always return revision=None to work
+        # Even though the problem was set as draft, we always return revision= KEY_REVISION_PUBLISHED to work
         # with old mongo/draft modulestores.
-        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
+        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
         prob_locator = BlockUsageLocator(
             prob_course_key.for_branch('production'),
             block_type='problem', block_id='problem2'
         )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
-        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
+        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
         # same for chapter except chapter cannot be draft in old system
         chap_locator = BlockUsageLocator(
             prob_course_key.for_branch('production'),
@@ -309,7 +310,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
         chap_location = loc_mapper().translate_locator_to_location(chap_locator)
         self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
         # explicit branch
-        chap_locator = chap_locator.for_branch('draft')
+        chap_locator = chap_locator.for_branch(BRANCH_NAME_DRAFT)
         chap_location = loc_mapper().translate_locator_to_location(chap_locator)
         self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
         chap_locator = BlockUsageLocator(
@@ -320,7 +321,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
 
         # look for non-existent problem
         prob_locator2 = BlockUsageLocator(
-            prob_course_key.for_branch('draft'),
+            prob_course_key.for_branch(BRANCH_NAME_DRAFT),
             block_type='problem', block_id='problem3'
         )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
@@ -335,7 +336,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
             block_map={'abc123': {'problem': 'problem3'}}
         )
         prob_location = loc_mapper().translate_locator_to_location(prob_locator)
-        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
+        self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
 
     def test_special_chars(self):
         """

common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py

@@ -63,8 +63,10 @@ def check_has_course_method(modulestore, locator, locator_key_fields):
             ]
             for changes in locator_case_changes:
                 search_locator = locator.replace(**changes)
+                # if ignore_case is true, the course would be found with a different-cased course locator.
+                # if ignore_case is false, the course should NOT found given an incorrectly-cased locator.
                 assert_equals(
-                    modulestore.has_course(search_locator, ignore_case),
+                    modulestore.has_course(search_locator, ignore_case) is not None,
                     ignore_case,
                     error_message.format(search_locator, ignore_case)
                 )

common/lib/xmodule/xmodule/modulestore/tests/test_publish.py

@@ -3,6 +3,7 @@ Test the publish code (mostly testing that publishing doesn't result in orphans)
 """
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
+from xmodule.modulestore.tests.factories import check_mongo_calls
 
 
 class TestPublish(SplitWMongoCourseBoostrapper):
@@ -14,53 +15,48 @@ class TestPublish(SplitWMongoCourseBoostrapper):
         Create the course, publish all verticals
         * some detached items
         """
-        super(TestPublish, self)._create_course(split=False)
+        # There should be 12 inserts and 11 updates (max_sends)
+        # Should be 1 to verify course unique, 11 parent fetches,
+        # and n per _create_item where n is the size of the course tree non-leaf nodes
+        # for inheritance computation (which is 7*4 + sum(1..4) = 38) (max_finds)
+        with check_mongo_calls(self.draft_mongo, 70, 27):
+            with check_mongo_calls(self.old_mongo, 70, 27):
+                super(TestPublish, self)._create_course(split=False)
 
-        self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
-        self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
-        self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
-        self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
-        self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
-        self._create_item(
-            'discussion', 'Discussion1',
-            "discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
-            {
-                "discussion_category": "Lecture 1",
-                "discussion_target": "Lecture 1",
-                "display_name": "Lecture 1 Discussion",
-                "discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
-            },
-            'vertical', 'Vert1',
-            split=False
-        )
-        self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
-        self._create_item(
-            'discussion', 'Discussion2',
-            "discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
-            {
-                "discussion_category": "Lecture 2",
-                "discussion_target": "Lecture 2",
-                "display_name": "Lecture 2 Discussion",
-                "discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
-            },
-            'vertical', 'Vert2',
-            split=False
-        )
-        self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
-        self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
-        self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
+                self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
+                self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
+                self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
+                self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
+                self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
+                self._create_item(
+                    'discussion', 'Discussion1',
+                    "discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
+                    {
+                        "discussion_category": "Lecture 1",
+                        "discussion_target": "Lecture 1",
+                        "display_name": "Lecture 1 Discussion",
+                        "discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
+                    },
+                    'vertical', 'Vert1',
+                    split=False
+                )
+                self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
+                self._create_item(
+                    'discussion', 'Discussion2',
+                    "discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
+                    {
+                        "discussion_category": "Lecture 2",
+                        "discussion_target": "Lecture 2",
+                        "display_name": "Lecture 2 Discussion",
+                        "discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
+                    },
+                    'vertical', 'Vert2',
+                    split=False
+                )
+                self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
+                self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
+                self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
 
-    def _xmodule_recurse(self, item, action):
-        """
-        Applies action depth-first down tree and to item last.
-
-        A copy of  cms.djangoapps.contentstore.views.helpers._xmodule_recurse to reproduce its use and behavior
-        outside of django.
-        """
-        for child in item.get_children():
-            self._xmodule_recurse(child, action)
-
-        action(item)
 
     def test_publish_draft_delete(self):
         """
@@ -69,29 +65,31 @@ class TestPublish(SplitWMongoCourseBoostrapper):
         """
         location = self.old_course_key.make_usage_key('vertical', name='Vert1')
         item = self.draft_mongo.get_item(location, 2)
-        self._xmodule_recurse(
-            item,
-            lambda i: self.draft_mongo.publish(i.location, self.userid)
-        )
+        # Vert1 has 3 children; so, publishes 4 nodes which may mean 4 inserts & 1 bulk remove
+        # 25-June-2014 find calls are 19. Probably due to inheritance recomputation?
+        with check_mongo_calls(self.draft_mongo, 19, 5):
+            self.draft_mongo.publish(item.location, self.userid)
+
         # verify status
         item = self.draft_mongo.get_item(location, 0)
         self.assertFalse(getattr(item, 'is_draft', False), "Item was published. Draft should not exist")
         # however, children are still draft, but I'm not sure that's by design
 
         # convert back to draft
-        self.draft_mongo.convert_to_draft(location)
+        self.draft_mongo.convert_to_draft(location, self.userid)
         # both draft and published should exist
         draft_vert = self.draft_mongo.get_item(location, 0)
         self.assertTrue(getattr(draft_vert, 'is_draft', False), "Item was converted to draft but doesn't say so")
         item = self.old_mongo.get_item(location, 0)
         self.assertFalse(getattr(item, 'is_draft', False), "Published item doesn't say so")
 
-        # delete the discussion (which oddly is not in draft mode)
+        # delete the draft version of the discussion
         location = self.old_course_key.make_usage_key('discussion', name='Discussion1')
-        self.draft_mongo.delete_item(location)
-        # remove pointer from draft vertical (verify presence first to ensure process is valid)
-        self.assertIn(location, draft_vert.children)
-        draft_vert.children.remove(location)
+        self.draft_mongo.delete_item(location, self.userid)
+
+        draft_vert = self.draft_mongo.get_item(draft_vert.location, 0)
+        # remove pointer from draft vertical (still there b/c not refetching vert)
+        self.assertNotIn(location, draft_vert.children)
         # move the other child
         other_child_loc = self.old_course_key.make_usage_key('html', name='Html2')
         draft_vert.children.remove(other_child_loc)
@@ -100,12 +98,10 @@ class TestPublish(SplitWMongoCourseBoostrapper):
         self.draft_mongo.update_item(draft_vert, self.userid)
         self.draft_mongo.update_item(other_vert, self.userid)
         # publish
-        self._xmodule_recurse(
-            draft_vert,
-            lambda i: self.draft_mongo.publish(i.location, self.userid)
-        )
+        self.draft_mongo.publish(draft_vert.location, self.userid)
         item = self.old_mongo.get_item(draft_vert.location, 0)
         self.assertNotIn(location, item.children)
+        self.assertIsNone(self.draft_mongo.get_parent_location(location))
         with self.assertRaises(ItemNotFoundError):
             self.draft_mongo.get_item(location)
         self.assertNotIn(other_child_loc, item.children)

common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py

@@ -5,9 +5,9 @@ Tests for split_migrator
 import uuid
 import random
 import mock
+from xmodule.modulestore import KEY_REVISION_PUBLISHED
 from xmodule.modulestore.loc_mapper_store import LocMapperStore
 from xmodule.modulestore.split_migrator import SplitMigrator
-from xmodule.modulestore.mongo import draft
 from xmodule.modulestore.tests import test_location_mapper
 from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
 from xblock.fields import Reference, ReferenceList, ReferenceValueDict
@@ -177,7 +177,9 @@ class TestMigration(SplitWMongoCourseBoostrapper):
         # check that locations match
         self.assertEqual(
             presplit_dag_root.location,
-            self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(revision=None)
+            self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(
+                revision=KEY_REVISION_PUBLISHED
+            )
         )
         # compare all fields but children
         for name, field in presplit_dag_root.fields.iteritems():

common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py

@@ -9,6 +9,8 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
 from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
 from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
 from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
+from xmodule.modulestore import BRANCH_DRAFT_PREFERRED, BRANCH_NAME_DRAFT
+from mock import Mock
 
 
 class SplitWMongoCourseBoostrapper(unittest.TestCase):
@@ -39,7 +41,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
         'xblock_mixins': (InheritanceMixin,)
     }
 
-    split_course_key = CourseLocator('test_org', 'test_course.runid', branch='draft')
+    split_course_key = CourseLocator('test_org', 'test_course.runid', branch=BRANCH_NAME_DRAFT)
 
     def setUp(self):
         self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
@@ -53,7 +55,9 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
         self.addCleanup(self.split_mongo.db.connection.close)
         self.addCleanup(self.tear_down_split)
         self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
-        self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
+        self.draft_mongo = DraftMongoModuleStore(
+            self.db_config, branch_setting_func=lambda: BRANCH_DRAFT_PREFERRED, **self.modulestore_options
+        )
         self.addCleanup(self.tear_down_mongo)
         self.old_course_key = None
         self.runtime = None
@@ -86,7 +90,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
             mongo = self.old_mongo
         else:
             mongo = self.draft_mongo
-        mongo.create_and_save_xmodule(location, data, metadata, self.runtime)
+        mongo.create_and_save_xmodule(location, self.userid, definition_data=data, metadata=metadata, runtime=self.runtime)
         if isinstance(data, basestring):
             fields = {'data': data}
         else:

common/lib/xmodule/xmodule/modulestore/tests/test_xml.py

@@ -43,7 +43,7 @@ class TestXMLModuleStore(unittest.TestCase):
 
     def test_xml_modulestore_type(self):
         store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
-        self.assertEqual(store.get_modulestore_type('foo/bar/baz'), XML_MODULESTORE_TYPE)
+        self.assertEqual(store.get_modulestore_type(), XML_MODULESTORE_TYPE)
 
     def test_unicode_chars_in_xml_content(self):
         # edX/full/6.002_Spring_2012 has non-ASCII chars, and during

common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py

@@ -8,7 +8,7 @@ from xblock.runtime import Runtime, KvsFieldData, DictKeyValueStore
 from xmodule.x_module import XModuleMixin
 from opaque_keys.edx.locations import Location
 from xmodule.modulestore.inheritance import InheritanceMixin
-from xmodule.modulestore.xml_importer import import_module
+from xmodule.modulestore.xml_importer import _import_module_and_update_references
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.tests import DATA_DIR
 from uuid import uuid4
@@ -141,9 +141,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Move to different runtime w/ different course id
         target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
-        new_version = import_module(
+        new_version = _import_module_and_update_references(
             self.xblock,
             modulestore(),
+            999,
             self.xblock.location.course_key,
             target_location_namespace,
             do_import_static=False
@@ -177,9 +178,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Remap the namespace
         target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
-        new_version = import_module(
+        new_version = _import_module_and_update_references(
             self.xblock,
             modulestore(),
+            999,
             self.xblock.location.course_key,
             target_location_namespace.course_key,
             do_import_static=False
@@ -208,9 +210,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Remap the namespace
         target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
-        new_version = import_module(
+        new_version = _import_module_and_update_references(
             self.xblock,
             modulestore(),
+            999,
             self.xblock.location.course_key,
             target_location_namespace.course_key,
             do_import_static=False

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -19,6 +19,7 @@ from xmodule.errortracker import make_error_tracker, exc_info_to_str
 from xmodule.mako_module import MakoDescriptorSystem
 from xmodule.x_module import XMLParsingSystem, policy_key
 from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
+from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY
 from xmodule.tabs import CourseTabList
 from opaque_keys.edx.keys import UsageKey
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
@@ -67,6 +68,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
         # cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
         self.course_id = course_id
         self.load_error_modules = load_error_modules
+        self.modulestore = xmlstore
 
         def process_xml(xml):
             """Takes an xml string, and returns a XBlock created from
@@ -332,7 +334,7 @@ class ParentTracker(object):
         """
         Init
         """
-        # location -> set(parents).  Not using defaultdict because we care about the empty case.
+        # location -> parent.  Not using defaultdict because we care about the empty case.
         self._parents = dict()
 
     def add_parent(self, child, parent):
@@ -341,8 +343,7 @@ class ParentTracker(object):
 
         child and parent must be :class:`.Location` instances.
         """
-        setp = self._parents.setdefault(child, set())
-        setp.add(parent)
+        self._parents[child] = parent
 
     def is_known(self, child):
         """
@@ -353,13 +354,13 @@ class ParentTracker(object):
     def make_known(self, location):
         """Tell the parent tracker about an object, without registering any
         parents for it.  Used for the top level course descriptor locations."""
-        self._parents.setdefault(location, set())
+        self._parents.setdefault(location, None)
 
-    def parents(self, child):
+    def parent(self, child):
         """
-        Return a list of the parents of this child.  If not is_known(child), will throw a KeyError
+        Return the parent of this child.  If not is_known(child), will throw a KeyError
         """
-        return list(self._parents[child])
+        return self._parents[child]
 
 
 class XMLModuleStore(ModuleStoreReadBase):
@@ -409,6 +410,9 @@ class XMLModuleStore(ModuleStoreReadBase):
 
         self.i18n_service = i18n_service
 
+        # The XML Module Store is a read-only store and only handles published content
+        self.branch_setting_func = lambda: REVISION_OPTION_PUBLISHED_ONLY
+
         # If we are specifically asked for missing courses, that should
         # be an error.  If we are asked for "all" courses, find the ones
         # that have a course.xml. We sort the dirs in alpha order so we always
@@ -785,25 +789,25 @@ class XMLModuleStore(ModuleStoreReadBase):
         # here just to quell the abstractmethod. someone could write the impl if needed
         raise NotImplementedError
 
-    def get_parent_locations(self, location):
-        '''Find all locations that are the parents of this location in this
+    def get_parent_location(self, location, **kwargs):
+        '''Find the location that is the parent of this location in this
         course.  Needed for path_to_location().
-
-        returns an iterable of things that can be passed to Location.  This may
-        be empty if there are no parents.
         '''
         if not self.parent_trackers[location.course_key].is_known(location):
             raise ItemNotFoundError("{0} not in {1}".format(location, location.course_key))
 
-        return self.parent_trackers[location.course_key].parents(location)
+        return self.parent_trackers[location.course_key].parent(location)
 
-    def get_modulestore_type(self, course_id):
+    def get_modulestore_type(self, course_key=None):
         """
         Returns an enumeration-like type reflecting the type of this modulestore
         The return can be one of:
         "xml" (for XML based courses),
         "mongo" for old-style MongoDB backed courses,
         "split" for new-style split MongoDB backed courses.
+
+        Args:
+            course_key: just for signature compatibility
         """
         return XML_MODULESTORE_TYPE
 

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

@@ -7,15 +7,18 @@ import lxml.etree
 from xblock.fields import Scope
 from xmodule.contentstore.content import StaticContent
 from xmodule.exceptions import NotFoundError
-from opaque_keys.edx.locations import Location
+from xmodule.modulestore import (
+    EdxJSONEncoder, BRANCH_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_PREFERRED, REVISION_OPTION_DRAFT_ONLY
+)
 from xmodule.modulestore.inheritance import own_metadata
+from xmodule.modulestore.mixed import store_branch_setting
 from fs.osfs import OSFS
 from json import dumps
 import json
-import datetime
 import os
 from path import path
 import shutil
+from xmodule.modulestore.mongo.base import DIRECT_ONLY_CATEGORIES
 
 DRAFT_DIR = "drafts"
 PUBLISHED_DIR = "published"
@@ -25,29 +28,7 @@ EXPORT_VERSION_KEY = "export_format"
 DEFAULT_CONTENT_FIELDS = ['metadata', 'data']
 
 
-class EdxJSONEncoder(json.JSONEncoder):
-    """
-    Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
-
-    `Location`s are encoded as their url string form, and `datetime`s as
-    ISO date strings
-    """
-    def default(self, obj):
-        if isinstance(obj, Location):
-            return obj.to_deprecated_string()
-        elif isinstance(obj, datetime.datetime):
-            if obj.tzinfo is not None:
-                if obj.utcoffset() is None:
-                    return obj.isoformat() + 'Z'
-                else:
-                    return obj.isoformat()
-            else:
-                return obj.isoformat()
-        else:
-            return super(EdxJSONEncoder, self).default(obj)
-
-
-def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
+def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
     """
     Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
 
@@ -56,8 +37,6 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
     `course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
     `root_dir`: The directory to write the exported xml to
     `course_dir`: The name of the directory inside `root_dir` to write the course content to
-    `draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
-        alongside the public content in the course.
     """
 
     course = modulestore.get_course(course_key)
@@ -66,7 +45,10 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
     export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
 
     root = lxml.etree.Element('unknown')
-    course.add_xml_to_node(root)
+
+    # export only the published content
+    with store_branch_setting(course.runtime.modulestore, BRANCH_PUBLISHED_ONLY):
+        course.add_xml_to_node(root)
 
     with export_fs.open('course.xml', 'w') as course_xml:
         lxml.etree.ElementTree(root).write(course_xml)
@@ -121,26 +103,26 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
         policy = {'course/' + course.location.name: own_metadata(course)}
         course_policy.write(dumps(policy, cls=EdxJSONEncoder))
 
-    # export draft content
     # NOTE: this code assumes that verticals are the top most draftable container
-    # should we change the application, then this assumption will no longer
-    # be valid
-    if draft_modulestore is not None:
-        draft_verticals = draft_modulestore.get_items(course_key, category='vertical', revision='draft')
-        if len(draft_verticals) > 0:
-            draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
-            for draft_vertical in draft_verticals:
-                parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
-                # Don't try to export orphaned items.
-                if len(parent_locs) > 0:
-                    logging.debug('parent_locs = {0}'.format(parent_locs))
-                    draft_vertical.xml_attributes['parent_sequential_url'] = parent_locs[0].to_deprecated_string()
-                    sequential = modulestore.get_item(parent_locs[0])
+    # should we change the application, then this assumption will no longer be valid
+    # NOTE: we need to explicitly implement the logic for setting the vertical's parent
+    # and index here since the XML modulestore cannot load draft modules
+    draft_verticals = modulestore.get_items(course_key, category='vertical', revision=REVISION_OPTION_DRAFT_ONLY)
+    if len(draft_verticals) > 0:
+        draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
+        for draft_vertical in draft_verticals:
+            parent_loc = modulestore.get_parent_location(draft_vertical.location, revision=REVISION_OPTION_DRAFT_PREFERRED)
+            # Don't try to export orphaned items.
+            if parent_loc is not None:
+                logging.debug('parent_loc = {0}'.format(parent_loc))
+                if parent_loc.category in DIRECT_ONLY_CATEGORIES:
+                    draft_vertical.xml_attributes['parent_sequential_url'] = parent_loc.to_deprecated_string()
+                    sequential = modulestore.get_item(parent_loc)
                     index = sequential.children.index(draft_vertical.location)
                     draft_vertical.xml_attributes['index_in_children_list'] = str(index)
-                    draft_vertical.runtime.export_fs = draft_course_dir
-                    node = lxml.etree.Element('unknown')
-                    draft_vertical.add_xml_to_node(node)
+                draft_vertical.runtime.export_fs = draft_course_dir
+                node = lxml.etree.Element('unknown')
+                draft_vertical.add_xml_to_node(node)
 
 
 def _export_field_content(xblock_item, item_dir):
@@ -205,7 +187,7 @@ def convert_between_versions(source_dir, target_dir):
 
         shutil.copytree(published_dir, copy_root)
 
-        # If there is a "draft" branch, copy it. All other branches are ignored.
+        # If there is a DRAFT branch, copy it. All other branches are ignored.
         copy_drafts()
 
     def copy_drafts():

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -10,12 +10,14 @@ from xmodule.x_module import XModuleDescriptor
 from opaque_keys.edx.keys import UsageKey
 from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
 from xmodule.contentstore.content import StaticContent
+from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
 from .inheritance import own_metadata
 from xmodule.errortracker import make_error_tracker
 from .store_utilities import rewrite_nonportable_content_links
 import xblock
 from xmodule.tabs import CourseTabList
 from xmodule.modulestore.exceptions import InvalidLocationError
+from xmodule.modulestore import KEY_REVISION_PUBLISHED, KEY_REVISION_DRAFT
 
 log = logging.getLogger(__name__)
 
@@ -107,10 +109,10 @@ def import_static_content(
 
 
 def import_from_xml(
-        store, data_dir, course_dirs=None,
+        store, user_id, data_dir, course_dirs=None,
         default_class='xmodule.raw_module.RawDescriptor',
         load_error_modules=True, static_content_store=None,
-        target_course_id=None, verbose=False, draft_store=None,
+        target_course_id=None, verbose=False,
         do_import_static=True, create_new_course_if_not_present=False):
     """
     Import the specified xml data_dir into the "store" modulestore,
@@ -171,18 +173,13 @@ def import_from_xml(
                 store.create_course(dest_course_id.org, dest_course_id.offering)
             except InvalidLocationError:
                 # course w/ same org and course exists 
-                     log.debug(
-                         "Skipping import of course with id, {0},"
-                         "since it collides with an existing one".format(dest_course_id)
-                     )
-                     continue
-
-        try:
-            # turn off all write signalling while importing as this
-            # is a high volume operation on stores that need it
-            if hasattr(store, 'ignore_write_events_on_courses'):
-                store.ignore_write_events_on_courses.add(dest_course_id)
+                log.debug(
+                    "Skipping import of course with id, {0},"
+                    "since it collides with an existing one".format(dest_course_id)
+                )
+                continue
 
+        with store_bulk_write_operations_on_course(store, dest_course_id):
             course_data_path = None
 
             if verbose:
@@ -209,8 +206,8 @@ def import_from_xml(
 
                     log.debug('course data_dir={0}'.format(module.data_dir))
 
-                    course = import_module(
-                        module, store,
+                    course = _import_module_and_update_references(
+                        module, store, user_id,
                         course_key,
                         dest_course_id,
                         do_import_static=do_import_static
@@ -249,9 +246,12 @@ def import_from_xml(
                     if course.tabs is None or len(course.tabs) == 0:
                         CourseTabList.initialize_default(course)
 
-                    store.update_item(course)
+                    store.update_item(course, user_id)
 
                     course_items.append(course)
+                    break
+
+            # TODO: shouldn't this raise an exception if course wasn't found?
 
             # then import all the static content
             if static_content_store is not None and do_import_static:
@@ -284,7 +284,7 @@ def import_from_xml(
                     dest_course_id, subpath=simport, verbose=verbose
                 )
 
-            # finally loop through all the modules
+            # now loop through all the modules
             for module in xml_module_store.modules[course_key].itervalues():
                 if module.scope_ids.block_type == 'course':
                     # we've already saved the course module up at the top
@@ -296,41 +296,36 @@ def import_from_xml(
                         loc=module.location
                     ))
 
-                import_module(
+                _import_module_and_update_references(
                     module, store,
+                    user_id,
                     course_key,
                     dest_course_id,
                     do_import_static=do_import_static,
-                    system=course.runtime
+                    runtime=course.runtime
                 )
 
-            # now import any 'draft' items
-            if draft_store is not None:
-                import_course_draft(
-                    xml_module_store,
-                    store,
-                    draft_store,
-                    course_data_path,
-                    static_content_store,
-                    course_key,
-                    dest_course_id,
-                    course.runtime
-                )
-
-        finally:
-            # turn back on all write signalling on stores that need it
-            if (hasattr(store, 'ignore_write_events_on_courses') and
-                    dest_course_id in store.ignore_write_events_on_courses):
-                store.ignore_write_events_on_courses.remove(dest_course_id)
-                store.refresh_cached_metadata_inheritance_tree(dest_course_id)
+            # finally, publish the course
+            store.publish(course.location, user_id)
+
+            # now import any DRAFT items
+            _import_course_draft(
+                xml_module_store,
+                store,
+                user_id,
+                course_data_path,
+                course_key,
+                dest_course_id,
+                course.runtime
+            )
 
     return xml_module_store, course_items
 
 
-def import_module(
-        module, store,
+def _import_module_and_update_references(
+        module, store, user_id,
         source_course_id, dest_course_id,
-        do_import_static=True, system=None):
+        do_import_static=True, runtime=None):
 
     logging.debug(u'processing import of module {}...'.format(module.location.to_deprecated_string()))
 
@@ -347,7 +342,7 @@ def import_module(
     new_usage_key = module.scope_ids.usage_id.map_into_course(dest_course_id)
     if new_usage_key.category == 'course':
         new_usage_key = new_usage_key.replace(name=dest_course_id.run)
-    new_module = store.create_xmodule(new_usage_key, system=system)
+    new_module = store.create_xmodule(new_usage_key, runtime=runtime)
 
     def _convert_reference_fields_to_new_namespace(reference):
         """
@@ -391,14 +386,19 @@ def import_module(
                 setattr(new_module, field_name, value)
             else:
                 setattr(new_module, field_name, getattr(module, field_name))
-    store.update_item(new_module, '**replace_user**', allow_not_found=True)
+    store.update_item(new_module, user_id, allow_not_found=True)
     return new_module
 
 
-def import_course_draft(
-        xml_module_store, store, draft_store, course_data_path,
-        static_content_store, source_course_id,
-        target_course_id, mongo_runtime):
+def _import_course_draft(
+        xml_module_store,
+        store,
+        user_id,
+        course_data_path,
+        source_course_id,
+        target_course_id,
+        mongo_runtime
+):
     '''
     This will import all the content inside of the 'drafts' folder, if it exists
     NOTE: This is not a full course import, basically in our current
@@ -502,17 +502,17 @@ def import_course_draft(
                 course_key = descriptor.location.course_key
                 try:
                     def _import_module(module):
-                        # Update the module's location to "draft" revision
+                        # Update the module's location to DRAFT revision
                         # We need to call this method (instead of updating the location directly)
                         # to ensure that pure XBlock field data is updated correctly.
-                        _update_module_location(module, module.location.replace(revision='draft'))
+                        _update_module_location(module, module.location.replace(revision=KEY_REVISION_DRAFT))
 
                         # make sure our parent has us in its list of children
                         # this is to make sure private only verticals show up
                         # in the list of children since they would have been
                         # filtered out from the non-draft store export
                         if module.location.category == 'vertical':
-                            non_draft_location = module.location.replace(revision=None)
+                            non_draft_location = module.location.replace(revision=KEY_REVISION_PUBLISHED)
                             sequential_url = module.xml_attributes['parent_sequential_url']
                             index = int(module.xml_attributes['index_in_children_list'])
 
@@ -525,12 +525,13 @@ def import_course_draft(
 
                             if non_draft_location not in sequential.children:
                                 sequential.children.insert(index, non_draft_location)
-                                store.update_item(sequential, '**replace_user**')
+                                store.update_item(sequential, user_id)
 
-                        import_module(
-                            module, draft_store,
+                        _import_module_and_update_references(
+                            module, store, user_id,
                             source_course_id,
-                            target_course_id, system=mongo_runtime
+                            target_course_id,
+                            runtime=mongo_runtime,
                         )
                         for child in module.get_children():
                             _import_module(child)

common/lib/xmodule/xmodule/split_test_module.py

@@ -420,7 +420,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
             if selected_partition is not None:
                 self.group_id_mapping = {}  # pylint: disable=attribute-defined-outside-init
                 for group in selected_partition.groups:
-                    self._create_vertical_for_group(group)
+                    self._create_vertical_for_group(group, user.id)
                 # Don't need to call update_item in the modulestore because the caller of this method will do it.
         else:
             # If children referenced in group_id_to_child have been deleted, remove them from the map.
@@ -553,7 +553,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
         for group in user_partition.groups:
             str_group_id = unicode(group.id)
             if str_group_id not in self.group_id_to_child:
-                self._create_vertical_for_group(group)
+                self._create_vertical_for_group(group, request.user.id)
                 changed = True
 
         if changed:
@@ -561,7 +561,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
             self.system.modulestore.update_item(self, None)
         return Response()
 
-    def _create_vertical_for_group(self, group):
+    def _create_vertical_for_group(self, group, user_id):
         """
         Creates a vertical to associate with the group.
 
@@ -576,9 +576,10 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
         metadata = {'display_name': group.name}
         modulestore.create_and_save_xmodule(
             dest_usage_key,
+            user_id,
             definition_data=None,
             metadata=metadata,
-            system=self.system,
+            runtime=self.system,
         )
         self.children.append(dest_usage_key)  # pylint: disable=no-member
         self.group_id_to_child[unicode(group.id)] = dest_usage_key

common/lib/xmodule/xmodule/tests/test_export.py

@@ -23,9 +23,10 @@ from xblock.fields import String, Scope, Integer
 from xblock.test.tools import blocks_are_equivalent
 
 from opaque_keys.edx.locations import Location
+from xmodule.modulestore import EdxJSONEncoder
 from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.xml_exporter import (
-    EdxJSONEncoder, convert_between_versions, get_version
+    convert_between_versions, get_version
 )
 from xmodule.tests import DATA_DIR
 from xmodule.tests.helpers import directories_equal

common/lib/xmodule/xmodule/xml_module.py

@@ -8,7 +8,7 @@ from lxml import etree
 from xblock.fields import Dict, Scope, ScopeIds
 from xmodule.x_module import XModuleDescriptor
 from xmodule.modulestore.inheritance import own_metadata, InheritanceKeyValueStore
-from xmodule.modulestore.xml_exporter import EdxJSONEncoder
+from xmodule.modulestore import EdxJSONEncoder
 from xblock.runtime import KvsFieldData
 
 log = logging.getLogger(__name__)

lms/djangoapps/branding/tests.py

@@ -10,7 +10,7 @@ from django.test.utils import override_settings
 from django.test.client import RequestFactory
 
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.django import editable_modulestore
+from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.factories import CourseFactory
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 import student.views
@@ -28,12 +28,12 @@ class AnonymousIndexPageTest(ModuleStoreTestCase):
     Tests that anonymous users can access the '/' page,  Need courses with start date
     """
     def setUp(self):
-        self.store = editable_modulestore()
+        self.store = modulestore()
         self.factory = RequestFactory()
         self.course = CourseFactory.create()
         self.course.days_early_for_beta = 5
         self.course.enrollment_start = datetime.datetime.now(UTC) + datetime.timedelta(days=3)
-        self.store.update_item(self.course)
+        self.store.update_item(self.course, '**replace_user**')
 
     @override_settings(FEATURES=FEATURES_WITH_STARTDATE)
     def test_none_user_index_access_with_startdate_fails(self):

lms/djangoapps/courseware/features/annotatable.py

@@ -3,7 +3,7 @@ import textwrap
 from lettuce import world, steps
 from nose.tools import assert_in, assert_equals, assert_true
 
-from common import i_am_registered_for_the_course, visit_scenario_item
+from common import i_am_registered_for_the_course, visit_scenario_item, publish
 
 DATA_TEMPLATE = textwrap.dedent("""\
     <annotatable>
@@ -79,6 +79,8 @@ class AnnotatableSteps(object):
             data=DATA_TEMPLATE.format("\n".join(ANNOTATION_TEMPLATE.format(i) for i in xrange(count)))
         )
 
+        publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
+
         self.annotations_count = count
 
     def view_component(self, step):
@@ -123,6 +125,7 @@ class AnnotatableSteps(object):
                     )
                 )
             )
+        publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
 
     def click_reply(self, step, problem):
         r"""I click "Reply to annotation" on passage (?P<problem>\d+)$"""