Skip to content

E
edx-platform
Commit b5d1ec08 by Nimisha Asthagiri
Options

Merge pull request #3915 from edx/studio/uses-mixed-modulestore 

Enable mixed modulestore
parents 580de552 a9213509
Showing with 867 additions and 948 deletions
cms/djangoapps/contentstore/course_info_model.py
......@@ -28,16 +28,15 @@ from xmodule.html_module import CourseInfoModule
log = logging.getLogger(__name__)
def get_course_updates(location, provided_id):
def get_course_updates(location, provided_id, user_id):
"""
Retrieve the relevant course_info updates and unpack into the model which the client expects:
[{id : index, date : string, content : html string}]
"""
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, user_id)
course_update_items = get_course_update_items(course_updates, provided_id)
return _get_visible_update(course_update_items)
......@@ -50,10 +49,9 @@ def update_course_updates(location, update, passed_id=None, user=None):
into the html structure.
"""
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, user.id)
course_update_items = list(reversed(get_course_update_items(course_updates)))
......@@ -135,7 +133,7 @@ def delete_course_update(location, update, passed_id, user):
return HttpResponseBadRequest()
try:
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
except ItemNotFoundError:
return HttpResponseBadRequest()
......@@ -239,6 +237,6 @@ def save_course_update_items(location, course_updates, course_update_items, user
course_updates.data = _get_html(course_update_items)
# update db record
modulestore('direct').update_item(course_updates, user.id)
modulestore().update_item(course_updates, user.id)
return course_updates
cms/djangoapps/contentstore/features/video.py
# pylint: disable=C0111
from lettuce import world, step
from contentstore.utils import get_modulestore
from selenium.webdriver.common.keys import Keys
from xmodule.modulestore.django import modulestore
VIDEO_BUTTONS = {
'CC': '.hide-subtitles',
......@@ -137,9 +137,9 @@ def xml_only_video(step):
world.wait(1)
course = world.scenario_dict['COURSE']
store = get_modulestore(course.location)
store = modulestore()
parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location
parent_location = store.get_items(course.id, category='vertical')[0].location
youtube_id = 'ABCDEFG'
world.scenario_dict['YOUTUBE_ID'] = youtube_id
......
cms/djangoapps/contentstore/git_export_utils.py
......@@ -128,8 +128,8 @@ def export_to_git(course_id, repo, user='', rdir=None):
root_dir = os.path.dirname(rdirp)
course_dir = os.path.splitext(os.path.basename(rdirp))[0]
try:
export_to_xml(modulestore('direct'), contentstore(), course_id,
root_dir, course_dir, modulestore())
export_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
......
cms/djangoapps/contentstore/management/commands/clone_course.py
......@@ -4,6 +4,7 @@ Script for cloning a course
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from xmodule.contentstore.django import contentstore
from student.roles import CourseInstructorRole, CourseStaffRole
from opaque_keys.edx.keys import CourseKey
......@@ -35,19 +36,18 @@ class Command(BaseCommand):
source_course_id = self.course_key_from_arg(args[0])
dest_course_id = self.course_key_from_arg(args[1])
mstore = modulestore('direct')
mstore = modulestore()
cstore = contentstore()
mstore.ignore_write_events_on_courses.add(dest_course_id)
print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))
if clone_course(mstore, cstore, source_course_id, dest_course_id):
print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_course_id).add_users(
*CourseInstructorRole(source_course_id).users_with_role()
)
CourseStaffRole(dest_course_id).add_users(
*CourseStaffRole(source_course_id).users_with_role()
)
with store_bulk_write_operations_on_course(mstore, dest_course_id):
if clone_course(mstore, cstore, source_course_id, dest_course_id, None):
print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_course_id).add_users(
*CourseInstructorRole(source_course_id).users_with_role()
)
CourseStaffRole(dest_course_id).add_users(
*CourseStaffRole(source_course_id).users_with_role()
)
cms/djangoapps/contentstore/management/commands/course_id_clash.py
......@@ -3,6 +3,7 @@ Script for finding all courses whose org/name pairs == other courses when ignori
"""
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
#
......@@ -10,12 +11,12 @@ from xmodule.modulestore.django import modulestore
#
class Command(BaseCommand):
"""
Script for finding all courses whose org/name pairs == other courses when ignoring case
Script for finding all courses in the Mongo Modulestore whose org/name pairs == other courses when ignoring case
"""
help = 'List all courses ids which may collide when ignoring case'
help = 'List all courses ids in the Mongo Modulestore which may collide when ignoring case'
def handle(self, *args, **options):
mstore = modulestore()
mstore = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE) # pylint: disable=protected-access
if hasattr(mstore, 'collection'):
map_fn = '''
function () {
......
cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py
......@@ -22,7 +22,7 @@ class Command(BaseCommand):
course_ids = [course_key]
else:
course_ids = [course.id for course in modulestore('direct').get_courses()]
course_ids = [course.id for course in modulestore().get_courses()]
if query_yes_no("Emptying trashcan. Confirm?", default="no"):
empty_asset_trashcan(course_ids)
cms/djangoapps/contentstore/management/commands/export.py
......@@ -35,4 +35,4 @@ class Command(BaseCommand):
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
cms/djangoapps/contentstore/management/commands/export_all_courses.py
......@@ -19,7 +19,7 @@ class Command(BaseCommand):
output_path = args[0]
cs = contentstore()
ms = modulestore('direct')
ms = modulestore()
root_dir = output_path
courses = ms.get_courses()
......@@ -35,7 +35,7 @@ class Command(BaseCommand):
if 1:
try:
course_dir = course_id.replace('/', '...')
export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
export_to_xml(ms, cs, course_id, root_dir, course_dir)
except Exception as err:
print("="*30 + "> Oops, failed to export %s" % course_id)
print("Error:")
......
cms/djangoapps/contentstore/management/commands/import.py
......@@ -37,15 +37,10 @@ class Command(BaseCommand):
data=data_dir,
courses=course_dirs,
dis=do_import_static))
try:
mstore = modulestore('direct')
except KeyError:
self.stdout.write('Unable to load direct modulestore, trying '
'default\n')
mstore = modulestore('default')
mstore = modulestore()
_, course_items = import_from_xml(
mstore, data_dir, course_dirs, load_error_modules=False,
mstore, "**replace_user**", data_dir, course_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static,
create_new_course_if_not_present=True,
......
cms/djangoapps/contentstore/management/commands/tests/test_import.py
......@@ -8,16 +8,13 @@ import shutil
import tempfile
from django.core.management import call_command
from django.test.utils import override_settings
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from django_comment_common.utils import are_permissions_roles_seeded
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from opaque_keys.edx.locations import SlashSeparatedCourseKey
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestImport(ModuleStoreTestCase):
"""
Unit tests for importing a course from command line
......
cms/djangoapps/contentstore/management/commands/tests/test_migrate_to_split.py
......@@ -5,9 +5,8 @@ import unittest
from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.migrate_to_split import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, REVISION_OPTION_PUBLISHED_ONLY
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, clear_existing_modulestores
......@@ -45,7 +44,6 @@ class TestArgParsing(unittest.TestCase):
@unittest.skip("Not fixing split mongo until we land this long branch")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestMigrateToSplit(ModuleStoreTestCase):
"""
Unit tests for migrating a course from old mongo to split mongo
......@@ -58,7 +56,7 @@ class TestMigrateToSplit(ModuleStoreTestCase):
password = 'foo'
self.user = User.objects.create_user(uname, email, password)
self.course = CourseFactory()
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, SPLIT_MONGO_MODULESTORE_TYPE)
self.addCleanup(clear_existing_modulestores)
def test_user_email(self):
......@@ -86,6 +84,6 @@ class TestMigrateToSplit(ModuleStoreTestCase):
str(self.user.id),
"org.dept+name.run",
)
locator = CourseLocator(org="org.dept", offering="name.run", branch="published")
locator = CourseLocator(org="org.dept", offering="name.run", branch=REVISION_OPTION_PUBLISHED_ONLY)
course_from_split = modulestore('split').get_course(locator)
self.assertIsNotNone(course_from_split)
cms/djangoapps/contentstore/management/commands/tests/test_rollback_split_course.py
......@@ -7,9 +7,7 @@ from mock import patch
from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.rollback_split_course import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.persistent_factories import PersistentCourseFactory
from xmodule.modulestore.tests.factories import CourseFactory
......@@ -39,7 +37,6 @@ class TestArgParsing(unittest.TestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line,
......@@ -58,7 +55,6 @@ class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line,
......@@ -77,7 +73,6 @@ class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourse(ModuleStoreTestCase):
"""
Unit tests for rolling back a split-mongo course from command line
......
cms/djangoapps/contentstore/tests/modulestore_config.py deleted 100644 → 0
"""
Define test configuration for modulestores.
"""
from xmodule.modulestore.tests.django_utils import studio_store_config
from django.conf import settings
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
cms/djangoapps/contentstore/tests/test_course_listing.py
......@@ -16,7 +16,8 @@ from contentstore.tests.utils import AjaxEnabledTestClient
from student.tests.factories import UserFactory
from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff, OrgStaffRole, OrgInstructorRole
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore
from xmodule.error_module import ErrorDescriptor
......@@ -197,6 +198,14 @@ class TestCourseListing(ModuleStoreTestCase):
self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed)
self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed)
# Now count the db queries
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
with check_mongo_calls(store.collection, USER_COURSES_COUNT):
courses_list = _accessible_courses_list_from_groups(self.request)
with check_mongo_calls(store.collection, 1):
courses_list = _accessible_courses_list(self.request)
def test_get_course_list_with_same_course_id(self):
"""
Test getting courses with same id but with different name case. Then try to delete one of them and
......@@ -253,18 +262,20 @@ class TestCourseListing(ModuleStoreTestCase):
Create good courses, courses that won't load, and deleted courses which still have
roles. Test course listing.
"""
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
course_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location, self.user)
course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location, self.user)
modulestore().delete_course(course_location)
store.delete_course(course_location)
course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
course = self._create_course_with_access_groups(course_location, self.user)
course_db_record = modulestore()._find_one(course.location)
course_db_record = store._find_one(course.location)
course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
modulestore().collection.update(
store.collection.update(
{'_id': course.location.to_deprecated_son()},
{'$set': {
'metadata.tabs': course_db_record['metadata']['tabs'],
......
cms/djangoapps/contentstore/tests/test_course_settings.py
......@@ -11,7 +11,7 @@ from django.test.utils import override_settings
from models.settings.course_details import (CourseDetails, CourseSettingsEncoder)
from models.settings.course_grading import CourseGradingModel
from contentstore.utils import get_modulestore, EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from contentstore.utils import EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from xmodule.modulestore.tests.factories import CourseFactory
from models.settings.course_metadata import CourseMetadata
......@@ -335,7 +335,7 @@ class CourseGradingTest(CourseTestCase):
def test_update_section_grader_type(self):
# Get the descriptor and the section_grader_type and assert they are the default values
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('notgraded', section_grader_type['graderType'])
......@@ -344,7 +344,7 @@ class CourseGradingTest(CourseTestCase):
# Change the default grader type to Homework, which should also mark the section as graded
CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user)
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('Homework', section_grader_type['graderType'])
......@@ -353,7 +353,7 @@ class CourseGradingTest(CourseTestCase):
# Change the grader type back to notgraded, which should also unmark the section as graded
CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user)
descriptor = get_modulestore(self.course.location).get_item(self.course.location)
descriptor = modulestore().get_item(self.course.location)
section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)
self.assertEqual('notgraded', section_grader_type['graderType'])
......@@ -413,8 +413,7 @@ class CourseGradingTest(CourseTestCase):
Populate the course, grab a section, get the url for the assignment type access
"""
self.populate_course()
sequential_usage_key = self.course.id.make_usage_key("sequential", None)
sections = get_modulestore(self.course.id).get_items(sequential_usage_key)
sections = modulestore().get_items(self.course.id, category="sequential")
# see if test makes sense
self.assertGreater(len(sections), 0, "No sections found")
section = sections[0] # just take the first one
......@@ -470,7 +469,7 @@ class CourseMetadataEditingTest(CourseTestCase):
)
self.update_check(test_model)
# try fresh fetch to ensure persistence
fresh = modulestore('direct').get_course(self.course.id)
fresh = modulestore().get_course(self.course.id)
test_model = CourseMetadata.fetch(fresh)
self.update_check(test_model)
# now change some of the existing metadata
......
cms/djangoapps/contentstore/tests/test_export_git.py
......@@ -16,7 +16,7 @@ from .utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
......@@ -66,7 +66,7 @@ class TestExportGit(CourseTestCase):
Test failed course export response.
"""
self.course_module.giturl = 'foobar'
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertIn('Export Failed:', response.content)
......@@ -76,7 +76,7 @@ class TestExportGit(CourseTestCase):
Regression test for making sure errors are properly stringified
"""
self.course_module.giturl = 'foobar'
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertNotIn('django.utils.functional.__proxy__', response.content)
......@@ -99,7 +99,7 @@ class TestExportGit(CourseTestCase):
self.populate_course()
self.course_module.giturl = 'file://{}'.format(bare_repo_dir)
get_modulestore(self.course_module.location).update_item(self.course_module)
modulestore().update_item(self.course_module, '**replace_user**')
response = self.client.get('{}?action=push'.format(self.test_url))
self.assertIn('Export Succeeded', response.content)
cms/djangoapps/contentstore/tests/test_i18n.py
from unittest import skip
from django.contrib.auth.models import User
from django.test.utils import override_settings
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
@override_settings(MODULESTORE=TEST_MODULESTORE)
class InternationalizationTest(ModuleStoreTestCase):
"""
Tests to validate Internationalization.
......
cms/djangoapps/contentstore/tests/test_import.py
......@@ -13,13 +13,11 @@ import copy
from django.contrib.auth.models import User
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.exceptions import NotFoundError
......@@ -30,7 +28,7 @@ TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreImportTest(ModuleStoreTestCase):
"""
Tests that rely on the toy and test_import_course courses.
......@@ -38,8 +36,6 @@ class ContentStoreImportTest(ModuleStoreTestCase):
"""
def setUp(self):
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
uname = 'testuser'
email = 'test+courses@edx.org'
password = 'foo'
......@@ -69,9 +65,10 @@ class ContentStoreImportTest(ModuleStoreTestCase):
(for do_import_static=False behavior).
'''
content_store = contentstore()
module_store = modulestore('direct')
module_store = modulestore()
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['test_import_course'],
static_content_store=content_store,
......@@ -91,6 +88,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
module_store, __, course = self.load_test_import_course()
__, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data',
['test_import_course_2'],
target_course_id=course.id,
......@@ -102,10 +100,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
"""
# Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
"""
module_store = modulestore('direct')
module_store = modulestore()
course_id = SlashSeparatedCourseKey(u'Юникода', u'unicode_course', u'échantillon')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['2014_Uni'],
target_course_id=course_id
......@@ -150,8 +149,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
'''
content_store = contentstore()
module_store = modulestore('direct')
import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
module_store = modulestore()
import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
......@@ -161,8 +160,8 @@ class ContentStoreImportTest(ModuleStoreTestCase):
self.assertEqual(count, 0)
def test_no_static_link_rewrites_on_import(self):
module_store = modulestore('direct')
_, courses = import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
module_store = modulestore()
_, courses = import_from_xml(module_store, '**replace_user**', 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
course_key = courses[0].id
handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
......@@ -177,10 +176,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
self.assertEqual(course.tabs[2]['name'], 'Syllabus')
def test_rewrite_reference_list(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'conditional_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['conditional'],
target_course_id=target_course_id
......@@ -206,10 +206,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
)
def test_rewrite_reference(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'peergrading_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['open_ended'],
target_course_id=target_course_id
......@@ -224,10 +225,11 @@ class ContentStoreImportTest(ModuleStoreTestCase):
)
def test_rewrite_reference_value_dict(self):
module_store = modulestore('direct')
module_store = modulestore()
target_course_id = SlashSeparatedCourseKey('testX', 'split_test_copy', 'copy_run')
import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['split_test_module'],
target_course_id=target_course_id
......
cms/djangoapps/contentstore/tests/test_import_draft_order.py
from django.test.utils import override_settings
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from contentstore.tests.modulestore_config import TEST_MODULESTORE
# This test is in the CMS module because the test configuration to use a draft
# modulestore is dependent on django.
@override_settings(MODULESTORE=TEST_MODULESTORE)
class DraftReorderTestCase(ModuleStoreTestCase):
def test_order(self):
store = modulestore('direct')
draft_store = modulestore('default')
_, course_items = import_from_xml(store, 'common/test/data/', ['import_draft_order'], draft_store=draft_store)
store = modulestore()
_, course_items = import_from_xml(store, '**replace_user**', 'common/test/data/', ['import_draft_order'])
course_key = course_items[0].id
sequential = draft_store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
sequential = store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
verticals = sequential.children
# The order that files are read in from the file system is not guaranteed (cannot rely on
......@@ -39,7 +33,7 @@ class DraftReorderTestCase(ModuleStoreTestCase):
self.assertEqual(course_key.make_usage_key('vertical', 'c'), verticals[6])
# Now also test that the verticals in a second sequential are correct.
sequential = draft_store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
sequential = store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
verticals = sequential.children
# 'asecond' and 'zsecond' are drafts with 'index_in_children_list' 0 and 2, respectively.
# 'secondsubsection' is a public vertical.
......
cms/djangoapps/contentstore/tests/test_import_pure_xblock.py
......@@ -2,8 +2,6 @@
Integration tests for importing courses containing pure XBlocks.
"""
from django.test.utils import override_settings
from xblock.core import XBlock
from xblock.fields import String
......@@ -11,7 +9,6 @@ from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.draft import as_draft
from contentstore.tests.modulestore_config import TEST_MODULESTORE
class StubXBlock(XBlock):
......@@ -29,12 +26,10 @@ class StubXBlock(XBlock):
test_field = String(default="default")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class XBlockImportTest(ModuleStoreTestCase):
def setUp(self):
self.store = modulestore('direct')
self.draft_store = modulestore('default')
self.store = modulestore()
@XBlock.register_temp_plugin(StubXBlock)
def test_import_public(self):
......@@ -67,8 +62,7 @@ class XBlockImportTest(ModuleStoreTestCase):
"""
_, courses = import_from_xml(
self.store, 'common/test/data', [course_dir],
draft_store=self.draft_store
self.store, '**replace_user**', 'common/test/data', [course_dir]
)
xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test')
......@@ -81,6 +75,7 @@ class XBlockImportTest(ModuleStoreTestCase):
self.assertEqual(xblock.test_field, expected_field_val)
if has_draft:
draft_xblock = self.draft_store.get_item(xblock_location)
draft_xblock = self.store.get_item(xblock_location)
self.assertTrue(getattr(draft_xblock, 'is_draft', False))
self.assertTrue(isinstance(draft_xblock, StubXBlock))
self.assertEqual(draft_xblock.test_field, expected_field_val)
cms/djangoapps/contentstore/tests/test_orphan.py
......@@ -32,8 +32,10 @@ class TestOrphan(CourseTestCase):
def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
location = self.course.location.replace(category=category, name=name)
store = modulestore('direct')
store.create_and_save_xmodule(location, data, metadata, runtime)
store = modulestore()
store.create_and_save_xmodule(
location, self.user.id, definition_data=data, metadata=metadata, runtime=runtime
)
if parent_name:
# add child to parent in mongo
parent_location = self.course.location.replace(category=parent_category, name=parent_name)
......
cms/djangoapps/contentstore/tests/test_permissions.py
......@@ -3,11 +3,9 @@ Test CRUD for authorization.
"""
import copy
from django.test.utils import override_settings
from django.contrib.auth.models import User
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_url, reverse_course_url
......@@ -16,7 +14,6 @@ from contentstore.views.access import has_course_access
from student import auth
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseAccess(ModuleStoreTestCase):
"""
Course-based access (as opposed to access of a non-course xblock)
......
cms/djangoapps/contentstore/tests/test_transcripts_utils.py
......@@ -21,7 +21,6 @@ from xmodule.exceptions import NotFoundError
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.video_module import transcripts_utils
from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
......@@ -76,7 +75,7 @@ class TestGenerateSubs(unittest.TestCase):
)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestSaveSubsToStore(ModuleStoreTestCase):
"""Tests for `save_subs_to_store` function."""
......@@ -156,7 +155,7 @@ class TestSaveSubsToStore(ModuleStoreTestCase):
_CONTENTSTORE.clear()
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestDownloadYoutubeSubs(ModuleStoreTestCase):
"""Tests for `download_youtube_subs` function."""
......
cms/djangoapps/contentstore/tests/tests.py
......@@ -15,14 +15,12 @@ from contentstore.tests.utils import parse_json, user, registration, AjaxEnabled
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.test_course_settings import CourseTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
import datetime
from pytz import UTC
from freezegun import freeze_time
@override_settings(MODULESTORE=TEST_MODULESTORE)
class ContentStoreTestCase(ModuleStoreTestCase):
def _login(self, email, password):
"""
......
cms/djangoapps/contentstore/tests/utils.py
......@@ -6,12 +6,10 @@ import json
from django.contrib.auth.models import User
from django.test.client import Client
from django.test.utils import override_settings
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.utils import get_modulestore
from student.models import Registration
......@@ -58,8 +56,6 @@ class AjaxEnabledTestClient(Client):
return self.get(path, data or {}, follow, HTTP_ACCEPT="application/json", **extra)
@override_settings(MODULESTORE=TEST_MODULESTORE)
class CourseTestCase(ModuleStoreTestCase):
def setUp(self):
"""
......@@ -91,7 +87,7 @@ class CourseTestCase(ModuleStoreTestCase):
number='999',
display_name='Robot Super Course',
)
self.store = get_modulestore(self.course.location)
self.store = modulestore()
def create_non_staff_authed_user_client(self, authenticate=True):
"""
......
cms/djangoapps/contentstore/utils.py
......@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.store_utilities import delete_course
from student.roles import CourseInstructorRole, CourseStaffRole
......@@ -30,31 +30,22 @@ def delete_course_and_groups(course_id, commit=False):
This deletes the courseware associated with a course_id as well as cleaning update_item
the various user table stuff (groups, permissions, etc.)
"""
module_store = modulestore('direct')
module_store = modulestore()
content_store = contentstore()
module_store.ignore_write_events_on_courses.add(course_id)
with store_bulk_write_operations_on_course(module_store, course_id):
if delete_course(module_store, content_store, course_id, commit):
if delete_course(module_store, content_store, course_id, commit):
print 'removing User permissions from course....'
# in the django layer, we need to remove all the user permissions groups associated with this course
if commit:
try:
staff_role = CourseStaffRole(course_id)
staff_role.remove_users(*staff_role.users_with_role())
instructor_role = CourseInstructorRole(course_id)
instructor_role.remove_users(*instructor_role.users_with_role())
except Exception as err:
log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
def get_modulestore(category_or_location):
"""
This function no longer does anything more than just calling `modulestore()`. It used
to select 'direct' v 'draft' based on the category.
"""
return modulestore()
print 'removing User permissions from course....'
# in the django layer, we need to remove all the user permissions groups associated with this course
if commit:
try:
staff_role = CourseStaffRole(course_id)
staff_role.remove_users(*staff_role.users_with_role())
instructor_role = CourseInstructorRole(course_id)
instructor_role.remove_users(*instructor_role.users_with_role())
except Exception as err:
log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))
def get_lms_link_for_item(location, preview=False):
......@@ -124,35 +115,18 @@ def course_image_url(course):
return path
class PublishState(object):
"""
The publish state for a given xblock-- either 'draft', 'private', or 'public'.
Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
"""
draft = 'draft'
private = 'private'
public = 'public'
def compute_publish_state(xblock):
"""
Returns whether this xblock is 'draft', 'public', or 'private'.
Returns whether this xblock is draft, public, or private.
'draft' content is in the process of being edited, but still has a previous
version visible in the LMS
'public' content is locked and visible in the LMS
'private' content is editable and not visible in the LMS
Returns:
PublishState.draft - content is in the process of being edited, but still has a previous
version deployed to LMS
PublishState.public - content is locked and deployed to LMS
PublishState.private - content is editable and not deployed to LMS
"""
if getattr(xblock, 'is_draft', False):
try:
modulestore('direct').get_item(xblock.location)
return PublishState.draft
except ItemNotFoundError:
return PublishState.private
else:
return PublishState.public
return modulestore().compute_publish_state(xblock)
def add_extra_panel_tab(tab_type, course):
......
cms/djangoapps/contentstore/views/checklist.py
......@@ -11,7 +11,7 @@ from django.http import HttpResponseNotFound
from django.core.exceptions import PermissionDenied
from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
from .access import has_course_access
from xmodule.course_module import CourseDescriptor
......@@ -47,7 +47,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
# from the template.
if not course_module.checklists:
course_module.checklists = CourseDescriptor.checklists.default
get_modulestore(course_module.location).update_item(course_module, request.user.id)
modulestore().update_item(course_module, request.user.id)
expanded_checklists = expand_all_action_urls(course_module)
if json_request:
......@@ -76,7 +76,7 @@ def checklists_handler(request, course_key_string, checklist_index=None):
# not default
course_module.checklists = course_module.checklists
course_module.save()
get_modulestore(course_module.location).update_item(course_module, request.user.id)
modulestore().update_item(course_module, request.user.id)
expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
return JsonResponse(localize_checklist_text(expanded_checklist))
else:
......
cms/djangoapps/contentstore/views/component.py
......@@ -13,6 +13,7 @@ from edxmako.shortcuts import render_to_response
from util.date_utils import get_default_time_display
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import PublishState
from xblock.core import XBlock
from xblock.django.request import webob_to_django_response, django_to_webob_request
......@@ -21,7 +22,7 @@ from xblock.fields import Scope
from xblock.plugin import PluginMissingError
from xblock.runtime import Mixologist
from contentstore.utils import get_lms_link_for_item, compute_publish_state, PublishState, get_modulestore
from contentstore.utils import get_lms_link_for_item, compute_publish_state
from contentstore.views.helpers import get_parent_xblock
from models.settings.course_grading import CourseGradingModel
......@@ -413,7 +414,7 @@ def _get_item_in_course(request, usage_key):
raise PermissionDenied()
course = modulestore().get_course(course_key)
item = get_modulestore(usage_key).get_item(usage_key, depth=1)
item = modulestore().get_item(usage_key, depth=1)
lms_link = get_lms_link_for_item(usage_key)
return course, item, lms_link
......@@ -436,7 +437,7 @@ def component_handler(request, usage_key_string, handler, suffix=''):
usage_key = UsageKey.from_string(usage_key_string)
descriptor = get_modulestore(usage_key).get_item(usage_key)
descriptor = modulestore().get_item(usage_key)
# Let the module handle the AJAX
req = django_to_webob_request(request)
......@@ -449,6 +450,6 @@ def component_handler(request, usage_key_string, handler, suffix=''):
# unintentional update to handle any side effects of handle call; so, request user didn't author
# the change
get_modulestore(usage_key).update_item(descriptor, None)
modulestore().update_item(descriptor, None)
return webob_to_django_response(resp)
cms/djangoapps/contentstore/views/course.py
......@@ -31,7 +31,6 @@ from contentstore.utils import (
get_lms_link_for_item,
add_extra_panel_tab,
remove_extra_panel_tab,
get_modulestore,
reverse_course_url
)
from models.settings.course_details import CourseDetails, CourseSettingsEncoder
......@@ -165,7 +164,7 @@ def _accessible_courses_list(request):
"""
List all courses available to the logged in user by iterating through all the courses
"""
courses = modulestore('direct').get_courses()
courses = modulestore().get_courses()
# filter out courses that we don't have access to
def course_filter(course):
......@@ -202,14 +201,15 @@ def _accessible_courses_list_from_groups(request):
if course_key is None:
# If the course_access does not have a course_id, it's an org-based role, so we fall back
raise AccessListFallback
try:
course = modulestore('direct').get_course(course_key)
except ItemNotFoundError:
# If a user has access to a course that doesn't exist, don't do anything with that course
pass
if course is not None and not isinstance(course, ErrorDescriptor):
# ignore deleted or errored courses
courses_list[course_key] = course
if course_key not in courses_list:
try:
course = modulestore().get_course(course_key)
except ItemNotFoundError:
# If a user has access to a course that doesn't exist, don't do anything with that course
pass
if course is not None and not isinstance(course, ErrorDescriptor):
# ignore deleted or errored courses
courses_list[course_key] = course
return courses_list.values()
......@@ -333,7 +333,7 @@ def create_new_course(request):
fields.update(metadata)
# Creating the course raises InvalidLocationError if an existing course with this org/name is found
new_course = modulestore('direct').create_course(
new_course = modulestore().create_course(
course_key.org,
course_key.offering,
fields=fields,
......@@ -440,7 +440,7 @@ def course_info_update_handler(request, course_key_string, provided_id=None):
raise PermissionDenied()
if request.method == 'GET':
course_updates = get_course_updates(usage_key, provided_id)
course_updates = get_course_updates(usage_key, provided_id, request.user.id)
if isinstance(course_updates, dict) and course_updates.get('error'):
return JsonResponse(course_updates, course_updates.get('status', 400))
else:
......@@ -739,7 +739,7 @@ def textbooks_list_handler(request, course_key_string):
"""
course_key = CourseKey.from_string(course_key_string)
course = _get_course_module(course_key, request.user)
store = get_modulestore(course.location)
store = modulestore()
if not "application/json" in request.META.get('HTTP_ACCEPT', 'text/html'):
# return HTML page
......@@ -814,7 +814,7 @@ def textbooks_detail_handler(request, course_key_string, textbook_id):
"""
course_key = CourseKey.from_string(course_key_string)
course_module = _get_course_module(course_key, request.user)
store = get_modulestore(course_module.location)
store = modulestore()
matching_id = [tb for tb in course_module.pdf_textbooks
if unicode(tb.get("id")) == unicode(textbook_id)]
if matching_id:
......
cms/djangoapps/contentstore/views/helpers.py
......@@ -39,34 +39,16 @@ def render_from_lms(template_name, dictionary, context=None, namespace='main'):
return render_to_string(template_name, dictionary, context, namespace="lms." + namespace)
def _xmodule_recurse(item, action, ignore_exception=()):
"""
Recursively apply provided action on item and its children
ignore_exception (Exception Object): A optional argument; when passed ignores the corresponding
exception raised during xmodule recursion,
"""
for child in item.get_children():
_xmodule_recurse(child, action, ignore_exception)
try:
return action(item)
except ignore_exception:
return
def get_parent_xblock(xblock):
"""
Returns the xblock that is the parent of the specified xblock, or None if it has no parent.
"""
locator = xblock.location
parent_locations = modulestore().get_parent_locations(locator,)
parent_location = modulestore().get_parent_location(locator)
if len(parent_locations) == 0:
if parent_location is None:
return None
elif len(parent_locations) > 1:
logging.error('Multiple parents have been found for %s', unicode(locator))
return modulestore().get_item(parent_locations[0])
return modulestore().get_item(parent_location)
def is_unit(xblock):
......
cms/djangoapps/contentstore/views/import_export.py
......@@ -217,13 +217,13 @@ def import_handler(request, course_key_string):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
modulestore(),
request.user.id,
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_course_id=course_key,
draft_store=modulestore()
)
new_location = course_items[0].location
......@@ -322,7 +322,7 @@ def export_handler(request, course_key_string):
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore())
export_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug('tar file being generated at {0}'.format(export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
......@@ -334,10 +334,10 @@ def export_handler(request, course_key_string):
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_locs = modulestore().get_parent_locations(failed_item.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
......
cms/djangoapps/contentstore/views/tabs.py
......@@ -117,7 +117,7 @@ def reorder_tabs_handler(course_item, request):
# persist the new order of the tabs
course_item.tabs = new_tab_list
modulestore('direct').update_item(course_item, request.user.id)
modulestore().update_item(course_item, request.user.id)
return JsonResponse()
......@@ -140,7 +140,7 @@ def edit_tab_handler(course_item, request):
if 'is_hidden' in request.json:
# set the is_hidden attribute on the requested tab
tab.is_hidden = request.json['is_hidden']
modulestore('direct').update_item(course_item, request.user.id)
modulestore().update_item(course_item, request.user.id)
else:
raise NotImplementedError('Unsupported request to edit tab: {0}'.format(request.json))
......@@ -163,7 +163,7 @@ def get_tab_by_locator(tab_list, usage_key_string):
Look for a tab with the specified locator. Returns the first matching tab.
"""
tab_location = UsageKey.from_string(usage_key_string)
item = modulestore('direct').get_item(tab_location)
item = modulestore().get_item(tab_location)
static_tab = StaticTab(
name=item.display_name,
url_slug=item.location.name,
......@@ -192,7 +192,7 @@ def primitive_delete(course, num):
# Note for future implementations: if you delete a static_tab, then Chris Dodge
# points out that there's other stuff to delete beyond this element.
# This code happens to not delete static_tab so it doesn't come up.
modulestore('direct').update_item(course, '**replace_user**')
modulestore().update_item(course, '**replace_user**')
def primitive_insert(course, num, tab_type, name):
......@@ -201,5 +201,5 @@ def primitive_insert(course, num, tab_type, name):
new_tab = CourseTab.from_json({u'type': unicode(tab_type), u'name': unicode(name)})
tabs = course.tabs
tabs.insert(num, new_tab)
modulestore('direct').update_item(course, '**replace_user**')
modulestore().update_item(course, '**replace_user**')
cms/djangoapps/contentstore/views/tests/test_assets.py
......@@ -48,9 +48,10 @@ class BasicAssetsTestCase(AssetsTestCase):
self.assertEquals(path, '/static/my_file_name.jpg')
def test_pdf_asset(self):
module_store = modulestore('direct')
module_store = modulestore()
_, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['toy'],
static_content_store=contentstore(),
......@@ -191,9 +192,10 @@ class LockAssetTestCase(AssetsTestCase):
return json.loads(resp.content)
# Load the toy course.
module_store = modulestore('direct')
module_store = modulestore()
_, course_items = import_from_xml(
module_store,
'**replace_user**',
'common/test/data/',
['toy'],
static_content_store=contentstore(),
......
cms/djangoapps/contentstore/views/tests/test_checklists.py
""" Unit tests for checklist methods in views.py. """
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.utils import reverse_course_url
from contentstore.views.checklist import expand_checklist_action_url
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore
import json
from contentstore.tests.utils import CourseTestCase
......@@ -21,8 +22,7 @@ class ChecklistTestCase(CourseTestCase):
def get_persisted_checklists(self):
""" Returns the checklists as persisted in the modulestore. """
modulestore = get_modulestore(self.course.location)
return modulestore.get_item(self.course.location).checklists
return modulestore().get_item(self.course.location).checklists
def compare_checklists(self, persisted, request):
"""
......@@ -54,8 +54,7 @@ class ChecklistTestCase(CourseTestCase):
self.course.checklists = None
# Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore
self.course.save()
modulestore = get_modulestore(self.course.location)
modulestore.update_item(self.course, self.user.id)
modulestore().update_item(self.course, self.user.id)
self.assertEqual(self.get_persisted_checklists(), None)
response = self.client.get(self.checklists_url)
self.assertEqual(payload, response.content)
......
cms/djangoapps/contentstore/views/tests/test_container_page.py
......@@ -3,8 +3,9 @@ Unit tests for the container page.
"""
import re
from contentstore.utils import compute_publish_state, PublishState
from contentstore.utils import compute_publish_state
from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore import PublishState
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory
......@@ -29,6 +30,7 @@ class ContainerPageTestCase(StudioPageTestCase):
category='vertical', display_name='Child Vertical')
self.video = ItemFactory.create(parent_location=self.child_vertical.location,
category="video", display_name="My Video")
self.store = modulestore()
def test_container_html(self):
self._test_html_content(
......@@ -49,12 +51,12 @@ class ContainerPageTestCase(StudioPageTestCase):
Create the scenario of an xblock with children (non-vertical) on the container page.
This should create a container page that is a child of another container page.
"""
published_container = ItemFactory.create(
draft_container = ItemFactory.create(
parent_location=self.child_container.location,
category="wrapper", display_name="Wrapper"
)
ItemFactory.create(
parent_location=published_container.location,
parent_location=draft_container.location,
category="html", display_name="Child HTML"
)
......@@ -63,7 +65,7 @@ class ContainerPageTestCase(StudioPageTestCase):
xblock,
expected_section_tag=(
'<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
'data-locator="{0}" data-course-key="{0.course_key}">'.format(published_container.location)
'data-locator="{0}" data-course-key="{0.course_key}">'.format(draft_container.location)
),
expected_breadcrumbs=(
r'<a href="/unit/{unit}"\s*'
......@@ -77,13 +79,12 @@ class ContainerPageTestCase(StudioPageTestCase):
)
)
# Test the published version of the container
test_container_html(published_container)
# Test the draft version of the container
test_container_html(draft_container)
# Now make the unit and its children into a draft and validate the container again
modulestore('draft').convert_to_draft(self.vertical.location)
modulestore('draft').convert_to_draft(self.child_vertical.location)
draft_container = modulestore('draft').convert_to_draft(published_container.location)
# Now publish the unit and validate again
self.store.publish(self.vertical.location, self.user.id)
draft_container = self.store.get_item(draft_container.location)
test_container_html(draft_container)
def _test_html_content(self, xblock, expected_section_tag, expected_breadcrumbs):
......@@ -98,7 +99,6 @@ class ContainerPageTestCase(StudioPageTestCase):
self.assertRegexpMatches(html, expected_breadcrumbs)
# Verify the link that allows users to change publish status.
expected_message = None
if publish_state == PublishState.public:
expected_message = 'you need to edit unit <a href="/unit/{}">Unit</a> as a draft.'
else:
......@@ -110,25 +110,25 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
Verify that a public xblock's container preview returns the expected HTML.
"""
self.validate_preview_html(self.vertical, self.container_view,
published_unit = self.store.publish(self.vertical.location, self.user.id)
published_child_container = self.store.get_item(self.child_container.location)
published_child_vertical = self.store.get_item(self.child_vertical.location)
self.validate_preview_html(published_unit, self.container_view,
can_edit=False, can_reorder=False, can_add=False)
self.validate_preview_html(self.child_container, self.container_view,
self.validate_preview_html(published_child_container, self.container_view,
can_edit=False, can_reorder=False, can_add=False)
self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
self.validate_preview_html(published_child_vertical, self.reorderable_child_view,
can_edit=False, can_reorder=False, can_add=False)
def test_draft_container_preview_html(self):
"""
Verify that a draft xblock's container preview returns the expected HTML.
"""
draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
draft_child_container = modulestore('draft').convert_to_draft(self.child_container.location)
draft_child_vertical = modulestore('draft').convert_to_draft(self.child_vertical.location)
self.validate_preview_html(draft_unit, self.container_view,
self.validate_preview_html(self.vertical, self.container_view,
can_edit=True, can_reorder=True, can_add=True)
self.validate_preview_html(draft_child_container, self.container_view,
self.validate_preview_html(self.child_container, self.container_view,
can_edit=True, can_reorder=True, can_add=True)
self.validate_preview_html(draft_child_vertical, self.reorderable_child_view,
self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
can_edit=True, can_reorder=True, can_add=True)
def test_public_child_container_preview_html(self):
......@@ -137,7 +137,8 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
category='split_test', display_name='Split Test')
self.validate_preview_html(empty_child_container, self.reorderable_child_view,
published_empty_child_container = self.store.publish(empty_child_container.location, '**replace_user**')
self.validate_preview_html(published_empty_child_container, self.reorderable_child_view,
can_reorder=False, can_edit=False, can_add=False)
def test_draft_child_container_preview_html(self):
......@@ -146,7 +147,5 @@ class ContainerPageTestCase(StudioPageTestCase):
"""
empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
category='split_test', display_name='Split Test')
modulestore('draft').convert_to_draft(self.vertical.location)
draft_empty_child_container = modulestore('draft').convert_to_draft(empty_child_container.location)
self.validate_preview_html(draft_empty_child_container, self.reorderable_child_view,
self.validate_preview_html(empty_child_container, self.reorderable_child_view,
can_reorder=True, can_edit=True, can_add=False)
cms/djangoapps/contentstore/views/tests/test_course_updates.py
......@@ -129,13 +129,12 @@ class CourseUpdateTest(CourseTestCase):
'''
# get the updates and populate 'data' field with some data.
location = self.course.id.make_usage_key('course_info', 'updates')
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().create_and_save_xmodule(location, self.user.id)
update_date = u"January 23, 2014"
update_content = u"Hello world!"
update_data = u"<ol><li><h2>" + update_date + "</h2>" + update_content + "</li></ol>"
course_updates.data = update_data
modulestore('direct').update_item(course_updates, self.user.id)
modulestore().update_item(course_updates, self.user.id)
# test getting all updates list
course_update_url = self.create_update_url()
......@@ -155,7 +154,7 @@ class CourseUpdateTest(CourseTestCase):
# test that while updating it converts old data (with string format in 'data' field)
# to new data (with list format in 'items' field) and respectively updates 'data' field.
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [])
# now try to update first update item
update_content = 'Testing'
......@@ -164,20 +163,20 @@ class CourseUpdateTest(CourseTestCase):
course_update_url + '1', payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
)
self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
# course_updates 'data' field should update accordingly
update_data = u"<section><article><h2>{date}</h2>{content}</article></section>".format(date=update_date, content=update_content)
self.assertEqual(course_updates.data, update_data)
# test delete course update item (soft delete)
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
# now try to delete first update item
resp = self.client.delete(course_update_url + '1')
self.assertEqual(json.loads(resp.content), [])
# confirm that course update is soft deleted ('status' flag set to 'deleted') in db
course_updates = modulestore('direct').get_item(location)
course_updates = modulestore().get_item(location)
self.assertEqual(course_updates.items,
[{u'date': update_date, u'content': update_content, u'id': 1, u'status': 'deleted'}])
......@@ -204,10 +203,10 @@ class CourseUpdateTest(CourseTestCase):
'''Test trying to add to a saved course_update which is not an ol.'''
# get the updates and set to something wrong
location = self.course.id.make_usage_key('course_info', 'updates')
modulestore('direct').create_and_save_xmodule(location)
course_updates = modulestore('direct').get_item(location)
modulestore().create_and_save_xmodule(location, self.user.id)
course_updates = modulestore().get_item(location)
course_updates.data = 'bad news'
modulestore('direct').update_item(course_updates, self.user.id)
modulestore().update_item(course_updates, self.user.id)
init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
content = init_content + '</iframe>'
......
cms/djangoapps/contentstore/views/tests/test_import_export.py
......@@ -294,7 +294,8 @@ class ExportTestCase(CourseTestCase):
"""
Export failure.
"""
ItemFactory.create(parent_location=self.course.location, category='aawefawef')
fake_xblock = ItemFactory.create(parent_location=self.course.location, category='aawefawef')
self.store.publish(fake_xblock.location, self.user.id)
self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')
def test_export_failure_subsection_level(self):
......
cms/djangoapps/contentstore/views/tests/test_transcripts.py
......@@ -21,12 +21,11 @@ from xmodule.exceptions import NotFoundError
from opaque_keys.edx.keys import UsageKey
from xmodule.video_module import transcripts_utils
from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class Basetranscripts(CourseTestCase):
"""Base test class for transcripts tests."""
......
cms/djangoapps/contentstore/views/tests/test_unit_page.py
......@@ -19,11 +19,13 @@ class UnitPageTestCase(StudioPageTestCase):
category='vertical', display_name='Unit')
self.video = ItemFactory.create(parent_location=self.vertical.location,
category="video", display_name="My Video")
self.store = modulestore()
def test_public_unit_page_html(self):
"""
Verify that an xblock returns the expected HTML for a public unit page.
"""
html = self.get_page_html(self.vertical)
self.validate_html_for_add_buttons(html)
......@@ -31,14 +33,14 @@ class UnitPageTestCase(StudioPageTestCase):
"""
Verify that an xblock returns the expected HTML for a draft unit page.
"""
draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
html = self.get_page_html(draft_unit)
html = self.get_page_html(self.vertical)
self.validate_html_for_add_buttons(html)
def test_public_component_preview_html(self):
"""
Verify that a public xblock's preview returns the expected HTML.
"""
published_video = self.store.publish(self.video.location, '**replace_user**')
self.validate_preview_html(self.video, STUDENT_VIEW,
can_edit=True, can_reorder=True, can_add=False)
......@@ -46,9 +48,7 @@ class UnitPageTestCase(StudioPageTestCase):
"""
Verify that a draft xblock's preview returns the expected HTML.
"""
modulestore('draft').convert_to_draft(self.vertical.location)
draft_video = modulestore('draft').convert_to_draft(self.video.location)
self.validate_preview_html(draft_video, STUDENT_VIEW,
self.validate_preview_html(self.video, STUDENT_VIEW,
can_edit=True, can_reorder=True, can_add=False)
def test_public_child_container_preview_html(self):
......@@ -60,7 +60,8 @@ class UnitPageTestCase(StudioPageTestCase):
category='split_test', display_name='Split Test')
ItemFactory.create(parent_location=child_container.location,
category='html', display_name='grandchild')
self.validate_preview_html(child_container, STUDENT_VIEW,
published_child_container = self.store.publish(child_container.location, '**replace_user**')
self.validate_preview_html(published_child_container, STUDENT_VIEW,
can_reorder=True, can_edit=True, can_add=False)
def test_draft_child_container_preview_html(self):
......@@ -72,7 +73,6 @@ class UnitPageTestCase(StudioPageTestCase):
category='split_test', display_name='Split Test')
ItemFactory.create(parent_location=child_container.location,
category='html', display_name='grandchild')
modulestore('draft').convert_to_draft(self.vertical.location)
draft_child_container = modulestore('draft').get_item(child_container.location)
draft_child_container = self.store.get_item(child_container.location)
self.validate_preview_html(draft_child_container, STUDENT_VIEW,
can_reorder=True, can_edit=True, can_add=False)
cms/djangoapps/models/settings/course_details.py
......@@ -6,7 +6,7 @@ from json.encoder import JSONEncoder
from opaque_keys.edx.locations import Location
from xmodule.modulestore.exceptions import ItemNotFoundError
from contentstore.utils import get_modulestore, course_image_url
from contentstore.utils import course_image_url
from models.settings import course_grading
from xmodule.fields import Date
from xmodule.modulestore.django import modulestore
......@@ -34,7 +34,7 @@ class CourseDetails(object):
"""
Fetch the course details for the given course from persistence and return a CourseDetails model.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
course_details = cls(course_key.org, course_key.course, course_key.run)
course_details.start_date = descriptor.start
......@@ -46,31 +46,31 @@ class CourseDetails(object):
temploc = course_key.make_usage_key('about', 'syllabus')
try:
course_details.syllabus = get_modulestore(temploc).get_item(temploc).data
course_details.syllabus = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'short_description')
try:
course_details.short_description = get_modulestore(temploc).get_item(temploc).data
course_details.short_description = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'overview')
try:
course_details.overview = get_modulestore(temploc).get_item(temploc).data
course_details.overview = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'effort')
try:
course_details.effort = get_modulestore(temploc).get_item(temploc).data
course_details.effort = modulestore().get_item(temploc).data
except ItemNotFoundError:
pass
temploc = course_key.make_usage_key('about', 'video')
try:
raw_video = get_modulestore(temploc).get_item(temploc).data
raw_video = modulestore().get_item(temploc).data
course_details.intro_video = CourseDetails.parse_video_tag(raw_video)
except ItemNotFoundError:
pass
......@@ -84,14 +84,14 @@ class CourseDetails(object):
delete the about item.
"""
temploc = course_key.make_usage_key('about', about_key)
store = get_modulestore(temploc)
store = modulestore()
if data is None:
store.delete_item(temploc)
store.delete_item(temploc, user.id)
else:
try:
about_item = store.get_item(temploc)
except ItemNotFoundError:
about_item = store.create_xmodule(temploc, system=course.runtime)
about_item = store.create_xmodule(temploc, runtime=course.runtime)
about_item.data = data
store.update_item(about_item, user.id)
......@@ -100,7 +100,7 @@ class CourseDetails(object):
"""
Decode the json into CourseDetails and save any changed attrs to the db
"""
module_store = modulestore('direct')
module_store = modulestore()
descriptor = module_store.get_course(course_key)
dirty = False
......
cms/djangoapps/models/settings/course_grading.py
......@@ -21,7 +21,7 @@ class CourseGradingModel(object):
"""
Fetch the course grading policy for the given course from persistence and return a CourseGradingModel.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
model = cls(descriptor)
return model
......@@ -31,7 +31,7 @@ class CourseGradingModel(object):
Fetch the course's nth grader
Returns an empty dict if there's no such grader.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
index = int(index)
if len(descriptor.raw_grader) > index:
return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
......@@ -52,14 +52,14 @@ class CourseGradingModel(object):
Decode the json into CourseGradingModel and save any changes. Returns the modified model.
Probably not the usual path for updates as it's too coarse grained.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]
descriptor.raw_grader = graders_parsed
descriptor.grade_cutoffs = jsondict['grade_cutoffs']
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
CourseGradingModel.update_grace_period_from_json(course_key, jsondict['grace_period'], user)
......@@ -71,7 +71,7 @@ class CourseGradingModel(object):
Create or update the grader of the given type (string key) for the given course. Returns the modified
grader which is a full model on the client but not on the server (just a dict)
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
# parse removes the id; so, grab it before parse
index = int(grader.get('id', len(descriptor.raw_grader)))
......@@ -82,7 +82,7 @@ class CourseGradingModel(object):
else:
descriptor.raw_grader.append(grader)
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])
......@@ -92,10 +92,10 @@ class CourseGradingModel(object):
Create or update the grade cutoffs for the given course. Returns sent in cutoffs (ie., no extra
db fetch).
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
descriptor.grade_cutoffs = cutoffs
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return cutoffs
......@@ -106,7 +106,7 @@ class CourseGradingModel(object):
grace_period entry in an enclosing dict. It is also safe to call this method with a value of
None for graceperiodjson.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
# Before a graceperiod has ever been created, it will be None (once it has been
# created, it cannot be set back to None).
......@@ -117,14 +117,14 @@ class CourseGradingModel(object):
grace_timedelta = timedelta(**graceperiodjson)
descriptor.graceperiod = grace_timedelta
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def delete_grader(course_key, index, user):
"""
Delete the grader of the given type from the given course.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
index = int(index)
if index < len(descriptor.raw_grader):
......@@ -132,22 +132,22 @@ class CourseGradingModel(object):
# force propagation to definition
descriptor.raw_grader = descriptor.raw_grader
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def delete_grace_period(course_key, user):
"""
Delete the course's grace period.
"""
descriptor = modulestore('direct').get_course(course_key)
descriptor = modulestore().get_course(course_key)
del descriptor.graceperiod
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
@staticmethod
def get_section_grader_type(location):
descriptor = modulestore('direct').get_item(location)
descriptor = modulestore().get_item(location)
return {
"graderType": descriptor.format if descriptor.format is not None else 'notgraded',
"location": unicode(location),
......@@ -162,7 +162,7 @@ class CourseGradingModel(object):
del descriptor.format
del descriptor.graded
modulestore('direct').update_item(descriptor, user.id)
modulestore().update_item(descriptor, user.id)
return {'graderType': grader_type}
@staticmethod
......
cms/djangoapps/models/settings/course_metadata.py
from xblock.fields import Scope
from contentstore.utils import get_modulestore
from xmodule.modulestore.django import modulestore
from django.utils.translation import ugettext as _
class CourseMetadata(object):
'''
For CRUD operations on metadata fields which do not have specific editors
......@@ -86,6 +84,6 @@ class CourseMetadata(object):
setattr(descriptor, key, value)
if len(key_values) > 0:
get_modulestore(descriptor.location).update_item(descriptor, user.id if user else None)
modulestore().update_item(descriptor, user.id if user else None)
return cls.fetch(descriptor)
cms/envs/acceptance.py
......@@ -34,35 +34,17 @@ logging.getLogger('track.middleware').setLevel(logging.CRITICAL)
# use, so these are not set up for TOS and PRIVACY
logging.getLogger('edxmako.shortcuts').setLevel(logging.ERROR)
DOC_STORE_CONFIG = {
'host': 'localhost',
'db': 'acceptance_xmodule',
'collection': 'acceptance_modulestore_%s' % seed(),
}
MODULESTORE_OPTIONS = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
update_module_store_settings(
MODULESTORE,
doc_store_settings={
'db': 'acceptance_xmodule',
'collection': 'acceptance_modulestore_%s' % seed(),
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
}
}
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
......
cms/envs/aws.py
......@@ -229,7 +229,7 @@ if AWS_SECRET_ACCESS_KEY == "":
AWS_SECRET_ACCESS_KEY = None
DATABASES = AUTH_TOKENS['DATABASES']
MODULESTORE = AUTH_TOKENS['MODULESTORE']
MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS['MODULESTORE'])
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
# Datadog for events!
......
cms/envs/bok_choy.auth.json
......@@ -48,55 +48,47 @@
},
"MODULESTORE": {
"default": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
}
},
"direct": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
"mappings": {},
"reference_type": "Location",
"stores": [
{
"NAME": "draft",
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"user": "edxapp"
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"localhost"
],
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
}
},
{
"NAME": "xml",
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "** OVERRIDDEN **",
"default_class": "xmodule.hidden_module.HiddenDescriptor"
}
}
]
}
}
},
......
cms/envs/bok_choy.py
......@@ -27,9 +27,16 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=E11
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()
# Configure Mongo modulestore to use the test folder within the repo
for store in ["default", "direct"]:
MODULESTORE[store]['OPTIONS']['fs_root'] = (TEST_ROOT / "data").abspath() # pylint: disable=E1120
# Configure modulestore to use the test folder within the repo
update_module_store_settings(
MODULESTORE,
module_store_options={
'fs_root': (TEST_ROOT / "data").abspath(), # pylint: disable=E1120
},
xml_store_options={
'data_dir': (TEST_ROOT / "data").abspath(),
},
)
# Enable django-pipeline and staticfiles
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
......
cms/envs/common.py
......@@ -27,10 +27,12 @@ Longer TODO:
import imp
import sys
import lms.envs.common
# Although this module itself may not use these imported variables, other dependent modules may.
from lms.envs.common import (
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE
)
from path import path
from lms.envs.modulestore_settings import *
from lms.lib.xblock.mixin import LmsBlockMixin
from dealer.git import git
......@@ -244,6 +246,9 @@ XBLOCK_MIXINS = (LmsBlockMixin, InheritanceMixin, XModuleMixin)
# xblocks can be added via advanced settings
XBLOCK_SELECT_FUNCTION = prefer_xmodules
############################ Modulestore Configuration ################################
MODULESTORE_BRANCH = 'draft'
############################ DJANGO_BUILTINS ################################
# Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
DEBUG = False
......
cms/envs/dev.py
......@@ -19,30 +19,13 @@ LOGGING = get_logger_config(ENV_ROOT / "log",
dev_env=True,
debug=True)
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': GITHUB_REPO_ROOT,
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
update_module_store_settings(
MODULESTORE,
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': GITHUB_REPO_ROOT,
}
}
)
# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)
......
cms/envs/test.py
......@@ -16,6 +16,7 @@ from .common import *
import os
from path import path
from warnings import filterwarnings
from uuid import uuid4
# import settings from LMS for consistent behavior with CMS
from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME)
......@@ -58,40 +59,29 @@ STATICFILES_DIRS += [
if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
]
DOC_STORE_CONFIG = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'test_modulestore',
}
MODULESTORE_OPTIONS = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
'render_template': 'edxmako.shortcuts.render_to_string',
}
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
# Add split as another store for testing
MODULESTORE['default']['OPTIONS']['stores'].append(
{
'NAME': 'split',
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
'OPTIONS': {
'render_template': 'edxmako.shortcuts.render_to_string',
}
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
)
# Update module store settings per defaults for tests
update_module_store_settings(
MODULESTORE,
module_store_options={
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': TEST_ROOT / "data",
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
doc_store_settings={
'db': 'test_xmodule',
'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': MODULESTORE_OPTIONS
}
}
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
......
cms/static/coffee/src/views/unit.coffee
......@@ -159,7 +159,7 @@ define ["jquery", "jquery.ui", "gettext", "backbone",
@wait(true)
$.ajax({
type: 'DELETE',
url: @model.url() + "?" + $.param({recurse: true})
url: @model.url()
}).success(=>
analytics.track "Deleted Draft",
......
cms/static/js/base.js
......@@ -284,7 +284,7 @@ function _deleteItem($el, type) {
$.ajax({
type: 'DELETE',
url: ModuleUtils.getUpdateUrl(locator) +'?'+ $.param({recurse: true, all_versions: true}),
url: ModuleUtils.getUpdateUrl(locator),
success: function () {
$el.remove();
deleting.hide();
......
cms/static/js/views/pages/container.js
......@@ -161,8 +161,7 @@ define(["jquery", "underscore", "gettext", "js/views/baseview", "js/views/contai
return $.ajax({
type: 'DELETE',
url: self.getURLRoot() + "/" +
xblockElement.data('locator') + "?" +
$.param({recurse: true, all_versions: false})
xblockElement.data('locator')
}).success(_.bind(self.onDelete, self, xblockElement));
});
});
......
cms/templates/container.html
......@@ -2,7 +2,7 @@
<%!
import json
from contentstore.utils import PublishState
from xmodule.modulestore import PublishState
from contentstore.views.helpers import xblock_studio_url, EDITING_TEMPLATES
from django.utils.translation import ugettext as _
%>
......
common/djangoapps/contentserver/tests/test.py
......@@ -17,8 +17,7 @@ from student.models import CourseEnrollment
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import (studio_store_config,
ModuleStoreTestCase)
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_from_xml
log = logging.getLogger(__name__)
......@@ -26,10 +25,8 @@ log = logging.getLogger(__name__)
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Tests that use the toy course.
......@@ -39,16 +36,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
"""
Create user and login.
"""
settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
self.client = Client()
self.contentstore = contentstore()
self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
import_from_xml(modulestore(), '**replace_user**', 'common/test/data/', ['toy'],
static_content_store=self.contentstore, verbose=True)
# A locked asset
......
common/djangoapps/external_auth/tests/test_shib.py
......@@ -18,7 +18,7 @@ from django.utils.importlib import import_module
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from external_auth.models import ExternalAuthMap
......@@ -80,7 +80,7 @@ class ShibSPTest(ModuleStoreTestCase):
request_factory = RequestFactory()
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
@unittest.skipUnless(settings.FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
def test_exception_shib_login(self):
......
common/djangoapps/heartbeat/tests/test_heartbeat.py
......@@ -9,9 +9,9 @@ import mock
from django.test.utils import override_settings
from django.conf import settings
from django.test.testcases import TestCase
from xmodule.modulestore.tests.django_utils import mongo_store_config
from xmodule.modulestore.tests.django_utils import draft_mongo_store_config
TEST_MODULESTORE = mongo_store_config(settings.TEST_ROOT / "data")
TEST_MODULESTORE = draft_mongo_store_config(settings.TEST_ROOT / "data")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class HeartbeatTestCase(TestCase):
......
common/djangoapps/student/migrations/0036_access_roles_orgless.py
......@@ -24,27 +24,11 @@ class Migration(DataMigration):
"""
Converts group table entries for write access and beta_test roles to course access roles table.
"""
def get_modulestore(ms_type, key):
"""
Find the modulestore of the given type trying the key first
"""
try:
store = modulestore(key)
if isinstance(store, MixedModuleStore):
store = store.modulestores[key]
if store.get_modulestore_type(None) == ms_type:
return store
else:
return None
except KeyError:
return None
# Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
loc_map_collection = loc_mapper().location_map
xml_ms = get_modulestore(XML_MODULESTORE_TYPE, 'xml')
mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'default')
if mongo_ms is None:
mongo_ms = get_modulestore(MONGO_MODULESTORE_TYPE, 'direct')
mixed_ms = modulestore()
xml_ms = mixed_ms._get_modulestore_by_type(XML_MODULESTORE_TYPE)
mongo_ms = mixed_ms._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
query = Q(name__startswith='staff') | Q(name__startswith='instructor') | Q(name__startswith='beta_testers')
for group in orm['auth.Group'].objects.filter(query).exclude(name__contains="/").all():
......
common/djangoapps/student/tests/test_course_listing.py
......@@ -6,7 +6,8 @@ from mock import patch, Mock
from student.tests.factories import UserFactory
from student.roles import GlobalStaff
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, studio_store_config
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore
......@@ -14,13 +15,8 @@ from xmodule.error_module import ErrorDescriptor
from django.test.client import Client
from student.models import CourseEnrollment
from student.views import get_course_enrollment_pairs
from django.conf import settings
from django.test.utils import override_settings
TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseListing(ModuleStoreTestCase):
"""
Unit tests for getting the list of courses for a logged in user
......@@ -44,8 +40,7 @@ class TestCourseListing(ModuleStoreTestCase):
course = CourseFactory.create(
org=course_location.org,
number=course_location.course,
run=course_location.run,
modulestore=modulestore('direct'),
run=course_location.run
)
CourseEnrollment.enroll(self.student, course.id)
......@@ -84,7 +79,7 @@ class TestCourseListing(ModuleStoreTestCase):
self._create_course_with_access_groups(course_key)
with patch('xmodule.modulestore.mongo.base.MongoKeyValueStore', Mock(side_effect=Exception)):
self.assertIsInstance(modulestore('direct').get_course(course_key), ErrorDescriptor)
self.assertIsInstance(modulestore().get_course(course_key), ErrorDescriptor)
# get courses through iterating all courses
courses_list = list(get_course_enrollment_pairs(self.student, None, []))
......@@ -95,18 +90,20 @@ class TestCourseListing(ModuleStoreTestCase):
Create good courses, courses that won't load, and deleted courses which still have
roles. Test course listing.
"""
mongo_store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
good_location = SlashSeparatedCourseKey('testOrg', 'testCourse', 'RunBabyRun')
self._create_course_with_access_groups(good_location)
course_location = SlashSeparatedCourseKey('testOrg', 'doomedCourse', 'RunBabyRun')
self._create_course_with_access_groups(course_location)
modulestore('direct').delete_course(course_location)
mongo_store.delete_course(course_location)
course_location = SlashSeparatedCourseKey('testOrg', 'erroredCourse', 'RunBabyRun')
course = self._create_course_with_access_groups(course_location)
course_db_record = modulestore('direct')._find_one(course.location)
course_db_record = mongo_store._find_one(course.location)
course_db_record.setdefault('metadata', {}).get('tabs', []).append({"type": "wiko", "name": "Wiki" })
modulestore('direct').collection.update(
mongo_store.collection.update(
{'_id': course.location.to_deprecated_son()},
{'$set': {
'metadata.tabs': course_db_record['metadata']['tabs'],
......
common/djangoapps/student/tests/test_login.py
......@@ -17,7 +17,7 @@ from student.views import _parse_course_id_from_string, _get_course_enrollment_d
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from external_auth.models import ExternalAuthMap
from opaque_keys.edx.locations import SlashSeparatedCourseKey
......@@ -289,7 +289,7 @@ class ExternalAuthShibTest(ModuleStoreTestCase):
Tests how login_user() interacts with ExternalAuth, in particular Shib
"""
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
self.course = CourseFactory.create(org='Stanford', number='456', display_name='NO SHIB')
self.shib_course = CourseFactory.create(org='Stanford', number='123', display_name='Shib Only')
self.shib_course.enrollment_domain = 'shib:https://idp.stanford.edu/'
......
common/djangoapps/terrain/browser.py
......@@ -19,6 +19,7 @@ from json import dumps
from pymongo import MongoClient
import xmodule.modulestore.django
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
# There is an import issue when using django-staticfiles with lettuce
# Lettuce assumes that we are using django.contrib.staticfiles,
......@@ -189,7 +190,7 @@ def reset_databases(scenario):
mongo.drop_database(settings.CONTENTSTORE['DOC_STORE_CONFIG']['db'])
_CONTENTSTORE.clear()
modulestore = xmodule.modulestore.django.editable_modulestore()
modulestore = xmodule.modulestore.django.modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
modulestore.collection.drop()
xmodule.modulestore.django.clear_existing_modulestores()
......
common/djangoapps/terrain/course_helpers.py
......@@ -5,7 +5,8 @@ import urllib
from lettuce import world
from django.contrib.auth.models import User, Group
from student.models import CourseEnrollment
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.contentstore.django import contentstore
......@@ -71,5 +72,6 @@ def clear_courses():
# (though it shouldn't), do this manually
# from the bash shell to drop it:
# $ mongo test_xmodule --eval "db.dropDatabase()"
editable_modulestore().collection.drop()
store = modulestore()._get_modulestore_by_type(MONGO_MODULESTORE_TYPE)
store.collection.drop()
contentstore().fs_files.drop()
common/lib/xmodule/xmodule/contentstore/content.py
......@@ -48,7 +48,7 @@ class StaticContent(object):
- course_key: the course that this asset belongs to
- path: is the name of the static asset
- revision: is the object's revision information
- is_tumbnail: is whether or not we want the thumbnail version of this
- is_thumbnail: is whether or not we want the thumbnail version of this
asset
"""
path = path.replace('/', '_')
......
common/lib/xmodule/xmodule/contentstore/mongo.py
......@@ -12,7 +12,6 @@ from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
import json
import bson.son
from bson.son import SON
from opaque_keys.edx.locations import AssetLocation
......
common/lib/xmodule/xmodule/error_module.py
......@@ -13,7 +13,7 @@ from xmodule.x_module import XModule, XModuleDescriptor
from xmodule.errortracker import exc_info_to_str
from xblock.fields import String, Scope, ScopeIds
from xblock.field_data import DictFieldData
from xmodule.modulestore.xml_exporter import EdxJSONEncoder
from xmodule.modulestore import EdxJSONEncoder
log = logging.getLogger(__name__)
......
common/lib/xmodule/xmodule/modulestore/__init__.py
......@@ -5,6 +5,8 @@ that are stored in a database an accessible using their Location as an identifie
import logging
import re
import json
import datetime
from collections import namedtuple, defaultdict
import collections
......@@ -20,15 +22,60 @@ from opaque_keys import InvalidKeyError
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xblock.runtime import Mixologist
from xblock.core import XBlock
import datetime
log = logging.getLogger('edx.modulestore')
# Modulestore Types
SPLIT_MONGO_MODULESTORE_TYPE = 'split'
MONGO_MODULESTORE_TYPE = 'mongo'
XML_MODULESTORE_TYPE = 'xml'
# Key Revision constants to use for Location and Usage Keys
# Note: These values are persisted in the database, so should not be changed without migrations
KEY_REVISION_DRAFT = 'draft'
KEY_REVISION_PUBLISHED = None
# Revision constants to use for Module Store operations
# Note: These values are passed into store APIs and only used at run time
# both DRAFT and PUBLISHED versions are queried, with preference to DRAFT versions
REVISION_OPTION_DRAFT_PREFERRED = 'rev-opt-draft-preferred'
# only DRAFT versions are queried and no PUBLISHED versions
REVISION_OPTION_DRAFT_ONLY = 'rev-opt-draft-only'
# # only PUBLISHED versions are queried and no DRAFT versions
REVISION_OPTION_PUBLISHED_ONLY = 'rev-opt-published-only'
# all revisions are queried
REVISION_OPTION_ALL = 'rev-opt-all'
# Branch constants to use for stores, such as Mongo, that have only 2 branches: DRAFT and PUBLISHED
# Note: These values are taken from server configuration settings, so should not be changed without alerting DevOps
BRANCH_DRAFT_PREFERRED = 'draft'
BRANCH_PUBLISHED_ONLY = 'published'
# Branch constants to use for stores, such as Split, that have named branches
BRANCH_NAME_DRAFT = 'draft'
BRANCH_NAME_PUBLISHED = 'published'
class PublishState(object):
"""
The publish state for a given xblock-- either 'draft', 'private', or 'public'.
Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
"""
draft = 'draft'
private = 'private'
public = 'public'
class ModuleStoreRead(object):
"""
An abstract interface for a database backend that stores XModuleDescriptor
......@@ -184,11 +231,9 @@ class ModuleStoreRead(object):
pass
@abstractmethod
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
def get_parent_location(self, location, **kwargs):
'''Find the location that is the parent of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location.
'''
pass
......@@ -245,11 +290,14 @@ class ModuleStoreWrite(ModuleStoreRead):
@abstractmethod
def delete_item(self, location, user_id=None, **kwargs):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
Delete an item and its subtree from persistence. Remove the item from any parents (Note, does not
affect parents from other branches or logical branches; thus, in old mongo, deleting something
whose parent cannot be draft, deletes it from both but deleting a component under a draft vertical
only deletes it from the draft.
Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
......@@ -346,7 +394,7 @@ class ModuleStoreReadBase(ModuleStoreRead):
def has_course(self, course_id, ignore_case=False):
"""
Look for a specific course id. Returns whether it exists.
Returns the course_id of the course if it was found, else None
Args:
course_id (CourseKey):
ignore_case (boolean): some modulestores are case-insensitive. Use this flag
......@@ -355,12 +403,18 @@ class ModuleStoreReadBase(ModuleStoreRead):
# linear search through list
assert(isinstance(course_id, CourseKey))
if ignore_case:
return any(
(c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower())
for c in self.get_courses()
return next(
(
c.id for c in self.get_courses()
if c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower()
),
None
)
else:
return any(c.id == course_id for c in self.get_courses())
return next(
(c.id for c in self.get_courses() if c.id == course_id),
None
)
def heartbeat(self):
"""
......@@ -411,13 +465,12 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError
def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
def delete_item(self, location, user_id=None, force=False):
"""
Delete an item from persistence. Pass the user's unique id which the persistent store
should save with the update if it has that ability.
:param delete_all_versions: removes both the draft and published version of this item from
the course if using draft and old mongo. Split may or may not implement this.
:param user_id: ID of the user deleting the item
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
......@@ -426,6 +479,21 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
"""
raise NotImplementedError
def create_and_save_xmodule(self, location, user_id, definition_data=None, metadata=None, runtime=None, fields={}):
"""
Create the new xmodule and save it.
:param location: a Location--must have a category
:param user_id: ID of the user creating and saving the xmodule
:param definition_data: can be empty. The initial definition_data for the kvs
:param metadata: can be empty, the initial metadata for the kvs
:param runtime: if you already have an xblock from the course, the xblock.runtime value
:param fields: a dictionary of field names and values for the new xmodule
"""
new_object = self.create_xmodule(location, definition_data, metadata, runtime, fields)
self.update_item(new_object, user_id, allow_not_found=True)
return new_object
def only_xmodules(identifier, entry_points):
"""Only use entry_points that are supplied by the xmodule package"""
......@@ -441,3 +509,25 @@ def prefer_xmodules(identifier, entry_points):
return default_select(identifier, from_xmodule)
else:
return default_select(identifier, entry_points)
class EdxJSONEncoder(json.JSONEncoder):
"""
Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
`Location`s are encoded as their url string form, and `datetime`s as
ISO date strings
"""
def default(self, obj):
if isinstance(obj, Location):
return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
return obj.isoformat() + 'Z'
else:
return obj.isoformat()
else:
return obj.isoformat()
else:
return super(EdxJSONEncoder, self).default(obj)
common/lib/xmodule/xmodule/modulestore/django.py
......@@ -5,16 +5,18 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
"""
from __future__ import absolute_import
from importlib import import_module
import re
from importlib import import_module
from django.conf import settings
from django.core.cache import get_cache, InvalidCacheBackendError
import django.utils
import re
import threading
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.util.django import get_current_request_hostname
import xmodule.modulestore # pylint: disable=unused-import
# We may not always have the request_cache module available
try:
......@@ -23,10 +25,6 @@ try:
except ImportError:
HAS_REQUEST_CACHE = False
_MODULESTORES = {}
FUNCTION_KEYS = ['render_template']
def load_function(path):
"""
......@@ -48,6 +46,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
_options = {}
_options.update(options)
FUNCTION_KEYS = ['render_template']
for key in FUNCTION_KEYS:
if key in _options and isinstance(_options[key], basestring):
_options[key] = load_function(_options[key])
......@@ -69,59 +68,50 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
doc_store_config=doc_store_config,
i18n_service=i18n_service or ModuleI18nService(),
branch_setting_func=_get_modulestore_branch_setting,
**_options
)
def get_default_store_name_for_current_request():
"""
This method will return the appropriate default store mapping for the current Django request,
else 'default' which is the system default
"""
store_name = 'default'
# A singleton instance of the Mixed Modulestore
_MIXED_MODULESTORE = None
# see what request we are currently processing - if any at all - and get hostname for the request
hostname = get_current_request_hostname()
# get mapping information which is defined in configurations
mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
# compare hostname against the regex expressions set of mappings
# which will tell us which store name to use
if hostname and mappings:
for key in mappings.keys():
if re.match(key, hostname):
store_name = mappings[key]
return store_name
def modulestore():
"""
Returns the Mixed modulestore
"""
global _MIXED_MODULESTORE # pylint: disable=global-statement
if _MIXED_MODULESTORE is None:
_MIXED_MODULESTORE = create_modulestore_instance(
settings.MODULESTORE['default']['ENGINE'],
settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE['default'].get('OPTIONS', {})
)
return store_name
return _MIXED_MODULESTORE
def modulestore(name=None):
"""
This returns an instance of a modulestore of given name. This will wither return an existing
modulestore or create a new one
def clear_existing_modulestores():
"""
Clear the existing modulestore instances, causing
them to be re-created when accessed again.
if not name:
# If caller did not specify name then we should
# determine what should be the default
name = get_default_store_name_for_current_request()
This is useful for flushing state between unit tests.
"""
global _MIXED_MODULESTORE, _loc_singleton # pylint: disable=global-statement
_MIXED_MODULESTORE = None
# pylint: disable=W0603
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None
if name not in _MODULESTORES:
_MODULESTORES[name] = create_modulestore_instance(
settings.MODULESTORE[name]['ENGINE'],
settings.MODULESTORE[name].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE[name].get('OPTIONS', {})
)
# inject loc_mapper into newly created modulestore if it needs it
if name == 'split' and _loc_singleton is not None:
_MODULESTORES['split'].loc_mapper = _loc_singleton
return _MODULESTORES[name]
# singleton instance of the loc_mapper
_loc_singleton = None
_loc_singleton = None
def loc_mapper():
"""
Get the loc mapper which bidirectionally maps Locations to Locators. Used like modulestore() as
......@@ -137,56 +127,8 @@ def loc_mapper():
loc_cache = get_cache('default')
# instantiate
_loc_singleton = LocMapperStore(loc_cache, **settings.DOC_STORE_CONFIG)
# inject into split mongo modulestore
if 'split' in _MODULESTORES:
_MODULESTORES['split'].loc_mapper = _loc_singleton
return _loc_singleton
def clear_existing_modulestores():
"""
Clear the existing modulestore instances, causing
them to be re-created when accessed again.
This is useful for flushing state between unit tests.
"""
_MODULESTORES.clear()
# pylint: disable=W0603
global _loc_singleton
cache = getattr(_loc_singleton, "cache", None)
if cache:
cache.clear()
_loc_singleton = None
def editable_modulestore(name='default'):
"""
Retrieve a modulestore that we can modify.
This is useful for tests that need to insert test
data into the modulestore.
Currently, only Mongo-backed modulestores can be modified.
Returns `None` if no editable modulestore is available.
"""
# Try to retrieve the ModuleStore
# Depending on the settings, this may or may not
# be editable.
store = modulestore(name)
# If this is a `MixedModuleStore`, then we will need
# to retrieve the actual Mongo instance.
# We assume that the default is Mongo.
if hasattr(store, 'modulestores'):
store = store.modulestores['default']
# At this point, we either have the ability to create
# items in the store, or we do not.
if hasattr(store, 'create_course'):
return store
else:
return None
return _loc_singleton
class ModuleI18nService(object):
......@@ -217,3 +159,40 @@ class ModuleI18nService(object):
# then Cale was a liar.
from util.date_utils import strftime_localized
return strftime_localized(*args, **kwargs)
# thread local cache
_THREAD_CACHE = threading.local()
def _get_modulestore_branch_setting():
"""
Returns the branch setting for the module store from the current Django request if configured,
else returns the branch value from the configuration settings if set,
else returns None
The value of the branch setting is cached in a thread-local variable so it is not repeatedly recomputed
"""
def get_branch_setting():
"""
Finds and returns the branch setting based on the Django request and the configuration settings
"""
branch = None
hostname = get_current_request_hostname()
if hostname:
# get mapping information which is defined in configurations
mappings = getattr(settings, 'HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', None)
# compare hostname against the regex expressions set of mappings which will tell us which branch to use
if mappings:
for key in mappings.iterkeys():
if re.match(key, hostname):
return mappings[key]
if branch is None:
branch = getattr(settings, 'MODULESTORE_BRANCH', None)
return branch
# cache the branch setting for this thread so we don't have to recompute it each time
if not hasattr(_THREAD_CACHE, 'branch_setting'):
_THREAD_CACHE.branch_setting = get_branch_setting()
return _THREAD_CACHE.branch_setting
common/lib/xmodule/xmodule/modulestore/exceptions.py
......@@ -27,6 +27,15 @@ class NoPathToItem(Exception):
pass
class ReferentialIntegrityError(Exception):
"""
An incorrect pointer to an object exists. For example, 2 parents point to the same child, an
xblock points to a nonexistent child (which probably raises ItemNotFoundError instead depending
on context).
"""
pass
class DuplicateItemError(Exception):
"""
Attempted to create an item which already exists.
......@@ -66,3 +75,13 @@ class DuplicateCourseError(Exception):
super(DuplicateCourseError, self).__init__()
self.course_id = course_id
self.existing_entry = existing_entry
class InvalidBranchSetting(Exception):
"""
Raised when the process' branch setting did not match the required setting for the attempted operation on a store.
"""
def __init__(self, expected_setting, actual_setting):
super(InvalidBranchSetting, self).__init__()
self.expected_setting = expected_setting
self.actual_setting = actual_setting
common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
......@@ -7,6 +7,7 @@ import pymongo
import bson.son
import urllib
from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from opaque_keys.edx.locations import SlashSeparatedCourseKey
......@@ -53,8 +54,8 @@ class LocMapperStore(object):
self.cache = cache
# location_map functions
def create_map_entry(self, course_key, org=None, offering=None, draft_branch='draft', prod_branch='published',
block_map=None):
def create_map_entry(self, course_key, org=None, offering=None,
draft_branch=BRANCH_NAME_DRAFT, prod_branch=BRANCH_NAME_PUBLISHED, block_map=None):
"""
Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & offering. If
org and offering are not provided, it defaults them based on course_key.
......@@ -244,7 +245,7 @@ class LocMapperStore(object):
for old_name, cat_to_usage in entry['block_map'].iteritems():
for category, block_id in cat_to_usage.iteritems():
# cache all entries and then figure out if we have the one we want
# Always return revision=None because the
# Always return revision=KEY_REVISION_PUBLISHED because the
# old draft module store wraps locations as draft before
# trying to access things.
location = old_course_id.make_usage_key(
......
common/lib/xmodule/xmodule/modulestore/search.py
from itertools import repeat
from .exceptions import (ItemNotFoundError, NoPathToItem)
......@@ -53,11 +52,8 @@ def path_to_location(modulestore, usage_key):
while len(queue) > 0:
(next_usage, path) = queue.pop() # Takes from the end
# get_parent_locations should raise ItemNotFoundError if location
# isn't found so we don't have to do it explicitly. Call this
# first to make sure the location is there (even if it's a course, and
# we would otherwise immediately exit).
parents = modulestore.get_parent_locations(next_usage)
# get_parent_location raises ItemNotFoundError if location isn't found
parent = modulestore.get_parent_location(next_usage)
# print 'Processing loc={0}, path={1}'.format(next_usage, path)
if next_usage.definition_key.block_type == "course":
......@@ -67,7 +63,7 @@ def path_to_location(modulestore, usage_key):
# otherwise, add parent locations at the end
newpath = (next_usage, path)
queue.extend(zip(parents, repeat(newpath)))
queue.append((parent, newpath))
# If we're here, there is no path
return None
......
common/lib/xmodule/xmodule/modulestore/store_utilities.py
......@@ -2,6 +2,7 @@ import re
import logging
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_ONLY
def _prefix_only_url_replace_regex(prefix):
......@@ -87,10 +88,12 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
return text
def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
def _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id):
for module in modules:
original_loc = module.location
module.location = module.location.map_into_course(dest_course_id)
if module.location.category == 'course':
module.location = module.location.replace(name=module.location.run)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
......@@ -108,10 +111,10 @@ def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
module.children = new_children
modulestore.update_item(module, '**replace_user**')
modulestore.update_item(module, user_id, allow_not_found=True)
def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
def clone_course(modulestore, contentstore, source_course_id, dest_course_id, user_id):
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_course(dest_course_id):
......@@ -120,29 +123,26 @@ def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
dest_modules = modulestore.get_items(dest_course_id)
basically_empty = True
for module in dest_modules:
if module.location.category == 'course' or (module.location.category == 'about'
and module.location.name == 'overview'):
if module.location.category == 'course' or (
module.location.category == 'about' and module.location.name == 'overview'
):
continue
basically_empty = False
break
if not basically_empty:
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# only course and about overview allowed
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_course_id))
# check to see if the source course is actually there
if not modulestore.has_course(source_course_id):
raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
# Get all modules under this namespace which is (tag, org, course) tuple
modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_PUBLISHED_ONLY)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
course_location = dest_course_id.make_usage_key('course', dest_course_id.run)
modulestore.publish(course_location, user_id)
modules = modulestore.get_items(source_course_id, revision=None)
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
modules = modulestore.get_items(source_course_id, revision='draft')
_clone_modules(modulestore, modules, source_course_id, dest_course_id)
modules = modulestore.get_items(source_course_id, revision=REVISION_OPTION_DRAFT_ONLY)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
# now iterate through all of the assets and clone them
# first the thumbnails
......
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
......@@ -5,7 +5,8 @@ Modulestore configuration for test cases.
from uuid import uuid4
from django.test import TestCase
from xmodule.modulestore.django import (
editable_modulestore, clear_existing_modulestores, loc_mapper)
modulestore, clear_existing_modulestores, loc_mapper)
from xmodule.modulestore import MONGO_MODULESTORE_TYPE
from xmodule.contentstore.django import contentstore
......@@ -25,7 +26,7 @@ def mixed_store_config(data_dir, mappings):
where 'xml' and 'default' are the two options provided by this configuration,
mapping (respectively) to XML-backed and Mongo-backed modulestores..
"""
mongo_config = mongo_store_config(data_dir)
draft_mongo_config = draft_mongo_store_config(data_dir)
xml_config = xml_store_config(data_dir)
store = {
......@@ -33,40 +34,13 @@ def mixed_store_config(data_dir, mappings):
'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
'OPTIONS': {
'mappings': mappings,
'stores': {
'default': mongo_config['default'],
'xml': xml_config['default']
}
'stores': [
draft_mongo_config['default'],
xml_config['default']
]
}
}
}
store['direct'] = store['default']
return store
def mongo_store_config(data_dir):
"""
Defines default module store using MongoModuleStore.
Use of this config requires mongo to be running.
"""
store = {
'default': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
},
'OPTIONS': {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': data_dir,
'render_template': 'edxmako.shortcuts.render_to_string'
}
}
}
store['direct'] = store['default']
return store
......@@ -83,6 +57,7 @@ def draft_mongo_store_config(data_dir):
store = {
'default': {
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': {
'host': 'localhost',
......@@ -93,7 +68,6 @@ def draft_mongo_store_config(data_dir):
}
}
store['direct'] = store['default']
return store
......@@ -103,6 +77,7 @@ def xml_store_config(data_dir):
"""
store = {
'default': {
'NAME': 'xml',
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': data_dir,
......@@ -111,48 +86,6 @@ def xml_store_config(data_dir):
}
}
store['direct'] = store['default']
return store
def studio_store_config(data_dir):
"""
Defines modulestore structure used by Studio tests.
"""
store_config = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
}
options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': data_dir,
'render_template': 'edxmako.shortcuts.render_to_string',
}
store = {
'default': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'draft': {
'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': store_config,
'OPTIONS': options
}
}
return store
......@@ -203,20 +136,19 @@ class ModuleStoreTestCase(TestCase):
'course' is an instance of CourseDescriptor for which we want
to update metadata.
"""
store = editable_modulestore()
store = modulestore()
store.update_item(course, '**replace_user**')
updated_course = store.get_course(course.id)
return updated_course
@staticmethod
def drop_mongo_collections(store_name='default'):
def drop_mongo_collections(modulestore_type=MONGO_MODULESTORE_TYPE):
"""
If using a Mongo-backed modulestore & contentstore, drop the collections.
"""
# This will return the mongo-backed modulestore
# even if we're using a mixed modulestore
store = editable_modulestore(store_name)
store = modulestore()
if hasattr(store, '_get_modulestore_by_type'):
store = store._get_modulestore_by_type(modulestore_type) # pylint: disable=W0212
if hasattr(store, 'collection'):
connection = store.collection.database.connection
store.collection.drop()
......@@ -268,7 +200,7 @@ class ModuleStoreTestCase(TestCase):
def _pre_setup(self):
"""
Flush the ModuleStore before each test.
Flush the ModuleStore.
"""
# Flush the Mongo modulestore
......
common/lib/xmodule/xmodule/modulestore/tests/factories.py
......@@ -5,6 +5,10 @@ from uuid import uuid4
from xmodule.modulestore import prefer_xmodules
from opaque_keys.edx.locations import Location
from xblock.core import XBlock
from xmodule.tabs import StaticTab
from decorator import contextmanager
from mock import Mock, patch
from nose.tools import assert_less_equal
class Dummy(object):
......@@ -23,10 +27,8 @@ class XModuleFactory(Factory):
@lazy_attribute
def modulestore(self):
# Delayed import so that we only depend on django if the caller
# hasn't provided their own modulestore
from xmodule.modulestore.django import editable_modulestore
return editable_modulestore('direct')
from xmodule.modulestore.django import modulestore
return modulestore()
class CourseFactory(XModuleFactory):
......@@ -63,7 +65,7 @@ class CourseFactory(XModuleFactory):
# Save the attributes we just set
new_course.save()
# Update the data in the mongo datastore
store.update_item(new_course)
store.update_item(new_course, '**replace_user**')
return new_course
......@@ -141,6 +143,7 @@ class ItemFactory(XModuleFactory):
display_name = kwargs.pop('display_name', None)
metadata = kwargs.pop('metadata', {})
location = kwargs.pop('location')
user_id = kwargs.pop('user_id', 999)
assert isinstance(location, Location)
assert location != parent_location
......@@ -162,7 +165,8 @@ class ItemFactory(XModuleFactory):
# replace the display name with an optional parameter passed in from the caller
if display_name is not None:
metadata['display_name'] = display_name
store.create_and_save_xmodule(location, metadata=metadata, definition_data=data)
runtime = parent.runtime if parent else None
store.create_and_save_xmodule(location, user_id, metadata=metadata, definition_data=data, runtime=runtime)
module = store.get_item(location)
......@@ -171,10 +175,53 @@ class ItemFactory(XModuleFactory):
# Save the attributes we just set
module.save()
store.update_item(module)
store.update_item(module, '**replace_user**')
if 'detached' not in module._class_tags:
parent.children.append(location)
store.update_item(parent, '**replace_user**')
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
# if we add one then we need to also add it to the policy information (i.e. metadata)
# we should remove this once we can break this reference from the course to static tabs
if category == 'static_tab':
course = store.get_course(location.course_key)
course.tabs.append(
StaticTab(
name=display_name,
url_slug=location.name,
)
)
store.update_item(course, '**replace_user**')
return store.get_item(location)
@contextmanager
def check_mongo_calls(mongo_store, max_finds=0, max_sends=None):
"""
Instruments the given store to count the number of calls to find (incl find_one) and the number
of calls to send_message which is for insert, update, and remove (if you provide max_sends). At the
end of the with statement, it compares the counts to the max_finds and max_sends using a simple
assertLessEqual.
:param mongo_store: the MongoModulestore or subclass to watch
:param max_finds: the maximum number of find calls to allow
:param max_sends: If none, don't instrument the send calls. If non-none, count and compare to
the given int value.
"""
try:
find_wrap = Mock(wraps=mongo_store.collection.find)
wrap_patch = patch.object(mongo_store.collection, 'find', find_wrap)
wrap_patch.start()
if max_sends:
sends_wrap = Mock(wraps=mongo_store.database.connection._send_message)
sends_patch = patch.object(mongo_store.database.connection, '_send_message', sends_wrap)
sends_patch.start()
yield
finally:
wrap_patch.stop()
if max_sends:
sends_patch.stop()
assert_less_equal(sends_wrap.call_count, max_sends)
assert_less_equal(find_wrap.call_count, max_finds)
common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py
from xmodule.modulestore import SPLIT_MONGO_MODULESTORE_TYPE, BRANCH_NAME_DRAFT
from xmodule.course_module import CourseDescriptor
from xmodule.x_module import XModuleDescriptor
import factory
......@@ -14,7 +15,7 @@ class SplitFactory(factory.Factory):
# Delayed import so that we only depend on django if the caller
# hasn't provided their own modulestore
from xmodule.modulestore.django import modulestore
return modulestore('split')
return modulestore()._get_modulestore_by_type(SPLIT_MONGO_MODULESTORE_TYPE)
class PersistentCourseFactory(SplitFactory):
......@@ -24,7 +25,7 @@ class PersistentCourseFactory(SplitFactory):
keywords: any xblock field plus (note, the below are filtered out; so, if they
become legitimate xblock fields, they won't be settable via this factory)
* org: defaults to textX
* master_branch: (optional) defaults to 'draft'
* master_branch: (optional) defaults to BRANCH_NAME_DRAFT
* user_id: (optional) defaults to 'test_user'
* display_name (xblock field): will default to 'Robot Super Course' unless provided
"""
......@@ -33,7 +34,7 @@ class PersistentCourseFactory(SplitFactory):
# pylint: disable=W0613
@classmethod
def _create(cls, target_class, offering='999', org='testX', user_id='test_user',
master_branch='draft', **kwargs):
master_branch=BRANCH_NAME_DRAFT, **kwargs):
modulestore = kwargs.pop('modulestore')
root_block_id = kwargs.pop('root_block_id', 'course')
......
common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
......@@ -5,6 +5,7 @@ import unittest
import uuid
from opaque_keys.edx.locations import Location
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore import BRANCH_NAME_PUBLISHED, BRANCH_NAME_DRAFT, KEY_REVISION_PUBLISHED
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from mock import Mock
......@@ -62,8 +63,8 @@ class TestLocationMapper(LocMapperSetupSansDjango):
self.assertIsNotNone(entry, "Didn't find entry")
self.assertEqual(entry['org'], org)
self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
self.assertEqual(entry['draft_branch'], 'draft')
self.assertEqual(entry['prod_branch'], 'published')
self.assertEqual(entry['draft_branch'], BRANCH_NAME_DRAFT)
self.assertEqual(entry['prod_branch'], BRANCH_NAME_PUBLISHED)
self.assertEqual(entry['block_map'], {})
course2 = 'quux_course'
......@@ -123,7 +124,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
"""
prob_locator = loc_mapper().translate_location(
location,
published=(branch == 'published'),
published=(branch == BRANCH_NAME_PUBLISHED),
add_entry_if_missing=add_entry
)
self.assertEqual(prob_locator.org, org)
......@@ -133,7 +134,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
course_locator = loc_mapper().translate_location_to_course_locator(
location.course_key,
published=(branch == 'published'),
published=(branch == BRANCH_NAME_PUBLISHED),
)
self.assertEqual(course_locator.org, org)
self.assertEqual(course_locator.offering, offering)
......@@ -168,7 +169,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
)
test_problem_locn = Location(org, course, run, 'problem', 'abc123')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published')
self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED)
# look for non-existent problem
with self.assertRaises(ItemNotFoundError):
loc_mapper().translate_location(
......@@ -183,7 +184,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
test_no_cat_locn = test_no_cat_locn.replace(name='def456')
self.translate_n_check(
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', 'published'
test_no_cat_locn, new_style_org, new_style_offering, 'problem4', BRANCH_NAME_PUBLISHED
)
# add a distractor course (note that abc123 has a different translation in this one)
......@@ -202,12 +203,12 @@ class TestLocationMapper(LocMapperSetupSansDjango):
)
# test that old translation still works
self.translate_n_check(
test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published'
test_problem_locn, new_style_org, new_style_offering, 'problem2', BRANCH_NAME_PUBLISHED
)
# and new returns new id
self.translate_n_check(
test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
'problem3', 'published'
'problem3', BRANCH_NAME_PUBLISHED
)
def test_translate_location_dwim(self):
......@@ -221,11 +222,11 @@ class TestLocationMapper(LocMapperSetupSansDjango):
problem_name = 'abc123abc123abc123abc123abc123f9'
location = Location(org, course, run, 'problem', problem_name)
new_offering = '{}.{}'.format(course, run)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
# create an entry w/o a guid name
other_location = Location(org, course, run, 'chapter', 'intro')
self.translate_n_check(other_location, org, new_offering, 'intro', 'published', True)
self.translate_n_check(other_location, org, new_offering, 'intro', BRANCH_NAME_PUBLISHED, True)
# add a distractor course
delta_new_org = '{}.geek_dept'.format(org)
......@@ -237,7 +238,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
delta_new_org, delta_new_offering,
block_map={problem_name: {'problem': 'problem3'}}
)
self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
self.translate_n_check(location, org, new_offering, 'problemabc', BRANCH_NAME_PUBLISHED, True)
# add a new one to both courses (ensure name doesn't have same beginning)
new_prob_name = uuid.uuid4().hex
......@@ -245,10 +246,10 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_prob_name = uuid.uuid4().hex
new_prob_locn = location.replace(name=new_prob_name)
new_usage_id = 'problem{}'.format(new_prob_name[:3])
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, 'published', True)
self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True)
new_prob_locn = new_prob_locn.replace(run=run)
self.translate_n_check(
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, 'published', True
new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, BRANCH_NAME_PUBLISHED, True
)
def test_translate_locator(self):
......@@ -263,7 +264,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_style_offering = '{}.{}'.format(course, run)
prob_course_key = CourseLocator(
org=new_style_org, offering=new_style_offering,
branch='published',
branch=BRANCH_NAME_PUBLISHED,
)
prob_locator = BlockUsageLocator(
prob_course_key,
......@@ -285,22 +286,22 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# only one course matches
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# default branch
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
# test get_course keyword
prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
# explicit branch
prob_locator = prob_locator.for_branch('draft')
prob_locator = prob_locator.for_branch(BRANCH_NAME_DRAFT)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# Even though the problem was set as draft, we always return revision=None to work
# Even though the problem was set as draft, we always return revision= KEY_REVISION_PUBLISHED to work
# with old mongo/draft modulestores.
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
prob_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
block_type='problem', block_id='problem2'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
# same for chapter except chapter cannot be draft in old system
chap_locator = BlockUsageLocator(
prob_course_key.for_branch('production'),
......@@ -309,7 +310,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# explicit branch
chap_locator = chap_locator.for_branch('draft')
chap_locator = chap_locator.for_branch(BRANCH_NAME_DRAFT)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
chap_locator = BlockUsageLocator(
......@@ -320,7 +321,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# look for non-existent problem
prob_locator2 = BlockUsageLocator(
prob_course_key.for_branch('draft'),
prob_course_key.for_branch(BRANCH_NAME_DRAFT),
block_type='problem', block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
......@@ -335,7 +336,7 @@ class TestLocationMapper(LocMapperSetupSansDjango):
block_map={'abc123': {'problem': 'problem3'}}
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', KEY_REVISION_PUBLISHED))
def test_special_chars(self):
"""
......
common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
......@@ -63,8 +63,10 @@ def check_has_course_method(modulestore, locator, locator_key_fields):
]
for changes in locator_case_changes:
search_locator = locator.replace(**changes)
# if ignore_case is true, the course would be found with a different-cased course locator.
# if ignore_case is false, the course should NOT found given an incorrectly-cased locator.
assert_equals(
modulestore.has_course(search_locator, ignore_case),
modulestore.has_course(search_locator, ignore_case) is not None,
ignore_case,
error_message.format(search_locator, ignore_case)
)
common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
......@@ -3,6 +3,7 @@ Test the publish code (mostly testing that publishing doesn't result in orphans)
"""
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
from xmodule.modulestore.tests.factories import check_mongo_calls
class TestPublish(SplitWMongoCourseBoostrapper):
......@@ -14,53 +15,48 @@ class TestPublish(SplitWMongoCourseBoostrapper):
Create the course, publish all verticals
* some detached items
"""
super(TestPublish, self)._create_course(split=False)
# There should be 12 inserts and 11 updates (max_sends)
# Should be 1 to verify course unique, 11 parent fetches,
# and n per _create_item where n is the size of the course tree non-leaf nodes
# for inheritance computation (which is 7*4 + sum(1..4) = 38) (max_finds)
with check_mongo_calls(self.draft_mongo, 70, 27):
with check_mongo_calls(self.old_mongo, 70, 27):
super(TestPublish, self)._create_course(split=False)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
{
"discussion_category": "Lecture 1",
"discussion_target": "Lecture 1",
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert1',
split=False
)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
{
"discussion_category": "Lecture 2",
"discussion_target": "Lecture 2",
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert2',
split=False
)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
{
"discussion_category": "Lecture 1",
"discussion_target": "Lecture 1",
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert1',
split=False
)
self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
{
"discussion_category": "Lecture 2",
"discussion_target": "Lecture 2",
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
'vertical', 'Vert2',
split=False
)
self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
def _xmodule_recurse(self, item, action):
"""
Applies action depth-first down tree and to item last.
A copy of cms.djangoapps.contentstore.views.helpers._xmodule_recurse to reproduce its use and behavior
outside of django.
"""
for child in item.get_children():
self._xmodule_recurse(child, action)
action(item)
def test_publish_draft_delete(self):
"""
......@@ -69,29 +65,31 @@ class TestPublish(SplitWMongoCourseBoostrapper):
"""
location = self.old_course_key.make_usage_key('vertical', name='Vert1')
item = self.draft_mongo.get_item(location, 2)
self._xmodule_recurse(
item,
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
# Vert1 has 3 children; so, publishes 4 nodes which may mean 4 inserts & 1 bulk remove
# 25-June-2014 find calls are 19. Probably due to inheritance recomputation?
with check_mongo_calls(self.draft_mongo, 19, 5):
self.draft_mongo.publish(item.location, self.userid)
# verify status
item = self.draft_mongo.get_item(location, 0)
self.assertFalse(getattr(item, 'is_draft', False), "Item was published. Draft should not exist")
# however, children are still draft, but I'm not sure that's by design
# convert back to draft
self.draft_mongo.convert_to_draft(location)
self.draft_mongo.convert_to_draft(location, self.userid)
# both draft and published should exist
draft_vert = self.draft_mongo.get_item(location, 0)
self.assertTrue(getattr(draft_vert, 'is_draft', False), "Item was converted to draft but doesn't say so")
item = self.old_mongo.get_item(location, 0)
self.assertFalse(getattr(item, 'is_draft', False), "Published item doesn't say so")
# delete the discussion (which oddly is not in draft mode)
# delete the draft version of the discussion
location = self.old_course_key.make_usage_key('discussion', name='Discussion1')
self.draft_mongo.delete_item(location)
# remove pointer from draft vertical (verify presence first to ensure process is valid)
self.assertIn(location, draft_vert.children)
draft_vert.children.remove(location)
self.draft_mongo.delete_item(location, self.userid)
draft_vert = self.draft_mongo.get_item(draft_vert.location, 0)
# remove pointer from draft vertical (still there b/c not refetching vert)
self.assertNotIn(location, draft_vert.children)
# move the other child
other_child_loc = self.old_course_key.make_usage_key('html', name='Html2')
draft_vert.children.remove(other_child_loc)
......@@ -100,12 +98,10 @@ class TestPublish(SplitWMongoCourseBoostrapper):
self.draft_mongo.update_item(draft_vert, self.userid)
self.draft_mongo.update_item(other_vert, self.userid)
# publish
self._xmodule_recurse(
draft_vert,
lambda i: self.draft_mongo.publish(i.location, self.userid)
)
self.draft_mongo.publish(draft_vert.location, self.userid)
item = self.old_mongo.get_item(draft_vert.location, 0)
self.assertNotIn(location, item.children)
self.assertIsNone(self.draft_mongo.get_parent_location(location))
with self.assertRaises(ItemNotFoundError):
self.draft_mongo.get_item(location)
self.assertNotIn(other_child_loc, item.children)
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
......@@ -5,9 +5,9 @@ Tests for split_migrator
import uuid
import random
import mock
from xmodule.modulestore import KEY_REVISION_PUBLISHED
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.mongo import draft
from xmodule.modulestore.tests import test_location_mapper
from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
from xblock.fields import Reference, ReferenceList, ReferenceValueDict
......@@ -177,7 +177,9 @@ class TestMigration(SplitWMongoCourseBoostrapper):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(revision=None)
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(
revision=KEY_REVISION_PUBLISHED
)
)
# compare all fields but children
for name, field in presplit_dag_root.fields.iteritems():
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
......@@ -9,6 +9,8 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore import BRANCH_DRAFT_PREFERRED, BRANCH_NAME_DRAFT
from mock import Mock
class SplitWMongoCourseBoostrapper(unittest.TestCase):
......@@ -39,7 +41,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
'xblock_mixins': (InheritanceMixin,)
}
split_course_key = CourseLocator('test_org', 'test_course.runid', branch='draft')
split_course_key = CourseLocator('test_org', 'test_course.runid', branch=BRANCH_NAME_DRAFT)
def setUp(self):
self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
......@@ -53,7 +55,9 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
self.addCleanup(self.split_mongo.db.connection.close)
self.addCleanup(self.tear_down_split)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore(
self.db_config, branch_setting_func=lambda: BRANCH_DRAFT_PREFERRED, **self.modulestore_options
)
self.addCleanup(self.tear_down_mongo)
self.old_course_key = None
self.runtime = None
......@@ -86,7 +90,7 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
mongo = self.old_mongo
else:
mongo = self.draft_mongo
mongo.create_and_save_xmodule(location, data, metadata, self.runtime)
mongo.create_and_save_xmodule(location, self.userid, definition_data=data, metadata=metadata, runtime=self.runtime)
if isinstance(data, basestring):
fields = {'data': data}
else:
......
common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
......@@ -43,7 +43,7 @@ class TestXMLModuleStore(unittest.TestCase):
def test_xml_modulestore_type(self):
store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
self.assertEqual(store.get_modulestore_type('foo/bar/baz'), XML_MODULESTORE_TYPE)
self.assertEqual(store.get_modulestore_type(), XML_MODULESTORE_TYPE)
def test_unicode_chars_in_xml_content(self):
# edX/full/6.002_Spring_2012 has non-ASCII chars, and during
......
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
......@@ -8,7 +8,7 @@ from xblock.runtime import Runtime, KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleMixin
from opaque_keys.edx.locations import Location
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.xml_importer import import_module
from xmodule.modulestore.xml_importer import _import_module_and_update_references
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
from uuid import uuid4
......@@ -141,9 +141,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Move to different runtime w/ different course id
target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace,
do_import_static=False
......@@ -177,9 +178,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False
......@@ -208,9 +210,10 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
new_version = import_module(
new_version = _import_module_and_update_references(
self.xblock,
modulestore(),
999,
self.xblock.location.course_key,
target_location_namespace.course_key,
do_import_static=False
......
common/lib/xmodule/xmodule/modulestore/xml.py
......@@ -19,6 +19,7 @@ from xmodule.errortracker import make_error_tracker, exc_info_to_str
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.modulestore import REVISION_OPTION_PUBLISHED_ONLY
from xmodule.tabs import CourseTabList
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey
......@@ -67,6 +68,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
# cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
self.course_id = course_id
self.load_error_modules = load_error_modules
self.modulestore = xmlstore
def process_xml(xml):
"""Takes an xml string, and returns a XBlock created from
......@@ -332,7 +334,7 @@ class ParentTracker(object):
"""
Init
"""
# location -> set(parents). Not using defaultdict because we care about the empty case.
# location -> parent. Not using defaultdict because we care about the empty case.
self._parents = dict()
def add_parent(self, child, parent):
......@@ -341,8 +343,7 @@ class ParentTracker(object):
child and parent must be :class:`.Location` instances.
"""
setp = self._parents.setdefault(child, set())
setp.add(parent)
self._parents[child] = parent
def is_known(self, child):
"""
......@@ -353,13 +354,13 @@ class ParentTracker(object):
def make_known(self, location):
"""Tell the parent tracker about an object, without registering any
parents for it. Used for the top level course descriptor locations."""
self._parents.setdefault(location, set())
self._parents.setdefault(location, None)
def parents(self, child):
def parent(self, child):
"""
Return a list of the parents of this child. If not is_known(child), will throw a KeyError
Return the parent of this child. If not is_known(child), will throw a KeyError
"""
return list(self._parents[child])
return self._parents[child]
class XMLModuleStore(ModuleStoreReadBase):
......@@ -409,6 +410,9 @@ class XMLModuleStore(ModuleStoreReadBase):
self.i18n_service = i18n_service
# The XML Module Store is a read-only store and only handles published content
self.branch_setting_func = lambda: REVISION_OPTION_PUBLISHED_ONLY
# If we are specifically asked for missing courses, that should
# be an error. If we are asked for "all" courses, find the ones
# that have a course.xml. We sort the dirs in alpha order so we always
......@@ -785,25 +789,25 @@ class XMLModuleStore(ModuleStoreReadBase):
# here just to quell the abstractmethod. someone could write the impl if needed
raise NotImplementedError
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
def get_parent_location(self, location, **kwargs):
'''Find the location that is the parent of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location. This may
be empty if there are no parents.
'''
if not self.parent_trackers[location.course_key].is_known(location):
raise ItemNotFoundError("{0} not in {1}".format(location, location.course_key))
return self.parent_trackers[location.course_key].parents(location)
return self.parent_trackers[location.course_key].parent(location)
def get_modulestore_type(self, course_id):
def get_modulestore_type(self, course_key=None):
"""
Returns an enumeration-like type reflecting the type of this modulestore
The return can be one of:
"xml" (for XML based courses),
"mongo" for old-style MongoDB backed courses,
"split" for new-style split MongoDB backed courses.
Args:
course_key: just for signature compatibility
"""
return XML_MODULESTORE_TYPE
......
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
......@@ -7,15 +7,18 @@ import lxml.etree
from xblock.fields import Scope
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from opaque_keys.edx.locations import Location
from xmodule.modulestore import (
EdxJSONEncoder, BRANCH_PUBLISHED_ONLY, REVISION_OPTION_DRAFT_PREFERRED, REVISION_OPTION_DRAFT_ONLY
)
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.mixed import store_branch_setting
from fs.osfs import OSFS
from json import dumps
import json
import datetime
import os
from path import path
import shutil
from xmodule.modulestore.mongo.base import DIRECT_ONLY_CATEGORIES
DRAFT_DIR = "drafts"
PUBLISHED_DIR = "published"
......@@ -25,29 +28,7 @@ EXPORT_VERSION_KEY = "export_format"
DEFAULT_CONTENT_FIELDS = ['metadata', 'data']
class EdxJSONEncoder(json.JSONEncoder):
"""
Custom JSONEncoder that handles `Location` and `datetime.datetime` objects.
`Location`s are encoded as their url string form, and `datetime`s as
ISO date strings
"""
def default(self, obj):
if isinstance(obj, Location):
return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
return obj.isoformat() + 'Z'
else:
return obj.isoformat()
else:
return obj.isoformat()
else:
return super(EdxJSONEncoder, self).default(obj)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
......@@ -56,8 +37,6 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course = modulestore.get_course(course_key)
......@@ -66,7 +45,10 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
course.add_xml_to_node(root)
# export only the published content
with store_branch_setting(course.runtime.modulestore, BRANCH_PUBLISHED_ONLY):
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
......@@ -121,26 +103,26 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, d
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items(course_key, category='vertical', revision='draft')
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = parent_locs[0].to_deprecated_string()
sequential = modulestore.get_item(parent_locs[0])
# should we change the application, then this assumption will no longer be valid
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
draft_verticals = modulestore.get_items(course_key, category='vertical', revision=REVISION_OPTION_DRAFT_ONLY)
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_loc = modulestore.get_parent_location(draft_vertical.location, revision=REVISION_OPTION_DRAFT_PREFERRED)
# Don't try to export orphaned items.
if parent_loc is not None:
logging.debug('parent_loc = {0}'.format(parent_loc))
if parent_loc.category in DIRECT_ONLY_CATEGORIES:
draft_vertical.xml_attributes['parent_sequential_url'] = parent_loc.to_deprecated_string()
sequential = modulestore.get_item(parent_loc)
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
def _export_field_content(xblock_item, item_dir):
......@@ -205,7 +187,7 @@ def convert_between_versions(source_dir, target_dir):
shutil.copytree(published_dir, copy_root)
# If there is a "draft" branch, copy it. All other branches are ignored.
# If there is a DRAFT branch, copy it. All other branches are ignored.
copy_drafts()
def copy_drafts():
......
common/lib/xmodule/xmodule/modulestore/xml_importer.py
......@@ -10,12 +10,14 @@ from xmodule.x_module import XModuleDescriptor
from opaque_keys.edx.keys import UsageKey
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
from .store_utilities import rewrite_nonportable_content_links
import xblock
from xmodule.tabs import CourseTabList
from xmodule.modulestore.exceptions import InvalidLocationError
from xmodule.modulestore import KEY_REVISION_PUBLISHED, KEY_REVISION_DRAFT
log = logging.getLogger(__name__)
......@@ -107,10 +109,10 @@ def import_static_content(
def import_from_xml(
store, data_dir, course_dirs=None,
store, user_id, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None,
target_course_id=None, verbose=False, draft_store=None,
target_course_id=None, verbose=False,
do_import_static=True, create_new_course_if_not_present=False):
"""
Import the specified xml data_dir into the "store" modulestore,
......@@ -171,18 +173,13 @@ def import_from_xml(
store.create_course(dest_course_id.org, dest_course_id.offering)
except InvalidLocationError:
# course w/ same org and course exists
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
try:
# turn off all write signalling while importing as this
# is a high volume operation on stores that need it
if hasattr(store, 'ignore_write_events_on_courses'):
store.ignore_write_events_on_courses.add(dest_course_id)
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
with store_bulk_write_operations_on_course(store, dest_course_id):
course_data_path = None
if verbose:
......@@ -209,8 +206,8 @@ def import_from_xml(
log.debug('course data_dir={0}'.format(module.data_dir))
course = import_module(
module, store,
course = _import_module_and_update_references(
module, store, user_id,
course_key,
dest_course_id,
do_import_static=do_import_static
......@@ -249,9 +246,12 @@ def import_from_xml(
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
store.update_item(course)
store.update_item(course, user_id)
course_items.append(course)
break
# TODO: shouldn't this raise an exception if course wasn't found?
# then import all the static content
if static_content_store is not None and do_import_static:
......@@ -284,7 +284,7 @@ def import_from_xml(
dest_course_id, subpath=simport, verbose=verbose
)
# finally loop through all the modules
# now loop through all the modules
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
# we've already saved the course module up at the top
......@@ -296,41 +296,36 @@ def import_from_xml(
loc=module.location
))
import_module(
_import_module_and_update_references(
module, store,
user_id,
course_key,
dest_course_id,
do_import_static=do_import_static,
system=course.runtime
runtime=course.runtime
)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(
xml_module_store,
store,
draft_store,
course_data_path,
static_content_store,
course_key,
dest_course_id,
course.runtime
)
finally:
# turn back on all write signalling on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
dest_course_id in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.remove(dest_course_id)
store.refresh_cached_metadata_inheritance_tree(dest_course_id)
# finally, publish the course
store.publish(course.location, user_id)
# now import any DRAFT items
_import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
course_key,
dest_course_id,
course.runtime
)
return xml_module_store, course_items
def import_module(
module, store,
def _import_module_and_update_references(
module, store, user_id,
source_course_id, dest_course_id,
do_import_static=True, system=None):
do_import_static=True, runtime=None):
logging.debug(u'processing import of module {}...'.format(module.location.to_deprecated_string()))
......@@ -347,7 +342,7 @@ def import_module(
new_usage_key = module.scope_ids.usage_id.map_into_course(dest_course_id)
if new_usage_key.category == 'course':
new_usage_key = new_usage_key.replace(name=dest_course_id.run)
new_module = store.create_xmodule(new_usage_key, system=system)
new_module = store.create_xmodule(new_usage_key, runtime=runtime)
def _convert_reference_fields_to_new_namespace(reference):
"""
......@@ -391,14 +386,19 @@ def import_module(
setattr(new_module, field_name, value)
else:
setattr(new_module, field_name, getattr(module, field_name))
store.update_item(new_module, '**replace_user**', allow_not_found=True)
store.update_item(new_module, user_id, allow_not_found=True)
return new_module
def import_course_draft(
xml_module_store, store, draft_store, course_data_path,
static_content_store, source_course_id,
target_course_id, mongo_runtime):
def _import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
source_course_id,
target_course_id,
mongo_runtime
):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
......@@ -502,17 +502,17 @@ def import_course_draft(
course_key = descriptor.location.course_key
try:
def _import_module(module):
# Update the module's location to "draft" revision
# Update the module's location to DRAFT revision
# We need to call this method (instead of updating the location directly)
# to ensure that pure XBlock field data is updated correctly.
_update_module_location(module, module.location.replace(revision='draft'))
_update_module_location(module, module.location.replace(revision=KEY_REVISION_DRAFT))
# make sure our parent has us in its list of children
# this is to make sure private only verticals show up
# in the list of children since they would have been
# filtered out from the non-draft store export
if module.location.category == 'vertical':
non_draft_location = module.location.replace(revision=None)
non_draft_location = module.location.replace(revision=KEY_REVISION_PUBLISHED)
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
......@@ -525,12 +525,13 @@ def import_course_draft(
if non_draft_location not in sequential.children:
sequential.children.insert(index, non_draft_location)
store.update_item(sequential, '**replace_user**')
store.update_item(sequential, user_id)
import_module(
module, draft_store,
_import_module_and_update_references(
module, store, user_id,
source_course_id,
target_course_id, system=mongo_runtime
target_course_id,
runtime=mongo_runtime,
)
for child in module.get_children():
_import_module(child)
......
common/lib/xmodule/xmodule/split_test_module.py
......@@ -420,7 +420,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
if selected_partition is not None:
self.group_id_mapping = {} # pylint: disable=attribute-defined-outside-init
for group in selected_partition.groups:
self._create_vertical_for_group(group)
self._create_vertical_for_group(group, user.id)
# Don't need to call update_item in the modulestore because the caller of this method will do it.
else:
# If children referenced in group_id_to_child have been deleted, remove them from the map.
......@@ -553,7 +553,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
for group in user_partition.groups:
str_group_id = unicode(group.id)
if str_group_id not in self.group_id_to_child:
self._create_vertical_for_group(group)
self._create_vertical_for_group(group, request.user.id)
changed = True
if changed:
......@@ -561,7 +561,7 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
self.system.modulestore.update_item(self, None)
return Response()
def _create_vertical_for_group(self, group):
def _create_vertical_for_group(self, group, user_id):
"""
Creates a vertical to associate with the group.
......@@ -576,9 +576,10 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
metadata = {'display_name': group.name}
modulestore.create_and_save_xmodule(
dest_usage_key,
user_id,
definition_data=None,
metadata=metadata,
system=self.system,
runtime=self.system,
)
self.children.append(dest_usage_key) # pylint: disable=no-member
self.group_id_to_child[unicode(group.id)] = dest_usage_key
common/lib/xmodule/xmodule/tests/test_export.py
......@@ -23,9 +23,10 @@ from xblock.fields import String, Scope, Integer
from xblock.test.tools import blocks_are_equivalent
from opaque_keys.edx.locations import Location
from xmodule.modulestore import EdxJSONEncoder
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.modulestore.xml_exporter import (
EdxJSONEncoder, convert_between_versions, get_version
convert_between_versions, get_version
)
from xmodule.tests import DATA_DIR
from xmodule.tests.helpers import directories_equal
......
common/lib/xmodule/xmodule/xml_module.py
......@@ -8,7 +8,7 @@ from lxml import etree
from xblock.fields import Dict, Scope, ScopeIds
from xmodule.x_module import XModuleDescriptor
from xmodule.modulestore.inheritance import own_metadata, InheritanceKeyValueStore
from xmodule.modulestore.xml_exporter import EdxJSONEncoder
from xmodule.modulestore import EdxJSONEncoder
from xblock.runtime import KvsFieldData
log = logging.getLogger(__name__)
......
lms/djangoapps/branding/tests.py
......@@ -10,7 +10,7 @@ from django.test.utils import override_settings
from django.test.client import RequestFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import editable_modulestore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory
from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
import student.views
......@@ -28,12 +28,12 @@ class AnonymousIndexPageTest(ModuleStoreTestCase):
Tests that anonymous users can access the '/' page, Need courses with start date
"""
def setUp(self):
self.store = editable_modulestore()
self.store = modulestore()
self.factory = RequestFactory()
self.course = CourseFactory.create()
self.course.days_early_for_beta = 5
self.course.enrollment_start = datetime.datetime.now(UTC) + datetime.timedelta(days=3)
self.store.update_item(self.course)
self.store.update_item(self.course, '**replace_user**')
@override_settings(FEATURES=FEATURES_WITH_STARTDATE)
def test_none_user_index_access_with_startdate_fails(self):
......
lms/djangoapps/courseware/features/annotatable.py
......@@ -3,7 +3,7 @@ import textwrap
from lettuce import world, steps
from nose.tools import assert_in, assert_equals, assert_true
from common import i_am_registered_for_the_course, visit_scenario_item
from common import i_am_registered_for_the_course, visit_scenario_item, publish
DATA_TEMPLATE = textwrap.dedent("""\
<annotatable>
......@@ -79,6 +79,8 @@ class AnnotatableSteps(object):
data=DATA_TEMPLATE.format("\n".join(ANNOTATION_TEMPLATE.format(i) for i in xrange(count)))
)
publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
self.annotations_count = count
def view_component(self, step):
......@@ -123,6 +125,7 @@ class AnnotatableSteps(object):
)
)
)
publish(world.scenario_dict['ANNOTATION_VERTICAL'].location)
def click_reply(self, step, problem):
r"""I click "Reply to annotation" on passage (?P<problem>\d+)$"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment