Skip to content

E
edx-platform
Commit 211f0797 by Don Mitchell
Options

Merge pull request #4294 from edx/split_mongo_assets_cherry 

Split mongo assets cherry
parents a5a3f625 a74ba6b9
Showing with 1014 additions and 643 deletions
cms/djangoapps/contentstore/management/commands/clone_course.py
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
Script for cloning a course Script for cloning a course
""" """
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mixed import store_bulk_write_operations_on_course from xmodule.modulestore.mixed import store_bulk_write_operations_on_course
from xmodule.contentstore.django import contentstore
from student.roles import CourseInstructorRole, CourseStaffRole from student.roles import CourseInstructorRole, CourseStaffRole
from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.keys import CourseKey
from opaque_keys import InvalidKeyError from opaque_keys import InvalidKeyError
...@@ -37,12 +35,11 @@ class Command(BaseCommand): ...@@ -37,12 +35,11 @@ class Command(BaseCommand):
dest_course_id = self.course_key_from_arg(args[1]) dest_course_id = self.course_key_from_arg(args[1])
mstore = modulestore() mstore = modulestore()
cstore = contentstore()
print("Cloning course {0} to {1}".format(source_course_id, dest_course_id)) print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))
with store_bulk_write_operations_on_course(mstore, dest_course_id): with store_bulk_write_operations_on_course(mstore, dest_course_id):
if clone_course(mstore, cstore, source_course_id, dest_course_id, None): if mstore.clone_course(source_course_id, dest_course_id, None):
print("copying User permissions...") print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize # purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_course_id).add_users( CourseInstructorRole(dest_course_id).add_users(
......
cms/djangoapps/contentstore/management/commands/migrate_to_split.py
# pylint: disable=W0212
""" """
Django management command to migrate a course from the old Mongo modulestore Django management command to migrate a course from the old Mongo modulestore
to the new split-Mongo modulestore. to the new split-Mongo modulestore.
...@@ -10,6 +12,7 @@ from xmodule.modulestore.django import loc_mapper ...@@ -10,6 +12,7 @@ from xmodule.modulestore.django import loc_mapper
from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.keys import CourseKey
from opaque_keys import InvalidKeyError from opaque_keys import InvalidKeyError
from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore import ModuleStoreEnum
def user_from_str(identifier): def user_from_str(identifier):
...@@ -66,9 +69,8 @@ class Command(BaseCommand): ...@@ -66,9 +69,8 @@ class Command(BaseCommand):
course_key, user, org, offering = self.parse_args(*args) course_key, user, org, offering = self.parse_args(*args)
migrator = SplitMigrator( migrator = SplitMigrator(
draft_modulestore=modulestore('default'), draft_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo),
direct_modulestore=modulestore('direct'), split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(), loc_mapper=loc_mapper(),
) )
......
cms/djangoapps/contentstore/management/commands/tests/test_rollback_split_course.py
...@@ -14,7 +14,9 @@ from xmodule.modulestore.tests.factories import CourseFactory ...@@ -14,7 +14,9 @@ from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, loc_mapper from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.split_migrator import SplitMigrator from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore import ModuleStoreEnum
# pylint: disable=E1101 # pylint: disable=E1101
# pylint: disable=W0212
@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9") @unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
...@@ -87,9 +89,8 @@ class TestRollbackSplitCourse(ModuleStoreTestCase): ...@@ -87,9 +89,8 @@ class TestRollbackSplitCourse(ModuleStoreTestCase):
# migrate old course to split # migrate old course to split
migrator = SplitMigrator( migrator = SplitMigrator(
draft_modulestore=modulestore('default'), draft_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo),
direct_modulestore=modulestore('direct'), split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(), loc_mapper=loc_mapper(),
) )
migrator.migrate_mongo_course(self.old_course.location, self.user) migrator.migrate_mongo_course(self.old_course.location, self.user)
......
cms/djangoapps/contentstore/tests/test_clone_course.py 0 → 100644
"""
Unit tests for cloning a course between the same and different module stores.
"""
from django.utils.unittest.case import skipIf
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys.edx.locator import CourseLocator
from xmodule.modulestore import ModuleStoreEnum
from contentstore.tests.utils import CourseTestCase
@skipIf(
not 'run' in CourseLocator.KEY_FIELDS,
"Pending integration with latest opaque-keys library - need removal of offering, make_asset_key on CourseLocator, etc."
)
class CloneCourseTest(CourseTestCase):
"""
Unit tests for cloning a course
"""
def test_clone_course(self):
"""Tests cloning of a course as follows: XML -> Mongo (+ data) -> Mongo -> Split -> Split"""
# 1. import and populate test toy course
mongo_course1_id = self.import_and_populate_course()
self.check_populated_course(mongo_course1_id)
# 2. clone course (mongo -> mongo)
# TODO - This is currently failing since clone_course doesn't handle Private content - fails on Publish
mongo_course2_id = SlashSeparatedCourseKey('edX2', 'toy2', '2013_Fall')
self.store.clone_course(mongo_course1_id, mongo_course2_id, self.user.id)
self.assertCoursesEqual(mongo_course1_id, mongo_course2_id)
# 3. clone course (mongo -> split)
with self.store.set_default_store(ModuleStoreEnum.Type.split):
split_course3_id = CourseLocator(
org="edx3", course="split3", run="2013_Fall", branch=ModuleStoreEnum.BranchName.draft
)
self.store.clone_course(mongo_course2_id, split_course3_id, self.user.id)
self.assertCoursesEqual(mongo_course2_id, split_course3_id)
# 4. clone course (split -> split)
split_course4_id = CourseLocator(
org="edx4", course="split4", run="2013_Fall", branch=ModuleStoreEnum.BranchName.draft
)
self.store.clone_course(split_course3_id, split_course4_id, self.user.id)
self.assertCoursesEqual(split_course3_id, split_course4_id)
cms/djangoapps/contentstore/tests/test_contentstore.py
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# pylint: disable=E1101 # pylint: disable=E1101
# pylint: disable=W0212
import copy import copy
import mock import mock
import re
import shutil import shutil
from datetime import timedelta from datetime import timedelta
from fs.osfs import OSFS from fs.osfs import OSFS
from json import loads from json import loads
from path import path from path import path
from pymongo import MongoClient
from tempdir import mkdtemp_clean from tempdir import mkdtemp_clean
from textwrap import dedent from textwrap import dedent
from uuid import uuid4 from uuid import uuid4
...@@ -20,23 +19,19 @@ from django.contrib.auth.models import User ...@@ -20,23 +19,19 @@ from django.contrib.auth.models import User
from django.test import TestCase from django.test import TestCase
from django.test.utils import override_settings from django.test.utils import override_settings
from contentstore.tests.utils import parse_json, AjaxEnabledTestClient from contentstore.tests.utils import parse_json, AjaxEnabledTestClient, CourseTestCase
from contentstore.views.component import ADVANCED_COMPONENT_TYPES from contentstore.views.component import ADVANCED_COMPONENT_TYPES
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore, _CONTENTSTORE from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan
from xmodule.exceptions import NotFoundError, InvalidVersionError from xmodule.exceptions import NotFoundError, InvalidVersionError
from xmodule.modulestore import mongo, PublishState, ModuleStoreEnum from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.mongo.base import MongoRevisionKey
from xmodule.modulestore.mixed import store_branch_setting from xmodule.modulestore.mixed import store_branch_setting
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.inheritance import own_metadata from xmodule.modulestore.inheritance import own_metadata
from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.store_utilities import clone_course, delete_course from xmodule.modulestore.store_utilities import delete_course
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
...@@ -52,6 +47,7 @@ from student import auth ...@@ -52,6 +47,7 @@ from student import auth
from student.models import CourseEnrollment from student.models import CourseEnrollment
from student.roles import CourseCreatorRole, CourseInstructorRole from student.roles import CourseCreatorRole, CourseInstructorRole
from opaque_keys import InvalidKeyError from opaque_keys import InvalidKeyError
from contentstore.tests.utils import get_url
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
...@@ -68,37 +64,12 @@ class MongoCollectionFindWrapper(object): ...@@ -68,37 +64,12 @@ class MongoCollectionFindWrapper(object):
return self.original(query, *args, **kwargs) return self.original(query, *args, **kwargs)
def get_url(handler_name, key_value, key_name='usage_key_string', kwargs=None):
# Helper function for getting HTML for a page in Studio and
# checking that it does not error.
return reverse_url(handler_name, key_name, key_value, kwargs)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreTestCase(ModuleStoreTestCase): class ContentStoreTestCase(CourseTestCase):
""" """
Base class for Content Store Test Cases Base class for Content Store Test Cases
""" """
def setUp(self): pass
uname = 'testuser'
email = 'test+courses@edx.org'
password = 'foo'
# Create the use so we can log them in.
self.user = User.objects.create_user(uname, email, password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
self.user.is_active = True
# Staff has access to view all courses
self.user.is_staff = True
# Save the data that we've just changed to the db.
self.user.save()
self.client = AjaxEnabledTestClient()
self.client.login(username=uname, password=password)
class ContentStoreToyCourseTest(ContentStoreTestCase): class ContentStoreToyCourseTest(ContentStoreTestCase):
""" """
...@@ -106,7 +77,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -106,7 +77,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
TODO: refactor using CourseFactory so they do not. TODO: refactor using CourseFactory so they do not.
""" """
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def check_components_on_page(self, component_types, expected_types): def check_components_on_page(self, component_types, expected_types):
...@@ -121,7 +92,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -121,7 +92,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
exactly the same -- for example, 'video' in exactly the same -- for example, 'video' in
component_types should cause 'Video' to be present. component_types should cause 'Video' to be present.
""" """
store = modulestore() store = self.store
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple']) _, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course = course_items[0] course = course_items[0]
course.advanced_modules = component_types course.advanced_modules = component_types
...@@ -148,7 +119,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -148,7 +119,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.check_components_on_page(['word_cloud'], ['Word cloud']) self.check_components_on_page(['word_cloud'], ['Word cloud'])
def test_malformed_edit_unit_request(self): def test_malformed_edit_unit_request(self):
store = modulestore() store = self.store
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple']) _, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
# just pick one vertical # just pick one vertical
...@@ -158,23 +129,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -158,23 +129,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertEqual(resp.status_code, 400) self.assertEqual(resp.status_code, 400)
def check_edit_unit(self, test_course_name): def check_edit_unit(self, test_course_name):
_, course_items = import_from_xml(modulestore(), self.user.id, 'common/test/data/', [test_course_name]) """Verifies the editing HTML in all the verticals in the given test course"""
_, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', [test_course_name])
items = modulestore().get_items(course_items[0].id, category='vertical') items = self.store.get_items(course_items[0].id, category='vertical')
self._check_verticals(items) self._check_verticals(items)
def _lock_an_asset(self, content_store, course_id):
"""
Lock an arbitrary asset in the course
:param course_location:
"""
course_assets, __ = content_store.get_all_content_for_course(course_id)
self.assertGreater(len(course_assets), 0, "No assets to lock")
asset_id = course_assets[0]['_id']
asset_key = StaticContent.compute_location(course_id, asset_id['name'])
content_store.set_attr(asset_key, 'locked', True)
return asset_key
def test_edit_unit_toy(self): def test_edit_unit_toy(self):
self.check_edit_unit('toy') self.check_edit_unit('toy')
...@@ -191,7 +151,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -191,7 +151,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
Unfortunately, None = published for the revision field, so get_items() would return Unfortunately, None = published for the revision field, so get_items() would return
both draft and non-draft copies. both draft and non-draft copies.
''' '''
store = modulestore() store = self.store
_, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple']) _, course_items = import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course_key = course_items[0].id course_key = course_items[0].id
html_usage_key = course_key.make_usage_key('html', 'test_html') html_usage_key = course_key.make_usage_key('html', 'test_html')
...@@ -218,7 +178,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -218,7 +178,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
module as 'own-metadata' when publishing. Also verifies the metadata inheritance is module as 'own-metadata' when publishing. Also verifies the metadata inheritance is
properly computed properly computed
''' '''
draft_store = modulestore() draft_store = self.store
import_from_xml(draft_store, self.user.id, 'common/test/data/', ['simple']) import_from_xml(draft_store, self.user.id, 'common/test/data/', ['simple'])
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall') course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
...@@ -279,7 +239,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -279,7 +239,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertEqual(html_module.graceperiod, new_graceperiod) self.assertEqual(html_module.graceperiod, new_graceperiod)
def test_get_depth_with_drafts(self): def test_get_depth_with_drafts(self):
store = modulestore() store = self.store
import_from_xml(store, self.user.id, 'common/test/data/', ['simple']) import_from_xml(store, self.user.id, 'common/test/data/', ['simple'])
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall') course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
...@@ -307,16 +267,15 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -307,16 +267,15 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertEqual(num_drafts, 1) self.assertEqual(num_drafts, 1)
def test_no_static_link_rewrites_on_import(self): def test_no_static_link_rewrites_on_import(self):
module_store = modulestore() _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course = course_items[0] course = course_items[0]
handouts_usage_key = course.id.make_usage_key('course_info', 'handouts') handouts_usage_key = course.id.make_usage_key('course_info', 'handouts')
handouts = module_store.get_item(handouts_usage_key) handouts = self.store.get_item(handouts_usage_key)
self.assertIn('/static/', handouts.data) self.assertIn('/static/', handouts.data)
handouts_usage_key = course.id.make_usage_key('html', 'toyhtml') handouts_usage_key = course.id.make_usage_key('html', 'toyhtml')
handouts = module_store.get_item(handouts_usage_key) handouts = self.store.get_item(handouts_usage_key)
self.assertIn('/static/', handouts.data) self.assertIn('/static/', handouts.data)
@mock.patch('xmodule.course_module.requests.get') @mock.patch('xmodule.course_module.requests.get')
...@@ -327,17 +286,15 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -327,17 +286,15 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
</table_of_contents> </table_of_contents>
""").strip() """).strip()
module_store = modulestore() import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
self.assertGreater(len(course.textbooks), 0) self.assertGreater(len(course.textbooks), 0)
def test_import_polls(self): def test_import_polls(self):
module_store = modulestore() _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_key = course_items[0].id course_key = course_items[0].id
items = module_store.get_items(course_key, category='poll_question') items = self.store.get_items(course_key, category='poll_question')
found = len(items) > 0 found = len(items) > 0
self.assertTrue(found) self.assertTrue(found)
...@@ -353,7 +310,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -353,7 +310,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
""" """
Tests the ajax callback to render an XModule Tests the ajax callback to render an XModule
""" """
direct_store = modulestore() direct_store = self.store
_, course_items = import_from_xml(direct_store, self.user.id, 'common/test/data/', ['toy']) _, course_items = import_from_xml(direct_store, self.user.id, 'common/test/data/', ['toy'])
usage_key = course_items[0].id.make_usage_key('vertical', 'vertical_test') usage_key = course_items[0].id.make_usage_key('vertical', 'vertical_test')
...@@ -370,7 +327,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -370,7 +327,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertContains(resp, 'edX+toy+2012_Fall+poll_question+T1_changemind_poll_foo_2') self.assertContains(resp, 'edX+toy+2012_Fall+poll_question+T1_changemind_poll_foo_2')
def test_delete(self): def test_delete(self):
store = modulestore() store = self.store
course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course') course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
chapterloc = ItemFactory.create(parent_location=course.location, display_name="Chapter").location chapterloc = ItemFactory.create(parent_location=course.location, display_name="Chapter").location
...@@ -405,14 +362,13 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -405,14 +362,13 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html
while there is a base definition in /about/effort.html while there is a base definition in /about/effort.html
''' '''
module_store = modulestore() _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course_key = course_items[0].id course_key = course_items[0].id
effort = module_store.get_item(course_key.make_usage_key('about', 'effort')) effort = self.store.get_item(course_key.make_usage_key('about', 'effort'))
self.assertEqual(effort.data, '6 hours') self.assertEqual(effort.data, '6 hours')
# this one should be in a non-override folder # this one should be in a non-override folder
effort = module_store.get_item(course_key.make_usage_key('about', 'end_date')) effort = self.store.get_item(course_key.make_usage_key('about', 'end_date'))
self.assertEqual(effort.data, 'TBD') self.assertEqual(effort.data, 'TBD')
def test_asset_import(self): def test_asset_import(self):
...@@ -421,10 +377,9 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -421,10 +377,9 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
''' '''
content_store = contentstore() content_store = contentstore()
module_store = modulestore() import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')) course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
self.assertIsNotNone(course) self.assertIsNotNone(course)
...@@ -510,8 +465,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -510,8 +465,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
content_store = contentstore() content_store = contentstore()
trash_store = contentstore('trashcan') trash_store = contentstore('trashcan')
module_store = modulestore() _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
# look up original (and thumbnail) in content store, should be there after import # look up original (and thumbnail) in content store, should be there after import
location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt') location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
...@@ -542,17 +496,16 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -542,17 +496,16 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
Test that course info updates are imported and exported with all content fields ('data', 'items') Test that course info updates are imported and exported with all content fields ('data', 'items')
""" """
content_store = contentstore() content_store = contentstore()
module_store = modulestore()
data_dir = "common/test/data/" data_dir = "common/test/data/"
import_from_xml(module_store, self.user.id, data_dir, ['course_info_updates'], import_from_xml(self.store, self.user.id, data_dir, ['course_info_updates'],
static_content_store=content_store, verbose=True) static_content_store=content_store, verbose=True)
course_id = SlashSeparatedCourseKey('edX', 'course_info_updates', '2014_T1') course_id = SlashSeparatedCourseKey('edX', 'course_info_updates', '2014_T1')
course = module_store.get_course(course_id) course = self.store.get_course(course_id)
self.assertIsNotNone(course) self.assertIsNotNone(course)
course_updates = module_store.get_item(course_id.make_usage_key('course_info', 'updates')) course_updates = self.store.get_item(course_id.make_usage_key('course_info', 'updates'))
self.assertIsNotNone(course_updates) self.assertIsNotNone(course_updates)
...@@ -576,7 +529,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -576,7 +529,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
# with same content as in course 'info' directory # with same content as in course 'info' directory
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export') export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
# check that exported course has files 'updates.html' and 'updates.items.json' # check that exported course has files 'updates.html' and 'updates.items.json'
filesystem = OSFS(root_dir / 'test_export/info') filesystem = OSFS(root_dir / 'test_export/info')
...@@ -622,119 +575,8 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -622,119 +575,8 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id) all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id)
self.assertEqual(len(all_thumbnails), 0) self.assertEqual(len(all_thumbnails), 0)
def test_clone_course(self):
course_data = {
'org': 'MITx',
'number': '999',
'display_name': 'Robot Super Course',
'run': '2013_Spring',
}
module_store = modulestore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
source_course_id = course_items[0].id
dest_course_id = _get_course_id(course_data)
# get a vertical (and components in it) to put into DRAFT
# this is to assert that draft content is also cloned over
vertical = module_store.get_item(
source_course_id.make_usage_key('vertical', 'vertical_test'),
depth=1
)
module_store.convert_to_draft(vertical.location, self.user.id)
items = module_store.get_items(source_course_id)
self.assertGreater(len(items), 0)
_create_course(self, dest_course_id, course_data)
content_store = contentstore()
# now do the actual cloning
clone_course(module_store, content_store, source_course_id, dest_course_id, self.user.id)
# first assert that all draft content got cloned as well
draft_items = module_store.get_items(source_course_id, revision=ModuleStoreEnum.RevisionOption.draft_only)
self.assertGreater(len(draft_items), 0)
draft_clone_items = module_store.get_items(dest_course_id, revision=ModuleStoreEnum.RevisionOption.draft_only)
self.assertGreater(len(draft_clone_items), 0)
self.assertEqual(len(draft_items), len(draft_clone_items))
# now loop through all the units in the course and verify that the clone can render them, which
# means the objects are at least present
items = module_store.get_items(source_course_id)
self.assertGreater(len(items), 0)
clone_items = module_store.get_items(dest_course_id)
self.assertGreater(len(clone_items), 0)
for descriptor in items:
source_item = module_store.get_item(descriptor.location)
new_loc = descriptor.location.map_into_course(dest_course_id)
if descriptor.location.category == 'course':
new_loc = new_loc.replace(name=new_loc.run)
print "Checking {0} should now also be at {1}".format(descriptor.location, new_loc)
lookup_item = module_store.get_item(new_loc)
if hasattr(source_item, 'data') and hasattr(lookup_item, 'data'):
self.assertEqual(source_item.data, lookup_item.data)
# also make sure that metadata was cloned over and filtered with own_metadata, i.e. inherited
# values were not explicitly set
self.assertEqual(own_metadata(source_item), own_metadata(lookup_item))
# check that the children are as expected
self.assertEqual(source_item.has_children, lookup_item.has_children)
if source_item.has_children:
expected_children = []
for child_loc in source_item.children:
child_loc = child_loc.map_into_course(dest_course_id)
expected_children.append(child_loc)
self.assertEqual(expected_children, lookup_item.children)
def test_portable_link_rewrites_during_clone_course(self):
course_data = {
'org': 'MITx',
'number': '999',
'display_name': 'Robot Super Course',
'run': '2013_Spring'
}
module_store = modulestore()
content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
source_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
dest_course_id = _get_course_id(course_data)
# let's force a non-portable link in the clone source
# as a final check, make sure that any non-portable links are rewritten during cloning
html_module = module_store.get_item(source_course_id.make_usage_key('html', 'nonportable'))
self.assertIsInstance(html_module.data, basestring)
new_data = html_module.data = html_module.data.replace('/static/', '/c4x/{0}/{1}/asset/'.format(
source_course_id.org, source_course_id.run))
module_store.update_item(html_module, self.user.id)
html_module = module_store.get_item(html_module.location)
self.assertEqual(new_data, html_module.data)
# create the destination course
_create_course(self, dest_course_id, course_data)
# do the actual cloning
clone_course(module_store, content_store, source_course_id, dest_course_id, self.user.id)
# make sure that any non-portable links are rewritten during cloning
html_module = module_store.get_item(dest_course_id.make_usage_key('html', 'nonportable'))
self.assertIn('/asset/foo.jpg', html_module.data)
def test_illegal_draft_crud_ops(self): def test_illegal_draft_crud_ops(self):
draft_store = modulestore() draft_store = self.store
course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course') course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
...@@ -759,20 +601,19 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -759,20 +601,19 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertEqual(resp.status_code, 400) self.assertEqual(resp.status_code, 400)
def test_rewrite_nonportable_links_on_import(self): def test_rewrite_nonportable_links_on_import(self):
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
# first check a static asset link # first check a static asset link
course_key = SlashSeparatedCourseKey('edX', 'toy', 'run') course_key = SlashSeparatedCourseKey('edX', 'toy', 'run')
html_module_location = course_key.make_usage_key('html', 'nonportable') html_module_location = course_key.make_usage_key('html', 'nonportable')
html_module = module_store.get_item(html_module_location) html_module = self.store.get_item(html_module_location)
self.assertIn('/static/foo.jpg', html_module.data) self.assertIn('/static/foo.jpg', html_module.data)
# then check a intra courseware link # then check a intra courseware link
html_module_location = course_key.make_usage_key('html', 'nonportable_link') html_module_location = course_key.make_usage_key('html', 'nonportable_link')
html_module = module_store.get_item(html_module_location) html_module = self.store.get_item(html_module_location)
self.assertIn('/jump_to_id/nonportable_link', html_module.data) self.assertIn('/jump_to_id/nonportable_link', html_module.data)
def test_delete_course(self): def test_delete_course(self):
...@@ -780,24 +621,23 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -780,24 +621,23 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
This test will import a course, make a draft item, and delete it. This will also assert that the This test will import a course, make a draft item, and delete it. This will also assert that the
draft content is also deleted draft content is also deleted
""" """
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store) _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
course_id = course_items[0].id course_id = course_items[0].id
# get a vertical (and components in it) to put into DRAFT # get a vertical (and components in it) to put into DRAFT
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1) vertical = self.store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
module_store.convert_to_draft(vertical.location, self.user.id) self.store.convert_to_draft(vertical.location, self.user.id)
# delete the course # delete the course
delete_course(module_store, content_store, course_id, commit=True) delete_course(self.store, content_store, course_id, commit=True)
# assert that there's absolutely no non-draft modules in the course # assert that there's absolutely no non-draft modules in the course
# this should also include all draft items # this should also include all draft items
items = module_store.get_items(course_id) items = self.store.get_items(course_id)
self.assertEqual(len(items), 0) self.assertEqual(len(items), 0)
# assert that all content in the asset library is also deleted # assert that all content in the asset library is also deleted
...@@ -823,74 +663,26 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -823,74 +663,26 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
</table_of_contents> </table_of_contents>
""").strip() """).strip()
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
course_id = self.import_and_populate_course()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# get a vertical (and components in it) to copy into an orphan sub dag
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
# We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
vertical.location = vertical.location.replace(name='no_references')
module_store.update_item(vertical, self.user.id, allow_not_found=True)
orphan_vertical = module_store.get_item(vertical.location)
self.assertEqual(orphan_vertical.location.name, 'no_references')
# get the original vertical (and components in it) to put into DRAFT
vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
self.assertEqual(len(orphan_vertical.children), len(vertical.children))
draft_vertical = module_store.convert_to_draft(vertical.location, self.user.id)
self.assertEqual(module_store.compute_publish_state(draft_vertical), PublishState.draft)
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
# now create a new/different private (draft only) vertical
vertical.location = mongo.draft.as_draft(course_id.make_usage_key('vertical', 'a_private_vertical'))
vertical = module_store.create_and_save_xmodule(vertical.location, self.user.id)
self.assertEqual(module_store.compute_publish_state(vertical), PublishState.private)
private_vertical = module_store.get_item(vertical.location)
vertical = None # blank out b/c i destructively manipulated its location 2 lines above
# now create a new/different published (no draft) vertical
public_vertical_location = course_id.make_usage_key('vertical', 'a_published_vertical')
module_store.create_and_save_xmodule(public_vertical_location, self.user.id)
public_vertical = module_store.publish(public_vertical_location, self.user.id)
self.assertEqual(module_store.compute_publish_state(public_vertical), PublishState.public)
# add the new private and new public to list of children
sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
private_location_no_draft = private_vertical.location.replace(revision=MongoRevisionKey.published)
sequential.children.append(private_location_no_draft)
sequential.children.append(public_vertical_location)
module_store.update_item(sequential, self.user.id)
# read back the sequential, to make sure we have a pointer to
sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
self.assertIn(private_location_no_draft, sequential.children)
locked_asset_key = self._lock_an_asset(content_store, course_id)
locked_asset_attrs = content_store.get_attrs(locked_asset_key)
# the later import will reupload
del locked_asset_attrs['uploadDate']
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir # export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export') export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
# check for static tabs # check for static tabs
self.verify_content_existence(module_store, root_dir, course_id, 'tabs', 'static_tab', '.html') self.verify_content_existence(self.store, root_dir, course_id, 'tabs', 'static_tab', '.html')
# check for about content # check for about content
self.verify_content_existence(module_store, root_dir, course_id, 'about', 'about', '.html') self.verify_content_existence(self.store, root_dir, course_id, 'about', 'about', '.html')
# check for grading_policy.json # check for grading_policy.json
filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall') filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall')
self.assertTrue(filesystem.exists('grading_policy.json')) self.assertTrue(filesystem.exists('grading_policy.json'))
course = module_store.get_course(course_id) course = self.store.get_course(course_id)
# compare what's on disk compared to what we have in our course # compare what's on disk compared to what we have in our course
with filesystem.open('grading_policy.json', 'r') as grading_policy: with filesystem.open('grading_policy.json', 'r') as grading_policy:
on_disk = loads(grading_policy.read()) on_disk = loads(grading_policy.read())
...@@ -906,25 +698,23 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -906,25 +698,23 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course)) self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course))
# remove old course # remove old course
delete_course(module_store, content_store, course_id, commit=True) delete_course(self.store, content_store, course_id, commit=True)
# reimport over old course # reimport over old course
self.check_import( self.check_import(root_dir, content_store, course_id)
module_store, root_dir, content_store, course_id,
locked_asset_key, locked_asset_attrs
)
# import to different course id # import to different course id
self.check_import( new_course_id = SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday')
module_store, root_dir, content_store, SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday'), self.check_import(root_dir, content_store, new_course_id)
locked_asset_key, locked_asset_attrs self.assertCoursesEqual(course_id, new_course_id)
)
shutil.rmtree(root_dir) shutil.rmtree(root_dir)
def check_import(self, module_store, root_dir, content_store, course_id, def check_import(self, root_dir, content_store, course_id):
locked_asset_key, locked_asset_attrs): """Imports the course in root_dir into the given course_id and verifies its content"""
# reimport # reimport
import_from_xml( import_from_xml(
module_store, self.store,
self.user.id, self.user.id,
root_dir, root_dir,
['test_export'], ['test_export'],
...@@ -932,74 +722,33 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -932,74 +722,33 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
target_course_id=course_id, target_course_id=course_id,
) )
items = module_store.get_items( # verify content of the course
course_id, self.check_populated_course(course_id)
category='vertical',
revision=ModuleStoreEnum.RevisionOption.published_only
)
self._check_verticals(items)
def verify_item_publish_state(item, publish_state): # verify additional export attributes
if publish_state in (PublishState.private, PublishState.draft): def verify_export_attrs_removed(attributes):
self.assertTrue(getattr(item, 'is_draft', False)) """Verifies all temporary attributes added during export are removed"""
else: self.assertNotIn('index_in_children_list', attributes)
self.assertFalse(getattr(item, 'is_draft', False)) self.assertNotIn('parent_sequential_url', attributes)
self.assertEqual(module_store.compute_publish_state(item), publish_state)
def get_and_verify_item_publish_state(item_type, item_name, publish_state): vertical = self.store.get_item(course_id.make_usage_key('vertical', self.TEST_VERTICAL))
item = module_store.get_item(course_id.make_usage_key(item_type, item_name)) verify_export_attrs_removed(vertical.xml_attributes)
verify_item_publish_state(item, publish_state)
return item
# verify that the draft vertical is draft
vertical = get_and_verify_item_publish_state('vertical', 'vertical_test', PublishState.draft)
self.assertNotIn('index_in_children_list', vertical.xml_attributes)
self.assertNotIn('parent_sequential_url', vertical.xml_attributes)
for child in vertical.get_children(): for child in vertical.get_children():
verify_item_publish_state(child, PublishState.draft) verify_export_attrs_removed(child.xml_attributes)
self.assertNotIn('index_in_children_list', child.xml_attributes)
if hasattr(child, 'data'):
self.assertNotIn('index_in_children_list', child.data)
self.assertNotIn('parent_sequential_url', child.xml_attributes)
if hasattr(child, 'data'): if hasattr(child, 'data'):
self.assertNotIn('parent_sequential_url', child.data) verify_export_attrs_removed(child.data)
# make sure that we don't have a sequential that is not in draft mode
get_and_verify_item_publish_state('sequential', 'vertical_sequential', PublishState.public)
# verify that we have the private vertical
get_and_verify_item_publish_state('vertical', 'a_private_vertical', PublishState.private)
# verify that we have the public vertical
get_and_verify_item_publish_state('vertical', 'a_published_vertical', PublishState.public)
# make sure the textbook survived the export/import
course = module_store.get_course(course_id)
self.assertGreater(len(course.textbooks), 0)
locked_asset_key = locked_asset_key.map_into_course(course_id)
new_attrs = content_store.get_attrs(locked_asset_key)
for key, value in locked_asset_attrs.iteritems():
if key == '_id':
self.assertEqual(value['name'], new_attrs[key]['name'])
elif key == 'filename':
pass
else:
self.assertEqual(value, new_attrs[key])
def test_export_course_with_metadata_only_video(self): def test_export_course_with_metadata_only_video(self):
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# create a new video module and add it as a child to a vertical # create a new video module and add it as a child to a vertical
# this re-creates a bug whereby since the video template doesn't have # this re-creates a bug whereby since the video template doesn't have
# anything in 'data' field, the export was blowing up # anything in 'data' field, the export was blowing up
verticals = module_store.get_items(course_id, category='vertical') verticals = self.store.get_items(course_id, category='vertical')
self.assertGreater(len(verticals), 0) self.assertGreater(len(verticals), 0)
...@@ -1012,7 +761,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1012,7 +761,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir # export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export') export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
shutil.rmtree(root_dir) shutil.rmtree(root_dir)
...@@ -1020,13 +769,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1020,13 +769,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
""" """
Similar to `test_export_course_with_metadata_only_video`. Similar to `test_export_course_with_metadata_only_video`.
""" """
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['word_cloud']) import_from_xml(self.store, self.user.id, 'common/test/data/', ['word_cloud'])
course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring') course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring')
verticals = module_store.get_items(course_id, category='vertical') verticals = self.store.get_items(course_id, category='vertical')
self.assertGreater(len(verticals), 0) self.assertGreater(len(verticals), 0)
...@@ -1039,7 +787,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1039,7 +787,7 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir # export out to a tempdir
export_to_xml(module_store, content_store, course_id, root_dir, 'test_export') export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
shutil.rmtree(root_dir) shutil.rmtree(root_dir)
...@@ -1048,13 +796,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1048,13 +796,12 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
Test that an empty `data` field is preserved through Test that an empty `data` field is preserved through
export/import. export/import.
""" """
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
verticals = module_store.get_items(course_id, category='vertical') verticals = self.store.get_items(course_id, category='vertical')
self.assertGreater(len(verticals), 0) self.assertGreater(len(verticals), 0)
...@@ -1067,11 +814,11 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1067,11 +814,11 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
# Export the course # Export the course
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip') export_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
# Reimport and get the video back # Reimport and get the video back
import_from_xml(module_store, self.user.id, root_dir) import_from_xml(self.store, self.user.id, root_dir)
imported_word_cloud = module_store.get_item(course_id.make_usage_key('word_cloud', 'untitled')) imported_word_cloud = self.store.get_item(course_id.make_usage_key('word_cloud', 'untitled'))
# It should now contain empty data # It should now contain empty data
self.assertEquals(imported_word_cloud.data, '') self.assertEquals(imported_word_cloud.data, '')
...@@ -1080,33 +827,30 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1080,33 +827,30 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
""" """
Test that a course which has HTML that has style formatting is preserved in export/import Test that a course which has HTML that has style formatting is preserved in export/import
""" """
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# Export the course # Export the course
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip') export_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
# Reimport and get the video back # Reimport and get the video back
import_from_xml(module_store, self.user.id, root_dir) import_from_xml(self.store, self.user.id, root_dir)
# get the sample HTML with styling information # get the sample HTML with styling information
html_module = module_store.get_item(course_id.make_usage_key('html', 'with_styling')) html_module = self.store.get_item(course_id.make_usage_key('html', 'with_styling'))
self.assertIn('<p style="font:italic bold 72px/30px Georgia, serif; color: red; ">', html_module.data) self.assertIn('<p style="font:italic bold 72px/30px Georgia, serif; color: red; ">', html_module.data)
# get the sample HTML with just a simple <img> tag information # get the sample HTML with just a simple <img> tag information
html_module = module_store.get_item(course_id.make_usage_key('html', 'just_img')) html_module = self.store.get_item(course_id.make_usage_key('html', 'just_img'))
self.assertIn('<img src="/static/foo_bar.jpg" />', html_module.data) self.assertIn('<img src="/static/foo_bar.jpg" />', html_module.data)
def test_course_handouts_rewrites(self): def test_course_handouts_rewrites(self):
module_store = modulestore()
# import a test course # import a test course
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = course_items[0].id course_id = course_items[0].id
handouts_location = course_id.make_usage_key('course_info', 'handouts') handouts_location = course_id.make_usage_key('course_info', 'handouts')
...@@ -1121,8 +865,8 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1121,8 +865,8 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
self.assertContains(resp, '/c4x/edX/toy/asset/handouts_sample_handout.txt') self.assertContains(resp, '/c4x/edX/toy/asset/handouts_sample_handout.txt')
def test_prefetch_children(self): def test_prefetch_children(self):
mongo_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) mongo_store = self.store._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
import_from_xml(modulestore(), self.user.id, 'common/test/data/', ['toy']) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
wrapper = MongoCollectionFindWrapper(mongo_store.collection.find) wrapper = MongoCollectionFindWrapper(mongo_store.collection.find)
...@@ -1152,32 +896,31 @@ class ContentStoreToyCourseTest(ContentStoreTestCase): ...@@ -1152,32 +896,31 @@ class ContentStoreToyCourseTest(ContentStoreTestCase):
def test_export_course_without_content_store(self): def test_export_course_without_content_store(self):
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
# Create toy course # Create toy course
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy']) _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
course_id = course_items[0].id course_id = course_items[0].id
root_dir = path(mkdtemp_clean()) root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir) print 'Exporting to tempdir = {0}'.format(root_dir)
export_to_xml(module_store, None, course_id, root_dir, 'test_export_no_content_store') export_to_xml(self.store, None, course_id, root_dir, 'test_export_no_content_store')
# Delete the course from module store and reimport it # Delete the course from module store and reimport it
delete_course(module_store, content_store, course_id, commit=True) delete_course(self.store, content_store, course_id, commit=True)
import_from_xml( import_from_xml(
module_store, self.user.id, root_dir, ['test_export_no_content_store'], self.store, self.user.id, root_dir, ['test_export_no_content_store'],
static_content_store=None, static_content_store=None,
target_course_id=course_id target_course_id=course_id
) )
# Verify reimported course # Verify reimported course
items = module_store.get_items( items = self.store.get_items(
course_id, course_id,
category='sequential', category='sequential',
name='vertical_sequential' name='vertical_sequential'
...@@ -1198,39 +941,17 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1198,39 +941,17 @@ class ContentStoreTest(ContentStoreTestCase):
Tests for the CMS ContentStore application. Tests for the CMS ContentStore application.
""" """
def setUp(self): def setUp(self):
""" super(ContentStoreTest, self).setUp()
These tests need a user in the DB so that the django Test Client
can log them in.
They inherit from the ModuleStoreTestCase class so that the mongodb collection
will be cleared out before each test case execution and deleted
afterwards.
"""
uname = 'testuser'
email = 'test+courses@edx.org'
password = 'foo'
# Create the use so we can log them in.
self.user = User.objects.create_user(uname, email, password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
self.user.is_active = True
# Staff has access to view all courses
self.user.is_staff = True
self.user.save()
self.client = AjaxEnabledTestClient()
self.client.login(username=uname, password=password)
self.course_data = { self.course_data = {
'org': 'MITx', 'org': 'MITx',
'number': '999', 'number': '111',
'display_name': 'Robot Super Course', 'display_name': 'Robot Super Course',
'run': '2013_Spring' 'run': '2013_Spring'
} }
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def assert_created_course(self, number_suffix=None): def assert_created_course(self, number_suffix=None):
...@@ -1535,7 +1256,7 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1535,7 +1256,7 @@ class ContentStoreTest(ContentStoreTestCase):
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
payload = parse_json(resp) payload = parse_json(resp)
problem_loc = UsageKey.from_string(payload['locator']) problem_loc = UsageKey.from_string(payload['locator'])
problem = modulestore().get_item(problem_loc) problem = self.store.get_item(problem_loc)
# should be a CapaDescriptor # should be a CapaDescriptor
self.assertIsInstance(problem, CapaDescriptor, "New problem is not a CapaDescriptor") self.assertIsInstance(problem, CapaDescriptor, "New problem is not a CapaDescriptor")
context = problem.get_context() context = problem.get_context()
...@@ -1555,7 +1276,7 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1555,7 +1276,7 @@ class ContentStoreTest(ContentStoreTestCase):
) )
self.assertEqual(resp.status_code, 200) self.assertEqual(resp.status_code, 200)
_, course_items = import_from_xml(modulestore(), self.user.id, 'common/test/data/', ['simple']) _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['simple'])
course_key = course_items[0].id course_key = course_items[0].id
resp = self._show_course_overview(course_key) resp = self._show_course_overview(course_key)
...@@ -1604,13 +1325,12 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1604,13 +1325,12 @@ class ContentStoreTest(ContentStoreTestCase):
delete_item(category='chapter', name='chapter_2') delete_item(category='chapter', name='chapter_2')
def test_import_into_new_course_id(self): def test_import_into_new_course_id(self):
module_store = modulestore()
target_course_id = _get_course_id(self.course_data) target_course_id = _get_course_id(self.course_data)
_create_course(self, target_course_id, self.course_data) _create_course(self, target_course_id, self.course_data)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
modules = module_store.get_items(target_course_id) modules = self.store.get_items(target_course_id)
# we should have a number of modules in there # we should have a number of modules in there
# we can't specify an exact number since it'll always be changing # we can't specify an exact number since it'll always be changing
...@@ -1621,7 +1341,7 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1621,7 +1341,7 @@ class ContentStoreTest(ContentStoreTestCase):
# #
# first check PDF textbooks, to make sure the url paths got updated # first check PDF textbooks, to make sure the url paths got updated
course_module = module_store.get_course(target_course_id) course_module = self.store.get_course(target_course_id)
self.assertEqual(len(course_module.pdf_textbooks), 1) self.assertEqual(len(course_module.pdf_textbooks), 1)
self.assertEqual(len(course_module.pdf_textbooks[0]["chapters"]), 2) self.assertEqual(len(course_module.pdf_textbooks[0]["chapters"]), 2)
...@@ -1629,8 +1349,6 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1629,8 +1349,6 @@ class ContentStoreTest(ContentStoreTestCase):
self.assertEqual(course_module.pdf_textbooks[0]["chapters"][1]["url"], '/static/Chapter2.pdf') self.assertEqual(course_module.pdf_textbooks[0]["chapters"][1]["url"], '/static/Chapter2.pdf')
def test_import_into_new_course_id_wiki_slug_renamespacing(self): def test_import_into_new_course_id_wiki_slug_renamespacing(self):
module_store = modulestore()
# If reimporting into the same course do not change the wiki_slug. # If reimporting into the same course do not change the wiki_slug.
target_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') target_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course_data = { course_data = {
...@@ -1640,17 +1358,17 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1640,17 +1358,17 @@ class ContentStoreTest(ContentStoreTestCase):
'run': target_course_id.run 'run': target_course_id.run
} }
_create_course(self, target_course_id, course_data) _create_course(self, target_course_id, course_data)
course_module = module_store.get_course(target_course_id) course_module = self.store.get_course(target_course_id)
course_module.wiki_slug = 'toy' course_module.wiki_slug = 'toy'
course_module.save() course_module.save()
# Import a course with wiki_slug == location.course # Import a course with wiki_slug == location.course
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id) course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'toy') self.assertEquals(course_module.wiki_slug, 'toy')
# But change the wiki_slug if it is a different course. # But change the wiki_slug if it is a different course.
target_course_id = SlashSeparatedCourseKey('MITx', '999', '2013_Spring') target_course_id = SlashSeparatedCourseKey('MITx', '111', '2013_Spring')
course_data = { course_data = {
'org': target_course_id.org, 'org': target_course_id.org,
'number': target_course_id.course, 'number': target_course_id.course,
...@@ -1660,23 +1378,22 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1660,23 +1378,22 @@ class ContentStoreTest(ContentStoreTestCase):
_create_course(self, target_course_id, course_data) _create_course(self, target_course_id, course_data)
# Import a course with wiki_slug == location.course # Import a course with wiki_slug == location.course
import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id) import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id) course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring') self.assertEquals(course_module.wiki_slug, 'MITx.111.2013_Spring')
# Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run) # Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run)
import_from_xml(module_store, self.user.id, 'common/test/data/', ['two_toys'], target_course_id=target_course_id) import_from_xml(self.store, self.user.id, 'common/test/data/', ['two_toys'], target_course_id=target_course_id)
course_module = module_store.get_course(target_course_id) course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring') self.assertEquals(course_module.wiki_slug, 'MITx.111.2013_Spring')
def test_import_metadata_with_attempts_empty_string(self): def test_import_metadata_with_attempts_empty_string(self):
module_store = modulestore() import_from_xml(self.store, self.user.id, 'common/test/data/', ['simple'])
import_from_xml(module_store, self.user.id, 'common/test/data/', ['simple'])
did_load_item = False did_load_item = False
try: try:
course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem') course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem')
usage_key = course_key.make_usage_key('problem', 'ps01-simple') usage_key = course_key.make_usage_key('problem', 'ps01-simple')
module_store.get_item(usage_key) self.store.get_item(usage_key)
did_load_item = True did_load_item = True
except ItemNotFoundError: except ItemNotFoundError:
pass pass
...@@ -1685,23 +1402,21 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1685,23 +1402,21 @@ class ContentStoreTest(ContentStoreTestCase):
self.assertTrue(did_load_item) self.assertTrue(did_load_item)
def test_forum_id_generation(self): def test_forum_id_generation(self):
module_store = modulestore()
course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course') course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
new_component_location = course.id.make_usage_key('discussion', 'new_component') new_component_location = course.id.make_usage_key('discussion', 'new_component')
# crate a new module and add it as a child to a vertical # crate a new module and add it as a child to a vertical
module_store.create_and_save_xmodule(new_component_location, self.user.id) self.store.create_and_save_xmodule(new_component_location, self.user.id)
new_discussion_item = module_store.get_item(new_component_location) new_discussion_item = self.store.get_item(new_component_location)
self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$') self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')
def test_metadata_inheritance(self): def test_metadata_inheritance(self):
module_store = modulestore() _, course_items = import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'])
_, course_items = import_from_xml(module_store, self.user.id, 'common/test/data/', ['toy'])
course = course_items[0] course = course_items[0]
verticals = module_store.get_items(course.id, category='vertical') verticals = self.store.get_items(course.id, category='vertical')
# let's assert on the metadata_inheritance on an existing vertical # let's assert on the metadata_inheritance on an existing vertical
for vertical in verticals: for vertical in verticals:
...@@ -1713,14 +1428,14 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1713,14 +1428,14 @@ class ContentStoreTest(ContentStoreTestCase):
new_component_location = course.id.make_usage_key('html', 'new_component') new_component_location = course.id.make_usage_key('html', 'new_component')
# crate a new module and add it as a child to a vertical # crate a new module and add it as a child to a vertical
new_object = module_store.create_xmodule(new_component_location) new_object = self.store.create_xmodule(new_component_location)
module_store.update_item(new_object, self.user.id, allow_not_found=True) self.store.update_item(new_object, self.user.id, allow_not_found=True)
parent = verticals[0] parent = verticals[0]
parent.children.append(new_component_location) parent.children.append(new_component_location)
module_store.update_item(parent, self.user.id) self.store.update_item(parent, self.user.id)
# flush the cache # flush the cache
new_module = module_store.get_item(new_component_location) new_module = self.store.get_item(new_component_location)
# check for grace period definition which should be defined at the course level # check for grace period definition which should be defined at the course level
self.assertEqual(parent.graceperiod, new_module.graceperiod) self.assertEqual(parent.graceperiod, new_module.graceperiod)
...@@ -1733,10 +1448,10 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1733,10 +1448,10 @@ class ContentStoreTest(ContentStoreTestCase):
# now let's define an override at the leaf node level # now let's define an override at the leaf node level
# #
new_module.graceperiod = timedelta(1) new_module.graceperiod = timedelta(1)
module_store.update_item(new_module, self.user.id) self.store.update_item(new_module, self.user.id)
# flush the cache and refetch # flush the cache and refetch
new_module = module_store.get_item(new_component_location) new_module = self.store.get_item(new_component_location)
self.assertEqual(timedelta(1), new_module.graceperiod) self.assertEqual(timedelta(1), new_module.graceperiod)
...@@ -1753,9 +1468,8 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1753,9 +1468,8 @@ class ContentStoreTest(ContentStoreTestCase):
self.assertGreaterEqual(len(course.checklists), 4) self.assertGreaterEqual(len(course.checklists), 4)
# by fetching # by fetching
module_store = modulestore() fetched_course = self.store.get_item(course.location)
fetched_course = module_store.get_item(course.location) fetched_item = self.store.get_item(vertical.location)
fetched_item = module_store.get_item(vertical.location)
self.assertIsNotNone(fetched_course.start) self.assertIsNotNone(fetched_course.start)
self.assertEqual(course.start, fetched_course.start) self.assertEqual(course.start, fetched_course.start)
self.assertEqual(fetched_course.start, fetched_item.start) self.assertEqual(fetched_course.start, fetched_item.start)
...@@ -1765,20 +1479,18 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1765,20 +1479,18 @@ class ContentStoreTest(ContentStoreTestCase):
def test_image_import(self): def test_image_import(self):
"""Test backwards compatibilty of course image.""" """Test backwards compatibilty of course image."""
module_store = modulestore()
content_store = contentstore() content_store = contentstore()
# Use conditional_and_poll, as it's got an image already # Use conditional_and_poll, as it's got an image already
import_from_xml( import_from_xml(
module_store, self.store,
self.user.id, self.user.id,
'common/test/data/', 'common/test/data/',
['conditional_and_poll'], ['conditional_and_poll'],
static_content_store=content_store static_content_store=content_store
) )
course = module_store.get_courses()[0] course = self.store.get_courses()[0]
# Make sure the course image is set to the right place # Make sure the course image is set to the right place
self.assertEqual(course.course_image, 'images_course_image.jpg') self.assertEqual(course.course_image, 'images_course_image.jpg')
...@@ -1799,8 +1511,8 @@ class ContentStoreTest(ContentStoreTestCase): ...@@ -1799,8 +1511,8 @@ class ContentStoreTest(ContentStoreTestCase):
course_key = _get_course_id(self.course_data) course_key = _get_course_id(self.course_data)
_create_course(self, course_key, self.course_data) _create_course(self, course_key, self.course_data)
course_module = modulestore().get_course(course_key) course_module = self.store.get_course(course_key)
self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring') self.assertEquals(course_module.wiki_slug, 'MITx.111.2013_Spring')
class MetadataSaveTestCase(ContentStoreTestCase): class MetadataSaveTestCase(ContentStoreTestCase):
...@@ -1852,8 +1564,8 @@ class MetadataSaveTestCase(ContentStoreTestCase): ...@@ -1852,8 +1564,8 @@ class MetadataSaveTestCase(ContentStoreTestCase):
delattr(self.video_descriptor, field_name) delattr(self.video_descriptor, field_name)
self.assertNotIn('html5_sources', own_metadata(self.video_descriptor)) self.assertNotIn('html5_sources', own_metadata(self.video_descriptor))
modulestore().update_item(self.video_descriptor, self.user.id) self.store.update_item(self.video_descriptor, self.user.id)
module = modulestore().get_item(location) module = self.store.get_item(location)
self.assertNotIn('html5_sources', own_metadata(module)) self.assertNotIn('html5_sources', own_metadata(module))
......
cms/djangoapps/contentstore/tests/test_core_caching.py
from cache_toolbox.core import get_cached_content, set_cached_content, del_cached_content from cache_toolbox.core import get_cached_content, set_cached_content, del_cached_content
from opaque_keys.edx.locations import Location from opaque_keys.edx.locations import Location
from xmodule.contentstore.content import StaticContent
from django.test import TestCase from django.test import TestCase
......
cms/djangoapps/contentstore/tests/test_export_git.py
...@@ -10,7 +10,6 @@ from uuid import uuid4 ...@@ -10,7 +10,6 @@ from uuid import uuid4
from django.conf import settings from django.conf import settings
from django.test.utils import override_settings from django.test.utils import override_settings
from pymongo import MongoClient
from .utils import CourseTestCase from .utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils import contentstore.git_export_utils as git_export_utils
...@@ -37,7 +36,7 @@ class TestExportGit(CourseTestCase): ...@@ -37,7 +36,7 @@ class TestExportGit(CourseTestCase):
self.test_url = reverse_course_url('export_git', self.course.id) self.test_url = reverse_course_url('export_git', self.course.id)
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) modulestore().contentstore.drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def test_giturl_missing(self): def test_giturl_missing(self):
......
cms/djangoapps/contentstore/tests/test_import.py
...@@ -7,7 +7,6 @@ Tests for import_from_xml using the mongo modulestore. ...@@ -7,7 +7,6 @@ Tests for import_from_xml using the mongo modulestore.
from django.test.client import Client from django.test.client import Client
from django.test.utils import override_settings from django.test.utils import override_settings
from django.conf import settings from django.conf import settings
from path import path
import copy import copy
from django.contrib.auth.models import User from django.contrib.auth.models import User
...@@ -22,7 +21,6 @@ from xmodule.contentstore.django import _CONTENTSTORE ...@@ -22,7 +21,6 @@ from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.exceptions import NotFoundError from xmodule.exceptions import NotFoundError
from uuid import uuid4 from uuid import uuid4
from pymongo import MongoClient
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
...@@ -56,7 +54,7 @@ class ContentStoreImportTest(ModuleStoreTestCase): ...@@ -56,7 +54,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
self.client.login(username=uname, password=password) self.client.login(username=uname, password=password)
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def load_test_import_course(self): def load_test_import_course(self):
......
cms/djangoapps/contentstore/tests/test_transcripts_utils.py
...@@ -151,7 +151,7 @@ class TestSaveSubsToStore(ModuleStoreTestCase): ...@@ -151,7 +151,7 @@ class TestSaveSubsToStore(ModuleStoreTestCase):
def tearDown(self): def tearDown(self):
self.clear_subs_content() self.clear_subs_content()
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
...@@ -190,7 +190,7 @@ class TestDownloadYoutubeSubs(ModuleStoreTestCase): ...@@ -190,7 +190,7 @@ class TestDownloadYoutubeSubs(ModuleStoreTestCase):
org=self.org, number=self.number, display_name=self.display_name) org=self.org, number=self.number, display_name=self.display_name)
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def test_success_downloading_subs(self): def test_success_downloading_subs(self):
......
cms/djangoapps/contentstore/tests/utils.py
# pylint: disable=E1101
''' '''
Utilities for contentstore tests Utilities for contentstore tests
''' '''
import json import json
import re
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test.client import Client from django.test.client import Client
from xmodule.contentstore.django import contentstore
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import PublishState, ModuleStoreEnum
from xmodule.modulestore.django import modulestore from xmodule.modulestore.django import modulestore
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.xml_importer import import_from_xml
from student.models import Registration from student.models import Registration
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from contentstore.utils import reverse_url
def parse_json(response): def parse_json(response):
...@@ -133,3 +142,213 @@ class CourseTestCase(ModuleStoreTestCase): ...@@ -133,3 +142,213 @@ class CourseTestCase(ModuleStoreTestCase):
""" """
self.course.save() self.course.save()
self.store.update_item(self.course, self.user.id) self.store.update_item(self.course, self.user.id)
TEST_VERTICAL = 'vertical_test'
PRIVATE_VERTICAL = 'a_private_vertical'
PUBLISHED_VERTICAL = 'a_published_vertical'
SEQUENTIAL = 'vertical_sequential'
LOCKED_ASSET_KEY = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
def import_and_populate_course(self):
"""
Imports the test toy course and populates it with additional test data
"""
content_store = contentstore()
import_from_xml(self.store, self.user.id, 'common/test/data/', ['toy'], static_content_store=content_store)
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
# create an Orphan
# We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
vertical = self.store.get_item(course_id.make_usage_key('vertical', self.TEST_VERTICAL), depth=1)
vertical.location = vertical.location.replace(name='no_references')
self.store.update_item(vertical, self.user.id, allow_not_found=True)
orphan_vertical = self.store.get_item(vertical.location)
self.assertEqual(orphan_vertical.location.name, 'no_references')
self.assertEqual(len(orphan_vertical.children), len(vertical.children))
# create a Draft vertical
vertical = self.store.get_item(course_id.make_usage_key('vertical', self.TEST_VERTICAL), depth=1)
draft_vertical = self.store.convert_to_draft(vertical.location, self.user.id)
self.assertEqual(self.store.compute_publish_state(draft_vertical), PublishState.draft)
# create a Private (draft only) vertical
private_vertical = self.store.create_and_save_xmodule(course_id.make_usage_key('vertical', self.PRIVATE_VERTICAL), self.user.id)
self.assertEqual(self.store.compute_publish_state(private_vertical), PublishState.private)
# create a Published (no draft) vertical
public_vertical = self.store.create_and_save_xmodule(course_id.make_usage_key('vertical', self.PUBLISHED_VERTICAL), self.user.id)
public_vertical = self.store.publish(public_vertical.location, self.user.id)
self.assertEqual(self.store.compute_publish_state(public_vertical), PublishState.public)
# add the new private and new public as children of the sequential
sequential = self.store.get_item(course_id.make_usage_key('sequential', self.SEQUENTIAL))
sequential.children.append(private_vertical.location)
sequential.children.append(public_vertical.location)
self.store.update_item(sequential, self.user.id)
# lock an asset
content_store.set_attr(self.LOCKED_ASSET_KEY, 'locked', True)
# create a non-portable link - should be rewritten in new courses
html_module = self.store.get_item(course_id.make_usage_key('html', 'nonportable'))
new_data = html_module.data = html_module.data.replace(
'/static/',
'/c4x/{0}/{1}/asset/'.format(course_id.org, course_id.course)
)
self.store.update_item(html_module, self.user.id)
html_module = self.store.get_item(html_module.location)
self.assertEqual(new_data, html_module.data)
return course_id
def check_populated_course(self, course_id):
"""
Verifies the content of the given course, per data that was populated in import_and_populate_course
"""
items = self.store.get_items(
course_id,
category='vertical',
revision=ModuleStoreEnum.RevisionOption.published_only
)
self.check_verticals(items)
def verify_item_publish_state(item, publish_state):
"""Verifies the publish state of the item is as expected."""
if publish_state in (PublishState.private, PublishState.draft):
self.assertTrue(getattr(item, 'is_draft', False))
else:
self.assertFalse(getattr(item, 'is_draft', False))
self.assertEqual(self.store.compute_publish_state(item), publish_state)
def get_and_verify_publish_state(item_type, item_name, publish_state):
"""Gets the given item from the store and verifies the publish state of the item is as expected."""
item = self.store.get_item(course_id.make_usage_key(item_type, item_name))
verify_item_publish_state(item, publish_state)
return item
# verify that the draft vertical is draft
vertical = get_and_verify_publish_state('vertical', self.TEST_VERTICAL, PublishState.draft)
for child in vertical.get_children():
verify_item_publish_state(child, PublishState.draft)
# make sure that we don't have a sequential that is not in draft mode
sequential = get_and_verify_publish_state('sequential', self.SEQUENTIAL, PublishState.public)
# verify that we have the private vertical
private_vertical = get_and_verify_publish_state('vertical', self.PRIVATE_VERTICAL, PublishState.private)
# verify that we have the public vertical
public_vertical = get_and_verify_publish_state('vertical', self.PUBLISHED_VERTICAL, PublishState.public)
# verify verticals are children of sequential
for vert in [vertical, private_vertical, public_vertical]:
self.assertIn(vert.location, sequential.children)
# verify textbook exists
course = self.store.get_course(course_id)
self.assertGreater(len(course.textbooks), 0)
# verify asset attributes of locked asset key
self.assertAssetsEqual(self.LOCKED_ASSET_KEY, self.LOCKED_ASSET_KEY.course_key, course_id)
# verify non-portable links are rewritten
html_module = self.store.get_item(course_id.make_usage_key('html', 'nonportable'))
self.assertIn('/static/foo.jpg', html_module.data)
return course
def assertCoursesEqual(self, course1_id, course2_id):
"""
Verifies the content of the two given courses are equal
"""
course1_items = self.store.get_items(course1_id)
course2_items = self.store.get_items(course2_id)
self.assertGreater(len(course1_items), 0) # ensure it found content instead of [] == []
self.assertEqual(len(course1_items), len(course2_items))
for course1_item in course1_items:
course2_item_location = course1_item.location.map_into_course(course2_id)
if course1_item.location.category == 'course':
course2_item_location = course2_item_location.replace(name=course2_item_location.run)
course2_item = self.store.get_item(course2_item_location)
# compare published state
self.assertEqual(
self.store.compute_publish_state(course1_item),
self.store.compute_publish_state(course2_item)
)
# compare data
self.assertEqual(hasattr(course1_item, 'data'), hasattr(course2_item, 'data'))
if hasattr(course1_item, 'data'):
self.assertEqual(course1_item.data, course2_item.data)
# compare meta-data
self.assertEqual(own_metadata(course1_item), own_metadata(course2_item))
# compare children
self.assertEqual(course1_item.has_children, course2_item.has_children)
if course1_item.has_children:
expected_children = []
for course1_item_child in course1_item.children:
expected_children.append(
course1_item_child.map_into_course(course2_id)
)
self.assertEqual(expected_children, course2_item.children)
# compare assets
content_store = contentstore()
course1_assets, count_course1_assets = content_store.get_all_content_for_course(course1_id)
_, count_course2_assets = content_store.get_all_content_for_course(course2_id)
self.assertEqual(count_course1_assets, count_course2_assets)
for asset in course1_assets:
asset_id = asset.get('content_son', asset['_id'])
asset_key = StaticContent.compute_location(course1_id, asset_id['name'])
self.assertAssetsEqual(asset_key, course1_id, course2_id)
def check_verticals(self, items):
""" Test getting the editing HTML for each vertical. """
# assert is here to make sure that the course being tested actually has verticals (units) to check.
self.assertGreater(len(items), 0, "Course has no verticals (units) to check")
for descriptor in items:
resp = self.client.get_html(get_url('unit_handler', descriptor.location))
self.assertEqual(resp.status_code, 200)
test_no_locations(self, resp)
def assertAssetsEqual(self, asset_key, course1_id, course2_id):
"""Verifies the asset of the given key has the same attributes in both given courses."""
content_store = contentstore()
course1_asset_attrs = content_store.get_attrs(asset_key.map_into_course(course1_id))
course2_asset_attrs = content_store.get_attrs(asset_key.map_into_course(course2_id))
self.assertEqual(len(course1_asset_attrs), len(course2_asset_attrs))
for key, value in course1_asset_attrs.iteritems():
if key == '_id':
self.assertEqual(value['name'], course2_asset_attrs[key]['name'])
elif key == 'filename' or key == 'uploadDate' or key == 'content_son' or key == 'thumbnail_location':
pass
else:
self.assertEqual(value, course2_asset_attrs[key])
def test_no_locations(test, resp, status_code=200, html=True):
"""
Verifies that "i4x", which appears in old locations, but not
new locators, does not appear in the HTML response output.
Used to verify that database refactoring is complete.
"""
test.assertNotContains(resp, 'i4x', status_code=status_code, html=html)
if html:
# For HTML pages, it is nice to call the method with html=True because
# it checks that the HTML properly parses. However, it won't find i4x usages
# in JavaScript blocks.
content = resp.content
hits = len(re.findall(r"(?<!jump_to/)i4x://", content))
test.assertEqual(hits, 0, "i4x found outside of LMS jump-to links")
def get_url(handler_name, key_value, key_name='usage_key_string', kwargs=None):
"""
Helper function for getting HTML for a page in Studio and checking that it does not error.
"""
return reverse_url(handler_name, key_name, key_value, kwargs)
cms/djangoapps/contentstore/views/assets.py
...@@ -121,7 +121,7 @@ def _assets_json(request, course_key): ...@@ -121,7 +121,7 @@ def _assets_json(request, course_key):
asset_json = [] asset_json = []
for asset in assets: for asset in assets:
asset_id = asset['_id'] asset_id = asset.get('content_son', asset['_id'])
asset_location = StaticContent.compute_location(course_key, asset_id['name']) asset_location = StaticContent.compute_location(course_key, asset_id['name'])
# note, due to the schema change we may not have a 'thumbnail_location' in the result set # note, due to the schema change we may not have a 'thumbnail_location' in the result set
thumbnail_location = asset.get('thumbnail_location', None) thumbnail_location = asset.get('thumbnail_location', None)
......
cms/djangoapps/contentstore/views/tests/test_import_export.py
...@@ -17,12 +17,12 @@ from django.conf import settings ...@@ -17,12 +17,12 @@ from django.conf import settings
from contentstore.utils import reverse_course_url from contentstore.utils import reverse_course_url
from xmodule.contentstore.django import _CONTENTSTORE from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.factories import ItemFactory from xmodule.modulestore.tests.factories import ItemFactory
from contentstore.tests.utils import CourseTestCase from contentstore.tests.utils import CourseTestCase
from student import auth from student import auth
from student.roles import CourseInstructorRole, CourseStaffRole from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore.django import modulestore
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
...@@ -70,7 +70,7 @@ class ImportTestCase(CourseTestCase): ...@@ -70,7 +70,7 @@ class ImportTestCase(CourseTestCase):
def tearDown(self): def tearDown(self):
shutil.rmtree(self.content_dir) shutil.rmtree(self.content_dir)
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) modulestore().contentstore.drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def test_no_coursexml(self): def test_no_coursexml(self):
......
cms/djangoapps/contentstore/views/tests/test_transcripts.py
...@@ -81,7 +81,7 @@ class Basetranscripts(CourseTestCase): ...@@ -81,7 +81,7 @@ class Basetranscripts(CourseTestCase):
} }
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
......
common/djangoapps/contentserver/middleware.py
...@@ -50,7 +50,12 @@ class StaticContentServer(object): ...@@ -50,7 +50,12 @@ class StaticContentServer(object):
if getattr(content, "locked", False): if getattr(content, "locked", False):
if not hasattr(request, "user") or not request.user.is_authenticated(): if not hasattr(request, "user") or not request.user.is_authenticated():
return HttpResponseForbidden('Unauthorized') return HttpResponseForbidden('Unauthorized')
if not request.user.is_staff and not CourseEnrollment.is_enrolled_by_partial( if not request.user.is_staff:
if getattr(loc, 'deprecated', False) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key
):
return HttpResponseForbidden('Unauthorized')
if not getattr(loc, 'deprecated', False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key request.user, loc.course_key
): ):
return HttpResponseForbidden('Unauthorized') return HttpResponseForbidden('Unauthorized')
......
common/djangoapps/contentserver/tests/test.py
...@@ -4,8 +4,6 @@ Tests for StaticContentServer ...@@ -4,8 +4,6 @@ Tests for StaticContentServer
import copy import copy
import logging import logging
from uuid import uuid4 from uuid import uuid4
from path import path
from pymongo import MongoClient
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.conf import settings from django.conf import settings
...@@ -74,7 +72,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): ...@@ -74,7 +72,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
def tearDown(self): def tearDown(self):
MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db']) contentstore().drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
def test_unlocked_asset(self): def test_unlocked_asset(self):
......
common/djangoapps/terrain/browser.py
...@@ -186,11 +186,10 @@ def reset_databases(scenario): ...@@ -186,11 +186,10 @@ def reset_databases(scenario):
whereas modulestore data is in unique collection names. This data is created implicitly during the scenarios. whereas modulestore data is in unique collection names. This data is created implicitly during the scenarios.
If no data is created during the test, these lines equivilently do nothing. If no data is created during the test, these lines equivilently do nothing.
''' '''
mongo = MongoClient() modulestore = xmodule.modulestore.django.modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
mongo.drop_database(settings.CONTENTSTORE['DOC_STORE_CONFIG']['db']) modulestore.contentstore.drop_database()
_CONTENTSTORE.clear() _CONTENTSTORE.clear()
modulestore = xmodule.modulestore.django.modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
modulestore.collection.drop() modulestore.collection.drop()
xmodule.modulestore.django.clear_existing_modulestores() xmodule.modulestore.django.clear_existing_modulestores()
......
common/lib/xmodule/xmodule/contentstore/content.py
...@@ -10,8 +10,8 @@ import StringIO ...@@ -10,8 +10,8 @@ import StringIO
from urlparse import urlparse, urlunparse, parse_qsl from urlparse import urlparse, urlunparse, parse_qsl
from urllib import urlencode from urllib import urlencode
from opaque_keys.edx.locations import AssetLocation, SlashSeparatedCourseKey from opaque_keys.edx.locations import AssetLocation
from .django import contentstore from opaque_keys.edx.keys import CourseKey
from PIL import Image from PIL import Image
...@@ -103,7 +103,7 @@ class StaticContent(object): ...@@ -103,7 +103,7 @@ class StaticContent(object):
if course_key is None: if course_key is None:
return None return None
assert(isinstance(course_key, SlashSeparatedCourseKey)) assert(isinstance(course_key, CourseKey))
return course_key.make_asset_key('asset', '').to_deprecated_string() return course_key.make_asset_key('asset', '').to_deprecated_string()
@staticmethod @staticmethod
...@@ -111,6 +111,8 @@ class StaticContent(object): ...@@ -111,6 +111,8 @@ class StaticContent(object):
""" """
Generate an AssetKey for the given path (old c4x/org/course/asset/name syntax) Generate an AssetKey for the given path (old c4x/org/course/asset/name syntax)
""" """
# TODO OpaqueKey - change to from_string once opaque keys lands
# return AssetLocation.from_string(path)
return AssetLocation.from_deprecated_string(path) return AssetLocation.from_deprecated_string(path)
@staticmethod @staticmethod
...@@ -213,6 +215,12 @@ class ContentStore(object): ...@@ -213,6 +215,12 @@ class ContentStore(object):
""" """
raise NotImplementedError raise NotImplementedError
def copy_all_course_assets(self, source_course_key, dest_course_key):
"""
Copy all the course assets from source_course_key to dest_course_key
"""
raise NotImplementedError
def generate_thumbnail(self, content, tempfile_path=None): def generate_thumbnail(self, content, tempfile_path=None):
thumbnail_content = None thumbnail_content = None
# use a naming convention to associate originals with the thumbnail # use a naming convention to associate originals with the thumbnail
...@@ -248,7 +256,7 @@ class ContentStore(object): ...@@ -248,7 +256,7 @@ class ContentStore(object):
thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name,
'image/jpeg', thumbnail_file) 'image/jpeg', thumbnail_file)
contentstore().save(thumbnail_content) self.save(thumbnail_content)
except Exception, e: except Exception, e:
# log and continue as thumbnails are generally considered as optional # log and continue as thumbnails are generally considered as optional
......
common/lib/xmodule/xmodule/contentstore/mongo.py
...@@ -2,7 +2,6 @@ import pymongo ...@@ -2,7 +2,6 @@ import pymongo
import gridfs import gridfs
from gridfs.errors import NoFile from gridfs.errors import NoFile
from xmodule.modulestore.mongo.base import location_to_query, MongoModuleStore
from xmodule.contentstore.content import XASSET_LOCATION_TAG from xmodule.contentstore.content import XASSET_LOCATION_TAG
import logging import logging
...@@ -13,10 +12,11 @@ from fs.osfs import OSFS ...@@ -13,10 +12,11 @@ from fs.osfs import OSFS
import os import os
import json import json
from bson.son import SON from bson.son import SON
from opaque_keys.edx.locations import AssetLocation from opaque_keys.edx.locations import AssetLocation, SlashSeparatedCourseKey
class MongoContentStore(ContentStore): class MongoContentStore(ContentStore):
# pylint: disable=W0613 # pylint: disable=W0613
def __init__(self, host, db, port=27017, user=None, password=None, bucket='fs', collection=None, **kwargs): def __init__(self, host, db, port=27017, user=None, password=None, bucket='fs', collection=None, **kwargs):
""" """
...@@ -42,15 +42,29 @@ class MongoContentStore(ContentStore): ...@@ -42,15 +42,29 @@ class MongoContentStore(ContentStore):
self.fs_files = _db[bucket + ".files"] # the underlying collection GridFS uses self.fs_files = _db[bucket + ".files"] # the underlying collection GridFS uses
# TODO OpaqueKey - remove after merge of opaque urls
if not hasattr(AssetLocation, 'deprecated'):
setattr(AssetLocation, 'deprecated', True)
setattr(SlashSeparatedCourseKey, 'deprecated', True)
def drop_database(self):
"""
Only for use by test code. Removes the database!
"""
self.fs_files.database.connection.close()
self.fs_files.database.connection.drop_database(self.fs_files.database)
def save(self, content): def save(self, content):
content_id = self.asset_db_key(content.location) content_id, content_son = self.asset_db_key(content.location)
# Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair # The way to version files in gridFS is to not use the file id as the _id but just as the filename.
self.delete(content_id) # Then you can upload as many versions as you like and access by date or version. Because we use
# the location as the _id, we must delete before adding (there's no replace method in gridFS)
self.delete(content_id) # delete is a noop if the entry doesn't exist; so, don't waste time checking
thumbnail_location = content.thumbnail_location.to_deprecated_list_repr() if content.thumbnail_location else None thumbnail_location = content.thumbnail_location.to_deprecated_list_repr() if content.thumbnail_location else None
with self.fs.new_file(_id=content_id, filename=content.get_url_path(), content_type=content.content_type, with self.fs.new_file(_id=content_id, filename=content.get_url_path(), content_type=content.content_type,
displayname=content.name, displayname=content.name, content_son=content_son,
thumbnail_location=thumbnail_location, thumbnail_location=thumbnail_location,
import_path=content.import_path, import_path=content.import_path,
# getattr b/c caching may mean some pickled instances don't have attr # getattr b/c caching may mean some pickled instances don't have attr
...@@ -65,12 +79,13 @@ class MongoContentStore(ContentStore): ...@@ -65,12 +79,13 @@ class MongoContentStore(ContentStore):
def delete(self, location_or_id): def delete(self, location_or_id):
if isinstance(location_or_id, AssetLocation): if isinstance(location_or_id, AssetLocation):
location_or_id = self.asset_db_key(location_or_id) location_or_id, __ = self.asset_db_key(location_or_id)
# Deletes of non-existent files are considered successful # Deletes of non-existent files are considered successful
self.fs.delete(location_or_id) self.fs.delete(location_or_id)
def find(self, location, throw_on_not_found=True, as_stream=False): def find(self, location, throw_on_not_found=True, as_stream=False):
content_id = self.asset_db_key(location) content_id, __ = self.asset_db_key(location)
try: try:
if as_stream: if as_stream:
...@@ -101,21 +116,6 @@ class MongoContentStore(ContentStore): ...@@ -101,21 +116,6 @@ class MongoContentStore(ContentStore):
else: else:
return None return None
def get_stream(self, location):
content_id = self.asset_db_key(location)
try:
handle = self.fs.get(content_id)
except NoFile:
raise NotFoundError()
return handle
def close_stream(self, handle):
try:
handle.close()
except Exception: # pylint: disable=broad-except
pass
def export(self, location, output_directory): def export(self, location, output_directory):
content = self.find(location) content = self.find(location)
...@@ -145,7 +145,9 @@ class MongoContentStore(ContentStore): ...@@ -145,7 +145,9 @@ class MongoContentStore(ContentStore):
assets, __ = self.get_all_content_for_course(course_key) assets, __ = self.get_all_content_for_course(course_key)
for asset in assets: for asset in assets:
asset_location = AssetLocation._from_deprecated_son(asset['_id'], course_key.run) # pylint: disable=protected-access asset_id = asset.get('content_son', asset['_id'])
# assuming course_key's deprecated flag is controlling rather than presence or absence of 'run' in _id
asset_location = course_key.make_asset_key(asset_id['category'], asset_id['name'])
# TODO: On 6/19/14, I had to put a try/except around this # TODO: On 6/19/14, I had to put a try/except around this
# to export a course. The course failed on JSON files in # to export a course. The course failed on JSON files in
# the /static/ directory placed in it with an import. # the /static/ directory placed in it with an import.
...@@ -190,18 +192,15 @@ class MongoContentStore(ContentStore): ...@@ -190,18 +192,15 @@ class MongoContentStore(ContentStore):
] ]
''' '''
course_filter = course_key.make_asset_key(
"asset" if not get_thumbnails else "thumbnail",
None
)
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
if maxresults > 0: if maxresults > 0:
items = self.fs_files.find( items = self.fs_files.find(
location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), query_for_course(course_key, "asset" if not get_thumbnails else "thumbnail"),
skip=start, limit=maxresults, sort=sort skip=start, limit=maxresults, sort=sort
) )
else: else:
items = self.fs_files.find(location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), sort=sort) items = self.fs_files.find(
query_for_course(course_key, "asset" if not get_thumbnails else "thumbnail"), sort=sort
)
count = items.count() count = items.count()
return list(items), count return list(items), count
...@@ -243,7 +242,7 @@ class MongoContentStore(ContentStore): ...@@ -243,7 +242,7 @@ class MongoContentStore(ContentStore):
for attr in attr_dict.iterkeys(): for attr in attr_dict.iterkeys():
if attr in ['_id', 'md5', 'uploadDate', 'length']: if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError("{} is a protected attribute.".format(attr)) raise AttributeError("{} is a protected attribute.".format(attr))
asset_db_key = self.asset_db_key(location) asset_db_key, __ = self.asset_db_key(location)
# catch upsert error and raise NotFoundError if asset doesn't exist # catch upsert error and raise NotFoundError if asset doesn't exist
result = self.fs_files.update({'_id': asset_db_key}, {"$set": attr_dict}, upsert=False) result = self.fs_files.update({'_id': asset_db_key}, {"$set": attr_dict}, upsert=False)
if not result.get('updatedExisting', True): if not result.get('updatedExisting', True):
...@@ -259,30 +258,117 @@ class MongoContentStore(ContentStore): ...@@ -259,30 +258,117 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location :param location: a c4x asset location
""" """
asset_db_key = self.asset_db_key(location) asset_db_key, __ = self.asset_db_key(location)
item = self.fs_files.find_one({'_id': asset_db_key}) item = self.fs_files.find_one({'_id': asset_db_key})
if item is None: if item is None:
raise NotFoundError(asset_db_key) raise NotFoundError(asset_db_key)
return item return item
def copy_all_course_assets(self, source_course_key, dest_course_key):
"""
See :meth:`.ContentStore.copy_all_course_assets`
This implementation fairly expensively copies all of the data
"""
source_query = query_for_course(source_course_key)
# it'd be great to figure out how to do all of this on the db server and not pull the bits over
for asset in self.fs_files.find(source_query):
asset_key = self.make_id_son(asset)
# don't convert from string until fs access
source_content = self.fs.get(asset_key)
if isinstance(asset_key, basestring):
asset_key = AssetLocation.from_string(asset_key)
__, asset_key = self.asset_db_key(asset_key)
asset_key['org'] = dest_course_key.org
asset_key['course'] = dest_course_key.course
if getattr(dest_course_key, 'deprecated', False): # remove the run if exists
if 'run' in asset_key:
del asset_key['run']
asset_id = asset_key
else: # add the run, since it's the last field, we're golden
asset_key['run'] = dest_course_key.run
asset_id = unicode(dest_course_key.make_asset_key(asset_key['category'], asset_key['name']))
self.fs.put(
source_content.read(),
_id=asset_id, filename=asset['filename'], content_type=asset['contentType'],
displayname=asset['displayname'], content_son=asset_key,
# thumbnail is not technically correct but will be functionally correct as the code
# only looks at the name which is not course relative.
thumbnail_location=asset['thumbnail_location'],
import_path=asset['import_path'],
# getattr b/c caching may mean some pickled instances don't have attr
locked=asset.get('locked', False)
)
def delete_all_course_assets(self, course_key): def delete_all_course_assets(self, course_key):
""" """
Delete all assets identified via this course_key. Dangerous operation which may remove assets Delete all assets identified via this course_key. Dangerous operation which may remove assets
referenced by other runs or other courses. referenced by other runs or other courses.
:param course_key: :param course_key:
""" """
course_query = MongoModuleStore._course_key_to_son(course_key, tag=XASSET_LOCATION_TAG) # pylint: disable=protected-access course_query = query_for_course(course_key)
matching_assets = self.fs_files.find(course_query) matching_assets = self.fs_files.find(course_query)
for asset in matching_assets: for asset in matching_assets:
self.fs.delete(asset['_id']) asset_key = self.make_id_son(asset)
self.fs.delete(asset_key)
@staticmethod
def asset_db_key(location):
"""
Returns the database query to find the given asset location.
"""
# codifying the original order which pymongo used for the dicts coming out of location_to_dict # codifying the original order which pymongo used for the dicts coming out of location_to_dict
# stability of order is more important than sanity of order as any changes to order make things # stability of order is more important than sanity of order as any changes to order make things
# unfindable # unfindable
ordered_key_fields = ['category', 'name', 'course', 'tag', 'org', 'revision'] ordered_key_fields = ['category', 'name', 'course', 'tag', 'org', 'revision']
return SON((field_name, getattr(location, field_name)) for field_name in ordered_key_fields)
@classmethod
def asset_db_key(cls, location):
"""
Returns the database _id and son structured lookup to find the given asset location.
"""
dbkey = SON((field_name, getattr(location, field_name)) for field_name in cls.ordered_key_fields)
if getattr(location, 'deprecated', False):
content_id = dbkey
else:
# NOTE, there's no need to state that run doesn't exist in the negative case b/c access via
# SON requires equivalence (same keys and values in exact same order)
dbkey['run'] = location.run
content_id = unicode(location)
return content_id, dbkey
def make_id_son(self, fs_entry):
"""
Change the _id field in fs_entry into the properly ordered SON or string
Args:
fs_entry: the element returned by self.fs_files.find
"""
_id_field = fs_entry.get('_id', fs_entry)
if isinstance(_id_field, basestring):
return _id_field
dbkey = SON((field_name, _id_field.get(field_name)) for field_name in self.ordered_key_fields)
if 'run' in _id_field:
# NOTE, there's no need to state that run doesn't exist in the negative case b/c access via
# SON requires equivalence (same keys and values in exact same order)
dbkey['run'] = _id_field['run']
fs_entry['_id'] = dbkey
return dbkey
def query_for_course(course_key, category=None):
"""
Construct a SON object that will query for all assets possibly limited to the given type
(thumbnail v assets) in the course using the index in mongo_indexes.md
"""
if getattr(course_key, 'deprecated', False):
prefix = '_id'
else:
prefix = 'content_son'
dbkey = SON([
('{}.tag'.format(prefix), XASSET_LOCATION_TAG),
('{}.org'.format(prefix), course_key.org),
('{}.course'.format(prefix), course_key.course),
])
if category:
dbkey['{}.category'.format(prefix)] = category
if getattr(course_key, 'deprecated', False):
dbkey['{}.run'.format(prefix)] = {'$exists': False}
else:
dbkey['{}.run'.format(prefix)] = course_key.run
return dbkey
common/lib/xmodule/xmodule/modulestore/__init__.py
...@@ -330,6 +330,23 @@ class ModuleStoreWrite(ModuleStoreRead): ...@@ -330,6 +330,23 @@ class ModuleStoreWrite(ModuleStoreRead):
pass pass
@abstractmethod @abstractmethod
def clone_course(self, source_course_id, dest_course_id, user_id):
"""
Sets up source_course_id to point a course with the same content as the desct_course_id. This
operation may be cheap or expensive. It may have to copy all assets and all xblock content or
merely setup new pointers.
Backward compatibility: this method used to require in some modulestores that dest_course_id
pointed to an empty but already created course. Implementers should support this or should
enable creating the course from scratch.
Raises:
ItemNotFoundError: if the source course doesn't exist (or any of its xblocks aren't found)
DuplicateItemError: if the destination course already exists (with content in some cases)
"""
pass
@abstractmethod
def delete_course(self, course_key, user_id=None): def delete_course(self, course_key, user_id=None):
""" """
Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions
...@@ -434,8 +451,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite): ...@@ -434,8 +451,10 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
''' '''
Implement interface functionality that can be shared. Implement interface functionality that can be shared.
''' '''
def __init__(self, **kwargs): def __init__(self, contentstore, **kwargs):
super(ModuleStoreWriteBase, self).__init__(**kwargs) super(ModuleStoreWriteBase, self).__init__(**kwargs)
self.contentstore = contentstore
# TODO: Don't have a runtime just to generate the appropriate mixin classes (cpennington) # TODO: Don't have a runtime just to generate the appropriate mixin classes (cpennington)
# This is only used by partition_fields_by_scope, which is only needed because # This is only used by partition_fields_by_scope, which is only needed because
# the split mongo store is used for item creation as well as item persistence # the split mongo store is used for item creation as well as item persistence
...@@ -501,6 +520,16 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite): ...@@ -501,6 +520,16 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
self.update_item(new_object, user_id, allow_not_found=True) self.update_item(new_object, user_id, allow_not_found=True)
return new_object return new_object
def clone_course(self, source_course_id, dest_course_id, user_id):
"""
This base method just copies the assets. The lower level impls must do the actual cloning of
content.
"""
# copy the assets
self.contentstore.copy_all_course_assets(source_course_id, dest_course_id)
super(ModuleStoreWriteBase, self).clone_course(source_course_id, dest_course_id, user_id)
return dest_course_id
def only_xmodules(identifier, entry_points): def only_xmodules(identifier, entry_points):
"""Only use entry_points that are supplied by the xmodule package""" """Only use entry_points that are supplied by the xmodule package"""
......
common/lib/xmodule/xmodule/modulestore/django.py
...@@ -17,6 +17,7 @@ import threading ...@@ -17,6 +17,7 @@ import threading
from xmodule.modulestore.loc_mapper_store import LocMapperStore from xmodule.modulestore.loc_mapper_store import LocMapperStore
from xmodule.util.django import get_current_request_hostname from xmodule.util.django import get_current_request_hostname
import xmodule.modulestore # pylint: disable=unused-import import xmodule.modulestore # pylint: disable=unused-import
from xmodule.contentstore.django import contentstore
# We may not always have the request_cache module available # We may not always have the request_cache module available
try: try:
...@@ -37,7 +38,7 @@ def load_function(path): ...@@ -37,7 +38,7 @@ def load_function(path):
return getattr(import_module(module_path), name) return getattr(import_module(module_path), name)
def create_modulestore_instance(engine, doc_store_config, options, i18n_service=None): def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None):
""" """
This will return a new instance of a modulestore given an engine and options This will return a new instance of a modulestore given an engine and options
""" """
...@@ -62,6 +63,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service= ...@@ -62,6 +63,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
metadata_inheritance_cache = get_cache('default') metadata_inheritance_cache = get_cache('default')
return class_( return class_(
contentstore=content_store,
metadata_inheritance_cache_subsystem=metadata_inheritance_cache, metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
request_cache=request_cache, request_cache=request_cache,
xblock_mixins=getattr(settings, 'XBLOCK_MIXINS', ()), xblock_mixins=getattr(settings, 'XBLOCK_MIXINS', ()),
...@@ -85,6 +87,7 @@ def modulestore(): ...@@ -85,6 +87,7 @@ def modulestore():
if _MIXED_MODULESTORE is None: if _MIXED_MODULESTORE is None:
_MIXED_MODULESTORE = create_modulestore_instance( _MIXED_MODULESTORE = create_modulestore_instance(
settings.MODULESTORE['default']['ENGINE'], settings.MODULESTORE['default']['ENGINE'],
contentstore(),
settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}), settings.MODULESTORE['default'].get('DOC_STORE_CONFIG', {}),
settings.MODULESTORE['default'].get('OPTIONS', {}) settings.MODULESTORE['default'].get('OPTIONS', {})
) )
......
common/lib/xmodule/xmodule/modulestore/mixed.py
...@@ -11,7 +11,7 @@ from contextlib import contextmanager ...@@ -11,7 +11,7 @@ from contextlib import contextmanager
from opaque_keys import InvalidKeyError from opaque_keys import InvalidKeyError
from . import ModuleStoreWriteBase from . import ModuleStoreWriteBase
from xmodule.modulestore import PublishState from xmodule.modulestore import PublishState, ModuleStoreEnum, split_migrator
from xmodule.modulestore.django import create_modulestore_instance, loc_mapper from xmodule.modulestore.django import create_modulestore_instance, loc_mapper
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError
...@@ -29,12 +29,12 @@ class MixedModuleStore(ModuleStoreWriteBase): ...@@ -29,12 +29,12 @@ class MixedModuleStore(ModuleStoreWriteBase):
""" """
ModuleStore knows how to route requests to the right persistence ms ModuleStore knows how to route requests to the right persistence ms
""" """
def __init__(self, mappings, stores, i18n_service=None, **kwargs): def __init__(self, contentstore, mappings, stores, i18n_service=None, **kwargs):
""" """
Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
collection of other modulestore configuration information collection of other modulestore configuration information
""" """
super(MixedModuleStore, self).__init__(**kwargs) super(MixedModuleStore, self).__init__(contentstore, **kwargs)
self.modulestores = [] self.modulestores = []
self.mappings = {} self.mappings = {}
...@@ -61,6 +61,7 @@ class MixedModuleStore(ModuleStoreWriteBase): ...@@ -61,6 +61,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
] ]
store = create_modulestore_instance( store = create_modulestore_instance(
store_settings['ENGINE'], store_settings['ENGINE'],
self.contentstore,
store_settings.get('DOC_STORE_CONFIG', {}), store_settings.get('DOC_STORE_CONFIG', {}),
store_settings.get('OPTIONS', {}), store_settings.get('OPTIONS', {}),
i18n_service=i18n_service, i18n_service=i18n_service,
...@@ -295,6 +296,36 @@ class MixedModuleStore(ModuleStoreWriteBase): ...@@ -295,6 +296,36 @@ class MixedModuleStore(ModuleStoreWriteBase):
return store.create_course(org, offering, user_id, fields, **kwargs) return store.create_course(org, offering, user_id, fields, **kwargs)
def clone_course(self, source_course_id, dest_course_id, user_id):
"""
See the superclass for the general documentation.
If cloning w/in a store, delegates to that store's clone_course which, in order to be self-
sufficient, should handle the asset copying (call the same method as this one does)
If cloning between stores,
* copy the assets
* migrate the courseware
"""
source_modulestore = self._get_modulestore_for_courseid(source_course_id)
# for a temporary period of time, we may want to hardcode dest_modulestore as split if there's a split
# to have only course re-runs go to split. This code, however, uses the config'd priority
dest_modulestore = self._get_modulestore_for_courseid(dest_course_id)
if source_modulestore == dest_modulestore:
return source_modulestore.clone_course(source_course_id, dest_course_id, user_id)
# ensure super's only called once. The delegation above probably calls it; so, don't move
# the invocation above the delegation call
super(MixedModuleStore, self).clone_course(source_course_id, dest_course_id, user_id)
if dest_modulestore.get_modulestore_type() == ModuleStoreEnum.Type.split:
if not hasattr(self, 'split_migrator'):
self.split_migrator = split_migrator.SplitMigrator(
dest_modulestore, source_modulestore, loc_mapper()
)
self.split_migrator.migrate_mongo_course(
source_course_id, user_id, dest_course_id.org, dest_course_id.offering
)
def create_item(self, course_or_parent_loc, category, user_id=None, **kwargs): def create_item(self, course_or_parent_loc, category, user_id=None, **kwargs):
""" """
Create and return the item. If parent_loc is a specific location v a course id, Create and return the item. If parent_loc is a specific location v a course id,
...@@ -460,6 +491,24 @@ class MixedModuleStore(ModuleStoreWriteBase): ...@@ -460,6 +491,24 @@ class MixedModuleStore(ModuleStoreWriteBase):
else: else:
raise NotImplementedError(u"Cannot call {} on store {}".format(method, store)) raise NotImplementedError(u"Cannot call {} on store {}".format(method, store))
@contextmanager
def set_default_store(self, store_type):
"""
A context manager for temporarily changing the default store in the Mixed modulestore
"""
previous_store_list = self.modulestores
found = False
try:
for i, store in enumerate(self.modulestores):
if store.get_modulestore_type() == store_type:
self.modulestores.insert(0, self.modulestores.pop(i))
found = True
yield
if not found:
raise Exception(u"Cannot find store of type {}".format(store_type))
finally:
self.modulestores = previous_store_list
@contextmanager @contextmanager
def store_branch_setting(store, branch_setting): def store_branch_setting(store, branch_setting):
......
common/lib/xmodule/xmodule/modulestore/mongo/base.py
...@@ -332,12 +332,12 @@ class MongoModuleStore(ModuleStoreWriteBase): ...@@ -332,12 +332,12 @@ class MongoModuleStore(ModuleStoreWriteBase):
""" """
A Mongodb backed ModuleStore A Mongodb backed ModuleStore
""" """
reference_type = Location reference_type = SlashSeparatedCourseKey
# TODO (cpennington): Enable non-filesystem filestores # TODO (cpennington): Enable non-filesystem filestores
# pylint: disable=C0103 # pylint: disable=C0103
# pylint: disable=W0201 # pylint: disable=W0201
def __init__(self, doc_store_config, fs_root, render_template, def __init__(self, contentstore, doc_store_config, fs_root, render_template,
default_class=None, default_class=None,
error_tracker=null_error_tracker, error_tracker=null_error_tracker,
i18n_service=None, i18n_service=None,
...@@ -346,7 +346,7 @@ class MongoModuleStore(ModuleStoreWriteBase): ...@@ -346,7 +346,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
:param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
""" """
super(MongoModuleStore, self).__init__(**kwargs) super(MongoModuleStore, self).__init__(contentstore, **kwargs)
def do_connection( def do_connection(
db, collection, host, port=27017, tz_aware=True, user=None, password=None, **kwargs db, collection, host, port=27017, tz_aware=True, user=None, password=None, **kwargs
...@@ -857,7 +857,6 @@ class MongoModuleStore(ModuleStoreWriteBase): ...@@ -857,7 +857,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
Raises: Raises:
InvalidLocationError: If a course with the same org and offering already exists InvalidLocationError: If a course with the same org and offering already exists
""" """
course, _, run = offering.partition('/') course, _, run = offering.partition('/')
course_id = SlashSeparatedCourseKey(org, course, run) course_id = SlashSeparatedCourseKey(org, course, run)
......
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
...@@ -7,15 +7,21 @@ and otherwise returns i4x://org/course/cat/name). ...@@ -7,15 +7,21 @@ and otherwise returns i4x://org/course/cat/name).
""" """
import pymongo import pymongo
import logging
from opaque_keys.edx.locations import Location
from xmodule.exceptions import InvalidVersionError from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import PublishState, ModuleStoreEnum from xmodule.modulestore import PublishState, ModuleStoreEnum
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError, InvalidBranchSetting from xmodule.modulestore.exceptions import (
ItemNotFoundError, DuplicateItemError, InvalidBranchSetting, DuplicateCourseError
)
from xmodule.modulestore.mongo.base import ( from xmodule.modulestore.mongo.base import (
MongoModuleStore, MongoRevisionKey, as_draft, as_published, MongoModuleStore, MongoRevisionKey, as_draft, as_published,
DIRECT_ONLY_CATEGORIES, SORT_REVISION_FAVOR_DRAFT DIRECT_ONLY_CATEGORIES, SORT_REVISION_FAVOR_DRAFT
) )
from opaque_keys.edx.locations import Location from xmodule.modulestore.store_utilities import rewrite_nonportable_content_links
log = logging.getLogger(__name__)
def wrap_draft(item): def wrap_draft(item):
...@@ -138,6 +144,73 @@ class DraftModuleStore(MongoModuleStore): ...@@ -138,6 +144,73 @@ class DraftModuleStore(MongoModuleStore):
del key['_id.revision'] del key['_id.revision']
return self.collection.find(key).count() > 0 return self.collection.find(key).count() > 0
def clone_course(self, source_course_id, dest_course_id, user_id):
"""
Only called if cloning within this store or if env doesn't set up mixed.
* copy the courseware
"""
# check to see if the source course is actually there
if not self.has_course(source_course_id):
raise ItemNotFoundError("Cannot find a course at {0}. Aborting".format(source_course_id))
# verify that the dest_location really is an empty course
# b/c we don't want the payload, I'm copying the guts of get_items here
query = self._course_key_to_son(dest_course_id)
query['_id.category'] = {'$nin': ['course', 'about']}
if self.collection.find(query).limit(1).count() > 0:
raise DuplicateCourseError(
dest_course_id,
"Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(
dest_course_id
)
)
# clone the assets
super(DraftModuleStore, self).clone_course(source_course_id, dest_course_id, user_id)
# get the whole old course
new_course = self.get_course(dest_course_id)
if new_course is None:
# create_course creates the about overview
new_course = self.create_course(dest_course_id.org, dest_course_id.offering, user_id)
# Get all modules under this namespace which is (tag, org, course) tuple
modules = self.get_items(source_course_id, revision=ModuleStoreEnum.RevisionOption.published_only)
self._clone_modules(modules, dest_course_id, user_id)
course_location = dest_course_id.make_usage_key('course', dest_course_id.run)
self.publish(course_location, user_id)
modules = self.get_items(source_course_id, revision=ModuleStoreEnum.RevisionOption.draft_only)
self._clone_modules(modules, dest_course_id, user_id)
return True
def _clone_modules(self, modules, dest_course_id, user_id):
"""Clones each module into the given course"""
for module in modules:
original_loc = module.location
module.location = module.location.map_into_course(dest_course_id)
if module.location.category == 'course':
module.location = module.location.replace(name=module.location.run)
log.info("Cloning module %s to %s....", original_loc, module.location)
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, basestring):
module.data = rewrite_nonportable_content_links(
original_loc.course_key, dest_course_id, module.data
)
# repoint children
if module.has_children:
new_children = []
for child_loc in module.children:
child_loc = child_loc.map_into_course(dest_course_id)
new_children.append(child_loc)
module.children = new_children
self.update_item(module, user_id, allow_not_found=True)
def _get_raw_parent_locations(self, location, key_revision): def _get_raw_parent_locations(self, location, key_revision):
""" """
Get the parents but don't unset the revision in their locations. Get the parents but don't unset the revision in their locations.
......
common/lib/xmodule/xmodule/modulestore/split_migrator.py
...@@ -15,10 +15,9 @@ class SplitMigrator(object): ...@@ -15,10 +15,9 @@ class SplitMigrator(object):
Copies courses from old mongo to split mongo and sets up location mapping so any references to the old Copies courses from old mongo to split mongo and sets up location mapping so any references to the old
name will be able to find the new elements. name will be able to find the new elements.
""" """
def __init__(self, split_modulestore, direct_modulestore, draft_modulestore, loc_mapper): def __init__(self, split_modulestore, draft_modulestore, loc_mapper):
super(SplitMigrator, self).__init__() super(SplitMigrator, self).__init__()
self.split_modulestore = split_modulestore self.split_modulestore = split_modulestore
self.direct_modulestore = direct_modulestore
self.draft_modulestore = draft_modulestore self.draft_modulestore = draft_modulestore
self.loc_mapper = loc_mapper self.loc_mapper = loc_mapper
...@@ -43,7 +42,7 @@ class SplitMigrator(object): ...@@ -43,7 +42,7 @@ class SplitMigrator(object):
# locations are in location, children, conditionals, course.tab # locations are in location, children, conditionals, course.tab
# create the course: set fields to explicitly_set for each scope, id_root = new_course_locator, master_branch = 'production' # create the course: set fields to explicitly_set for each scope, id_root = new_course_locator, master_branch = 'production'
original_course = self.direct_modulestore.get_course(course_key) original_course = self.draft_modulestore.get_course(course_key)
new_course_root_locator = self.loc_mapper.translate_location(original_course.location) new_course_root_locator = self.loc_mapper.translate_location(original_course.location)
new_course = self.split_modulestore.create_course( new_course = self.split_modulestore.create_course(
new_course_root_locator.org, new_course_root_locator.offering, user.id, new_course_root_locator.org, new_course_root_locator.offering, user.id,
...@@ -65,7 +64,7 @@ class SplitMigrator(object): ...@@ -65,7 +64,7 @@ class SplitMigrator(object):
# iterate over published course elements. Wildcarding rather than descending b/c some elements are orphaned (e.g., # iterate over published course elements. Wildcarding rather than descending b/c some elements are orphaned (e.g.,
# course about pages, conditionals) # course about pages, conditionals)
for module in self.direct_modulestore.get_items(course_key): for module in self.draft_modulestore.get_items(course_key, revision=ModuleStoreEnum.RevisionOption.published_only):
# don't copy the course again. No drafts should get here # don't copy the course again. No drafts should get here
if module.location != old_course_loc: if module.location != old_course_loc:
# create split_xblock using split.create_item # create split_xblock using split.create_item
......
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
...@@ -105,7 +105,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): ...@@ -105,7 +105,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
SCHEMA_VERSION = 1 SCHEMA_VERSION = 1
reference_type = Locator reference_type = Locator
def __init__(self, doc_store_config, fs_root, render_template,
def __init__(self, contentstore, doc_store_config, fs_root, render_template,
default_class=None, default_class=None,
error_tracker=null_error_tracker, error_tracker=null_error_tracker,
loc_mapper=None, loc_mapper=None,
...@@ -115,7 +116,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): ...@@ -115,7 +116,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
""" """
super(SplitMongoModuleStore, self).__init__(**kwargs) super(SplitMongoModuleStore, self).__init__(contentstore, **kwargs)
self.loc_mapper = loc_mapper self.loc_mapper = loc_mapper
self.db_connection = MongoConnection(**doc_store_config) self.db_connection = MongoConnection(**doc_store_config)
...@@ -870,6 +871,20 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): ...@@ -870,6 +871,20 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# reconstruct the new_item from the cache # reconstruct the new_item from the cache
return self.get_item(item_loc) return self.get_item(item_loc)
def clone_course(self, source_course_id, dest_course_id, user_id):
"""
See :meth: `.ModuleStoreWrite.clone_course` for documentation.
In split, other than copying the assets, this is cheap as it merely creates a new version of the
existing course.
"""
super(SplitMongoModuleStore, self).clone_course(source_course_id, dest_course_id, user_id)
source_index = self.get_course_index_info(source_course_id)
return self.create_course(
dest_course_id.org, dest_course_id.offering, user_id, fields=None, # override start_date?
versions_dict=source_index['versions']
)
def create_course( def create_course(
self, org, offering, user_id, fields=None, self, org, offering, user_id, fields=None,
master_branch=ModuleStoreEnum.BranchName.draft, versions_dict=None, root_category='course', master_branch=ModuleStoreEnum.BranchName.draft, versions_dict=None, root_category='course',
......
common/lib/xmodule/xmodule/modulestore/store_utilities.py
...@@ -2,7 +2,6 @@ import re ...@@ -2,7 +2,6 @@ import re
import logging import logging
from xmodule.contentstore.content import StaticContent from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import ModuleStoreEnum
def _prefix_only_url_replace_regex(prefix): def _prefix_only_url_replace_regex(prefix):
...@@ -88,91 +87,6 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text): ...@@ -88,91 +87,6 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
return text return text
def _clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id):
for module in modules:
original_loc = module.location
module.location = module.location.map_into_course(dest_course_id)
if module.location.category == 'course':
module.location = module.location.replace(name=module.location.run)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, basestring):
module.data = rewrite_nonportable_content_links(
source_course_id, dest_course_id, module.data
)
# repoint children
if module.has_children:
new_children = []
for child_loc in module.children:
child_loc = child_loc.map_into_course(dest_course_id)
new_children.append(child_loc)
module.children = new_children
modulestore.update_item(module, user_id, allow_not_found=True)
def clone_course(modulestore, contentstore, source_course_id, dest_course_id, user_id):
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_course(dest_course_id):
raise Exception(u"An empty course at {0} must have already been created. Aborting...".format(dest_course_id))
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
dest_modules = modulestore.get_items(dest_course_id)
for module in dest_modules:
if module.location.category == 'course' or (
module.location.category == 'about' and module.location.name == 'overview'
):
continue
# only course and about overview allowed
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_course_id))
# check to see if the source course is actually there
if not modulestore.has_course(source_course_id):
raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
# Get all modules under this namespace which is (tag, org, course) tuple
modules = modulestore.get_items(source_course_id, revision=ModuleStoreEnum.RevisionOption.published_only)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
course_location = dest_course_id.make_usage_key('course', dest_course_id.run)
modulestore.publish(course_location, user_id)
modules = modulestore.get_items(source_course_id, revision=ModuleStoreEnum.RevisionOption.draft_only)
_clone_modules(modulestore, modules, source_course_id, dest_course_id, user_id)
# now iterate through all of the assets and clone them
# first the thumbnails
thumb_keys = contentstore.get_all_content_thumbnails_for_course(source_course_id)
for thumb_key in thumb_keys:
content = contentstore.find(thumb_key)
content.location = content.location.map_into_course(dest_course_id)
print "Cloning thumbnail {0} to {1}".format(thumb_key, content.location)
contentstore.save(content)
# now iterate through all of the assets, also updating the thumbnail pointer
asset_keys, __ = contentstore.get_all_content_for_course(source_course_id)
for asset_key in asset_keys:
content = contentstore.find(asset_key)
content.location = content.location.map_into_course(dest_course_id)
# be sure to update the pointer to the thumbnail
if content.thumbnail_location is not None:
content.thumbnail_location = content.thumbnail_location.map_into_course(dest_course_id)
print "Cloning asset {0} to {1}".format(asset_key, content.location)
contentstore.save(content)
return True
def delete_course(modulestore, contentstore, course_key, commit=False): def delete_course(modulestore, contentstore, course_key, commit=False):
""" """
This method will actually do the work to delete all content in a course in a MongoDB backed This method will actually do the work to delete all content in a course in a MongoDB backed
......
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
...@@ -7,7 +7,6 @@ from django.test import TestCase ...@@ -7,7 +7,6 @@ from django.test import TestCase
from xmodule.modulestore.django import ( from xmodule.modulestore.django import (
modulestore, clear_existing_modulestores, loc_mapper) modulestore, clear_existing_modulestores, loc_mapper)
from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore import ModuleStoreEnum
from xmodule.contentstore.django import contentstore
def mixed_store_config(data_dir, mappings): def mixed_store_config(data_dir, mappings):
...@@ -160,10 +159,8 @@ class ModuleStoreTestCase(TestCase): ...@@ -160,10 +159,8 @@ class ModuleStoreTestCase(TestCase):
connection.drop_database(store.db.name) connection.drop_database(store.db.name)
connection.close() connection.close()
if contentstore().fs_files: if hasattr(store, 'contentstore'):
db = contentstore().fs_files.database store.contentstore.drop_database()
db.connection.drop_database(db)
db.connection.close()
location_mapper = loc_mapper() location_mapper = loc_mapper()
if location_mapper.db: if location_mapper.db:
......
common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py 0 → 100644
"""
Test contentstore.mongo functionality
"""
import logging
from uuid import uuid4
import unittest
import mimetypes
from tempfile import mkdtemp
import path
import shutil
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.tests import DATA_DIR
from xmodule.contentstore.mongo import MongoContentStore
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
import ddt
from __builtin__ import delattr
log = logging.getLogger(__name__)
HOST = 'localhost'
PORT = 27017
DB = 'test_mongo_%s' % uuid4().hex[:5]
@ddt.ddt
class TestContentstore(unittest.TestCase):
"""
Test the methods in contentstore.mongo using deprecated and non-deprecated keys
"""
# don't use these 2 class vars as they restore behavior once the tests are done
asset_deprecated = None
ssck_deprecated = None
@classmethod
def tearDownClass(cls):
"""
Restores deprecated values
"""
if cls.asset_deprecated is not None:
setattr(AssetLocation, 'deprecated', cls.asset_deprecated)
else:
delattr(AssetLocation, 'deprecated')
if cls.ssck_deprecated is not None:
setattr(SlashSeparatedCourseKey, 'deprecated', cls.ssck_deprecated)
else:
delattr(SlashSeparatedCourseKey, 'deprecated')
return super(TestContentstore, cls).tearDownClass()
def set_up_assets(self, deprecated):
"""
Setup contentstore w/ proper overriding of deprecated.
"""
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
self.contentstore = MongoContentStore(HOST, DB, port=PORT)
self.addCleanup(self.contentstore.drop_database)
setattr(AssetLocation, 'deprecated', deprecated)
setattr(SlashSeparatedCourseKey, 'deprecated', deprecated)
self.course1_key = SlashSeparatedCourseKey('test', 'asset_test', '2014_07')
self.course2_key = SlashSeparatedCourseKey('test', 'asset_test2', '2014_07')
self.course1_files = ['contains.sh', 'picture1.jpg', 'picture2.jpg']
self.course2_files = ['picture1.jpg', 'picture3.jpg', 'door_2.ogg']
def load_assets(course_key, files):
locked = False
for filename in files:
asset_key = course_key.make_asset_key('asset', filename)
self.save_asset(filename, asset_key, filename, locked)
locked = not locked
load_assets(self.course1_key, self.course1_files)
load_assets(self.course2_key, self.course2_files)
def save_asset(self, filename, asset_key, displayname, locked):
"""
Load and save the given file.
"""
with open("{}/static/{}".format(DATA_DIR, filename), "rb") as f:
content = StaticContent(
asset_key, displayname, mimetypes.guess_type(filename)[0], f.read(),
locked=locked
)
self.contentstore.save(content)
@ddt.data(True, False)
def test_delete(self, deprecated):
"""
Test that deleting assets works
"""
self.set_up_assets(deprecated)
asset_key = self.course1_key.make_asset_key('asset', self.course1_files[0])
self.contentstore.delete(asset_key)
with self.assertRaises(NotFoundError):
self.contentstore.find(asset_key)
# ensure deleting a non-existent file is a noop
self.contentstore.delete(asset_key)
@ddt.data(True, False)
def test_find(self, deprecated):
"""
Test using find
"""
self.set_up_assets(deprecated)
asset_key = self.course1_key.make_asset_key('asset', self.course1_files[0])
self.assertIsNotNone(self.contentstore.find(asset_key), "Could not find {}".format(asset_key))
self.assertIsNotNone(self.contentstore.find(asset_key, as_stream=True), "Could not find {}".format(asset_key))
unknown_asset = self.course1_key.make_asset_key('asset', 'no_such_file.gif')
with self.assertRaises(NotFoundError):
self.contentstore.find(unknown_asset)
self.assertIsNone(
self.contentstore.find(unknown_asset, throw_on_not_found=False),
"Found unknown asset {}".format(unknown_asset)
)
@ddt.data(True, False)
def test_export_for_course(self, deprecated):
"""
Test export
"""
self.set_up_assets(deprecated)
root_dir = path.path(mkdtemp())
try:
self.contentstore.export_all_for_course(
self.course1_key, root_dir,
path.path(root_dir / "policy.json"),
)
for filename in self.course1_files:
filepath = path.path(root_dir / filename)
self.assertTrue(filepath.isfile(), "{} is not a file".format(filepath))
for filename in self.course2_files:
if filename not in self.course1_files:
filepath = path.path(root_dir / filename)
self.assertFalse(filepath.isfile(), "{} is unexpected exported a file".format(filepath))
finally:
shutil.rmtree(root_dir)
@ddt.data(True, False)
def test_get_all_content(self, deprecated):
"""
Test get_all_content_for_course
"""
self.set_up_assets(deprecated)
course1_assets, count = self.contentstore.get_all_content_for_course(self.course1_key)
self.assertEqual(count, len(self.course1_files), course1_assets)
for asset in course1_assets:
parsed = AssetLocation.from_deprecated_string(asset['filename'])
self.assertIn(parsed.name, self.course1_files)
course1_assets, __ = self.contentstore.get_all_content_for_course(self.course1_key, 1, 1)
self.assertEqual(len(course1_assets), 1, course1_assets)
fake_course = SlashSeparatedCourseKey('test', 'fake', 'non')
course_assets, count = self.contentstore.get_all_content_for_course(fake_course)
self.assertEqual(count, 0)
self.assertEqual(course_assets, [])
@ddt.data(True, False)
def test_attrs(self, deprecated):
"""
Test setting and getting attrs
"""
self.set_up_assets(deprecated)
for filename in self.course1_files:
asset_key = self.course1_key.make_asset_key('asset', filename)
prelocked = self.contentstore.get_attr(asset_key, 'locked', False)
self.contentstore.set_attr(asset_key, 'locked', not prelocked)
self.assertEqual(self.contentstore.get_attr(asset_key, 'locked', False), not prelocked)
@ddt.data(True, False)
def test_copy_assets(self, deprecated):
"""
copy_all_course_assets
"""
self.set_up_assets(deprecated)
dest_course = SlashSeparatedCourseKey('test', 'destination', 'copy')
self.contentstore.copy_all_course_assets(self.course1_key, dest_course)
for filename in self.course1_files:
asset_key = self.course1_key.make_asset_key('asset', filename)
dest_key = dest_course.make_asset_key('asset', filename)
source = self.contentstore.find(asset_key)
copied = self.contentstore.find(dest_key)
for propname in ['name', 'content_type', 'length', 'locked']:
self.assertEqual(getattr(source, propname), getattr(copied, propname))
__, count = self.contentstore.get_all_content_for_course(dest_course)
self.assertEqual(count, len(self.course1_files))
@ddt.data(True, False)
def test_delete_assets(self, deprecated):
"""
delete_all_course_assets
"""
self.set_up_assets(deprecated)
self.contentstore.delete_all_course_assets(self.course1_key)
__, count = self.contentstore.get_all_content_for_course(self.course1_key)
self.assertEqual(count, 0)
# ensure it didn't remove any from other course
__, count = self.contentstore.get_all_content_for_course(self.course2_key)
self.assertEqual(count, len(self.course2_files))
common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
...@@ -210,7 +210,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango): ...@@ -210,7 +210,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
if index > 0: if index > 0:
store_configs[index], store_configs[0] = store_configs[0], store_configs[index] store_configs[index], store_configs[0] = store_configs[0], store_configs[index]
break break
self.store = MixedModuleStore(**self.options) self.store = MixedModuleStore(None, **self.options)
self.addCleanup(self.store.close_all_connections) self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys # convert to CourseKeys
...@@ -518,7 +518,7 @@ def load_function(path): ...@@ -518,7 +518,7 @@ def load_function(path):
# pylint: disable=unused-argument # pylint: disable=unused-argument
def create_modulestore_instance(engine, doc_store_config, options, i18n_service=None): def create_modulestore_instance(engine, contentstore, doc_store_config, options, i18n_service=None):
""" """
This will return a new instance of a modulestore given an engine and options This will return a new instance of a modulestore given an engine and options
""" """
...@@ -526,6 +526,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service= ...@@ -526,6 +526,7 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
return class_( return class_(
doc_store_config=doc_store_config, doc_store_config=doc_store_config,
contentstore=contentstore,
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
**options **options
) )
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
# pylint: disable=E1101
# pylint: disable=W0212
# pylint: disable=E0611 # pylint: disable=E0611
from nose.tools import assert_equals, assert_raises, \ from nose.tools import assert_equals, assert_raises, \
assert_not_equals, assert_false, assert_true, assert_greater, assert_is_instance, assert_is_none assert_not_equals, assert_false, assert_true, assert_greater, assert_is_instance, assert_is_none
...@@ -86,8 +88,8 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -86,8 +88,8 @@ class TestMongoModuleStore(unittest.TestCase):
cls.connection.drop_database(DB) cls.connection.drop_database(DB)
cls.connection.close() cls.connection.close()
@staticmethod @classmethod
def initdb(): def initdb(cls):
# connect to the db # connect to the db
doc_store_config = { doc_store_config = {
'host': HOST, 'host': HOST,
...@@ -101,6 +103,7 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -101,6 +103,7 @@ class TestMongoModuleStore(unittest.TestCase):
# Also test draft store imports # Also test draft store imports
# #
draft_store = DraftModuleStore( draft_store = DraftModuleStore(
content_store,
doc_store_config, FS_ROOT, RENDER_TEMPLATE, doc_store_config, FS_ROOT, RENDER_TEMPLATE,
default_class=DEFAULT_CLASS, default_class=DEFAULT_CLASS,
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred
...@@ -109,7 +112,7 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -109,7 +112,7 @@ class TestMongoModuleStore(unittest.TestCase):
draft_store, draft_store,
999, 999,
DATA_DIR, DATA_DIR,
TestMongoModuleStore.courses, cls.courses,
static_content_store=content_store static_content_store=content_store
) )
...@@ -131,13 +134,6 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -131,13 +134,6 @@ class TestMongoModuleStore(unittest.TestCase):
# Destroy the test db. # Destroy the test db.
connection.drop_database(DB) connection.drop_database(DB)
def setUp(self):
# make a copy for convenience
self.connection = TestMongoModuleStore.connection
def tearDown(self):
pass
def test_init(self): def test_init(self):
'''Make sure the db loads''' '''Make sure the db loads'''
ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True})) ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
...@@ -145,6 +141,7 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -145,6 +141,7 @@ class TestMongoModuleStore(unittest.TestCase):
def test_mongo_modulestore_type(self): def test_mongo_modulestore_type(self):
store = MongoModuleStore( store = MongoModuleStore(
None,
{'host': HOST, 'db': DB, 'collection': COLLECTION}, {'host': HOST, 'db': DB, 'collection': COLLECTION},
FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS
) )
...@@ -284,54 +281,57 @@ class TestMongoModuleStore(unittest.TestCase): ...@@ -284,54 +281,57 @@ class TestMongoModuleStore(unittest.TestCase):
Test getting, setting, and defaulting the locked attr and arbitrary attrs. Test getting, setting, and defaulting the locked attr and arbitrary attrs.
""" """
location = Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall') location = Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')
course_content, __ = TestMongoModuleStore.content_store.get_all_content_for_course(location.course_key) course_content, __ = self.content_store.get_all_content_for_course(location.course_key)
assert_true(len(course_content) > 0) assert_true(len(course_content) > 0)
# a bit overkill, could just do for content[0] # a bit overkill, could just do for content[0]
for content in course_content: for content in course_content:
assert not content.get('locked', False) assert not content.get('locked', False)
asset_key = AssetLocation._from_deprecated_son(content['_id'], location.run) asset_key = AssetLocation._from_deprecated_son(content.get('content_son', content['_id']), location.run)
assert not TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False) assert not self.content_store.get_attr(asset_key, 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(asset_key) attrs = self.content_store.get_attrs(asset_key)
assert_in('uploadDate', attrs) assert_in('uploadDate', attrs)
assert not attrs.get('locked', False) assert not attrs.get('locked', False)
TestMongoModuleStore.content_store.set_attr(asset_key, 'locked', True) self.content_store.set_attr(asset_key, 'locked', True)
assert TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False) assert self.content_store.get_attr(asset_key, 'locked', False)
attrs = TestMongoModuleStore.content_store.get_attrs(asset_key) attrs = self.content_store.get_attrs(asset_key)
assert_in('locked', attrs) assert_in('locked', attrs)
assert attrs['locked'] is True assert attrs['locked'] is True
TestMongoModuleStore.content_store.set_attrs(asset_key, {'miscel': 99}) self.content_store.set_attrs(asset_key, {'miscel': 99})
assert_equals(TestMongoModuleStore.content_store.get_attr(asset_key, 'miscel'), 99) assert_equals(self.content_store.get_attr(asset_key, 'miscel'), 99)
asset_key = AssetLocation._from_deprecated_son(course_content[0]['_id'], location.run) asset_key = AssetLocation._from_deprecated_son(
course_content[0].get('content_son', course_content[0]['_id']),
location.run
)
assert_raises( assert_raises(
AttributeError, TestMongoModuleStore.content_store.set_attr, asset_key, AttributeError, self.content_store.set_attr, asset_key,
'md5', 'ff1532598830e3feac91c2449eaa60d6' 'md5', 'ff1532598830e3feac91c2449eaa60d6'
) )
assert_raises( assert_raises(
AttributeError, TestMongoModuleStore.content_store.set_attrs, asset_key, AttributeError, self.content_store.set_attrs, asset_key,
{'foo': 9, 'md5': 'ff1532598830e3feac91c2449eaa60d6'} {'foo': 9, 'md5': 'ff1532598830e3feac91c2449eaa60d6'}
) )
assert_raises( assert_raises(
NotFoundError, TestMongoModuleStore.content_store.get_attr, NotFoundError, self.content_store.get_attr,
Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'), Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'),
'displayname' 'displayname'
) )
assert_raises( assert_raises(
NotFoundError, TestMongoModuleStore.content_store.set_attr, NotFoundError, self.content_store.set_attr,
Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'), Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'),
'displayname', 'hello' 'displayname', 'hello'
) )
assert_raises( assert_raises(
NotFoundError, TestMongoModuleStore.content_store.get_attrs, NotFoundError, self.content_store.get_attrs,
Location('bogus', 'bogus', 'bogus', 'asset', 'bogus') Location('bogus', 'bogus', 'bogus', 'asset', 'bogus')
) )
assert_raises( assert_raises(
NotFoundError, TestMongoModuleStore.content_store.set_attrs, NotFoundError, self.content_store.set_attrs,
Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'), Location('bogus', 'bogus', 'bogus', 'asset', 'bogus'),
{'displayname': 'hello'} {'displayname': 'hello'}
) )
assert_raises( assert_raises(
NotFoundError, TestMongoModuleStore.content_store.set_attrs, NotFoundError, self.content_store.set_attrs,
Location('bogus', 'bogus', 'bogus', 'asset', None), Location('bogus', 'bogus', 'bogus', 'asset', None),
{'displayname': 'hello'} {'displayname': 'hello'}
) )
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
...@@ -23,7 +23,7 @@ class TestMigration(SplitWMongoCourseBoostrapper): ...@@ -23,7 +23,7 @@ class TestMigration(SplitWMongoCourseBoostrapper):
# pylint: disable=W0142 # pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config) self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
self.split_mongo.loc_mapper = self.loc_mapper self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper) self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo, self.loc_mapper)
def tearDown(self): def tearDown(self):
dbref = self.loc_mapper.db dbref = self.loc_mapper.db
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
...@@ -1759,6 +1759,7 @@ def modulestore(): ...@@ -1759,6 +1759,7 @@ def modulestore():
# pylint: disable=W0142 # pylint: disable=W0142
SplitModuleTest.modulestore = class_( SplitModuleTest.modulestore = class_(
None, # contentstore
SplitModuleTest.MODULESTORE['DOC_STORE_CONFIG'], SplitModuleTest.MODULESTORE['DOC_STORE_CONFIG'],
**options **options
) )
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
...@@ -49,14 +49,15 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase): ...@@ -49,14 +49,15 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
self.userid = random.getrandbits(32) self.userid = random.getrandbits(32)
super(SplitWMongoCourseBoostrapper, self).setUp() super(SplitWMongoCourseBoostrapper, self).setUp()
self.split_mongo = SplitMongoModuleStore( self.split_mongo = SplitMongoModuleStore(
None,
self.db_config, self.db_config,
**self.modulestore_options **self.modulestore_options
) )
self.addCleanup(self.split_mongo.db.connection.close) self.addCleanup(self.split_mongo.db.connection.close)
self.addCleanup(self.tear_down_split) self.addCleanup(self.tear_down_split)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options) self.old_mongo = MongoModuleStore(None, self.db_config, **self.modulestore_options)
self.draft_mongo = DraftMongoModuleStore( self.draft_mongo = DraftMongoModuleStore(
self.db_config, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, **self.modulestore_options None, self.db_config, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, **self.modulestore_options
) )
self.addCleanup(self.tear_down_mongo) self.addCleanup(self.tear_down_mongo)
self.old_course_key = None self.old_course_key = None
......
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
...@@ -85,6 +85,7 @@ def modulestore(): ...@@ -85,6 +85,7 @@ def modulestore():
# pylint: disable=W0142 # pylint: disable=W0142
ModuleStoreNoSettings.modulestore = class_( ModuleStoreNoSettings.modulestore = class_(
None, # contentstore
ModuleStoreNoSettings.MODULESTORE['DOC_STORE_CONFIG'], ModuleStoreNoSettings.MODULESTORE['DOC_STORE_CONFIG'],
**options **options
) )
......
common/lib/xmodule/xmodule/modulestore/xml.py
...@@ -19,15 +19,14 @@ from xmodule.errortracker import make_error_tracker, exc_info_to_str ...@@ -19,15 +19,14 @@ from xmodule.errortracker import make_error_tracker, exc_info_to_str
from xmodule.mako_module import MakoDescriptorSystem from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import XMLParsingSystem, policy_key from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore import ModuleStoreEnum, ModuleStoreReadBase
from xmodule.tabs import CourseTabList from xmodule.tabs import CourseTabList
from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
from xblock.field_data import DictFieldData from xblock.field_data import DictFieldData
from xblock.runtime import DictKeyValueStore, IdGenerator from xblock.runtime import DictKeyValueStore, IdGenerator
from . import ModuleStoreReadBase, Location, ModuleStoreEnum
from .exceptions import ItemNotFoundError from .exceptions import ItemNotFoundError
from .inheritance import compute_inherited_metadata, inheriting_field_data from .inheritance import compute_inherited_metadata, inheriting_field_data
...@@ -720,7 +719,7 @@ class XMLModuleStore(ModuleStoreReadBase): ...@@ -720,7 +719,7 @@ class XMLModuleStore(ModuleStoreReadBase):
except KeyError: except KeyError:
raise ItemNotFoundError(usage_key) raise ItemNotFoundError(usage_key)
def get_items(self, course_id, settings=None, content=None, **kwargs): def get_items(self, course_id, settings=None, content=None, revision=None, **kwargs):
""" """
Returns: Returns:
list of XModuleDescriptor instances for the matching items within the course with list of XModuleDescriptor instances for the matching items within the course with
...@@ -745,6 +744,9 @@ class XMLModuleStore(ModuleStoreReadBase): ...@@ -745,6 +744,9 @@ class XMLModuleStore(ModuleStoreReadBase):
you can search dates by providing either a datetime for == (probably you can search dates by providing either a datetime for == (probably
useless) or a tuple (">"|"<" datetime) for after or before, etc. useless) or a tuple (">"|"<" datetime) for after or before, etc.
""" """
if revision == ModuleStoreEnum.RevisionOption.draft_only:
return []
items = [] items = []
category = kwargs.pop('category', None) category = kwargs.pop('category', None)
......
common/lib/xmodule/xmodule/modulestore/xml_importer.py
...@@ -97,7 +97,7 @@ def import_static_content( ...@@ -97,7 +97,7 @@ def import_static_content(
try: try:
static_content_store.save(content) static_content_store.save(content)
except Exception as err: except Exception as err:
log.exception('Error importing {0}, error={1}'.format( log.exception(u'Error importing {0}, error={1}'.format(
fullname_with_subpath, err fullname_with_subpath, err
)) ))
......
common/lib/xmodule/xmodule/video_module/transcripts_utils.py
...@@ -500,8 +500,7 @@ class Transcript(object): ...@@ -500,8 +500,7 @@ class Transcript(object):
Delete asset by location and filename. Delete asset by location and filename.
""" """
try: try:
content = Transcript.get_asset(location, filename) contentstore().delete(Transcript.asset_location(location, filename))
contentstore().delete(content.get_id())
log.info("Transcript asset %s was removed from store.", filename) log.info("Transcript asset %s was removed from store.", filename)
except NotFoundError: except NotFoundError:
pass pass
......
common/test/data/static/contains.sh 0 → 100644
#!/usr/bin/env zsh
git log --all ^opaque-keys-merge-base --format=%H $1 | while read f; do git branch --contains $f; done | sort -u
lms/djangoapps/courseware/tests/test_video_handlers.py
...@@ -65,7 +65,10 @@ def _clear_assets(location): ...@@ -65,7 +65,10 @@ def _clear_assets(location):
assets, __ = store.get_all_content_for_course(location.course_key) assets, __ = store.get_all_content_for_course(location.course_key)
for asset in assets: for asset in assets:
asset_location = AssetLocation._from_deprecated_son(asset["_id"], location.course_key.run) asset_location = AssetLocation._from_deprecated_son(
asset.get('content_son', asset["_id"]),
location.course_key.run
)
del_cached_content(asset_location) del_cached_content(asset_location)
store.delete(asset_location) store.delete(asset_location)
......
mongo_indexes.md
...@@ -26,10 +26,16 @@ fs.files: ...@@ -26,10 +26,16 @@ fs.files:
Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort
which can be `uploadDate`, `display_name`, which can be `uploadDate`, `display_name`,
Replace existing index which leaves out `run` with this one:
``` ```
ensureIndex({'_id.tag': 1, '_id.org': 1, '_id.course': 1, '_id.category': 1}) ensureIndex({'_id.tag': 1, '_id.org': 1, '_id.course': 1, '_id.category': 1, '_id.run': 1})
ensureIndex({'content_son.tag': 1, 'content_son.org': 1, 'content_son.course': 1, 'content_son.category': 1, 'content_son.run': 1})
``` ```
Note: I'm not advocating adding one which leaves out `category` for now because that would only be
used for `delete_all_course_assets` which in the future should not actually delete the assets except
when doing garbage collection.
Remove index on `displayname` Remove index on `displayname`
modulestore: modulestore:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment