mattdrayer/api-migrate-courseids: Data migration script

lms/djangoapps/api_manager/management/commands/migrate_courseids.py

+"""
+One-time data migration script -- shoulen't need to run it again
+"""
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+from django.test import RequestFactory
+
+from api_manager import models as api_models
+from api_manager.courseware_access import get_course, get_course_child
+from opaque_keys import InvalidKeyError
+from projects import models as project_models
+
+
+class Command(BaseCommand):
+    """
+    Migrates legacy course/content identifiers across several models to the new format
+    """
+
+    def handle(self, *args, **options):
+        request_factory = RequestFactory()
+        request = request_factory.get('/')
+        request.user = User(email='migration@edx.org', username='migration', password='migration', is_active=True)
+
+        projects = project_models.Project.objects.all()
+        for project in projects:
+            course_descriptor, course_key, course_content = get_course(request, request.user, project.course_id)
+            project.course_id = unicode(course_key)
+            try:
+                project.content_id = course_key.make_usage_key_from_deprecated_string(project.content_id)
+            except InvalidKeyError:
+                pass  # If the key conversion fails it was either a new-style key or junk data
+            project.save()
+
+        workgroup_reviews = project_models.WorkgroupReview.objects.all()
+        for wr in workgroup_reviews:
+            course_id = wr.workgroup.project.course_id
+            course_descriptor, course_key, course_content = get_course(request, request.user, course_id)
+            try:
+                wr.content_id = course_key.make_usage_key_from_deprecated_string(wr.content_id)
+            except InvalidKeyError:
+                pass  # If the key conversion fails it was either a new-style key or junk data
+            wr.save()
+
+        workgroup_submission_reviews = project_models.WorkgroupSubmissionReview.objects.all()
+        for wsr in workgroup_submission_reviews:
+            course_id = wsr.submission.workgroup.project.course_id
+            course_descriptor, course_key, course_content = get_course(request, request.user, course_id)
+            try:
+                wsr.content_id = course_key.make_usage_key_from_deprecated_string(wsr.content_id)
+            except InvalidKeyError:
+                pass  # If the key conversion fails it was either a new-style key or junk data
+            wsr.save()
+
+
+        course_groups = api_models.CourseGroupRelationship.objects.all()
+        for cg in course_groups:
+            course_id = cg.course_id
+            course_descriptor, course_key, course_content = get_course(request, request.user, course_id)
+            cg.course_id = unicode(course_key)
+            cg.save()
+
+        course_content_groups = api_models.CourseContentGroupRelationship.objects.all()
+        for ccg in course_content_groups:
+            course_id = ccg.course_id
+            course_descriptor, course_key, course_content = get_course(request, request.user, course_id)
+            ccg.course_id = unicode(course_key)
+            try:
+                ccg.content_id = course_key.make_usage_key_from_deprecated_string(ccg.content_id)
+            except InvalidKeyError:
+                pass  # If the key conversion fails it was either a new-style key or junk data
+            ccg.save()
+
+        course_module_completions = api_models.CourseModuleCompletion.objects.all()
+        for cmc in course_module_completions:
+            course_id = cmc.course_id
+            course_descriptor, course_key, course_content = get_course(request, request.user, course_id)
+            cmc.course_id = unicode(course_key)
+            try:
+                cmc.content_id = course_key.make_usage_key_from_deprecated_string(cmc.content_id)
+            except InvalidKeyError:
+                pass  # If the key conversion fails it was either a new-style key or junk data
+            cmc.save()

lms/djangoapps/api_manager/management/commands/tests/test_migrate_courseids.py

+"""
+Run these tests @ Devstack:
+    rake fasttest_lms[common/djangoapps/api_manager/management/commands/tests/test_migrate_orgdata.py]
+"""
+from datetime import datetime
+import uuid
+
+from django.contrib.auth.models import Group, User
+
+from api_manager import models as api_models
+from api_manager.management.commands import migrate_courseids
+from projects import models as project_models
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+
+
+class MigrateCourseIdsTests(ModuleStoreTestCase):
+    """
+    Test suite for data migration script
+    """
+
+    def setUp(self):
+
+        self.course = CourseFactory.create(
+            start=datetime(2014, 6, 16, 14, 30),
+            end=datetime(2015, 1, 16)
+        )
+        self.test_data = '<html>{}</html>'.format(str(uuid.uuid4()))
+
+        self.chapter = ItemFactory.create(
+            category="chapter",
+            parent_location=self.course.location,
+            data=self.test_data,
+            due=datetime(2014, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+
+        self.old_style_course_id = self.course.id.to_deprecated_string()
+        self.new_style_course_id = unicode(self.course.id)
+        self.old_style_content_id = self.chapter.location.to_deprecated_string()
+        self.new_style_content_id = unicode(self.chapter.location)
+
+        self.course2 = CourseFactory.create(
+            org='TEST',
+            start=datetime(2014, 6, 16, 14, 30),
+            end=datetime(2015, 1, 16)
+        )
+        self.chapter2 = ItemFactory.create(
+            category="chapter",
+            parent_location=self.course2.location,
+            data=self.test_data,
+            due=datetime(2014, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+
+        self.new_style_course_id2 = unicode(self.course2.id)
+        self.new_style_content_id2 = unicode(self.chapter2.location)
+
+
+    def test_migrate_courseids(self):
+        """
+        Test the data migration
+        """
+        # Set up the data to be migrated
+        user = User.objects.create(email='testuser@edx.org', username='testuser', password='testpassword', is_active=True)
+        project = project_models.Project.objects.create(course_id=self.old_style_course_id, content_id=self.old_style_content_id)
+        workgroup = project_models.Workgroup.objects.create(name='Test Workgroup', project=project)
+        workgroup_review = project_models.WorkgroupReview.objects.create(workgroup=workgroup, content_id=self.old_style_content_id)
+        workgroup_submission = project_models.WorkgroupSubmission.objects.create(workgroup=workgroup, user=user)
+        workgroup_submission_review = project_models.WorkgroupSubmissionReview.objects.create(submission=workgroup_submission, content_id=self.old_style_content_id)
+        group = Group.objects.create(name='Test Group')
+        group_profile = api_models.GroupProfile.objects.create(group=group)
+        course_group = api_models.CourseGroupRelationship.objects.create(course_id=self.old_style_course_id, group=group)
+        course_content_group = api_models.CourseContentGroupRelationship.objects.create(course_id=self.old_style_course_id, content_id=self.old_style_content_id, group_profile=group_profile)
+        course_module_completion = api_models.CourseModuleCompletion.objects.create(user=user, course_id=self.old_style_course_id, content_id=self.old_style_content_id)
+
+        user2 = User.objects.create(email='testuser2@edx.org', username='testuser2', password='testpassword2', is_active=True)
+        project2 = project_models.Project.objects.create(course_id=self.new_style_course_id2, content_id=self.new_style_content_id2)
+        workgroup2 = project_models.Workgroup.objects.create(name='Test Workgroup2', project=project2)
+        workgroup_review2 = project_models.WorkgroupReview.objects.create(workgroup=workgroup2, content_id=self.new_style_content_id2)
+        workgroup_submission2 = project_models.WorkgroupSubmission.objects.create(workgroup=workgroup2, user=user2)
+        workgroup_submission_review2 = project_models.WorkgroupSubmissionReview.objects.create(submission=workgroup_submission2, content_id=self.new_style_content_id2)
+        group2 = Group.objects.create(name='Test Group2')
+        group_profile2 = api_models.GroupProfile.objects.create(group=group2)
+        course_group2 = api_models.CourseGroupRelationship.objects.create(course_id=self.new_style_course_id2, group=group2)
+        course_content_group2 = api_models.CourseContentGroupRelationship.objects.create(course_id=self.new_style_course_id2, content_id=self.new_style_content_id2, group_profile=group_profile2)
+        course_module_completion2 = api_models.CourseModuleCompletion.objects.create(user=user2, course_id=self.new_style_course_id2, content_id=self.new_style_content_id2)
+
+
+        # Run the data migration
+        migrate_courseids.Command().handle()
+
+
+        # Confirm that the data has been properly migrated
+        updated_project = project_models.Project.objects.get(id=project.id)
+        self.assertEqual(updated_project.course_id, self.new_style_course_id)
+        self.assertEqual(updated_project.content_id, self.new_style_content_id)
+        updated_project = project_models.Project.objects.get(id=project2.id)
+        self.assertEqual(updated_project.course_id, self.new_style_course_id2)
+        self.assertEqual(updated_project.content_id, self.new_style_content_id2)
+        print "Project Data Migration Passed"
+
+        updated_workgroup_review = project_models.WorkgroupReview.objects.get(id=workgroup_review.id)
+        self.assertEqual(updated_workgroup_review.content_id, self.new_style_content_id)
+        updated_workgroup_review = project_models.WorkgroupReview.objects.get(id=workgroup_review2.id)
+        self.assertEqual(updated_workgroup_review.content_id, self.new_style_content_id2)
+        print "Workgroup Review Data Migration Passed"
+
+        updated_workgroup_submission_review = project_models.WorkgroupSubmissionReview.objects.get(id=workgroup_submission_review.id)
+        self.assertEqual(updated_workgroup_submission_review.content_id, self.new_style_content_id)
+        updated_workgroup_submission_review = project_models.WorkgroupSubmissionReview.objects.get(id=workgroup_submission_review2.id)
+        self.assertEqual(updated_workgroup_submission_review.content_id, self.new_style_content_id2)
+        print "Workgroup Submission Review Data Migration Passed"
+
+        updated_course_group = api_models.CourseGroupRelationship.objects.get(id=course_group.id)
+        self.assertEqual(updated_course_group.course_id, self.new_style_course_id)
+        updated_course_group = api_models.CourseGroupRelationship.objects.get(id=course_group2.id)
+        self.assertEqual(updated_course_group.course_id, self.new_style_course_id2)
+        print "Course Group Data Migration Passed"
+
+        updated_course_content_group = api_models.CourseContentGroupRelationship.objects.get(id=course_content_group.id)
+        self.assertEqual(updated_course_content_group.course_id, self.new_style_course_id)
+        self.assertEqual(updated_course_content_group.content_id, self.new_style_content_id)
+        updated_course_content_group = api_models.CourseContentGroupRelationship.objects.get(id=course_content_group2.id)
+        self.assertEqual(updated_course_content_group.course_id, self.new_style_course_id2)
+        self.assertEqual(updated_course_content_group.content_id, self.new_style_content_id2)
+        print "Course Content Group Data Migration Passed"
+
+        updated_course_module_completion = api_models.CourseModuleCompletion.objects.get(id=course_module_completion.id)
+        self.assertEqual(updated_course_module_completion.course_id, self.new_style_course_id)
+        self.assertEqual(updated_course_module_completion.content_id, self.new_style_content_id)
+        updated_course_module_completion = api_models.CourseModuleCompletion.objects.get(id=course_module_completion2.id)
+        self.assertEqual(updated_course_module_completion.course_id, self.new_style_course_id2)
+        self.assertEqual(updated_course_module_completion.content_id, self.new_style_content_id2)
+        print "Course Module Completion Data Migration Passed"

pavelib/utils/test/suites/nose_suite.py

@@ -2,7 +2,6 @@
 Classes used for defining and running nose test suites
 """
 import os
-from paver.easy import call_task
 from pavelib.utils.test import utils as test_utils
 from pavelib.utils.test.suites.suite import TestSuite
 from pavelib.utils.envs import Env