Skip to content

E
edx-platform
Commit 8da5a0c0 by Matt Drayer
Options

Merge pull request #267 from edx-solutions/mattdrayer/api-course-key-conversion 

mattdrayer/api-course-key-conversion:  CourseKey data migrations, plus g...
parents 99f90c76 4c275cd5
Showing with 509 additions and 10 deletions
lms/djangoapps/api_manager/management/commands/migrate_courseids_v2.py 0 → 100644
"""
One-time data migration script -- shoulen't need to run it again
"""
import logging
from django.core.management.base import BaseCommand
from api_manager import models as api_models
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Migrates legacy course/content identifiers across several models to the new format
"""
def handle(self, *args, **options):
log.warning('Migrating Course Groups...')
course_groups = api_models.CourseGroupRelationship.objects.all()
for cg in course_groups:
current_course_id = cg.course_id
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
cg.course_id = oldstyle_course_id
cg.save()
log.warning('Complete!')
log.warning('Migrating Course Content Groups...')
course_content_groups = api_models.CourseContentGroupRelationship.objects.all()
for ccg in course_content_groups:
current_course_id = ccg.course_id
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
ccg.course_id = oldstyle_course_id
current_content_id = ccg.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
ccg.content_id = oldstyle_content_id
ccg.save()
log.warning('Complete!')
log.warning('Migrating Course Module Completions...')
course_module_completions = api_models.CourseModuleCompletion.objects.all()
for cmc in course_module_completions:
current_course_id = cmc.course_id
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
cmc.course_id = oldstyle_course_id
current_content_id = cmc.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
cmc.content_id = oldstyle_content_id
if cmc.stage is not None:
current_stage = cmc.stage
oldstyle_stage = current_stage.replace("slashes:", "")
oldstyle_stage = current_stage.replace("+", "/")
cmc.stage = oldstyle_stage
cmc.save()
log.warning('Complete!')
lms/djangoapps/api_manager/management/commands/tests/test_migrate_courseids_v2.py 0 → 100644
"""
Run these tests @ Devstack:
rake fasttest_lms[common/djangoapps/api_manager/management/commands/tests/test_migrate_orgdata.py]
"""
from datetime import datetime
import uuid
from django.contrib.auth.models import Group, User
from django.test import TestCase
from django.test.utils import override_settings
from api_manager import models as api_models
from api_manager.management.commands import migrate_courseids_v2
from courseware.tests.modulestore_config import TEST_DATA_MIXED_MODULESTORE
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from django.db import connection
@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class MigrateCourseIdsTests(TestCase):
"""
Test suite for data migration script
"""
def setUp(self):
self.course = CourseFactory.create(
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.test_data = '<html>{}</html>'.format(str(uuid.uuid4()))
self.chapter = ItemFactory.create(
category="chapter",
parent_location=self.course.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id = self.course.id.to_deprecated_string()
self.new_style_course_id = unicode(self.course.id)
self.old_style_content_id = self.chapter.location.to_deprecated_string()
self.new_style_content_id = unicode(self.chapter.location)
self.course2 = CourseFactory.create(
org='TEST',
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.chapter2 = ItemFactory.create(
category="chapter",
parent_location=self.course2.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id2 = self.course2.id.to_deprecated_string()
self.new_style_course_id2 = unicode(self.course2.id)
self.old_style_content_id2 = self.chapter2.location.to_deprecated_string()
self.new_style_content_id2 = unicode(self.chapter2.location)
def test_migrate_courseids_v2(self):
"""
Test the data migration
"""
# Set up the data to be migrated
user = User.objects.create(email='testuser@edx.org', username='testuser', password='testpassword', is_active=True)
group = Group.objects.create(name='Test Group')
group_profile = api_models.GroupProfile.objects.create(group=group)
course_group = api_models.CourseGroupRelationship.objects.create(course_id=self.old_style_course_id, group=group)
course_content_group = api_models.CourseContentGroupRelationship.objects.create(course_id=self.old_style_course_id, content_id=self.old_style_content_id, group_profile=group_profile)
course_module_completion = api_models.CourseModuleCompletion.objects.create(user=user, course_id=self.old_style_course_id, content_id=self.old_style_content_id)
user2 = User.objects.create(email='testuser2@edx.org', username='testuser2', password='testpassword2', is_active=True)
group2 = Group.objects.create(name='Test Group2')
group_profile2 = api_models.GroupProfile.objects.create(group=group2)
course_group2 = api_models.CourseGroupRelationship.objects.create(course_id=self.new_style_course_id2, group=group2)
course_content_group2 = api_models.CourseContentGroupRelationship.objects.create(course_id=self.new_style_course_id2, content_id=self.new_style_content_id2, group_profile=group_profile2)
course_module_completion2 = api_models.CourseModuleCompletion.objects.create(user=user2, course_id=self.new_style_course_id2, content_id=self.new_style_content_id2)
# Run the data migration
migrate_courseids_v2.Command().handle()
# Confirm that the data has been properly migrated
updated_course_group = api_models.CourseGroupRelationship.objects.get(id=course_group.id)
self.assertEqual(updated_course_group.course_id, self.old_style_course_id)
updated_course_group = api_models.CourseGroupRelationship.objects.get(id=course_group2.id)
self.assertEqual(updated_course_group.course_id, self.old_style_course_id2)
print "Course Group Data Migration Passed"
updated_course_content_group = api_models.CourseContentGroupRelationship.objects.get(id=course_content_group.id)
self.assertEqual(updated_course_content_group.course_id, self.old_style_course_id)
self.assertEqual(updated_course_content_group.content_id, self.old_style_content_id)
updated_course_content_group = api_models.CourseContentGroupRelationship.objects.get(id=course_content_group2.id)
self.assertEqual(updated_course_content_group.course_id, self.old_style_course_id2)
self.assertEqual(updated_course_content_group.content_id, self.old_style_content_id2)
print "Course Content Group Data Migration Passed"
updated_course_module_completion = api_models.CourseModuleCompletion.objects.get(id=course_module_completion.id)
self.assertEqual(updated_course_module_completion.course_id, self.old_style_course_id)
self.assertEqual(updated_course_module_completion.content_id, self.old_style_content_id)
updated_course_module_completion = api_models.CourseModuleCompletion.objects.get(id=course_module_completion2.id)
self.assertEqual(updated_course_module_completion.course_id, self.old_style_course_id2)
self.assertEqual(updated_course_module_completion.content_id, self.old_style_content_id2)
print "Course Module Completion Data Migration Passed"
lms/djangoapps/api_manager/organizations/tests.py
......@@ -318,11 +318,11 @@ class OrganizationsApiTests(TestCase):
user = User.objects.get(pk=user_id)
users.append(user_id)
if i < 2:
StudentGradebook.objects.create(user=user, grade=0.75, proforma_grade=0.85)
StudentGradebook.objects.create(user=user, course_id=self.course.id, grade=0.75, proforma_grade=0.85)
elif i < 4:
StudentGradebook.objects.create(user=user, grade=0.82, proforma_grade=0.82)
StudentGradebook.objects.create(user=user, course_id=self.course.id, grade=0.82, proforma_grade=0.82)
else:
StudentGradebook.objects.create(user=user, grade=0.90, proforma_grade=0.91)
StudentGradebook.objects.create(user=user, course_id=self.course.id, grade=0.90, proforma_grade=0.91)
data = {
'name': self.test_organization_name,
......
lms/djangoapps/gradebook/management/commands/migrate_courseids_v2.py 0 → 100644
"""
One-time data migration script -- shoulen't need to run it again
"""
import logging
from django.core.management.base import BaseCommand
from opaque_keys.edx.keys import CourseKey
from gradebook import models
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Migrates legacy course/content identifiers across several models to the new format
"""
def handle(self, *args, **options):
log.warning('Migrating Student Gradebook Entries...')
gradebook_entries = models.StudentGradebook.objects.all()
for gbe in gradebook_entries:
current_course_id = unicode(gbe.course_id)
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
gbe.course_id = CourseKey.from_string(oldstyle_course_id)
gbe.save()
log.warning('Complete!')
log.warning('Migrating Student Gradebook History Entries...')
history_entries = models.StudentGradebookHistory.objects.all()
for he in history_entries:
current_course_id = unicode(he.course_id)
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
he.course_id = oldstyle_course_id
he.save()
log.warning('Complete!')
lms/djangoapps/gradebook/management/commands/tests/test_migrate_courseids_v2.py 0 → 100644
"""
Run these tests @ Devstack:
rake fasttest_lms[common/djangoapps/api_manager/management/commands/tests/test_migrate_orgdata.py]
"""
from datetime import datetime
import uuid
from django.contrib.auth.models import Group, User
from django.test import TestCase
from django.test.utils import override_settings
from gradebook import models as gradebook_models
from gradebook.management.commands import migrate_courseids_v2
from courseware.tests.modulestore_config import TEST_DATA_MIXED_MODULESTORE
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from django.db import connection
@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class MigrateCourseIdsTests(TestCase):
"""
Test suite for data migration script
"""
def setUp(self):
self.course = CourseFactory.create(
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.test_data = '<html>{}</html>'.format(str(uuid.uuid4()))
self.chapter = ItemFactory.create(
category="chapter",
parent_location=self.course.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id = self.course.id.to_deprecated_string()
self.new_style_course_id = unicode(self.course.id)
self.old_style_content_id = self.chapter.location.to_deprecated_string()
self.new_style_content_id = unicode(self.chapter.location)
self.course2 = CourseFactory.create(
org='TEST',
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.chapter2 = ItemFactory.create(
category="chapter",
parent_location=self.course2.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id2 = self.course2.id.to_deprecated_string()
self.new_style_course_id2 = unicode(self.course2.id)
self.old_style_content_id2 = self.chapter2.location.to_deprecated_string()
self.new_style_content_id2 = unicode(self.chapter2.location)
def test_migrate_courseids(self):
"""
Test the data migration
"""
# Set up the data to be migrated
user = User.objects.create(email='testuser@edx.org', username='testuser', password='testpassword', is_active=True)
gradebook_entry = gradebook_models.StudentGradebook.objects.create(user=user, course_id=self.new_style_course_id, grade=0.85, proforma_grade=0.74)
user2 = User.objects.create(email='testuser2@edx.org', username='testuser2', password='testpassword2', is_active=True)
gradebook_entry2 = gradebook_models.StudentGradebook.objects.create(user=user2, course_id=self.new_style_course_id2, grade=0.95, proforma_grade=0.64)
# Run the data migration
migrate_courseids_v2.Command().handle()
# Confirm that the data has been properly migrated
updated_gradebook_entries = gradebook_models.StudentGradebook.objects.get(id=gradebook_entry.id)
updated_gradebook_entry = gradebook_models.StudentGradebook.objects.get(id=gradebook_entry2.id)
self.assertEqual(unicode(updated_gradebook_entry.course_id), self.old_style_course_id2)
print "Student Gradebook Data Migration Passed"
updated_history_entries = gradebook_models.StudentGradebookHistory.objects.filter(user=user.id)
for entry in updated_history_entries:
self.assertEqual(unicode(entry.course_id), self.old_style_course_id)
updated_history_entries = gradebook_models.StudentGradebookHistory.objects.filter(user=user2.id)
for entry in updated_history_entries:
self.assertEqual(unicode(entry.course_id), self.old_style_course_id2)
print "Student Gradebook History Data Migration Passed"
lms/djangoapps/gradebook/models.py
......@@ -129,10 +129,23 @@ class StudentGradebookHistory(TimeStampedModel):
"""
Event hook for creating gradebook entry copies
"""
history_entry = StudentGradebookHistory(
user=instance.user,
course_id=instance.course_id,
grade=instance.grade,
proforma_grade=instance.proforma_grade
)
history_entry.save()
history_entries = StudentGradebookHistory.objects.filter(user=instance.user, course_id=instance.course_id)
latest_history_entry = None
if len(history_entries):
latest_history_entry = history_entries[0]
create_history_entry = False
if latest_history_entry is not None:
if (latest_history_entry.grade != instance.grade) or (latest_history_entry.proforma_grade != instance.proforma_grade):
create_history_entry = True
else:
create_history_entry = True
if create_history_entry:
new_history_entry = StudentGradebookHistory(
user=instance.user,
course_id=instance.course_id,
grade=instance.grade,
proforma_grade=instance.proforma_grade
)
new_history_entry.save()
lms/djangoapps/projects/management/commands/migrate_project_courseids_v2.py 0 → 100644
"""
One-time data migration script -- shoulen't need to run it again
"""
import logging
from django.core.management.base import BaseCommand
from projects.models import Project, WorkgroupReview, WorkgroupPeerReview, WorkgroupSubmissionReview
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Migrates legacy course/content identifiers across several models to the new format
"""
def handle(self, *args, **options):
log.warning('Migrating Projects...')
projects = Project.objects.all()
for project in projects:
current_course_id = project.course_id
oldstyle_course_id = current_course_id.replace("slashes:", "")
oldstyle_course_id = current_course_id.replace("+", "/")
project.course_id = oldstyle_course_id
current_content_id = project.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
project.content_id = oldstyle_content_id
project.save()
log.warning('Complete!')
log.warning('Migrating Workgroup Reviews...')
workgroup_reviews = WorkgroupReview.objects.all()
for wr in workgroup_reviews:
current_content_id = wr.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
wr.content_id = oldstyle_content_id
wr.save()
log.warning('Complete!')
log.warning('Migrating Workgroup Peer Reviews...')
workgroup_peer_reviews = WorkgroupPeerReview.objects.all()
for wpr in workgroup_reviews:
current_content_id = wpr.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
wpr.content_id = oldstyle_content_id
wpr.save()
log.warning('Complete!')
log.warning('Migrating Workgroup Submission Reviews...')
workgroup_submission_reviews = WorkgroupSubmissionReview.objects.all()
for wsr in workgroup_submission_reviews:
current_content_id = wsr.content_id
oldstyle_content_id = current_content_id.replace("slashes:", "")
oldstyle_content_id = current_content_id.replace("+", "/")
wsr.content_id = oldstyle_content_id
wsr.save()
log.warning('Complete!')
lms/djangoapps/projects/management/commands/tests/test_migrate_project_courseids_v2.py 0 → 100644
"""
Run these tests @ Devstack:
rake fasttest_lms[common/djangoapps/api_manager/management/commands/tests/test_migrate_orgdata.py]
"""
from datetime import datetime
import uuid
from django.contrib.auth.models import Group, User
from django.test import TestCase
from django.test.utils import override_settings
from projects.management.commands import migrate_project_courseids_v2
from courseware.tests.modulestore_config import TEST_DATA_MIXED_MODULESTORE
from projects.models import Project, Workgroup, WorkgroupReview, WorkgroupPeerReview, WorkgroupSubmission, WorkgroupSubmissionReview
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from django.db import connection
@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class MigrateCourseIdsTests(TestCase):
"""
Test suite for data migration script
"""
def setUp(self):
self.course = CourseFactory.create(
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.test_data = '<html>{}</html>'.format(str(uuid.uuid4()))
self.chapter = ItemFactory.create(
category="chapter",
parent_location=self.course.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id = self.course.id.to_deprecated_string()
self.new_style_course_id = unicode(self.course.id)
self.old_style_content_id = self.chapter.location.to_deprecated_string()
self.new_style_content_id = unicode(self.chapter.location)
self.course2 = CourseFactory.create(
org='TEST',
start=datetime(2014, 6, 16, 14, 30),
end=datetime(2015, 1, 16)
)
self.chapter2 = ItemFactory.create(
category="chapter",
parent_location=self.course2.location,
data=self.test_data,
due=datetime(2014, 5, 16, 14, 30),
display_name="Overview"
)
self.old_style_course_id2 = self.course2.id.to_deprecated_string()
self.new_style_course_id2 = unicode(self.course2.id)
self.old_style_content_id2 = self.chapter2.location.to_deprecated_string()
self.new_style_content_id2 = unicode(self.chapter2.location)
def test_migrate_project_courseids_v2(self):
"""
Test the data migration
"""
# Set up the data to be migrated
user = User.objects.create(email='testuser@edx.org', username='testuser', password='testpassword', is_active=True)
reviewer = User.objects.create(email='testreviewer@edx.org', username='testreviewer', password='testpassword', is_active=True)
project = Project.objects.create(course_id=self.old_style_course_id, content_id=self.old_style_content_id)
workgroup = Workgroup.objects.create(name='Test Workgroup', project=project)
workgroup_review = WorkgroupReview.objects.create(workgroup=workgroup, content_id=self.old_style_content_id)
workgroup_peer_review = WorkgroupPeerReview.objects.create(workgroup=workgroup, user=user, reviewer=reviewer, question="Question?", answer="Answer!", content_id=self.new_style_content_id)
workgroup_submission = WorkgroupSubmission.objects.create(workgroup=workgroup, user=user)
workgroup_submission_review = WorkgroupSubmissionReview.objects.create(submission=workgroup_submission, content_id=self.old_style_content_id)
user2 = User.objects.create(email='testuser2@edx.org', username='testuser2', password='testpassword2', is_active=True)
reviewer2 = User.objects.create(email='testreviewer2@edx.org', username='testreviewer2', password='testpassword', is_active=True)
project2 = Project.objects.create(course_id=self.new_style_course_id2, content_id=self.new_style_content_id2)
workgroup2 = Workgroup.objects.create(name='Test Workgroup2', project=project2)
workgroup_review2 = WorkgroupReview.objects.create(workgroup=workgroup2, content_id=self.new_style_content_id2)
workgroup_peer_review2 = WorkgroupPeerReview.objects.create(workgroup=workgroup2, user=user2, reviewer=reviewer2, question="Question?", answer="Answer!", content_id=self.new_style_content_id2)
workgroup_submission2 = WorkgroupSubmission.objects.create(workgroup=workgroup2, user=user2)
workgroup_submission_review2 = WorkgroupSubmissionReview.objects.create(submission=workgroup_submission2, content_id=self.new_style_content_id2)
# Run the data migration
migrate_project_courseids_v2.Command().handle()
# Confirm that the data has been properly migrated
updated_project = Project.objects.get(id=project.id)
self.assertEqual(updated_project.course_id, self.old_style_course_id)
self.assertEqual(updated_project.content_id, self.old_style_content_id)
updated_project = Project.objects.get(id=project2.id)
self.assertEqual(updated_project.course_id, self.old_style_course_id2)
self.assertEqual(updated_project.content_id, self.old_style_content_id2)
print "Project Data Migration Passed"
updated_workgroup_review = WorkgroupReview.objects.get(id=workgroup_review.id)
self.assertEqual(updated_workgroup_review.content_id, self.old_style_content_id)
updated_workgroup_review = WorkgroupReview.objects.get(id=workgroup_review2.id)
self.assertEqual(updated_workgroup_review.content_id, self.old_style_content_id2)
print "Workgroup Review Data Migration Passed"
updated_workgroup_peer_review = WorkgroupPeerReview.objects.get(id=workgroup_peer_review.id)
self.assertEqual(updated_workgroup_peer_review.content_id, self.old_style_content_id)
updated_workgroup_peer_review = WorkgroupPeerReview.objects.get(id=workgroup_peer_review2.id)
self.assertEqual(updated_workgroup_peer_review.content_id, self.old_style_content_id2)
print "Workgroup Peer Review Data Migration Passed"
updated_workgroup_submission_review = WorkgroupSubmissionReview.objects.get(id=workgroup_submission_review.id)
self.assertEqual(updated_workgroup_submission_review.content_id, self.old_style_content_id)
updated_workgroup_submission_review = WorkgroupSubmissionReview.objects.get(id=workgroup_submission_review2.id)
self.assertEqual(updated_workgroup_submission_review.content_id, self.old_style_content_id2)
print "Workgroup Submission Review Data Migration Passed"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment