Skip to content

E
edx-platform
Commit dd90c556 by Calen Pennington Committed by Kevin Falcone
Options

Create a new CSMHExtended table to hold our new data 

This is a clone (copy) of CSMH's declaration and methods with an added
id of UnsignedBigInAutoField

We should be able to delete the save_history code, but needs testing.

Add error logging when capa failures happen

Put StudentModuleHistory into its own database

Bump out the primary key on CSMHE

This gives us a gap to backfill as needed.
Since the new table's pk is an unsigned bigint, even for people who don't
consolidate CSMH into CSMHE, the lost rows are unlikely to matter.

Remove StudentModuleHistory cleaner
parent e0407893
Showing with 412 additions and 331 deletions
cms/envs/acceptance.py
...@@ -80,6 +80,14 @@ DATABASES = { ...@@ -80,6 +80,14 @@ DATABASES = {
'timeout': 30, 'timeout': 30,
}, },
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'OPTIONS': {
'timeout': 30,
},
} }
} }
......
common/lib/xmodule/xmodule/capa_module.py
...@@ -93,11 +93,23 @@ class CapaModule(CapaMixin, XModule): ...@@ -93,11 +93,23 @@ class CapaModule(CapaMixin, XModule):
result = handlers[dispatch](data) result = handlers[dispatch](data)
except NotFoundError as err: except NotFoundError as err:
_, _, traceback_obj = sys.exc_info() log.exception(
"Unable to find data when dispatching %s to %s for user %s",
dispatch,
self.scope_ids.usage_id,
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info() # pylint: disable=redefined-outer-name
raise ProcessingError(not_found_error_message), None, traceback_obj raise ProcessingError(not_found_error_message), None, traceback_obj
except Exception as err: except Exception as err:
_, _, traceback_obj = sys.exc_info() log.exception(
"Unknown error when dispatching %s to %s for user %s",
dispatch,
self.scope_ids.usage_id,
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info() # pylint: disable=redefined-outer-name
raise ProcessingError(generic_error_message), None, traceback_obj raise ProcessingError(generic_error_message), None, traceback_obj
after = self.get_progress() after = self.get_progress()
......
common/test/db_cache/bok_choy_data.json deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
common/test/db_cache/bok_choy_data_default.json 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
common/test/db_cache/bok_choy_schema.sql → common/test/db_cache/bok_choy_schema_default.sql
...@@ -412,7 +412,7 @@ CREATE TABLE `auth_permission` ( ...@@ -412,7 +412,7 @@ CREATE TABLE `auth_permission` (
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `content_type_id` (`content_type_id`,`codename`), UNIQUE KEY `content_type_id` (`content_type_id`,`codename`),
CONSTRAINT `auth__content_type_id_508cf46651277a81_fk_django_content_type_id` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`) CONSTRAINT `auth__content_type_id_508cf46651277a81_fk_django_content_type_id` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=746 DEFAULT CHARSET=utf8; ) ENGINE=InnoDB AUTO_INCREMENT=740 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `auth_registration`; DROP TABLE IF EXISTS `auth_registration`;
/*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET @saved_cs_client = @@character_set_client */;
...@@ -444,7 +444,7 @@ CREATE TABLE `auth_user` ( ...@@ -444,7 +444,7 @@ CREATE TABLE `auth_user` (
`date_joined` datetime(6) NOT NULL, `date_joined` datetime(6) NOT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `username` (`username`) UNIQUE KEY `username` (`username`)
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `auth_user_groups`; DROP TABLE IF EXISTS `auth_user_groups`;
/*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET @saved_cs_client = @@character_set_client */;
...@@ -1353,24 +1353,6 @@ CREATE TABLE `courseware_xmoduleuserstatesummaryfield` ( ...@@ -1353,24 +1353,6 @@ CREATE TABLE `courseware_xmoduleuserstatesummaryfield` (
KEY `courseware_xmoduleuserstatesummaryfield_0528eb2a` (`usage_id`) KEY `courseware_xmoduleuserstatesummaryfield_0528eb2a` (`usage_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8; ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `credentials_credentialsapiconfig`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `credentials_credentialsapiconfig` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`change_date` datetime(6) NOT NULL,
`enabled` tinyint(1) NOT NULL,
`internal_service_url` varchar(200) NOT NULL,
`public_service_url` varchar(200) NOT NULL,
`enable_learner_issuance` tinyint(1) NOT NULL,
`enable_studio_authoring` tinyint(1) NOT NULL,
`cache_ttl` int(10) unsigned NOT NULL,
`changed_by_id` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `credentials_crede_changed_by_id_273a2e6b0649c861_fk_auth_user_id` (`changed_by_id`),
CONSTRAINT `credentials_crede_changed_by_id_273a2e6b0649c861_fk_auth_user_id` FOREIGN KEY (`changed_by_id`) REFERENCES `auth_user` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `credit_creditcourse`; DROP TABLE IF EXISTS `credit_creditcourse`;
/*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */; /*!40101 SET character_set_client = utf8 */;
...@@ -1618,7 +1600,7 @@ CREATE TABLE `django_content_type` ( ...@@ -1618,7 +1600,7 @@ CREATE TABLE `django_content_type` (
`model` varchar(100) NOT NULL, `model` varchar(100) NOT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `django_content_type_app_label_45f3b1d93ec8c61c_uniq` (`app_label`,`model`) UNIQUE KEY `django_content_type_app_label_45f3b1d93ec8c61c_uniq` (`app_label`,`model`)
) ENGINE=InnoDB AUTO_INCREMENT=248 DEFAULT CHARSET=utf8; ) ENGINE=InnoDB AUTO_INCREMENT=246 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `django_migrations`; DROP TABLE IF EXISTS `django_migrations`;
/*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET @saved_cs_client = @@character_set_client */;
...@@ -1629,7 +1611,7 @@ CREATE TABLE `django_migrations` ( ...@@ -1629,7 +1611,7 @@ CREATE TABLE `django_migrations` (
`name` varchar(255) NOT NULL, `name` varchar(255) NOT NULL,
`applied` datetime(6) NOT NULL, `applied` datetime(6) NOT NULL,
PRIMARY KEY (`id`) PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=107 DEFAULT CHARSET=utf8; ) ENGINE=InnoDB AUTO_INCREMENT=103 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `django_openid_auth_association`; DROP TABLE IF EXISTS `django_openid_auth_association`;
/*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET @saved_cs_client = @@character_set_client */;
......
common/test/db_cache/bok_choy_schema_student_module_history.sql 0 → 100644
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
lms/djangoapps/ccx/tests/test_field_override_performance.py
...@@ -151,7 +151,10 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin, ...@@ -151,7 +151,10 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
""" """
return check_sum_of_calls(XBlock, ['__init__'], instantiations, instantiations, include_arguments=False) return check_sum_of_calls(XBlock, ['__init__'], instantiations, instantiations, include_arguments=False)
def instrument_course_progress_render(self, course_width, enable_ccx, view_as_ccx, queries, reads, xblocks): def instrument_course_progress_render(
self, course_width, enable_ccx, view_as_ccx,
default_queries, history_queries, reads, xblocks
):
""" """
Renders the progress page, instrumenting Mongo reads and SQL queries. Renders the progress page, instrumenting Mongo reads and SQL queries.
""" """
...@@ -173,10 +176,11 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin, ...@@ -173,10 +176,11 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
# can actually take affect. # can actually take affect.
OverrideFieldData.provider_classes = None OverrideFieldData.provider_classes = None
with self.assertNumQueries(queries): with self.assertNumQueries(default_queries, using='default'):
with self.assertMongoCallCount(reads): with self.assertNumQueries(history_queries, using='student_module_history'):
with self.assertXBlockInstantiations(xblocks): with self.assertMongoCallCount(reads):
self.grade_course(self.course, view_as_ccx) with self.assertXBlockInstantiations(xblocks):
self.grade_course(self.course, view_as_ccx)
@ddt.data(*itertools.product(('no_overrides', 'ccx'), range(1, 4), (True, False), (True, False))) @ddt.data(*itertools.product(('no_overrides', 'ccx'), range(1, 4), (True, False), (True, False)))
@ddt.unpack @ddt.unpack
...@@ -201,8 +205,12 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin, ...@@ -201,8 +205,12 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
raise SkipTest("Can't use a MongoModulestore test as a CCX course") raise SkipTest("Can't use a MongoModulestore test as a CCX course")
with self.settings(FIELD_OVERRIDE_PROVIDERS=providers[overrides]): with self.settings(FIELD_OVERRIDE_PROVIDERS=providers[overrides]):
queries, reads, xblocks = self.TEST_DATA[(overrides, course_width, enable_ccx, view_as_ccx)] default_queries, history_queries, reads, xblocks = self.TEST_DATA[
self.instrument_course_progress_render(course_width, enable_ccx, view_as_ccx, queries, reads, xblocks) (overrides, course_width, enable_ccx, view_as_ccx)
]
self.instrument_course_progress_render(
course_width, enable_ccx, view_as_ccx, default_queries, history_queries, reads, xblocks
)
class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase): class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
...@@ -213,25 +221,30 @@ class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase): ...@@ -213,25 +221,30 @@ class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
__test__ = True __test__ = True
TEST_DATA = { TEST_DATA = {
# (providers, course_width, enable_ccx, view_as_ccx): # of sql queries, # of mongo queries, # of xblocks # (providers, course_width, enable_ccx, view_as_ccx): (
('no_overrides', 1, True, False): (48, 6, 13), # # of sql queries to default,
('no_overrides', 2, True, False): (135, 6, 84), # # sql queries to student_module_history,
('no_overrides', 3, True, False): (480, 6, 335), # # of mongo queries,
('ccx', 1, True, False): (48, 6, 13), # # of xblocks
('ccx', 2, True, False): (135, 6, 84), # )
('ccx', 3, True, False): (480, 6, 335), ('no_overrides', 1, True, False): (23, 1, 6, 13),
('ccx', 1, True, True): (48, 6, 13), ('no_overrides', 2, True, False): (53, 16, 6, 84),
('ccx', 2, True, True): (135, 6, 84), ('no_overrides', 3, True, False): (183, 81, 6, 335),
('ccx', 3, True, True): (480, 6, 335), ('ccx', 1, True, False): (23, 1, 6, 13),
('no_overrides', 1, False, False): (48, 6, 13), ('ccx', 2, True, False): (53, 16, 6, 84),
('no_overrides', 2, False, False): (135, 6, 84), ('ccx', 3, True, False): (183, 81, 6, 335),
('no_overrides', 3, False, False): (480, 6, 335), ('ccx', 1, True, True): (23, 1, 6, 13),
('ccx', 1, False, False): (48, 6, 13), ('ccx', 2, True, True): (53, 16, 6, 84),
('ccx', 2, False, False): (135, 6, 84), ('ccx', 3, True, True): (183, 81, 6, 335),
('ccx', 3, False, False): (480, 6, 335), ('no_overrides', 1, False, False): (23, 1, 6, 13),
('ccx', 1, False, True): (48, 6, 13), ('no_overrides', 2, False, False): (53, 16, 6, 84),
('ccx', 2, False, True): (135, 6, 84), ('no_overrides', 3, False, False): (183, 81, 6, 335),
('ccx', 3, False, True): (480, 6, 335), ('ccx', 1, False, False): (23, 1, 6, 13),
('ccx', 2, False, False): (53, 16, 6, 84),
('ccx', 3, False, False): (183, 81, 6, 335),
('ccx', 1, False, True): (23, 1, 6, 13),
('ccx', 2, False, True): (53, 16, 6, 84),
('ccx', 3, False, True): (183, 81, 6, 335),
} }
...@@ -243,22 +256,22 @@ class TestFieldOverrideSplitPerformance(FieldOverridePerformanceTestCase): ...@@ -243,22 +256,22 @@ class TestFieldOverrideSplitPerformance(FieldOverridePerformanceTestCase):
__test__ = True __test__ = True
TEST_DATA = { TEST_DATA = {
('no_overrides', 1, True, False): (48, 4, 9), ('no_overrides', 1, True, False): (23, 1, 4, 9),
('no_overrides', 2, True, False): (135, 19, 54), ('no_overrides', 2, True, False): (53, 16, 19, 54),
('no_overrides', 3, True, False): (480, 84, 215), ('no_overrides', 3, True, False): (183, 81, 84, 215),
('ccx', 1, True, False): (48, 4, 9), ('ccx', 1, True, False): (23, 1, 4, 9),
('ccx', 2, True, False): (135, 19, 54), ('ccx', 2, True, False): (53, 16, 19, 54),
('ccx', 3, True, False): (480, 84, 215), ('ccx', 3, True, False): (183, 81, 84, 215),
('ccx', 1, True, True): (50, 4, 13), ('ccx', 1, True, True): (25, 1, 4, 13),
('ccx', 2, True, True): (137, 19, 84), ('ccx', 2, True, True): (55, 16, 19, 84),
('ccx', 3, True, True): (482, 84, 335), ('ccx', 3, True, True): (185, 81, 84, 335),
('no_overrides', 1, False, False): (48, 4, 9), ('no_overrides', 1, False, False): (23, 1, 4, 9),
('no_overrides', 2, False, False): (135, 19, 54), ('no_overrides', 2, False, False): (53, 16, 19, 54),
('no_overrides', 3, False, False): (480, 84, 215), ('no_overrides', 3, False, False): (183, 81, 84, 215),
('ccx', 1, False, False): (48, 4, 9), ('ccx', 1, False, False): (23, 1, 4, 9),
('ccx', 2, False, False): (135, 19, 54), ('ccx', 2, False, False): (53, 16, 19, 54),
('ccx', 3, False, False): (480, 84, 215), ('ccx', 3, False, False): (183, 81, 84, 215),
('ccx', 1, False, True): (48, 4, 9), ('ccx', 1, False, True): (23, 1, 4, 9),
('ccx', 2, False, True): (135, 19, 54), ('ccx', 2, False, True): (53, 16, 19, 54),
('ccx', 3, False, True): (480, 84, 215), ('ccx', 3, False, True): (183, 81, 84, 215),
} }
lms/djangoapps/courseware/fields.py 0 → 100644
"""
Custom fields for use in the courseware django app.
"""
from django.db.models.fields import AutoField
class UnsignedBigIntAutoField(AutoField):
"""
An unsigned 8-byte integer for auto-incrementing primary keys.
"""
def db_type(self, connection):
if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
return "bigint UNSIGNED AUTO_INCREMENT"
elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
# Sqlite will only auto-increment the ROWID column. Any INTEGER PRIMARY KEY column
# is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
# isn't an alias for ROWID, so we have to give up on the unsigned part.
return "integer"
else:
return None
lms/djangoapps/courseware/management/commands/clean_history.py deleted 100644 → 0
"""A command to clean the StudentModuleHistory table.
When we added XBlock storage, each field modification wrote a new history row
to the db. Now that we have bulk saves to avoid that database hammering, we
need to clean out the unnecessary rows from the database.
This command that does that.
"""
import datetime
import json
import logging
import optparse
import time
import traceback
from django.core.management.base import NoArgsCommand
from django.db import transaction
from django.db.models import Max
from courseware.models import StudentModuleHistory
class Command(NoArgsCommand):
"""The actual clean_history command to clean history rows."""
help = "Deletes unneeded rows from the StudentModuleHistory table."
option_list = NoArgsCommand.option_list + (
optparse.make_option(
'--batch',
type='int',
default=100,
help="Batch size, number of module_ids to examine in a transaction.",
),
optparse.make_option(
'--dry-run',
action='store_true',
default=False,
help="Don't change the database, just show what would be done.",
),
optparse.make_option(
'--sleep',
type='float',
default=0,
help="Seconds to sleep between batches.",
),
)
def handle_noargs(self, **options):
# We don't want to see the SQL output from the db layer.
logging.getLogger("django.db.backends").setLevel(logging.INFO)
smhc = StudentModuleHistoryCleaner(
dry_run=options["dry_run"],
)
smhc.main(batch_size=options["batch"], sleep=options["sleep"])
class StudentModuleHistoryCleaner(object):
"""Logic to clean rows from the StudentModuleHistory table."""
DELETE_GAP_SECS = 0.5 # Rows this close can be discarded.
STATE_FILE = "clean_history.json"
BATCH_SIZE = 100
def __init__(self, dry_run=False):
self.dry_run = dry_run
self.next_student_module_id = 0
self.last_student_module_id = 0
def main(self, batch_size=None, sleep=0):
"""Invoked from the management command to do all the work."""
batch_size = batch_size or self.BATCH_SIZE
self.last_student_module_id = self.get_last_student_module_id()
self.load_state()
while self.next_student_module_id <= self.last_student_module_id:
with transaction.atomic():
for smid in self.module_ids_to_check(batch_size):
try:
self.clean_one_student_module(smid)
except Exception: # pylint: disable=broad-except
trace = traceback.format_exc()
self.say("Couldn't clean student_module_id {}:\n{}".format(smid, trace))
if self.dry_run:
transaction.set_rollback(True)
else:
self.say("Committing")
self.save_state()
if sleep:
time.sleep(sleep)
def say(self, message):
"""
Display a message to the user.
The message will have a trailing newline added to it.
"""
print message
def load_state(self):
"""
Load the latest state from disk.
"""
try:
state_file = open(self.STATE_FILE)
except IOError:
self.say("No stored state")
self.next_student_module_id = 0
else:
with state_file:
state = json.load(state_file)
self.say(
"Loaded stored state: {}".format(
json.dumps(state, sort_keys=True)
)
)
self.next_student_module_id = state['next_student_module_id']
def save_state(self):
"""
Save the state to disk.
"""
state = {
'next_student_module_id': self.next_student_module_id,
}
with open(self.STATE_FILE, "w") as state_file:
json.dump(state, state_file)
self.say("Saved state: {}".format(json.dumps(state, sort_keys=True)))
def get_last_student_module_id(self):
"""
Return the id of the last student_module.
"""
last = StudentModuleHistory.objects.all() \
.aggregate(Max('student_module'))['student_module__max']
self.say("Last student_module_id is {}".format(last))
return last
def module_ids_to_check(self, batch_size):
"""Produce a sequence of student module ids to check.
`batch_size` is how many module ids to produce, max.
The sequence starts with `next_student_module_id`, and goes up to
and including `last_student_module_id`.
`next_student_module_id` is updated as each id is yielded.
"""
start = self.next_student_module_id
for smid in range(start, start + batch_size):
if smid > self.last_student_module_id:
break
yield smid
self.next_student_module_id = smid + 1
def get_history_for_student_modules(self, student_module_id):
"""
Get the history rows for a student module.
```student_module_id```: the id of the student module we're
interested in.
Return a list: [(id, created), ...], all the rows of history.
"""
history = StudentModuleHistory.objects \
.filter(student_module=student_module_id) \
.order_by('created', 'id')
return [(row.id, row.created) for row in history]
def delete_history(self, ids_to_delete):
"""
Delete history rows.
```ids_to_delete```: a non-empty list (or set...) of history row ids to delete.
"""
assert ids_to_delete
StudentModuleHistory.objects.filter(id__in=ids_to_delete).delete()
def clean_one_student_module(self, student_module_id):
"""Clean one StudentModule's-worth of history.
`student_module_id`: the id of the StudentModule to process.
"""
delete_gap = datetime.timedelta(seconds=self.DELETE_GAP_SECS)
history = self.get_history_for_student_modules(student_module_id)
if not history:
self.say("No history for student_module_id {}".format(student_module_id))
return
ids_to_delete = []
next_created = None
for history_id, created in reversed(history):
if next_created is not None:
# Compare this timestamp with the next one.
if (next_created - created) < delete_gap:
# This row is followed closely by another, we can discard
# this one.
ids_to_delete.append(history_id)
next_created = created
verb = "Would have deleted" if self.dry_run else "Deleting"
self.say("{verb} {to_delete} rows of {total} for student_module_id {id}".format(
verb=verb,
to_delete=len(ids_to_delete),
total=len(history),
id=student_module_id,
))
if ids_to_delete and not self.dry_run:
self.delete_history(ids_to_delete)
lms/djangoapps/courseware/migrations/0002_csmh-extended-keyspace.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import courseware.fields
from django.conf import settings
def bump_pk_start(apps, schema_editor):
if not schema_editor.connection.alias == 'student_module_history':
return
StudentModuleHistory = apps.get_model("courseware", "StudentModuleHistory")
initial_id = settings.STUDENTMODULEHISTORYEXTENDED_OFFSET + StudentModuleHistory.objects.all().latest('id').id
if schema_editor.connection.vendor == 'mysql':
schema_editor.execute('ALTER TABLE courseware_studentmodulehistoryextended AUTO_INCREMENT=%s', [initial_id])
elif schema_editor.connection.vendor == 'sqlite3':
# This is a hack to force sqlite to add new rows after the earlier rows we
# want to migrate.
StudentModuleHistory(
id=initial_id,
course_key=None,
usage_key=None,
username="",
version="",
created=datetime.datetime.now(),
).save()
class Migration(migrations.Migration):
dependencies = [
('courseware', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='StudentModuleHistoryExtended',
fields=[
('id', courseware.fields.UnsignedBigIntAutoField(serialize=False, primary_key=True)),
('version', models.CharField(db_index=True, max_length=255, null=True, blank=True)),
('created', models.DateTimeField(db_index=True)),
('state', models.TextField(null=True, blank=True)),
('grade', models.FloatField(null=True, blank=True)),
('max_grade', models.FloatField(null=True, blank=True)),
('student_module', models.ForeignKey(to='courseware.StudentModule', db_constraint=False)),
],
options={
'get_latest_by': 'created',
},
),
migrations.RunPython(bump_pk_start, reverse_code=migrations.RunPython.noop),
]
lms/djangoapps/courseware/models.py
...@@ -26,7 +26,7 @@ from student.models import user_by_anonymous_id ...@@ -26,7 +26,7 @@ from student.models import user_by_anonymous_id
from submissions.models import score_set, score_reset from submissions.models import score_set, score_reset
from xmodule_django.models import CourseKeyField, LocationKeyField, BlockTypeKeyField from xmodule_django.models import CourseKeyField, LocationKeyField, BlockTypeKeyField
log = logging.getLogger(__name__) from courseware.fields import UnsignedBigIntAutoField
log = logging.getLogger("edx.courseware") log = logging.getLogger("edx.courseware")
...@@ -188,6 +188,50 @@ class StudentModuleHistory(models.Model): ...@@ -188,6 +188,50 @@ class StudentModuleHistory(models.Model):
def __unicode__(self): def __unicode__(self):
return unicode(repr(self)) return unicode(repr(self))
class StudentModuleHistoryExtended(models.Model):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size.
This new extended CSMH has a larger primary key that won't run out of space
so quickly."""
objects = ChunkingManager()
HISTORY_SAVING_TYPES = {'problem'}
class Meta(object):
app_label = "courseware"
get_latest_by = "created"
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False)
version = models.CharField(max_length=255, null=True, blank=True, db_index=True)
# This should be populated from the modified field in StudentModule
created = models.DateTimeField(db_index=True)
state = models.TextField(null=True, blank=True)
grade = models.FloatField(null=True, blank=True)
max_grade = models.FloatField(null=True, blank=True)
@receiver(post_save, sender=StudentModule)
def save_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Checks the instance's module_type, and creates & saves a
StudentModuleHistory entry if the module_type is one that
we save.
"""
if instance.module_type in StudentModuleHistoryExtended.HISTORY_SAVING_TYPES:
history_entry = StudentModuleHistoryExtended(student_module=instance,
version=None,
created=instance.modified,
state=instance.state,
grade=instance.grade,
max_grade=instance.max_grade)
history_entry.save()
def __unicode__(self):
return unicode(repr(self))
class XBlockFieldBase(models.Model): class XBlockFieldBase(models.Model):
""" """
......
lms/djangoapps/courseware/routers.py 0 → 100644
"""
Database Routers for use with the courseware django app.
"""
class StudentModuleHistoryRouter(object):
"""
A Database Router that separates StudentModuleHistory into its own database.
"""
DATABASE_NAME = 'student_module_history'
def _is_csmh(self, model):
"""
Return True if ``model`` is courseware.StudentModuleHistory.
"""
return (
model._meta.app_label == 'courseware' and # pylint: disable=protected-access
model.__name__ == 'StudentModuleHistory'
)
def db_for_read(self, model, **hints): # pylint: disable=unused-argument
"""
Use the StudentModuleHistoryRouter.DATABASE_NAME if the model is StudentModuleHistory.
"""
if self._is_csmh(model):
return self.DATABASE_NAME
else:
return None
def db_for_write(self, model, **hints): # pylint: disable=unused-argument
"""
Use the StudentModuleHistoryRouter.DATABASE_NAME if the model is StudentModuleHistory.
"""
if self._is_csmh(model):
return self.DATABASE_NAME
else:
return None
def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument
"""
Disable relations if the model is StudentModuleHistory.
"""
if self._is_csmh(obj1) or self._is_csmh(obj2):
return False
return None
def allow_migrate(self, db, model): # pylint: disable=unused-argument
"""
Only sync StudentModuleHistory to StudentModuleHistoryRouter.DATABASE_Name
"""
if self._is_csmh(model):
return db == self.DATABASE_NAME
elif db == self.DATABASE_NAME:
return False
return None
lms/djangoapps/courseware/tests/test_model_data.py
...@@ -249,8 +249,9 @@ class TestMissingStudentModule(TestCase): ...@@ -249,8 +249,9 @@ class TestMissingStudentModule(TestCase):
# to discover if something other than the DjangoXBlockUserStateClient # to discover if something other than the DjangoXBlockUserStateClient
# has written to the StudentModule (such as UserStateCache setting the score # has written to the StudentModule (such as UserStateCache setting the score
# on the StudentModule). # on the StudentModule).
with self.assertNumQueries(5): with self.assertNumQueries(2, using='default'):
self.kvs.set(user_state_key('a_field'), 'a_value') with self.assertNumQueries(1, using='student_module_history'):
self.kvs.set(user_state_key('a_field'), 'a_value')
self.assertEquals(1, sum(len(cache) for cache in self.field_data_cache.cache.values())) self.assertEquals(1, sum(len(cache) for cache in self.field_data_cache.cache.values()))
self.assertEquals(1, StudentModule.objects.all().count()) self.assertEquals(1, StudentModule.objects.all().count())
......
lms/envs/acceptance.py
...@@ -72,6 +72,14 @@ DATABASES = { ...@@ -72,6 +72,14 @@ DATABASES = {
'timeout': 30, 'timeout': 30,
}, },
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'OPTIONS': {
'timeout': 30,
},
} }
} }
......
lms/envs/aws.py
...@@ -759,6 +759,11 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get( ...@@ -759,6 +759,11 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get(
# Course Content Bookmarks Settings # Course Content Bookmarks Settings
MAX_BOOKMARKS_PER_COURSE = ENV_TOKENS.get('MAX_BOOKMARKS_PER_COURSE', MAX_BOOKMARKS_PER_COURSE) MAX_BOOKMARKS_PER_COURSE = ENV_TOKENS.get('MAX_BOOKMARKS_PER_COURSE', MAX_BOOKMARKS_PER_COURSE)
# Offset for pk of courseware.StudentModuleHistoryExtended
STUDENTMODULEHISTORYEXTENDED_OFFSET = ENV_TOKENS.get(
'STUDENTMODULEHISTORYEXTENDED_OFFSET', STUDENTMODULEHISTORYEXTENDED_OFFSET
)
# Cutoff date for granting audit certificates # Cutoff date for granting audit certificates
if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None): if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None):
AUDIT_CERT_CUTOFF_DATE = dateutil.parser.parse(ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE')) AUDIT_CERT_CUTOFF_DATE = dateutil.parser.parse(ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE'))
......
lms/envs/aws_migrate.py
...@@ -13,18 +13,25 @@ from .aws import * ...@@ -13,18 +13,25 @@ from .aws import *
import os import os
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
DB_OVERRIDES = dict(
PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
USER=os.environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
)
if DB_OVERRIDES['PASSWORD'] is None: def get_db_overrides(db_name):
raise ImproperlyConfigured("No database password was provided for running " """
"migrations. This is fatal.") Now that we have multiple databases, we want to look up from the environment
for both databases.
"""
db_overrides = dict(
PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES[db_name]['ENGINE']),
USER=os.environ.get('DB_MIGRATION_USER', DATABASES[db_name]['USER']),
NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES[db_name]['NAME']),
HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES[db_name]['HOST']),
PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES[db_name]['PORT']),
)
for override, value in DB_OVERRIDES.iteritems(): if db_overrides['PASSWORD'] is None:
DATABASES['default'][override] = value raise ImproperlyConfigured("No database password was provided for running "
"migrations. This is fatal.")
return db_overrides
for db in ['default', 'student_module_history']:
DATABASES[db].update(get_db_overrides(db))
lms/envs/bok_choy.auth.json
...@@ -39,6 +39,14 @@ ...@@ -39,6 +39,14 @@
"PASSWORD": "", "PASSWORD": "",
"PORT": "3306", "PORT": "3306",
"USER": "root" "USER": "root"
},
"student_module_history": {
"ENGINE": "django.db.backends.mysql",
"HOST": "localhost",
"NAME": "student_module_history_test",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
} }
}, },
"DOC_STORE_CONFIG": { "DOC_STORE_CONFIG": {
......
lms/envs/common.py
...@@ -413,6 +413,12 @@ GEOIPV6_PATH = REPO_ROOT / "common/static/data/geoip/GeoIPv6.dat" ...@@ -413,6 +413,12 @@ GEOIPV6_PATH = REPO_ROOT / "common/static/data/geoip/GeoIPv6.dat"
# Where to look for a status message # Where to look for a status message
STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json" STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json"
############################ Global Database Configuration #####################
DATABASE_ROUTERS = [
'courseware.routers.StudentModuleHistoryRouter',
]
############################ OpenID Provider ################################## ############################ OpenID Provider ##################################
OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net'] OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net']
...@@ -2759,6 +2765,10 @@ MOBILE_APP_USER_AGENT_REGEXES = [ ...@@ -2759,6 +2765,10 @@ MOBILE_APP_USER_AGENT_REGEXES = [
r'edX/org.edx.mobile', r'edX/org.edx.mobile',
] ]
# Offset for courseware.StudentModuleHistoryExtended which is used to
# calculate the starting primary key for the underlying table.
STUDENTMODULEHISTORYEXTENDED_OFFSET = 10000
# Deprecated xblock types # Deprecated xblock types
DEPRECATED_ADVANCED_COMPONENT_TYPES = [] DEPRECATED_ADVANCED_COMPONENT_TYPES = []
......
lms/envs/dev.py
...@@ -48,6 +48,11 @@ DATABASES = { ...@@ -48,6 +48,11 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "edx.db", 'NAME': ENV_ROOT / "db" / "edx.db",
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "student_module_history.db",
'ATOMIC_REQUESTS': True,
} }
} }
......
lms/envs/devplus.py
...@@ -31,6 +31,15 @@ DATABASES = { ...@@ -31,6 +31,15 @@ DATABASES = {
'HOST': '127.0.0.1', 'HOST': '127.0.0.1',
'PORT': '3306', 'PORT': '3306',
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'student_module_history',
'USER': 'root',
'PASSWORD': '',
'HOST': '127.0.0.1',
'PORT': '3306',
'ATOMIC_REQUESTS': True,
} }
} }
......
lms/envs/static.py
...@@ -27,6 +27,11 @@ DATABASES = { ...@@ -27,6 +27,11 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "edx.db", 'NAME': ENV_ROOT / "db" / "edx.db",
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "student_module_history.db",
'ATOMIC_REQUESTS': True,
} }
} }
......
lms/envs/test.py
...@@ -186,7 +186,10 @@ DATABASES = { ...@@ -186,7 +186,10 @@ DATABASES = {
'NAME': TEST_ROOT / 'db' / 'edx.db', 'NAME': TEST_ROOT / 'db' / 'edx.db',
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
}, },
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / 'db' / 'student_module_history.db'
},
} }
if os.environ.get('DISABLE_MIGRATIONS'): if os.environ.get('DISABLE_MIGRATIONS'):
......
lms/envs/test_static_optimized.py
...@@ -19,7 +19,9 @@ DATABASES = { ...@@ -19,7 +19,9 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.sqlite3',
'ATOMIC_REQUESTS': True, 'ATOMIC_REQUESTS': True,
}, },
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
},
} }
# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up # Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up
......
scripts/reset-test-db.sh
...@@ -24,29 +24,54 @@ ...@@ -24,29 +24,54 @@
DB_CACHE_DIR="common/test/db_cache" DB_CACHE_DIR="common/test/db_cache"
declare -A databases
databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
# Ensure the test database exists. # Ensure the test database exists.
echo "CREATE DATABASE IF NOT EXISTS edxtest;" | mysql -u root for db in "${!databases[@]}"; do
echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root
# Clear out the test database
#
# We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
# "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
# or a jenkins worker environment) that already ran tests on another commit that had
# different migrations that created, dropped, or altered tables.
echo "Issuing a reset_db command to the bok_choy MySQL database."
./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db
# If there are cached database schemas/data, load them
if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json ]]; then
echo "Missing $DB_CACHE_DIR/bok_choy_schema_$db.sql or $DB_CACHE_DIR/bok_choy_data_$db.json, rebuilding cache"
REBUILD_CACHE=true
fi
done
# migrations are only stored in the default database
if [[ ! -f $DB_CACHE_DIR/bok_choy_migrations_data.sql ]]; then
REBUILD_CACHE=true
fi
# Clear out the test database
#
# We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
# "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
# or a jenkins worker environment) that already ran tests on another commit that had
# different migrations that created, dropped, or altered tables.
echo "Issuing a reset_db command to the bok_choy MySQL database."
./manage.py lms --settings bok_choy reset_db --traceback --noinput
# If there are cached database schemas/data, load them # If there are cached database schemas/data, load them
if [[ -f $DB_CACHE_DIR/bok_choy_schema.sql && -f $DB_CACHE_DIR/bok_choy_migrations_data.sql && -f $DB_CACHE_DIR/bok_choy_data.json ]]; then if [[ -z $REBUILD_CACHE ]]; then
echo "Found the bok_choy DB cache files. Loading them into the database..." echo "Found the bok_choy DB cache files. Loading them into the database..."
# Load the schema, then the data (including the migration history)
echo "Loading the schema from the filesystem into the MySQL DB." for db in "${!databases[@]}"; do
mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_schema.sql # Load the schema, then the data (including the migration history)
echo "Loading the schema from the filesystem into the MySQL DB."
mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
echo "Loading the fixture data from the filesystem into the MySQL DB."
./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
done
# Migrations are stored in the default database
echo "Loading the migration data from the filesystem into the MySQL DB." echo "Loading the migration data from the filesystem into the MySQL DB."
mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_migrations_data.sql mysql -u root "${databases['default']}" < $DB_CACHE_DIR/bok_choy_migrations_data.sql
echo "Loading the fixture data from the filesystem into the MySQL DB."
./manage.py lms --settings bok_choy loaddata $DB_CACHE_DIR/bok_choy_data.json
# Re-run migrations to ensure we are up-to-date # Re-run migrations to ensure we are up-to-date
echo "Running the lms migrations on the bok_choy DB." echo "Running the lms migrations on the bok_choy DB."
...@@ -66,13 +91,15 @@ else ...@@ -66,13 +91,15 @@ else
echo "Issuing a migrate command to the bok_choy MySQL database for the cms django apps." echo "Issuing a migrate command to the bok_choy MySQL database for the cms django apps."
./manage.py cms --settings bok_choy migrate --traceback --noinput ./manage.py cms --settings bok_choy migrate --traceback --noinput
# Dump the schema and data to the cache for db in "${!databases[@]}"; do
echo "Using the dumpdata command to save the fixture data to the filesystem." # Dump the schema and data to the cache
./manage.py lms --settings bok_choy dumpdata > $DB_CACHE_DIR/bok_choy_data.json echo "Using the dumpdata command to save the fixture data to the filesystem."
./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json
echo "Saving the schema of the bok_choy DB to the filesystem."
mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
done
# dump_data does not dump the django_migrations table so we do it separately. # dump_data does not dump the django_migrations table so we do it separately.
echo "Saving the django_migrations table of the bok_choy DB to the filesystem." echo "Saving the django_migrations table of the bok_choy DB to the filesystem."
mysqldump -u root --no-create-info edxtest django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data.sql mysqldump -u root --no-create-info "${databases['default']}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data.sql
echo "Saving the schema of the bok_choy DB to the filesystem."
mysqldump -u root --no-data --skip-comments --skip-dump-date edxtest > $DB_CACHE_DIR/bok_choy_schema.sql
fi fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment