Merge pull request #11569 from edx/jibsheet/csmh-extended

Student Module History Extension

cms/envs/acceptance.py

@@ -80,6 +80,14 @@ DATABASES = {
             'timeout': 30,
         },
         'ATOMIC_REQUESTS': True,
+    },
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
+        'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
+        'OPTIONS': {
+            'timeout': 30,
+        },
     }
 }
 

cms/envs/aws_migrate.py

@@ -26,5 +26,5 @@ if DB_OVERRIDES['PASSWORD'] is None:
     raise ImproperlyConfigured("No database password was provided for running "
                                "migrations.  This is fatal.")
 
-for override, value in DB_OVERRIDES.iteritems():
-    DATABASES['default'][override] = value
+DATABASES['default'].update(DB_OVERRIDES)
+DATABASES['student_module_history'].update(DB_OVERRIDES)

cms/envs/bok_choy.auth.json

@@ -30,6 +30,14 @@
             "PASSWORD": "",
             "PORT": "3306",
             "USER": "root"
+        },
+        "student_module_history": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "localhost",
+            "NAME": "student_module_history_test",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
         }
     },
     "DOC_STORE_CONFIG": {

cms/envs/common.py

@@ -1114,6 +1114,11 @@ PROCTORING_BACKEND_PROVIDER = {
 }
 PROCTORING_SETTINGS = {}
 
+############################ Global Database Configuration #####################
+
+DATABASE_ROUTERS = [
+    'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter',
+]
 
 ############################ OAUTH2 Provider ###################################
 

common/lib/xmodule/xmodule/capa_module.py

@@ -93,11 +93,23 @@ class CapaModule(CapaMixin, XModule):
             result = handlers[dispatch](data)
 
         except NotFoundError as err:
-            _, _, traceback_obj = sys.exc_info()
+            log.exception(
+                "Unable to find data when dispatching %s to %s for user %s",
+                dispatch,
+                self.scope_ids.usage_id,
+                self.scope_ids.user_id
+            )
+            _, _, traceback_obj = sys.exc_info()  # pylint: disable=redefined-outer-name
             raise ProcessingError(not_found_error_message), None, traceback_obj
 
         except Exception as err:
-            _, _, traceback_obj = sys.exc_info()
+            log.exception(
+                "Unknown error when dispatching %s to %s for user %s",
+                dispatch,
+                self.scope_ids.usage_id,
+                self.scope_ids.user_id
+            )
+            _, _, traceback_obj = sys.exc_info()  # pylint: disable=redefined-outer-name
             raise ProcessingError(generic_error_message), None, traceback_obj
 
         after = self.get_progress()

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -265,6 +265,8 @@ class SharedModuleStoreTestCase(TestCase):
     for Django ORM models that will get cleaned up properly.
     """
     MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     @classmethod
     def setUpClass(cls):
@@ -392,6 +394,8 @@ class ModuleStoreTestCase(TestCase):
     """
 
     MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     def setUp(self, **kwargs):
         """

common/test/db_cache/bok_choy_data_student_module_history.json

+[]
\ No newline at end of file

common/test/db_cache/bok_choy_migrations_data.sql

--- MySQL dump 10.13  Distrib 5.6.24, for debian-linux-gnu (x86_64)
---
--- Host: localhost    Database: edxtest
--- ------------------------------------------------------
--- Server version	5.6.24-2+deb.sury.org~precise+2
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Dumping data for table `django_migrations`
---
-
-LOCK TABLES `django_migrations` WRITE;
-/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
-INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-01-22 20:04:03.218179'),(2,'auth','0001_initial','2016-01-22 20:04:03.554767'),(3,'admin','0001_initial','2016-01-22 20:04:03.665239'),(4,'assessment','0001_initial','2016-01-22 20:04:06.917971'),(5,'assessment','0002_staffworkflow','2016-01-22 20:04:07.125841'),(6,'contenttypes','0002_remove_content_type_name','2016-01-22 20:04:07.307570'),(7,'auth','0002_alter_permission_name_max_length','2016-01-22 20:04:07.383568'),(8,'auth','0003_alter_user_email_max_length','2016-01-22 20:04:07.464880'),(9,'auth','0004_alter_user_username_opts','2016-01-22 20:04:07.496608'),(10,'auth','0005_alter_user_last_login_null','2016-01-22 20:04:07.584818'),(11,'auth','0006_require_contenttypes_0002','2016-01-22 20:04:07.591565'),(12,'bookmarks','0001_initial','2016-01-22 20:04:07.932691'),(13,'branding','0001_initial','2016-01-22 20:04:08.105513'),(14,'bulk_email','0001_initial','2016-01-22 20:04:08.469986'),(15,'bulk_email','0002_data__load_course_email_template','2016-01-22 20:04:08.538284'),(16,'instructor_task','0001_initial','2016-01-22 20:04:08.743012'),(17,'certificates','0001_initial','2016-01-22 20:04:09.838773'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-01-22 20:04:09.865157'),(19,'certificates','0003_data__default_modes','2016-01-22 20:04:09.943086'),(20,'certificates','0004_certificategenerationhistory','2016-01-22 20:04:10.116067'),(21,'certificates','0005_auto_20151208_0801','2016-01-22 20:04:10.232215'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-01-22 20:04:10.307271'),(23,'certificates','0007_certificateinvalidation','2016-01-22 20:04:10.510089'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-01-22 20:04:10.538558'),(25,'cors_csrf','0001_initial','2016-01-22 20:04:10.680353'),(26,'course_action_state','0001_initial','2016-01-22 20:04:11.030484'),(27,'course_groups','0001_initial','2016-01-22 20:04:12.283946'),(28,'course_modes','0001_initial','2016-01-22 20:04:12.452546'),(29,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-01-22 20:04:12.534953'),(30,'course_modes','0003_auto_20151113_1443','2016-01-22 20:04:12.569936'),(31,'course_modes','0004_auto_20151113_1457','2016-01-22 20:04:12.776505'),(32,'course_modes','0005_auto_20151217_0958','2016-01-22 20:04:12.807317'),(33,'course_overviews','0001_initial','2016-01-22 20:04:12.927012'),(34,'course_overviews','0002_add_course_catalog_fields','2016-01-22 20:04:13.220589'),(35,'course_overviews','0003_courseoverviewgeneratedhistory','2016-01-22 20:04:13.258023'),(36,'course_overviews','0004_courseoverview_org','2016-01-22 20:04:13.346700'),(37,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-01-22 20:04:13.375580'),(38,'course_overviews','0006_courseoverviewimageset','2016-01-22 20:04:13.464516'),(39,'course_overviews','0007_courseoverviewimageconfig','2016-01-22 20:04:13.651285'),(40,'course_structures','0001_initial','2016-01-22 20:04:13.689179'),(41,'courseware','0001_initial','2016-01-22 20:04:17.245019'),(42,'credentials','0001_initial','2016-01-22 20:04:17.454975'),(43,'credentials','0002_data__add_service_user','2016-01-22 20:04:17.494478'),(44,'credit','0001_initial','2016-01-22 20:04:19.593607'),(45,'dark_lang','0001_initial','2016-01-22 20:04:19.851812'),(46,'dark_lang','0002_data__enable_on_install','2016-01-22 20:04:19.886816'),(47,'default','0001_initial','2016-01-22 20:04:20.656170'),(48,'default','0002_add_related_name','2016-01-22 20:04:20.920330'),(49,'default','0003_alter_email_max_length','2016-01-22 20:04:21.038154'),(50,'django_comment_common','0001_initial','2016-01-22 20:04:21.777797'),(51,'django_notify','0001_initial','2016-01-22 20:04:23.074494'),(52,'django_openid_auth','0001_initial','2016-01-22 20:04:23.435834'),(53,'edx_proctoring','0001_initial','2016-01-22 20:04:28.470618'),(54,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-01-22 20:04:28.834557'),(55,'edx_proctoring','0003_auto_20160101_0525','2016-01-22 20:04:29.402026'),(56,'edxval','0001_initial','2016-01-22 20:04:30.249661'),(57,'edxval','0002_data__default_profiles','2016-01-22 20:04:30.307115'),(58,'embargo','0001_initial','2016-01-22 20:04:31.462353'),(59,'embargo','0002_data__add_countries','2016-01-22 20:04:32.017366'),(60,'external_auth','0001_initial','2016-01-22 20:04:32.753375'),(61,'lms_xblock','0001_initial','2016-01-22 20:04:33.090177'),(62,'sites','0001_initial','2016-01-22 20:04:33.156273'),(63,'microsite_configuration','0001_initial','2016-01-22 20:04:35.468782'),(64,'milestones','0001_initial','2016-01-22 20:04:36.698849'),(65,'milestones','0002_data__seed_relationship_types','2016-01-22 20:04:36.760089'),(66,'mobile_api','0001_initial','2016-01-22 20:04:38.189431'),(67,'notes','0001_initial','2016-01-22 20:04:38.639288'),(68,'oauth2','0001_initial','2016-01-22 20:04:40.968535'),(69,'oauth2_provider','0001_initial','2016-01-22 20:04:41.475836'),(70,'oauth_provider','0001_initial','2016-01-22 20:04:42.704759'),(71,'organizations','0001_initial','2016-01-22 20:04:42.985226'),(72,'organizations','0002_auto_20151119_2048','2016-01-22 20:04:43.049898'),(73,'problem_builder','0001_initial','2016-01-22 20:04:43.244682'),(74,'programs','0001_initial','2016-01-22 20:04:43.776265'),(75,'programs','0002_programsapiconfig_cache_ttl','2016-01-22 20:04:44.329122'),(76,'programs','0003_auto_20151120_1613','2016-01-22 20:04:46.494468'),(77,'rss_proxy','0001_initial','2016-01-22 20:04:46.572259'),(78,'self_paced','0001_initial','2016-01-22 20:04:47.142336'),(79,'sessions','0001_initial','2016-01-22 20:04:47.246566'),(80,'student','0001_initial','2016-01-22 20:05:04.804775'),(81,'shoppingcart','0001_initial','2016-01-22 20:05:20.059855'),(82,'shoppingcart','0002_auto_20151208_1034','2016-01-22 20:05:21.457216'),(83,'shoppingcart','0003_auto_20151217_0958','2016-01-22 20:05:22.922754'),(84,'splash','0001_initial','2016-01-22 20:05:23.712228'),(85,'static_replace','0001_initial','2016-01-22 20:05:24.489567'),(86,'status','0001_initial','2016-01-22 20:05:26.131108'),(87,'student','0002_auto_20151208_1034','2016-01-22 20:05:27.780831'),(88,'submissions','0001_initial','2016-01-22 20:05:29.846153'),(89,'submissions','0002_auto_20151119_0913','2016-01-22 20:05:30.098711'),(90,'survey','0001_initial','2016-01-22 20:05:31.137171'),(91,'teams','0001_initial','2016-01-22 20:05:33.436431'),(92,'third_party_auth','0001_initial','2016-01-22 20:05:38.010830'),(93,'track','0001_initial','2016-01-22 20:05:38.097314'),(94,'user_api','0001_initial','2016-01-22 20:05:44.556502'),(95,'util','0001_initial','2016-01-22 20:05:45.512595'),(96,'util','0002_data__default_rate_limit_config','2016-01-22 20:05:45.583331'),(97,'verify_student','0001_initial','2016-01-22 20:05:56.456343'),(98,'verify_student','0002_auto_20151124_1024','2016-01-22 20:05:58.780629'),(99,'verify_student','0003_auto_20151113_1443','2016-01-22 20:05:59.622634'),(100,'wiki','0001_initial','2016-01-22 20:06:30.273480'),(101,'wiki','0002_remove_article_subscription','2016-01-22 20:06:30.358472'),(102,'workflow','0001_initial','2016-01-22 20:06:30.741041'),(103,'xblock_django','0001_initial','2016-01-22 20:06:31.645852'),(104,'contentstore','0001_initial','2016-01-22 20:07:00.042399'),(105,'course_creators','0001_initial','2016-01-22 20:07:00.153769'),(106,'xblock_config','0001_initial','2016-01-22 20:07:00.552942');
-/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
-UNLOCK TABLES;
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
-/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2016-01-22 20:07:05

common/test/db_cache/bok_choy_migrations_data_default.sql

+-- MySQL dump 10.13  Distrib 5.6.14, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: edxtest
+-- ------------------------------------------------------
+-- Server version	5.6.14-1+debphp.org~precise+1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Dumping data for table `django_migrations`
+--
+
+LOCK TABLES `django_migrations` WRITE;
+/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
+INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-02-29 17:58:04.910265'),(2,'auth','0001_initial','2016-02-29 17:58:05.155297'),(3,'admin','0001_initial','2016-02-29 17:58:05.261098'),(4,'assessment','0001_initial','2016-02-29 17:58:08.458980'),(5,'assessment','0002_staffworkflow','2016-02-29 17:58:08.659081'),(6,'contenttypes','0002_remove_content_type_name','2016-02-29 17:58:08.818265'),(7,'auth','0002_alter_permission_name_max_length','2016-02-29 17:58:08.855835'),(8,'auth','0003_alter_user_email_max_length','2016-02-29 17:58:08.894682'),(9,'auth','0004_alter_user_username_opts','2016-02-29 17:58:08.918567'),(10,'auth','0005_alter_user_last_login_null','2016-02-29 17:58:08.971615'),(11,'auth','0006_require_contenttypes_0002','2016-02-29 17:58:08.976069'),(12,'bookmarks','0001_initial','2016-02-29 17:58:09.243246'),(13,'branding','0001_initial','2016-02-29 17:58:09.364920'),(14,'bulk_email','0001_initial','2016-02-29 17:58:09.630882'),(15,'bulk_email','0002_data__load_course_email_template','2016-02-29 17:58:09.715512'),(16,'instructor_task','0001_initial','2016-02-29 17:58:09.872596'),(17,'certificates','0001_initial','2016-02-29 17:58:10.722743'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-02-29 17:58:10.736641'),(19,'certificates','0003_data__default_modes','2016-02-29 17:58:10.775334'),(20,'certificates','0004_certificategenerationhistory','2016-02-29 17:58:10.911498'),(21,'certificates','0005_auto_20151208_0801','2016-02-29 17:58:11.002303'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-02-29 17:58:11.050549'),(23,'certificates','0007_certificateinvalidation','2016-02-29 17:58:11.166297'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-02-29 17:58:11.198023'),(25,'commerce','0002_commerceconfiguration','2016-02-29 17:58:11.291907'),(26,'contentserver','0001_initial','2016-02-29 17:58:11.375676'),(27,'cors_csrf','0001_initial','2016-02-29 17:58:11.462189'),(28,'course_action_state','0001_initial','2016-02-29 17:58:11.783744'),(29,'course_groups','0001_initial','2016-02-29 17:58:12.790006'),(30,'course_modes','0001_initial','2016-02-29 17:58:12.902796'),(31,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-02-29 17:58:12.957209'),(32,'course_modes','0003_auto_20151113_1443','2016-02-29 17:58:12.979234'),(33,'course_modes','0004_auto_20151113_1457','2016-02-29 17:58:13.095320'),(34,'course_modes','0005_auto_20151217_0958','2016-02-29 17:58:13.116644'),(35,'course_modes','0006_auto_20160208_1407','2016-02-29 17:58:13.208193'),(36,'course_overviews','0001_initial','2016-02-29 17:58:13.283088'),(37,'course_overviews','0002_add_course_catalog_fields','2016-02-29 17:58:13.499090'),(38,'course_overviews','0003_courseoverviewgeneratedhistory','2016-02-29 17:58:13.525866'),(39,'course_overviews','0004_courseoverview_org','2016-02-29 17:58:13.567577'),(40,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-02-29 17:58:13.585458'),(41,'course_overviews','0006_courseoverviewimageset','2016-02-29 17:58:13.636894'),(42,'course_overviews','0007_courseoverviewimageconfig','2016-02-29 17:58:13.764616'),(43,'course_overviews','0008_remove_courseoverview_facebook_url','2016-02-29 17:58:13.815867'),(44,'course_overviews','0009_readd_facebook_url','2016-02-29 17:58:13.872971'),(45,'course_structures','0001_initial','2016-02-29 17:58:13.899706'),(46,'courseware','0001_initial','2016-02-29 17:58:16.765503'),(47,'coursewarehistoryextended','0001_initial','2016-02-29 17:58:16.912454'),(48,'credentials','0001_initial','2016-02-29 17:58:17.072454'),(49,'credit','0001_initial','2016-02-29 17:58:18.478297'),(50,'dark_lang','0001_initial','2016-02-29 17:58:18.635638'),(51,'dark_lang','0002_data__enable_on_install','2016-02-29 17:58:18.654315'),(52,'default','0001_initial','2016-02-29 17:58:19.121037'),(53,'default','0002_add_related_name','2016-02-29 17:58:19.287646'),(54,'default','0003_alter_email_max_length','2016-02-29 17:58:19.355367'),(55,'django_comment_common','0001_initial','2016-02-29 17:58:19.826406'),(56,'django_notify','0001_initial','2016-02-29 17:58:20.691115'),(57,'django_openid_auth','0001_initial','2016-02-29 17:58:20.948196'),(58,'edx_proctoring','0001_initial','2016-02-29 17:58:25.028310'),(59,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-02-29 17:58:25.270722'),(60,'edx_proctoring','0003_auto_20160101_0525','2016-02-29 17:58:25.651659'),(61,'edx_proctoring','0004_auto_20160201_0523','2016-02-29 17:58:25.881209'),(62,'edxval','0001_initial','2016-02-29 17:58:26.490038'),(63,'edxval','0002_data__default_profiles','2016-02-29 17:58:26.519947'),(64,'embargo','0001_initial','2016-02-29 17:58:27.388295'),(65,'embargo','0002_data__add_countries','2016-02-29 17:58:27.716022'),(66,'external_auth','0001_initial','2016-02-29 17:58:28.339793'),(67,'lms_xblock','0001_initial','2016-02-29 17:58:28.600358'),(68,'sites','0001_initial','2016-02-29 17:58:28.639305'),(69,'microsite_configuration','0001_initial','2016-02-29 17:58:30.470563'),(70,'microsite_configuration','0002_auto_20160202_0228','2016-02-29 17:58:31.089884'),(71,'milestones','0001_initial','2016-02-29 17:58:32.206321'),(72,'milestones','0002_data__seed_relationship_types','2016-02-29 17:58:32.242905'),(73,'milestones','0003_coursecontentmilestone_requirements','2016-02-29 17:58:33.317573'),(74,'milestones','0004_auto_20151221_1445','2016-02-29 17:58:33.593210'),(75,'mobile_api','0001_initial','2016-02-29 17:58:33.849089'),(76,'notes','0001_initial','2016-02-29 17:58:34.185220'),(77,'oauth2','0001_initial','2016-02-29 17:58:35.892898'),(78,'oauth2_provider','0001_initial','2016-02-29 17:58:36.227236'),(79,'oauth_provider','0001_initial','2016-02-29 17:58:37.052991'),(80,'organizations','0001_initial','2016-02-29 17:58:37.290331'),(81,'programs','0001_initial','2016-02-29 17:58:37.667708'),(82,'programs','0002_programsapiconfig_cache_ttl','2016-02-29 17:58:38.067119'),(83,'programs','0003_auto_20151120_1613','2016-02-29 17:58:39.752602'),(84,'programs','0004_programsapiconfig_enable_certification','2016-02-29 17:58:40.179508'),(85,'programs','0005_programsapiconfig_max_retries','2016-02-29 17:58:40.596412'),(86,'rss_proxy','0001_initial','2016-02-29 17:58:40.641620'),(87,'self_paced','0001_initial','2016-02-29 17:58:41.078194'),(88,'sessions','0001_initial','2016-02-29 17:58:41.133704'),(89,'student','0001_initial','2016-02-29 17:58:53.980867'),(90,'shoppingcart','0001_initial','2016-02-29 17:59:06.233175'),(91,'shoppingcart','0002_auto_20151208_1034','2016-02-29 17:59:07.059000'),(92,'shoppingcart','0003_auto_20151217_0958','2016-02-29 17:59:07.907660'),(93,'splash','0001_initial','2016-02-29 17:59:08.351083'),(94,'static_replace','0001_initial','2016-02-29 17:59:08.818049'),(95,'static_replace','0002_assetexcludedextensionsconfig','2016-02-29 17:59:09.319193'),(96,'status','0001_initial','2016-02-29 17:59:10.500113'),(97,'student','0002_auto_20151208_1034','2016-02-29 17:59:11.650009'),(98,'submissions','0001_initial','2016-02-29 17:59:12.491480'),(99,'submissions','0002_auto_20151119_0913','2016-02-29 17:59:12.683289'),(100,'submissions','0003_submission_status','2016-02-29 17:59:12.811034'),(101,'survey','0001_initial','2016-02-29 17:59:13.767659'),(102,'teams','0001_initial','2016-02-29 17:59:16.603866'),(103,'third_party_auth','0001_initial','2016-02-29 17:59:19.924001'),(104,'track','0001_initial','2016-02-29 17:59:19.974203'),(105,'user_api','0001_initial','2016-02-29 17:59:24.197607'),(106,'util','0001_initial','2016-02-29 17:59:24.978364'),(107,'util','0002_data__default_rate_limit_config','2016-02-29 17:59:25.017205'),(108,'verify_student','0001_initial','2016-02-29 17:59:33.329519'),(109,'verify_student','0002_auto_20151124_1024','2016-02-29 17:59:34.238392'),(110,'verify_student','0003_auto_20151113_1443','2016-02-29 17:59:35.001393'),(111,'wiki','0001_initial','2016-02-29 17:59:56.418517'),(112,'wiki','0002_remove_article_subscription','2016-02-29 17:59:56.476734'),(113,'workflow','0001_initial','2016-02-29 17:59:56.760505'),(114,'xblock_django','0001_initial','2016-02-29 17:59:57.572216'),(115,'xblock_django','0002_auto_20160204_0809','2016-02-29 17:59:58.380438'),(116,'contentstore','0001_initial','2016-02-29 18:00:19.534458'),(117,'course_creators','0001_initial','2016-02-29 18:00:19.597219'),(118,'xblock_config','0001_initial','2016-02-29 18:00:19.855097');
+/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-02-29 18:00:23

common/test/db_cache/bok_choy_migrations_data_student_module_history.sql

+-- MySQL dump 10.13  Distrib 5.6.14, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: student_module_history_test
+-- ------------------------------------------------------
+-- Server version	5.6.14-1+debphp.org~precise+1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Dumping data for table `django_migrations`
+--
+
+LOCK TABLES `django_migrations` WRITE;
+/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
+INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-02-29 18:01:39.511327'),(2,'auth','0001_initial','2016-02-29 18:01:39.547129'),(3,'admin','0001_initial','2016-02-29 18:01:39.568351'),(4,'assessment','0001_initial','2016-02-29 18:01:40.149754'),(5,'assessment','0002_staffworkflow','2016-02-29 18:01:40.160082'),(6,'contenttypes','0002_remove_content_type_name','2016-02-29 18:01:40.231907'),(7,'auth','0002_alter_permission_name_max_length','2016-02-29 18:01:40.251926'),(8,'auth','0003_alter_user_email_max_length','2016-02-29 18:01:40.276419'),(9,'auth','0004_alter_user_username_opts','2016-02-29 18:01:40.297224'),(10,'auth','0005_alter_user_last_login_null','2016-02-29 18:01:40.317098'),(11,'auth','0006_require_contenttypes_0002','2016-02-29 18:01:40.319374'),(12,'bookmarks','0001_initial','2016-02-29 18:01:40.396366'),(13,'branding','0001_initial','2016-02-29 18:01:40.453783'),(14,'bulk_email','0001_initial','2016-02-29 18:01:40.565081'),(15,'bulk_email','0002_data__load_course_email_template','2016-02-29 18:01:40.574000'),(16,'instructor_task','0001_initial','2016-02-29 18:01:40.613011'),(17,'certificates','0001_initial','2016-02-29 18:01:40.980762'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-02-29 18:01:40.991190'),(19,'certificates','0003_data__default_modes','2016-02-29 18:01:41.003742'),(20,'certificates','0004_certificategenerationhistory','2016-02-29 18:01:41.061308'),(21,'certificates','0005_auto_20151208_0801','2016-02-29 18:01:41.113837'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-02-29 18:01:41.131141'),(23,'certificates','0007_certificateinvalidation','2016-02-29 18:01:41.189247'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-02-29 18:01:41.200385'),(25,'commerce','0002_commerceconfiguration','2016-02-29 18:01:41.260287'),(26,'contentserver','0001_initial','2016-02-29 18:01:41.323580'),(27,'cors_csrf','0001_initial','2016-02-29 18:01:41.388714'),(28,'course_action_state','0001_initial','2016-02-29 18:01:41.514544'),(29,'course_groups','0001_initial','2016-02-29 18:01:42.036647'),(30,'course_modes','0001_initial','2016-02-29 18:01:42.068946'),(31,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-02-29 18:01:42.084746'),(32,'course_modes','0003_auto_20151113_1443','2016-02-29 18:01:42.100516'),(33,'course_modes','0004_auto_20151113_1457','2016-02-29 18:01:42.187139'),(34,'course_modes','0005_auto_20151217_0958','2016-02-29 18:01:42.205556'),(35,'course_modes','0006_auto_20160208_1407','2016-02-29 18:01:42.289731'),(36,'course_overviews','0001_initial','2016-02-29 18:01:42.319879'),(37,'course_overviews','0002_add_course_catalog_fields','2016-02-29 18:01:42.402948'),(38,'course_overviews','0003_courseoverviewgeneratedhistory','2016-02-29 18:01:42.417563'),(39,'course_overviews','0004_courseoverview_org','2016-02-29 18:01:42.435373'),(40,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-02-29 18:01:42.448847'),(41,'course_overviews','0006_courseoverviewimageset','2016-02-29 18:01:42.472337'),(42,'course_overviews','0007_courseoverviewimageconfig','2016-02-29 18:01:42.561497'),(43,'course_overviews','0008_remove_courseoverview_facebook_url','2016-02-29 18:01:42.582078'),(44,'course_overviews','0009_readd_facebook_url','2016-02-29 18:01:42.608691'),(45,'course_structures','0001_initial','2016-02-29 18:01:42.625296'),(46,'courseware','0001_initial','2016-02-29 18:01:44.499738'),(47,'coursewarehistoryextended','0001_initial','2016-02-29 18:01:44.691094'),(48,'credentials','0001_initial','2016-02-29 18:01:44.788912'),(49,'credit','0001_initial','2016-02-29 18:01:45.621214'),(50,'dark_lang','0001_initial','2016-02-29 18:01:45.752882'),(51,'dark_lang','0002_data__enable_on_install','2016-02-29 18:01:45.765638'),(52,'default','0001_initial','2016-02-29 18:01:46.107319'),(53,'default','0002_add_related_name','2016-02-29 18:01:46.247368'),(54,'default','0003_alter_email_max_length','2016-02-29 18:01:46.265241'),(55,'django_comment_common','0001_initial','2016-02-29 18:01:46.569556'),(56,'django_notify','0001_initial','2016-02-29 18:01:47.200935'),(57,'django_openid_auth','0001_initial','2016-02-29 18:01:47.399851'),(58,'edx_proctoring','0001_initial','2016-02-29 18:01:50.659784'),(59,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-02-29 18:01:50.845795'),(60,'edx_proctoring','0003_auto_20160101_0525','2016-02-29 18:01:51.213992'),(61,'edx_proctoring','0004_auto_20160201_0523','2016-02-29 18:01:51.397904'),(62,'edxval','0001_initial','2016-02-29 18:01:51.631665'),(63,'edxval','0002_data__default_profiles','2016-02-29 18:01:51.651781'),(64,'embargo','0001_initial','2016-02-29 18:01:52.230500'),(65,'embargo','0002_data__add_countries','2016-02-29 18:01:52.401900'),(66,'external_auth','0001_initial','2016-02-29 18:01:52.851531'),(67,'lms_xblock','0001_initial','2016-02-29 18:01:53.081369'),(68,'sites','0001_initial','2016-02-29 18:01:53.104608'),(69,'microsite_configuration','0001_initial','2016-02-29 18:01:54.497782'),(70,'microsite_configuration','0002_auto_20160202_0228','2016-02-29 18:01:55.048275'),(71,'milestones','0001_initial','2016-02-29 18:01:55.459781'),(72,'milestones','0002_data__seed_relationship_types','2016-02-29 18:01:55.481439'),(73,'milestones','0003_coursecontentmilestone_requirements','2016-02-29 18:01:55.522487'),(74,'milestones','0004_auto_20151221_1445','2016-02-29 18:01:55.688330'),(75,'mobile_api','0001_initial','2016-02-29 18:01:56.844055'),(76,'notes','0001_initial','2016-02-29 18:01:57.066857'),(77,'oauth2','0001_initial','2016-02-29 18:01:58.412080'),(78,'oauth2_provider','0001_initial','2016-02-29 18:01:58.699157'),(79,'oauth_provider','0001_initial','2016-02-29 18:01:59.448662'),(80,'organizations','0001_initial','2016-02-29 18:01:59.538595'),(81,'programs','0001_initial','2016-02-29 18:01:59.922304'),(82,'programs','0002_programsapiconfig_cache_ttl','2016-02-29 18:02:00.317781'),(83,'programs','0003_auto_20151120_1613','2016-02-29 18:02:01.977886'),(84,'programs','0004_programsapiconfig_enable_certification','2016-02-29 18:02:02.413721'),(85,'programs','0005_programsapiconfig_max_retries','2016-02-29 18:02:02.799700'),(86,'rss_proxy','0001_initial','2016-02-29 18:02:02.828648'),(87,'self_paced','0001_initial','2016-02-29 18:02:03.260889'),(88,'sessions','0001_initial','2016-02-29 18:02:03.285722'),(89,'student','0001_initial','2016-02-29 18:02:14.964735'),(90,'shoppingcart','0001_initial','2016-02-29 18:02:26.167709'),(91,'shoppingcart','0002_auto_20151208_1034','2016-02-29 18:02:26.959607'),(92,'shoppingcart','0003_auto_20151217_0958','2016-02-29 18:02:27.772373'),(93,'splash','0001_initial','2016-02-29 18:02:28.180056'),(94,'static_replace','0001_initial','2016-02-29 18:02:28.600901'),(95,'static_replace','0002_assetexcludedextensionsconfig','2016-02-29 18:02:29.072670'),(96,'status','0001_initial','2016-02-29 18:02:30.149381'),(97,'student','0002_auto_20151208_1034','2016-02-29 18:02:31.383085'),(98,'submissions','0001_initial','2016-02-29 18:02:31.704655'),(99,'submissions','0002_auto_20151119_0913','2016-02-29 18:02:31.818741'),(100,'submissions','0003_submission_status','2016-02-29 18:02:31.875924'),(101,'survey','0001_initial','2016-02-29 18:02:32.613867'),(102,'teams','0001_initial','2016-02-29 18:02:35.208964'),(103,'third_party_auth','0001_initial','2016-02-29 18:02:38.385039'),(104,'track','0001_initial','2016-02-29 18:02:38.420093'),(105,'user_api','0001_initial','2016-02-29 18:02:42.631297'),(106,'util','0001_initial','2016-02-29 18:02:43.444988'),(107,'util','0002_data__default_rate_limit_config','2016-02-29 18:02:43.472520'),(108,'verify_student','0001_initial','2016-02-29 18:02:51.517974'),(109,'verify_student','0002_auto_20151124_1024','2016-02-29 18:02:52.305832'),(110,'verify_student','0003_auto_20151113_1443','2016-02-29 18:02:53.133517'),(111,'wiki','0001_initial','2016-02-29 18:03:14.972792'),(112,'wiki','0002_remove_article_subscription','2016-02-29 18:03:15.007100'),(113,'workflow','0001_initial','2016-02-29 18:03:15.147297'),(114,'xblock_django','0001_initial','2016-02-29 18:03:15.981916'),(115,'xblock_django','0002_auto_20160204_0809','2016-02-29 18:03:16.844433'),(116,'contentstore','0001_initial','2016-02-29 18:03:37.119213'),(117,'course_creators','0001_initial','2016-02-29 18:03:37.149132'),(118,'xblock_config','0001_initial','2016-02-29 18:03:37.381904');
+/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-02-29 18:03:40

common/test/db_cache/bok_choy_schema_student_module_history.sql

+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+DROP TABLE IF EXISTS `coursewarehistoryextended_studentmodulehistoryextended`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `coursewarehistoryextended_studentmodulehistoryextended` (
+  `version` varchar(255) DEFAULT NULL,
+  `created` datetime(6) NOT NULL,
+  `state` longtext,
+  `grade` double DEFAULT NULL,
+  `max_grade` double DEFAULT NULL,
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `student_module_id` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `coursewarehistoryextended_studentmodulehistoryextended_2af72f10` (`version`),
+  KEY `coursewarehistoryextended_studentmodulehistoryextended_e2fa5388` (`created`)
+) ENGINE=InnoDB AUTO_INCREMENT=10000 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+DROP TABLE IF EXISTS `django_migrations`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `django_migrations` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `app` varchar(255) NOT NULL,
+  `name` varchar(255) NOT NULL,
+  `applied` datetime(6) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=119 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+

lms/djangoapps/ccx/tests/test_field_override_performance.py

@@ -46,6 +46,8 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
     providers.
     """
     __test__ = False
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     # TEST_DATA must be overridden by subclasses
     TEST_DATA = None
@@ -151,7 +153,10 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
         """
         return check_sum_of_calls(XBlock, ['__init__'], instantiations, instantiations, include_arguments=False)
 
-    def instrument_course_progress_render(self, course_width, enable_ccx, view_as_ccx, queries, reads, xblocks):
+    def instrument_course_progress_render(
+            self, course_width, enable_ccx, view_as_ccx,
+            default_queries, history_queries, reads, xblocks
+    ):
         """
         Renders the progress page, instrumenting Mongo reads and SQL queries.
         """
@@ -173,7 +178,8 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
             # can actually take affect.
             OverrideFieldData.provider_classes = None
 
-            with self.assertNumQueries(queries):
+            with self.assertNumQueries(default_queries, using='default'):
+                with self.assertNumQueries(history_queries, using='student_module_history'):
                     with self.assertMongoCallCount(reads):
                         with self.assertXBlockInstantiations(xblocks):
                             self.grade_course(self.course, view_as_ccx)
@@ -201,8 +207,12 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
             raise SkipTest("Can't use a MongoModulestore test as a CCX course")
 
         with self.settings(FIELD_OVERRIDE_PROVIDERS=providers[overrides]):
-            queries, reads, xblocks = self.TEST_DATA[(overrides, course_width, enable_ccx, view_as_ccx)]
-            self.instrument_course_progress_render(course_width, enable_ccx, view_as_ccx, queries, reads, xblocks)
+            default_queries, history_queries, reads, xblocks = self.TEST_DATA[
+                (overrides, course_width, enable_ccx, view_as_ccx)
+            ]
+            self.instrument_course_progress_render(
+                course_width, enable_ccx, view_as_ccx, default_queries, history_queries, reads, xblocks
+            )
 
 
 class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
@@ -213,25 +223,30 @@ class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
     __test__ = True
 
     TEST_DATA = {
-        # (providers, course_width, enable_ccx, view_as_ccx): # of sql queries, # of mongo queries, # of xblocks
-        ('no_overrides', 1, True, False): (48, 6, 13),
-        ('no_overrides', 2, True, False): (135, 6, 84),
-        ('no_overrides', 3, True, False): (480, 6, 335),
-        ('ccx', 1, True, False): (48, 6, 13),
-        ('ccx', 2, True, False): (135, 6, 84),
-        ('ccx', 3, True, False): (480, 6, 335),
-        ('ccx', 1, True, True): (48, 6, 13),
-        ('ccx', 2, True, True): (135, 6, 84),
-        ('ccx', 3, True, True): (480, 6, 335),
-        ('no_overrides', 1, False, False): (48, 6, 13),
-        ('no_overrides', 2, False, False): (135, 6, 84),
-        ('no_overrides', 3, False, False): (480, 6, 335),
-        ('ccx', 1, False, False): (48, 6, 13),
-        ('ccx', 2, False, False): (135, 6, 84),
-        ('ccx', 3, False, False): (480, 6, 335),
-        ('ccx', 1, False, True): (48, 6, 13),
-        ('ccx', 2, False, True): (135, 6, 84),
-        ('ccx', 3, False, True): (480, 6, 335),
+        # (providers, course_width, enable_ccx, view_as_ccx): (
+        #     # of sql queries to default,
+        #     # sql queries to student_module_history,
+        #     # of mongo queries,
+        #     # of xblocks
+        # )
+        ('no_overrides', 1, True, False): (47, 1, 6, 13),
+        ('no_overrides', 2, True, False): (119, 16, 6, 84),
+        ('no_overrides', 3, True, False): (399, 81, 6, 335),
+        ('ccx', 1, True, False): (47, 1, 6, 13),
+        ('ccx', 2, True, False): (119, 16, 6, 84),
+        ('ccx', 3, True, False): (399, 81, 6, 335),
+        ('ccx', 1, True, True): (47, 1, 6, 13),
+        ('ccx', 2, True, True): (119, 16, 6, 84),
+        ('ccx', 3, True, True): (399, 81, 6, 335),
+        ('no_overrides', 1, False, False): (47, 1, 6, 13),
+        ('no_overrides', 2, False, False): (119, 16, 6, 84),
+        ('no_overrides', 3, False, False): (399, 81, 6, 335),
+        ('ccx', 1, False, False): (47, 1, 6, 13),
+        ('ccx', 2, False, False): (119, 16, 6, 84),
+        ('ccx', 3, False, False): (399, 81, 6, 335),
+        ('ccx', 1, False, True): (47, 1, 6, 13),
+        ('ccx', 2, False, True): (119, 16, 6, 84),
+        ('ccx', 3, False, True): (399, 81, 6, 335),
     }
 
 
@@ -243,22 +258,22 @@ class TestFieldOverrideSplitPerformance(FieldOverridePerformanceTestCase):
     __test__ = True
 
     TEST_DATA = {
-        ('no_overrides', 1, True, False): (48, 4, 9),
-        ('no_overrides', 2, True, False): (135, 19, 54),
-        ('no_overrides', 3, True, False): (480, 84, 215),
-        ('ccx', 1, True, False): (48, 4, 9),
-        ('ccx', 2, True, False): (135, 19, 54),
-        ('ccx', 3, True, False): (480, 84, 215),
-        ('ccx', 1, True, True): (50, 4, 13),
-        ('ccx', 2, True, True): (137, 19, 84),
-        ('ccx', 3, True, True): (482, 84, 335),
-        ('no_overrides', 1, False, False): (48, 4, 9),
-        ('no_overrides', 2, False, False): (135, 19, 54),
-        ('no_overrides', 3, False, False): (480, 84, 215),
-        ('ccx', 1, False, False): (48, 4, 9),
-        ('ccx', 2, False, False): (135, 19, 54),
-        ('ccx', 3, False, False): (480, 84, 215),
-        ('ccx', 1, False, True): (48, 4, 9),
-        ('ccx', 2, False, True): (135, 19, 54),
-        ('ccx', 3, False, True): (480, 84, 215),
+        ('no_overrides', 1, True, False): (47, 1, 4, 9),
+        ('no_overrides', 2, True, False): (119, 16, 19, 54),
+        ('no_overrides', 3, True, False): (399, 81, 84, 215),
+        ('ccx', 1, True, False): (47, 1, 4, 9),
+        ('ccx', 2, True, False): (119, 16, 19, 54),
+        ('ccx', 3, True, False): (399, 81, 84, 215),
+        ('ccx', 1, True, True): (49, 1, 4, 13),
+        ('ccx', 2, True, True): (121, 16, 19, 84),
+        ('ccx', 3, True, True): (401, 81, 84, 335),
+        ('no_overrides', 1, False, False): (47, 1, 4, 9),
+        ('no_overrides', 2, False, False): (119, 16, 19, 54),
+        ('no_overrides', 3, False, False): (399, 81, 84, 215),
+        ('ccx', 1, False, False): (47, 1, 4, 9),
+        ('ccx', 2, False, False): (119, 16, 19, 54),
+        ('ccx', 3, False, False): (399, 81, 84, 215),
+        ('ccx', 1, False, True): (47, 1, 4, 9),
+        ('ccx', 2, False, True): (119, 16, 19, 54),
+        ('ccx', 3, False, True): (399, 81, 84, 215),
     }

lms/djangoapps/courseware/management/commands/clean_history.py

-"""A command to clean the StudentModuleHistory table.
-
-When we added XBlock storage, each field modification wrote a new history row
-to the db.  Now that we have bulk saves to avoid that database hammering, we
-need to clean out the unnecessary rows from the database.
-
-This command that does that.
-
-"""
-
-import datetime
-import json
-import logging
-import optparse
-import time
-import traceback
-
-from django.core.management.base import NoArgsCommand
-from django.db import transaction
-from django.db.models import Max
-from courseware.models import StudentModuleHistory
-
-
-class Command(NoArgsCommand):
-    """The actual clean_history command to clean history rows."""
-
-    help = "Deletes unneeded rows from the StudentModuleHistory table."
-
-    option_list = NoArgsCommand.option_list + (
-        optparse.make_option(
-            '--batch',
-            type='int',
-            default=100,
-            help="Batch size, number of module_ids to examine in a transaction.",
-        ),
-        optparse.make_option(
-            '--dry-run',
-            action='store_true',
-            default=False,
-            help="Don't change the database, just show what would be done.",
-        ),
-        optparse.make_option(
-            '--sleep',
-            type='float',
-            default=0,
-            help="Seconds to sleep between batches.",
-        ),
-    )
-
-    def handle_noargs(self, **options):
-        # We don't want to see the SQL output from the db layer.
-        logging.getLogger("django.db.backends").setLevel(logging.INFO)
-
-        smhc = StudentModuleHistoryCleaner(
-            dry_run=options["dry_run"],
-        )
-        smhc.main(batch_size=options["batch"], sleep=options["sleep"])
-
-
-class StudentModuleHistoryCleaner(object):
-    """Logic to clean rows from the StudentModuleHistory table."""
-
-    DELETE_GAP_SECS = 0.5   # Rows this close can be discarded.
-    STATE_FILE = "clean_history.json"
-    BATCH_SIZE = 100
-
-    def __init__(self, dry_run=False):
-        self.dry_run = dry_run
-        self.next_student_module_id = 0
-        self.last_student_module_id = 0
-
-    def main(self, batch_size=None, sleep=0):
-        """Invoked from the management command to do all the work."""
-
-        batch_size = batch_size or self.BATCH_SIZE
-
-        self.last_student_module_id = self.get_last_student_module_id()
-        self.load_state()
-
-        while self.next_student_module_id <= self.last_student_module_id:
-            with transaction.atomic():
-                for smid in self.module_ids_to_check(batch_size):
-                    try:
-                        self.clean_one_student_module(smid)
-                    except Exception:       # pylint: disable=broad-except
-                        trace = traceback.format_exc()
-                        self.say("Couldn't clean student_module_id {}:\n{}".format(smid, trace))
-                if self.dry_run:
-                    transaction.set_rollback(True)
-                else:
-                    self.say("Committing")
-            self.save_state()
-            if sleep:
-                time.sleep(sleep)
-
-    def say(self, message):
-        """
-        Display a message to the user.
-
-        The message will have a trailing newline added to it.
-
-        """
-        print message
-
-    def load_state(self):
-        """
-        Load the latest state from disk.
-        """
-        try:
-            state_file = open(self.STATE_FILE)
-        except IOError:
-            self.say("No stored state")
-            self.next_student_module_id = 0
-        else:
-            with state_file:
-                state = json.load(state_file)
-            self.say(
-                "Loaded stored state: {}".format(
-                    json.dumps(state, sort_keys=True)
-                )
-            )
-            self.next_student_module_id = state['next_student_module_id']
-
-    def save_state(self):
-        """
-        Save the state to disk.
-        """
-        state = {
-            'next_student_module_id': self.next_student_module_id,
-        }
-        with open(self.STATE_FILE, "w") as state_file:
-            json.dump(state, state_file)
-        self.say("Saved state: {}".format(json.dumps(state, sort_keys=True)))
-
-    def get_last_student_module_id(self):
-        """
-        Return the id of the last student_module.
-        """
-        last = StudentModuleHistory.objects.all() \
-            .aggregate(Max('student_module'))['student_module__max']
-        self.say("Last student_module_id is {}".format(last))
-        return last
-
-    def module_ids_to_check(self, batch_size):
-        """Produce a sequence of student module ids to check.
-
-        `batch_size` is how many module ids to produce, max.
-
-        The sequence starts with `next_student_module_id`, and goes up to
-        and including `last_student_module_id`.
-
-        `next_student_module_id` is updated as each id is yielded.
-
-        """
-        start = self.next_student_module_id
-        for smid in range(start, start + batch_size):
-            if smid > self.last_student_module_id:
-                break
-            yield smid
-            self.next_student_module_id = smid + 1
-
-    def get_history_for_student_modules(self, student_module_id):
-        """
-        Get the history rows for a student module.
-
-        ```student_module_id```: the id of the student module we're
-        interested in.
-
-        Return a list: [(id, created), ...], all the rows of history.
-
-        """
-        history = StudentModuleHistory.objects \
-            .filter(student_module=student_module_id) \
-            .order_by('created', 'id')
-
-        return [(row.id, row.created) for row in history]
-
-    def delete_history(self, ids_to_delete):
-        """
-        Delete history rows.
-
-        ```ids_to_delete```: a non-empty list (or set...) of history row ids to delete.
-
-        """
-        assert ids_to_delete
-        StudentModuleHistory.objects.filter(id__in=ids_to_delete).delete()
-
-    def clean_one_student_module(self, student_module_id):
-        """Clean one StudentModule's-worth of history.
-
-        `student_module_id`: the id of the StudentModule to process.
-
-        """
-        delete_gap = datetime.timedelta(seconds=self.DELETE_GAP_SECS)
-
-        history = self.get_history_for_student_modules(student_module_id)
-        if not history:
-            self.say("No history for student_module_id {}".format(student_module_id))
-            return
-
-        ids_to_delete = []
-        next_created = None
-        for history_id, created in reversed(history):
-            if next_created is not None:
-                # Compare this timestamp with the next one.
-                if (next_created - created) < delete_gap:
-                    # This row is followed closely by another, we can discard
-                    # this one.
-                    ids_to_delete.append(history_id)
-
-            next_created = created
-
-        verb = "Would have deleted" if self.dry_run else "Deleting"
-        self.say("{verb} {to_delete} rows of {total} for student_module_id {id}".format(
-            verb=verb,
-            to_delete=len(ids_to_delete),
-            total=len(history),
-            id=student_module_id,
-        ))
-
-        if ids_to_delete and not self.dry_run:
-            self.delete_history(ids_to_delete)

lms/djangoapps/courseware/models.py

@@ -24,9 +24,9 @@ from django.dispatch import receiver, Signal
 from model_utils.models import TimeStampedModel
 from student.models import user_by_anonymous_id
 from submissions.models import score_set, score_reset
+import coursewarehistoryextended
 
 from xmodule_django.models import CourseKeyField, LocationKeyField, BlockTypeKeyField
-log = logging.getLogger(__name__)
 
 log = logging.getLogger("edx.courseware")
 
@@ -149,18 +149,15 @@ class StudentModule(models.Model):
         return unicode(repr(self))
 
 
-class StudentModuleHistory(models.Model):
-    """Keeps a complete history of state changes for a given XModule for a given
-    Student. Right now, we restrict this to problems so that the table doesn't
-    explode in size."""
+class BaseStudentModuleHistory(models.Model):
+    """Abstract class containing most fields used by any class
+    storing Student Module History"""
     objects = ChunkingManager()
     HISTORY_SAVING_TYPES = {'problem'}
 
     class Meta(object):
-        app_label = "courseware"
-        get_latest_by = "created"
+        abstract = True
 
-    student_module = models.ForeignKey(StudentModule, db_index=True)
     version = models.CharField(max_length=255, null=True, blank=True, db_index=True)
 
     # This should be populated from the modified field in StudentModule
@@ -169,11 +166,59 @@ class StudentModuleHistory(models.Model):
     grade = models.FloatField(null=True, blank=True)
     max_grade = models.FloatField(null=True, blank=True)
 
-    @receiver(post_save, sender=StudentModule)
+    @property
+    def csm(self):
+        """
+        Finds the StudentModule object for this history record, even if our data is split
+        across multiple data stores.  Django does not handle this correctly with the built-in
+        student_module property.
+        """
+        return StudentModule.objects.get(pk=self.student_module_id)
+
+    @staticmethod
+    def get_history(student_modules):
+        """
+        Find history objects across multiple backend stores for a given StudentModule
+        """
+
+        history_entries = []
+
+        if settings.FEATURES.get('ENABLE_CSMH_EXTENDED'):
+            history_entries += coursewarehistoryextended.models.StudentModuleHistoryExtended.objects.filter(
+                # Django will sometimes try to join to courseware_studentmodule
+                # so just do an in query
+                student_module__in=[module.id for module in student_modules]
+            ).order_by('-id')
+
+        # If we turn off reading from multiple history tables, then we don't want to read from
+        # StudentModuleHistory anymore, we believe that all history is in the Extended table.
+        if settings.FEATURES.get('ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES'):
+            # we want to save later SQL queries on the model which allows us to prefetch
+            history_entries += StudentModuleHistory.objects.prefetch_related('student_module').filter(
+                student_module__in=student_modules
+            ).order_by('-id')
+
+        return history_entries
+
+
+class StudentModuleHistory(BaseStudentModuleHistory):
+    """Keeps a complete history of state changes for a given XModule for a given
+    Student. Right now, we restrict this to problems so that the table doesn't
+    explode in size."""
+
+    class Meta(object):
+        app_label = "courseware"
+        get_latest_by = "created"
+
+    student_module = models.ForeignKey(StudentModule, db_index=True)
+
+    def __unicode__(self):
+        return unicode(repr(self))
+
     def save_history(sender, instance, **kwargs):  # pylint: disable=no-self-argument, unused-argument
         """
         Checks the instance's module_type, and creates & saves a
-        StudentModuleHistory entry if the module_type is one that
+        StudentModuleHistoryExtended entry if the module_type is one that
         we save.
         """
         if instance.module_type in StudentModuleHistory.HISTORY_SAVING_TYPES:
@@ -185,8 +230,11 @@ class StudentModuleHistory(models.Model):
                                                  max_grade=instance.max_grade)
             history_entry.save()
 
-    def __unicode__(self):
-        return unicode(repr(self))
+    # When the extended studentmodulehistory table exists, don't save
+    # duplicate history into courseware_studentmodulehistory, just retain
+    # data for reading.
+    if not settings.FEATURES.get('ENABLE_CSMH_EXTENDED'):
+        post_save.connect(save_history, sender=StudentModule)
 
 
 class XBlockFieldBase(models.Model):

lms/djangoapps/courseware/tests/test_access.py

@@ -520,6 +520,7 @@ class UserRoleTestCase(TestCase):
     """
     Tests for user roles.
     """
+
     def setUp(self):
         super(UserRoleTestCase, self).setUp()
         self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

lms/djangoapps/courseware/tests/test_grades.py

@@ -240,6 +240,9 @@ class TestProgressSummary(TestCase):
                    (2/5) (3/5) (0/1)   -   (1/3)   -   (3/10)
 
     """
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
+
     def setUp(self):
         super(TestProgressSummary, self).setUp()
         self.course_key = CourseLocator(

lms/djangoapps/courseware/tests/test_i18n.py

@@ -20,6 +20,7 @@ class BaseI18nTestCase(TestCase):
     """
     Base utilities for i18n test classes to derive from
     """
+
     def assert_tag_has_attr(self, content, tag, attname, value):
         """Assert that a tag in `content` has a certain value in a certain attribute."""
         regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d\- ]+)['"][^>]*>""".format(tag=tag, attname=attname)

lms/djangoapps/courseware/tests/test_model_data.py

@@ -103,6 +103,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
     """Tests for user_state storage via StudentModule"""
     other_key_factory = partial(DjangoKeyValueStore.Key, Scope.user_state, 2, location('usage_id'))  # user_id=2, not 1
     existing_field_name = "a_field"
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     def setUp(self):
         super(TestStudentModuleStorage, self).setUp()
@@ -137,7 +139,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
         # to discover if something other than the DjangoXBlockUserStateClient
         # has written to the StudentModule (such as UserStateCache setting the score
         # on the StudentModule).
-        with self.assertNumQueries(3):
+        with self.assertNumQueries(2, using='default'):
+            with self.assertNumQueries(1, using='student_module_history'):
                 self.kvs.set(user_state_key('a_field'), 'new_value')
         self.assertEquals(1, StudentModule.objects.all().count())
         self.assertEquals({'b_field': 'b_value', 'a_field': 'new_value'}, json.loads(StudentModule.objects.all()[0].state))
@@ -149,7 +152,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
         # to discover if something other than the DjangoXBlockUserStateClient
         # has written to the StudentModule (such as UserStateCache setting the score
         # on the StudentModule).
-        with self.assertNumQueries(3):
+        with self.assertNumQueries(2, using='default'):
+            with self.assertNumQueries(1, using='student_module_history'):
                 self.kvs.set(user_state_key('not_a_field'), 'new_value')
         self.assertEquals(1, StudentModule.objects.all().count())
         self.assertEquals({'b_field': 'b_value', 'a_field': 'a_value', 'not_a_field': 'new_value'}, json.loads(StudentModule.objects.all()[0].state))
@@ -161,7 +165,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
         # to discover if something other than the DjangoXBlockUserStateClient
         # has written to the StudentModule (such as UserStateCache setting the score
         # on the StudentModule).
-        with self.assertNumQueries(3):
+        with self.assertNumQueries(2, using='default'):
+            with self.assertNumQueries(1, using='student_module_history'):
                 self.kvs.delete(user_state_key('a_field'))
         self.assertEquals(1, StudentModule.objects.all().count())
         self.assertRaises(KeyError, self.kvs.get, user_state_key('not_a_field'))
@@ -201,7 +206,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
         # We also need to read the database to discover if something other than the
         # DjangoXBlockUserStateClient has written to the StudentModule (such as
         # UserStateCache setting the score on the StudentModule).
-        with self.assertNumQueries(3):
+        with self.assertNumQueries(2, using="default"):
+            with self.assertNumQueries(1, using="student_module_history"):
                 self.kvs.set_many(kv_dict)
 
         for key in kv_dict:
@@ -223,6 +229,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
 
 @attr('shard_1')
 class TestMissingStudentModule(TestCase):
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
+
     def setUp(self):
         super(TestMissingStudentModule, self).setUp()
 
@@ -244,12 +253,14 @@ class TestMissingStudentModule(TestCase):
         self.assertEquals(0, len(self.field_data_cache))
         self.assertEquals(0, StudentModule.objects.all().count())
 
-        # We are updating a problem, so we write to courseware_studentmodulehistory
+        # We are updating a problem, so we write to courseware_studentmodulehistoryextended
         # as well as courseware_studentmodule. We also need to read the database
         # to discover if something other than the DjangoXBlockUserStateClient
         # has written to the StudentModule (such as UserStateCache setting the score
         # on the StudentModule).
-        with self.assertNumQueries(5):
+        # Django 1.8 also has a number of other BEGIN and SAVESTATE queries.
+        with self.assertNumQueries(4, using='default'):
+            with self.assertNumQueries(1, using='student_module_history'):
                 self.kvs.set(user_state_key('a_field'), 'a_value')
 
         self.assertEquals(1, sum(len(cache) for cache in self.field_data_cache.cache.values()))

lms/djangoapps/courseware/tests/test_submitting_problems.py

@@ -19,7 +19,7 @@ from capa.tests.response_xml_factory import (
     CodeResponseXMLFactory,
 )
 from courseware import grades
-from courseware.models import StudentModule, StudentModuleHistory
+from courseware.models import StudentModule, BaseStudentModuleHistory
 from courseware.tests.helpers import LoginEnrollmentTestCase
 from lms.djangoapps.lms_xblock.runtime import quote_slashes
 from student.tests.factories import UserFactory
@@ -121,6 +121,8 @@ class TestSubmittingProblems(ModuleStoreTestCase, LoginEnrollmentTestCase, Probl
     Check that a course gets graded properly.
     """
 
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
     # arbitrary constant
     COURSE_SLUG = "100"
     COURSE_NAME = "test_course"
@@ -319,6 +321,9 @@ class TestCourseGrader(TestSubmittingProblems):
     """
     Suite of tests for the course grader.
     """
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
+
     def basic_setup(self, late=False, reset=False, showanswer=False):
         """
         Set up a simple course for testing basic grading functionality.
@@ -451,26 +456,20 @@ class TestCourseGrader(TestSubmittingProblems):
         self.submit_question_answer('p1', {'2_1': u'Correct'})
 
         # Now fetch the state entry for that problem.
-        student_module = StudentModule.objects.get(
+        student_module = StudentModule.objects.filter(
             course_id=self.course.id,
             student=self.student_user
         )
         # count how many state history entries there are
-        baseline = StudentModuleHistory.objects.filter(
-            student_module=student_module
-        )
-        baseline_count = baseline.count()
-        self.assertEqual(baseline_count, 3)
+        baseline = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEqual(len(baseline), 3)
 
         # now click "show answer"
         self.show_question_answer('p1')
 
         # check that we don't have more state history entries
-        csmh = StudentModuleHistory.objects.filter(
-            student_module=student_module
-        )
-        current_count = csmh.count()
-        self.assertEqual(current_count, 3)
+        csmh = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEqual(len(csmh), 3)
 
     def test_grade_with_max_score_cache(self):
         """
@@ -713,6 +712,8 @@ class TestCourseGrader(TestSubmittingProblems):
 @attr('shard_1')
 class ProblemWithUploadedFilesTest(TestSubmittingProblems):
     """Tests of problems with uploaded files."""
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     def setUp(self):
         super(ProblemWithUploadedFilesTest, self).setUp()
@@ -768,6 +769,8 @@ class TestPythonGradedResponse(TestSubmittingProblems):
     """
     Check that we can submit a schematic and custom response, and it answers properly.
     """
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     SCHEMATIC_SCRIPT = dedent("""
         # for a schematic response, submission[i] is the json representation

lms/djangoapps/courseware/tests/test_user_state_client.py

@@ -18,6 +18,8 @@ class TestDjangoUserStateClient(UserStateClientTestBase, TestCase):
     Tests of the DjangoUserStateClient backend.
     """
     __test__ = True
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
 
     def _user(self, user_idx):
         return self.users[user_idx].username

lms/djangoapps/courseware/user_state_client.py

@@ -15,7 +15,7 @@ except ImportError:
 import dogstats_wrapper as dog_stats_api
 from django.contrib.auth.models import User
 from xblock.fields import Scope, ScopeBase
-from courseware.models import StudentModule, StudentModuleHistory
+from courseware.models import StudentModule, BaseStudentModuleHistory
 from edx_user_state_client.interface import XBlockUserStateClient, XBlockUserState
 
 
@@ -312,9 +312,7 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
         if len(student_modules) == 0:
             raise self.DoesNotExist()
 
-        history_entries = StudentModuleHistory.objects.prefetch_related('student_module').filter(
-            student_module__in=student_modules
-        ).order_by('-id')
+        history_entries = BaseStudentModuleHistory.get_history(student_modules)
 
         # If no history records exist, raise an error
         if not history_entries:
@@ -332,9 +330,9 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
             if state == {}:
                 state = None
 
-            block_key = history_entry.student_module.module_state_key
+            block_key = history_entry.csm.module_state_key
             block_key = block_key.map_into_course(
-                history_entry.student_module.course_id
+                history_entry.csm.course_id
             )
 
             yield XBlockUserState(username, block_key, state, history_entry.created, scope)

lms/djangoapps/courseware/views.py

@@ -59,7 +59,7 @@ from courseware.courses import (
 )
 from courseware.masquerade import setup_masquerade
 from courseware.model_data import FieldDataCache, ScoresClient
-from courseware.models import StudentModuleHistory
+from courseware.models import StudentModule, BaseStudentModuleHistory
 from courseware.url_helpers import get_redirect_url
 from courseware.user_state_client import DjangoXBlockUserStateClient
 from edxmako.shortcuts import render_to_response, render_to_string, marketing_link
@@ -1173,11 +1173,12 @@ def submission_history(request, course_id, student_username, location):
 
     # This is ugly, but until we have a proper submissions API that we can use to provide
     # the scores instead, it will have to do.
-    scores = list(StudentModuleHistory.objects.filter(
-        student_module__module_state_key=usage_key,
-        student_module__student__username=student_username,
-        student_module__course_id=course_key
-    ).order_by('-id'))
+    csm = StudentModule.objects.filter(
+        module_state_key=usage_key,
+        student__username=student_username,
+        course_id=course_key)
+
+    scores = BaseStudentModuleHistory.get_history(csm)
 
     if len(scores) != len(history_entries):
         log.warning(

lms/djangoapps/coursewarehistoryextended/fields.py

+"""
+Custom fields for use in the coursewarehistoryextended django app.
+"""
+
+from django.db.models.fields import AutoField
+
+
+class UnsignedBigIntAutoField(AutoField):
+    """
+    An unsigned 8-byte integer for auto-incrementing primary keys.
+    """
+    def db_type(self, connection):
+        if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
+            return "bigint UNSIGNED AUTO_INCREMENT"
+        elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
+            # Sqlite will only auto-increment the ROWID column. Any INTEGER PRIMARY KEY column
+            # is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
+            # isn't an alias for ROWID, so we have to give up on the unsigned part.
+            return "integer"
+        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
+            # Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and
+            # goes 1-9.2x10^18
+            return "BIGSERIAL"
+        else:
+            return None

lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import coursewarehistoryextended.fields
+from django.conf import settings
+
+def bump_pk_start(apps, schema_editor):
+    if not schema_editor.connection.alias == 'student_module_history':
+        return
+    StudentModuleHistory = apps.get_model("courseware", "StudentModuleHistory")
+    biggest_id = StudentModuleHistory.objects.all().order_by('-id').first()
+    initial_id = settings.STUDENTMODULEHISTORYEXTENDED_OFFSET
+    if biggest_id is not None:
+        initial_id += biggest_id.id
+
+    if schema_editor.connection.vendor == 'mysql':
+        schema_editor.execute('ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended AUTO_INCREMENT=%s', [initial_id])
+    elif schema_editor.connection.vendor == 'sqlite3':
+        # This is a hack to force sqlite to add new rows after the earlier rows we
+        # want to migrate.
+        StudentModuleHistory(
+            id=initial_id,
+            course_key=None,
+            usage_key=None,
+            username="",
+            version="",
+            created=datetime.datetime.now(),
+        ).save()
+    elif schema_editor.connection.vendor == 'postgresql':
+        schema_editor.execute("SELECT setval('coursewarehistoryextended_studentmodulehistoryextended_seq', %s)", [initial_id])
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('courseware', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StudentModuleHistoryExtended',
+            fields=[
+                ('version', models.CharField(db_index=True, max_length=255, null=True, blank=True)),
+                ('created', models.DateTimeField(db_index=True)),
+                ('state', models.TextField(null=True, blank=True)),
+                ('grade', models.FloatField(null=True, blank=True)),
+                ('max_grade', models.FloatField(null=True, blank=True)),
+                ('id', coursewarehistoryextended.fields.UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)),
+            ],
+            options={
+                'get_latest_by': 'created',
+            },
+        ),
+        migrations.RunPython(bump_pk_start, reverse_code=migrations.RunPython.noop),
+    ]

lms/djangoapps/coursewarehistoryextended/models.py

+"""
+WE'RE USING MIGRATIONS!
+
+If you make changes to this model, be sure to create an appropriate migration
+file and check it in at the same time as your model changes. To do that,
+
+1. Go to the edx-platform dir
+2. ./manage.py schemamigration courseware --auto description_of_your_change
+3. Add the migration file created in edx-platform/lms/djangoapps/coursewarehistoryextended/migrations/
+
+
+ASSUMPTIONS: modules have unique IDs, even across different module_types
+
+"""
+from django.db import models
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+
+from coursewarehistoryextended.fields import UnsignedBigIntAutoField
+from courseware.models import StudentModule, BaseStudentModuleHistory
+
+
+class StudentModuleHistoryExtended(BaseStudentModuleHistory):
+    """Keeps a complete history of state changes for a given XModule for a given
+    Student. Right now, we restrict this to problems so that the table doesn't
+    explode in size.
+
+    This new extended CSMH has a larger primary key that won't run out of space
+    so quickly."""
+
+    class Meta(object):
+        app_label = 'coursewarehistoryextended'
+        get_latest_by = "created"
+
+    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+
+    student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False, on_delete=models.DO_NOTHING)
+
+    @receiver(post_save, sender=StudentModule)
+    def save_history(sender, instance, **kwargs):  # pylint: disable=no-self-argument, unused-argument
+        """
+        Checks the instance's module_type, and creates & saves a
+        StudentModuleHistoryExtended entry if the module_type is one that
+        we save.
+        """
+        if instance.module_type in StudentModuleHistoryExtended.HISTORY_SAVING_TYPES:
+            history_entry = StudentModuleHistoryExtended(student_module=instance,
+                                                         version=None,
+                                                         created=instance.modified,
+                                                         state=instance.state,
+                                                         grade=instance.grade,
+                                                         max_grade=instance.max_grade)
+            history_entry.save()
+
+    @receiver(post_delete, sender=StudentModule)
+    def delete_history(sender, instance, **kwargs):  # pylint: disable=no-self-argument, unused-argument
+        """
+        Django can't cascade delete across databases, so we tell it at the model level to
+        on_delete=DO_NOTHING and then listen for post_delete so we can clean up the CSMHE rows.
+        """
+        StudentModuleHistoryExtended.objects.filter(student_module=instance).all().delete()
+
+    def __unicode__(self):
+        return unicode(repr(self))

lms/djangoapps/coursewarehistoryextended/tests.py

+"""
+Tests for coursewarehistoryextended
+Many aspects of this app are covered by the courseware tests,
+but these are specific to the new storage model with multiple
+backend tables.
+"""
+
+import json
+from mock import patch
+from django.test import TestCase
+from django.conf import settings
+from unittest import skipUnless
+from nose.plugins.attrib import attr
+
+from courseware.models import BaseStudentModuleHistory, StudentModuleHistory, StudentModule
+
+from courseware.tests.factories import StudentModuleFactory, location, course_id
+
+
+@attr('shard_1')
+@skipUnless(settings.FEATURES["ENABLE_CSMH_EXTENDED"], "CSMH Extended needs to be enabled")
+class TestStudentModuleHistoryBackends(TestCase):
+    """ Tests of data in CSMH and CSMHE """
+    # Tell Django to clean out all databases, not just default
+    multi_db = True
+
+    def setUp(self):
+        super(TestStudentModuleHistoryBackends, self).setUp()
+        for record in (1, 2, 3):
+            # This will store into CSMHE via the post_save signal
+            csm = StudentModuleFactory.create(module_state_key=location('usage_id'),
+                                              course_id=course_id,
+                                              state=json.dumps({'type': 'csmhe', 'order': record}))
+            # This manually gets us a CSMH record to compare
+            csmh = StudentModuleHistory(student_module=csm,
+                                        version=None,
+                                        created=csm.modified,
+                                        state=json.dumps({'type': 'csmh', 'order': record}),
+                                        grade=csm.grade,
+                                        max_grade=csm.max_grade)
+            csmh.save()
+
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": True})
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": True})
+    def test_get_history_true_true(self):
+        student_module = StudentModule.objects.all()
+        history = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEquals(len(history), 6)
+        self.assertEquals({'type': 'csmhe', 'order': 3}, json.loads(history[0].state))
+        self.assertEquals({'type': 'csmhe', 'order': 2}, json.loads(history[1].state))
+        self.assertEquals({'type': 'csmhe', 'order': 1}, json.loads(history[2].state))
+        self.assertEquals({'type': 'csmh', 'order': 3}, json.loads(history[3].state))
+        self.assertEquals({'type': 'csmh', 'order': 2}, json.loads(history[4].state))
+        self.assertEquals({'type': 'csmh', 'order': 1}, json.loads(history[5].state))
+
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": True})
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": False})
+    def test_get_history_true_false(self):
+        student_module = StudentModule.objects.all()
+        history = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEquals(len(history), 3)
+        self.assertEquals({'type': 'csmhe', 'order': 3}, json.loads(history[0].state))
+        self.assertEquals({'type': 'csmhe', 'order': 2}, json.loads(history[1].state))
+        self.assertEquals({'type': 'csmhe', 'order': 1}, json.loads(history[2].state))
+
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": False})
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": True})
+    def test_get_history_false_true(self):
+        student_module = StudentModule.objects.all()
+        history = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEquals(len(history), 3)
+        self.assertEquals({'type': 'csmh', 'order': 3}, json.loads(history[0].state))
+        self.assertEquals({'type': 'csmh', 'order': 2}, json.loads(history[1].state))
+        self.assertEquals({'type': 'csmh', 'order': 1}, json.loads(history[2].state))
+
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": False})
+    @patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": False})
+    def test_get_history_false_false(self):
+        student_module = StudentModule.objects.all()
+        history = BaseStudentModuleHistory.get_history(student_module)
+        self.assertEquals(len(history), 0)

lms/envs/acceptance.py

@@ -72,6 +72,14 @@ DATABASES = {
             'timeout': 30,
         },
         'ATOMIC_REQUESTS': True,
+    },
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
+        'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
+        'OPTIONS': {
+            'timeout': 30,
+        },
     }
 }
 
@@ -145,7 +153,9 @@ LETTUCE_APPS = ('courseware', 'instructor')
 # This causes some pretty cryptic errors as lettuce tries
 # to parse files in `instructor_task` as features.
 # As a quick workaround, explicitly exclude the `instructor_task` app.
-LETTUCE_AVOID_APPS = ('instructor_task',)
+# The coursewarehistoryextended app also falls prey to this fuzzy
+# for the courseware app.
+LETTUCE_AVOID_APPS = ('instructor_task', 'coursewarehistoryextended')
 
 LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome')
 

lms/envs/aws.py

@@ -759,6 +759,11 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get(
 # Course Content Bookmarks Settings
 MAX_BOOKMARKS_PER_COURSE = ENV_TOKENS.get('MAX_BOOKMARKS_PER_COURSE', MAX_BOOKMARKS_PER_COURSE)
 
+# Offset for pk of courseware.StudentModuleHistoryExtended
+STUDENTMODULEHISTORYEXTENDED_OFFSET = ENV_TOKENS.get(
+    'STUDENTMODULEHISTORYEXTENDED_OFFSET', STUDENTMODULEHISTORYEXTENDED_OFFSET
+)
+
 # Cutoff date for granting audit certificates
 if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None):
     AUDIT_CERT_CUTOFF_DATE = dateutil.parser.parse(ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE'))
@@ -766,3 +771,7 @@ if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None):
 ################################ Settings for Credentials Service ################################
 
 CREDENTIALS_GENERATION_ROUTING_KEY = HIGH_PRIORITY_QUEUE
+
+# The extended StudentModule history table
+if FEATURES.get('ENABLE_CSMH_EXTENDED'):
+    INSTALLED_APPS += ('coursewarehistoryextended',)

lms/envs/aws_migrate.py

@@ -13,18 +13,25 @@ from .aws import *
 import os
 from django.core.exceptions import ImproperlyConfigured
 
-DB_OVERRIDES = dict(
+
+def get_db_overrides(db_name):
+    """
+    Now that we have multiple databases, we want to look up from the environment
+    for both databases.
+    """
+    db_overrides = dict(
         PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
-    ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
-    USER=os.environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
-    NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
-    HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
-    PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
-)
+        ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES[db_name]['ENGINE']),
+        USER=os.environ.get('DB_MIGRATION_USER', DATABASES[db_name]['USER']),
+        NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES[db_name]['NAME']),
+        HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES[db_name]['HOST']),
+        PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES[db_name]['PORT']),
+    )
 
-if DB_OVERRIDES['PASSWORD'] is None:
+    if db_overrides['PASSWORD'] is None:
         raise ImproperlyConfigured("No database password was provided for running "
                                    "migrations.  This is fatal.")
+    return db_overrides
 
-for override, value in DB_OVERRIDES.iteritems():
-    DATABASES['default'][override] = value
+for db in ['default', 'student_module_history']:
+    DATABASES[db].update(get_db_overrides(db))

lms/envs/bok_choy.auth.json

@@ -39,6 +39,14 @@
             "PASSWORD": "",
             "PORT": "3306",
             "USER": "root"
+        },
+        "student_module_history": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "localhost",
+            "NAME": "student_module_history_test",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
         }
     },
     "DOC_STORE_CONFIG": {

lms/envs/bok_choy.py

@@ -178,6 +178,11 @@ PROFILE_IMAGE_BACKEND = {
         'base_url': os.path.join(MEDIA_URL, 'profile-images/'),
     },
 }
+
+# Make sure we test with the extended history table
+FEATURES['ENABLE_CSMH_EXTENDED'] = True
+INSTALLED_APPS += ('coursewarehistoryextended',)
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

lms/envs/common.py

@@ -363,6 +363,18 @@ FEATURES = {
 
     # Show Language selector.
     'SHOW_LANGUAGE_SELECTOR': False,
+
+    # Write new CSM history to the extended table.
+    # This will eventually default to True and may be
+    # removed since all installs should have the separate
+    # extended history table.
+    'ENABLE_CSMH_EXTENDED': False,
+
+    # Read from both the CSMH and CSMHE history tables.
+    # This is the default, but can be disabled if all history
+    # lives in the Extended table, saving the frontend from
+    # making multiple queries.
+    'ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES': True
 }
 
 # Ignore static asset files on import which match this pattern
@@ -413,6 +425,12 @@ GEOIPV6_PATH = REPO_ROOT / "common/static/data/geoip/GeoIPv6.dat"
 # Where to look for a status message
 STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json"
 
+############################ Global Database Configuration #####################
+
+DATABASE_ROUTERS = [
+    'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter',
+]
+
 ############################ OpenID Provider  ##################################
 OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net']
 
@@ -2759,6 +2777,13 @@ MOBILE_APP_USER_AGENT_REGEXES = [
     r'edX/org.edx.mobile',
 ]
 
+# Offset for courseware.StudentModuleHistoryExtended which is used to
+# calculate the starting primary key for the underlying table.  This gap
+# should be large enough that you do not generate more than N courseware.StudentModuleHistory
+# records before you have deployed the app to write to coursewarehistoryextended.StudentModuleHistoryExtended
+# if you want to avoid an overlap in ids while searching for history across the two tables.
+STUDENTMODULEHISTORYEXTENDED_OFFSET = 10000
+
 # Deprecated xblock types
 DEPRECATED_ADVANCED_COMPONENT_TYPES = []
 

lms/envs/dev.py

@@ -48,6 +48,11 @@ DATABASES = {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': ENV_ROOT / "db" / "edx.db",
         'ATOMIC_REQUESTS': True,
+    },
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "student_module_history.db",
+        'ATOMIC_REQUESTS': True,
     }
 }
 

lms/envs/devplus.py

@@ -31,6 +31,15 @@ DATABASES = {
         'HOST': '127.0.0.1',
         'PORT': '3306',
         'ATOMIC_REQUESTS': True,
+    },
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'student_module_history',
+        'USER': 'root',
+        'PASSWORD': '',
+        'HOST': '127.0.0.1',
+        'PORT': '3306',
+        'ATOMIC_REQUESTS': True,
     }
 }
 

lms/envs/static.py

@@ -27,6 +27,11 @@ DATABASES = {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': ENV_ROOT / "db" / "edx.db",
         'ATOMIC_REQUESTS': True,
+    },
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "student_module_history.db",
+        'ATOMIC_REQUESTS': True,
     }
 }
 

lms/envs/test.py

@@ -186,7 +186,10 @@ DATABASES = {
         'NAME': TEST_ROOT / 'db' / 'edx.db',
         'ATOMIC_REQUESTS': True,
     },
-
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': TEST_ROOT / 'db' / 'student_module_history.db'
+    },
 }
 
 if os.environ.get('DISABLE_MIGRATIONS'):
@@ -194,6 +197,10 @@ if os.environ.get('DISABLE_MIGRATIONS'):
     # to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations.
     MIGRATION_MODULES = NoOpMigrationModules()
 
+# Make sure we test with the extended history table
+FEATURES['ENABLE_CSMH_EXTENDED'] = True
+INSTALLED_APPS += ('coursewarehistoryextended',)
+
 CACHES = {
     # This is the cache used for most things.
     # In staging/prod envs, the sessions also live here.

lms/envs/test_static_optimized.py

@@ -19,7 +19,9 @@ DATABASES = {
         'ENGINE': 'django.db.backends.sqlite3',
         'ATOMIC_REQUESTS': True,
     },
-
+    'student_module_history': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
 }
 
 # Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up

openedx/core/lib/django_courseware_routers.py

+"""
+Database Routers for use with the coursewarehistoryextended django app.
+"""
+
+
+class StudentModuleHistoryExtendedRouter(object):
+    """
+    A Database Router that separates StudentModuleHistoryExtended into its own database.
+    """
+
+    DATABASE_NAME = 'student_module_history'
+
+    def _is_csmh(self, model):
+        """
+        Return True if ``model`` is courseware.StudentModuleHistoryExtended.
+        """
+        return (
+            model._meta.app_label == 'coursewarehistoryextended' and  # pylint: disable=protected-access
+            model.__name__ == 'StudentModuleHistoryExtended'
+        )
+
+    def db_for_read(self, model, **hints):  # pylint: disable=unused-argument
+        """
+        Use the StudentModuleHistoryExtendedRouter.DATABASE_NAME if the model is StudentModuleHistoryExtended.
+        """
+        if self._is_csmh(model):
+            return self.DATABASE_NAME
+        else:
+            return None
+
+    def db_for_write(self, model, **hints):  # pylint: disable=unused-argument
+        """
+        Use the StudentModuleHistoryExtendedRouter.DATABASE_NAME if the model is StudentModuleHistoryExtended.
+        """
+        if self._is_csmh(model):
+            return self.DATABASE_NAME
+        else:
+            return None
+
+    def allow_relation(self, obj1, obj2, **hints):  # pylint: disable=unused-argument
+        """
+        Disable relations if the model is StudentModuleHistoryExtended.
+        """
+        if self._is_csmh(obj1) or self._is_csmh(obj2):
+            return False
+        return None
+
+    def allow_migrate(self, db, model):  # pylint: disable=unused-argument
+        """
+        Only sync StudentModuleHistoryExtended to StudentModuleHistoryExtendedRouter.DATABASE_Name
+        """
+        if self._is_csmh(model):
+            return db == self.DATABASE_NAME
+        elif db == self.DATABASE_NAME:
+            return False
+
+        return None

pavelib/utils/test/suites/acceptance_suite.py

@@ -69,8 +69,14 @@ class AcceptanceTestSuite(TestSuite):
     def __init__(self, *args, **kwargs):
         super(AcceptanceTestSuite, self).__init__(*args, **kwargs)
         self.root = 'acceptance'
-        self.db = Env.REPO_ROOT / 'test_root/db/test_edx.db'
-        self.db_cache = Env.REPO_ROOT / 'common/test/db_cache/lettuce.db'
+        self.dbs = {
+            'default': Env.REPO_ROOT / 'test_root/db/test_edx.db',
+            'student_module_history': Env.REPO_ROOT / 'test_root/db/test_student_module_history.db'
+        }
+        self.db_caches = {
+            'default': Env.REPO_ROOT / 'common/test/db_cache/lettuce.db',
+            'student_module_history': Env.REPO_ROOT / 'common/test/db_cache/lettuce_student_module_history.db'
+        }
         self.fasttest = kwargs.get('fasttest', False)
 
         if kwargs.get('system'):
@@ -114,24 +120,30 @@ class AcceptanceTestSuite(TestSuite):
         definitions to sync and migrate.
         """
 
-        if self.db.isfile():
+        for db in self.dbs.keys():
+            if self.dbs[db].isfile():
                 # Since we are using SQLLite, we can reset the database by deleting it on disk.
-            self.db.remove()
+                self.dbs[db].remove()
 
-        if self.db_cache.isfile():
+        if all(self.db_caches[cache].isfile() for cache in self.db_caches.keys()):
             # To speed up migrations, we check for a cached database file and start from that.
             # The cached database file should be checked into the repo
 
             # Copy the cached database to the test root directory
-            sh("cp {db_cache} {db}".format(db_cache=self.db_cache, db=self.db))
+            for db_alias in self.dbs.keys():
+                sh("cp {db_cache} {db}".format(db_cache=self.db_caches[db_alias], db=self.dbs[db_alias]))
 
             # Run migrations to update the db, starting from its cached state
-            sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial")
-            sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial")
+            for db_alias in sorted(self.dbs.keys()):
+                # pylint: disable=line-too-long
+                sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
+                sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
         else:
             # If no cached database exists, syncdb before migrating, then create the cache
-            sh("./manage.py lms --settings acceptance migrate --traceback --noinput")
-            sh("./manage.py cms --settings acceptance migrate --traceback --noinput")
+            for db_alias in sorted(self.dbs.keys()):
+                sh("./manage.py lms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
+                sh("./manage.py cms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
 
             # Create the cache if it doesn't already exist
-            sh("cp {db} {db_cache}".format(db_cache=self.db_cache, db=self.db))
+            for db_alias in self.dbs.keys():
+                sh("cp {db} {db_cache}".format(db_cache=self.db_caches[db_alias], db=self.dbs[db_alias]))

scripts/reset-test-db.sh

@@ -24,35 +24,55 @@
 
 DB_CACHE_DIR="common/test/db_cache"
 
+declare -A databases
+declare -a database_order
+databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
+database_order=("default" "student_module_history")
+
 # Ensure the test database exists.
-echo "CREATE DATABASE IF NOT EXISTS edxtest;" | mysql -u root
+for db in "${database_order[@]}"; do
+    echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root
 
-# Clear out the test database
-#
-# We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
-# "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
-# or a jenkins worker environment) that already ran tests on another commit that had
-# different migrations that created, dropped, or altered tables.
-echo "Issuing a reset_db command to the bok_choy MySQL database."
-./manage.py lms --settings bok_choy reset_db --traceback --noinput
+    # Clear out the test database
+    #
+    # We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
+    # "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
+    # or a jenkins worker environment) that already ran tests on another commit that had
+    # different migrations that created, dropped, or altered tables.
+    echo "Issuing a reset_db command to the $db bok_choy MySQL database."
+    ./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db
+
+    # If there are cached database schemas/data, load them
+    if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then
+        echo "Missing $DB_CACHE_DIR/bok_choy_schema_$db.sql or $DB_CACHE_DIR/bok_choy_data_$db.json, or $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql rebuilding cache"
+        REBUILD_CACHE=true
+    fi
+
+done
 
 # If there are cached database schemas/data, load them
-if [[ -f $DB_CACHE_DIR/bok_choy_schema.sql && -f $DB_CACHE_DIR/bok_choy_migrations_data.sql && -f $DB_CACHE_DIR/bok_choy_data.json ]]; then
+if [[ -z $REBUILD_CACHE ]]; then
 
     echo "Found the bok_choy DB cache files. Loading them into the database..."
+
+    for db in "${database_order[@]}"; do
         # Load the schema, then the data (including the migration history)
-    echo "Loading the schema from the filesystem into the MySQL DB."
-    mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_schema.sql
-    echo "Loading the migration data from the filesystem into the MySQL DB."
-    mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_migrations_data.sql
-    echo "Loading the fixture data from the filesystem into the MySQL DB."
-    ./manage.py lms --settings bok_choy loaddata $DB_CACHE_DIR/bok_choy_data.json
+        echo "Loading the schema from the filesystem into the $db MySQL DB."
+        mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
+        echo "Loading the fixture data from the filesystem into the $db MySQL DB."
+        ./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
+
+        # Migrations are stored in the default database
+        echo "Loading the migration data from the filesystem into the $db MySQL DB."
+        mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
 
         # Re-run migrations to ensure we are up-to-date
-    echo "Running the lms migrations on the bok_choy DB."
-    ./manage.py lms --settings bok_choy migrate --traceback --noinput
-    echo "Running the cms migrations on the bok_choy DB."
-    ./manage.py cms --settings bok_choy migrate --traceback --noinput
+        echo "Running the lms migrations on the $db bok_choy DB."
+        ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
+        echo "Running the cms migrations on the $db bok_choy DB."
+        ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
+
+    done
 
 # Otherwise, update the test database and update the cache
 else
@@ -60,19 +80,21 @@ else
     # Clean the cache directory
     mkdir -p $DB_CACHE_DIR && rm -f $DB_CACHE_DIR/bok_choy*
 
+    for db in "${database_order[@]}"; do
         # Re-run migrations on the test database
-    echo "Issuing a migrate command to the bok_choy MySQL database for the lms django apps."
-    ./manage.py lms --settings bok_choy migrate --traceback --noinput
-    echo "Issuing a migrate command to the bok_choy MySQL database for the cms django apps."
-    ./manage.py cms --settings bok_choy migrate --traceback --noinput
+        echo "Issuing a migrate command to the $db bok_choy MySQL database for the lms django apps."
+        ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
+        echo "Issuing a migrate command to the $db bok_choy MySQL database for the cms django apps."
+        ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
 
         # Dump the schema and data to the cache
-    echo "Using the dumpdata command to save the fixture data to the filesystem."
-    ./manage.py lms --settings bok_choy dumpdata > $DB_CACHE_DIR/bok_choy_data.json
+        echo "Using the dumpdata command to save the $db fixture data to the filesystem."
+        ./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json
+        echo "Saving the schema of the $dh bok_choy DB to the filesystem."
+        mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
+
         # dump_data does not dump the django_migrations table so we do it separately.
-    echo "Saving the django_migrations table of the bok_choy DB to the filesystem."
-    mysqldump -u root --no-create-info edxtest django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data.sql
-    echo "Saving the schema of the bok_choy DB to the filesystem."
-    mysqldump -u root --no-data --skip-comments --skip-dump-date edxtest > $DB_CACHE_DIR/bok_choy_schema.sql
+        echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem."
+        mysqldump -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
+    done
 fi
-