Skip to content

E
edx-platform
Commit 447c189d by Kevin Falcone
Options

Merge pull request #11569 from edx/jibsheet/csmh-extended 

Student Module History Extension
parents e0407893 b4ac588c
Showing with 815 additions and 420 deletions
cms/envs/acceptance.py
......@@ -80,6 +80,14 @@ DATABASES = {
'timeout': 30,
},
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'OPTIONS': {
'timeout': 30,
},
}
}
......
cms/envs/aws_migrate.py
......@@ -26,5 +26,5 @@ if DB_OVERRIDES['PASSWORD'] is None:
raise ImproperlyConfigured("No database password was provided for running "
"migrations. This is fatal.")
for override, value in DB_OVERRIDES.iteritems():
DATABASES['default'][override] = value
DATABASES['default'].update(DB_OVERRIDES)
DATABASES['student_module_history'].update(DB_OVERRIDES)
cms/envs/bok_choy.auth.json
......@@ -30,6 +30,14 @@
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
},
"student_module_history": {
"ENGINE": "django.db.backends.mysql",
"HOST": "localhost",
"NAME": "student_module_history_test",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
}
},
"DOC_STORE_CONFIG": {
......
cms/envs/common.py
......@@ -1114,6 +1114,11 @@ PROCTORING_BACKEND_PROVIDER = {
}
PROCTORING_SETTINGS = {}
############################ Global Database Configuration #####################
DATABASE_ROUTERS = [
'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter',
]
############################ OAUTH2 Provider ###################################
......
common/lib/xmodule/xmodule/capa_module.py
......@@ -93,11 +93,23 @@ class CapaModule(CapaMixin, XModule):
result = handlers[dispatch](data)
except NotFoundError as err:
_, _, traceback_obj = sys.exc_info()
log.exception(
"Unable to find data when dispatching %s to %s for user %s",
dispatch,
self.scope_ids.usage_id,
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info() # pylint: disable=redefined-outer-name
raise ProcessingError(not_found_error_message), None, traceback_obj
except Exception as err:
_, _, traceback_obj = sys.exc_info()
log.exception(
"Unknown error when dispatching %s to %s for user %s",
dispatch,
self.scope_ids.usage_id,
self.scope_ids.user_id
)
_, _, traceback_obj = sys.exc_info() # pylint: disable=redefined-outer-name
raise ProcessingError(generic_error_message), None, traceback_obj
after = self.get_progress()
......
common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
......@@ -265,6 +265,8 @@ class SharedModuleStoreTestCase(TestCase):
for Django ORM models that will get cleaned up properly.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
# Tell Django to clean out all databases, not just default
multi_db = True
@classmethod
def setUpClass(cls):
......@@ -392,6 +394,8 @@ class ModuleStoreTestCase(TestCase):
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self, **kwargs):
"""
......
common/test/db_cache/bok_choy_data.json deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
common/test/db_cache/bok_choy_data_default.json 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
common/test/db_cache/bok_choy_migrations_data.sql deleted 100644 → 0
-- MySQL dump 10.13 Distrib 5.6.24, for debian-linux-gnu (x86_64)
--
-- Host: localhost Database: edxtest
-- ------------------------------------------------------
-- Server version 5.6.24-2+deb.sury.org~precise+2
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Dumping data for table `django_migrations`
--
LOCK TABLES `django_migrations` WRITE;
/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-01-22 20:04:03.218179'),(2,'auth','0001_initial','2016-01-22 20:04:03.554767'),(3,'admin','0001_initial','2016-01-22 20:04:03.665239'),(4,'assessment','0001_initial','2016-01-22 20:04:06.917971'),(5,'assessment','0002_staffworkflow','2016-01-22 20:04:07.125841'),(6,'contenttypes','0002_remove_content_type_name','2016-01-22 20:04:07.307570'),(7,'auth','0002_alter_permission_name_max_length','2016-01-22 20:04:07.383568'),(8,'auth','0003_alter_user_email_max_length','2016-01-22 20:04:07.464880'),(9,'auth','0004_alter_user_username_opts','2016-01-22 20:04:07.496608'),(10,'auth','0005_alter_user_last_login_null','2016-01-22 20:04:07.584818'),(11,'auth','0006_require_contenttypes_0002','2016-01-22 20:04:07.591565'),(12,'bookmarks','0001_initial','2016-01-22 20:04:07.932691'),(13,'branding','0001_initial','2016-01-22 20:04:08.105513'),(14,'bulk_email','0001_initial','2016-01-22 20:04:08.469986'),(15,'bulk_email','0002_data__load_course_email_template','2016-01-22 20:04:08.538284'),(16,'instructor_task','0001_initial','2016-01-22 20:04:08.743012'),(17,'certificates','0001_initial','2016-01-22 20:04:09.838773'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-01-22 20:04:09.865157'),(19,'certificates','0003_data__default_modes','2016-01-22 20:04:09.943086'),(20,'certificates','0004_certificategenerationhistory','2016-01-22 20:04:10.116067'),(21,'certificates','0005_auto_20151208_0801','2016-01-22 20:04:10.232215'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-01-22 20:04:10.307271'),(23,'certificates','0007_certificateinvalidation','2016-01-22 20:04:10.510089'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-01-22 20:04:10.538558'),(25,'cors_csrf','0001_initial','2016-01-22 20:04:10.680353'),(26,'course_action_state','0001_initial','2016-01-22 20:04:11.030484'),(27,'course_groups','0001_initial','2016-01-22 20:04:12.283946'),(28,'course_modes','0001_initial','2016-01-22 20:04:12.452546'),(29,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-01-22 20:04:12.534953'),(30,'course_modes','0003_auto_20151113_1443','2016-01-22 20:04:12.569936'),(31,'course_modes','0004_auto_20151113_1457','2016-01-22 20:04:12.776505'),(32,'course_modes','0005_auto_20151217_0958','2016-01-22 20:04:12.807317'),(33,'course_overviews','0001_initial','2016-01-22 20:04:12.927012'),(34,'course_overviews','0002_add_course_catalog_fields','2016-01-22 20:04:13.220589'),(35,'course_overviews','0003_courseoverviewgeneratedhistory','2016-01-22 20:04:13.258023'),(36,'course_overviews','0004_courseoverview_org','2016-01-22 20:04:13.346700'),(37,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-01-22 20:04:13.375580'),(38,'course_overviews','0006_courseoverviewimageset','2016-01-22 20:04:13.464516'),(39,'course_overviews','0007_courseoverviewimageconfig','2016-01-22 20:04:13.651285'),(40,'course_structures','0001_initial','2016-01-22 20:04:13.689179'),(41,'courseware','0001_initial','2016-01-22 20:04:17.245019'),(42,'credentials','0001_initial','2016-01-22 20:04:17.454975'),(43,'credentials','0002_data__add_service_user','2016-01-22 20:04:17.494478'),(44,'credit','0001_initial','2016-01-22 20:04:19.593607'),(45,'dark_lang','0001_initial','2016-01-22 20:04:19.851812'),(46,'dark_lang','0002_data__enable_on_install','2016-01-22 20:04:19.886816'),(47,'default','0001_initial','2016-01-22 20:04:20.656170'),(48,'default','0002_add_related_name','2016-01-22 20:04:20.920330'),(49,'default','0003_alter_email_max_length','2016-01-22 20:04:21.038154'),(50,'django_comment_common','0001_initial','2016-01-22 20:04:21.777797'),(51,'django_notify','0001_initial','2016-01-22 20:04:23.074494'),(52,'django_openid_auth','0001_initial','2016-01-22 20:04:23.435834'),(53,'edx_proctoring','0001_initial','2016-01-22 20:04:28.470618'),(54,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-01-22 20:04:28.834557'),(55,'edx_proctoring','0003_auto_20160101_0525','2016-01-22 20:04:29.402026'),(56,'edxval','0001_initial','2016-01-22 20:04:30.249661'),(57,'edxval','0002_data__default_profiles','2016-01-22 20:04:30.307115'),(58,'embargo','0001_initial','2016-01-22 20:04:31.462353'),(59,'embargo','0002_data__add_countries','2016-01-22 20:04:32.017366'),(60,'external_auth','0001_initial','2016-01-22 20:04:32.753375'),(61,'lms_xblock','0001_initial','2016-01-22 20:04:33.090177'),(62,'sites','0001_initial','2016-01-22 20:04:33.156273'),(63,'microsite_configuration','0001_initial','2016-01-22 20:04:35.468782'),(64,'milestones','0001_initial','2016-01-22 20:04:36.698849'),(65,'milestones','0002_data__seed_relationship_types','2016-01-22 20:04:36.760089'),(66,'mobile_api','0001_initial','2016-01-22 20:04:38.189431'),(67,'notes','0001_initial','2016-01-22 20:04:38.639288'),(68,'oauth2','0001_initial','2016-01-22 20:04:40.968535'),(69,'oauth2_provider','0001_initial','2016-01-22 20:04:41.475836'),(70,'oauth_provider','0001_initial','2016-01-22 20:04:42.704759'),(71,'organizations','0001_initial','2016-01-22 20:04:42.985226'),(72,'organizations','0002_auto_20151119_2048','2016-01-22 20:04:43.049898'),(73,'problem_builder','0001_initial','2016-01-22 20:04:43.244682'),(74,'programs','0001_initial','2016-01-22 20:04:43.776265'),(75,'programs','0002_programsapiconfig_cache_ttl','2016-01-22 20:04:44.329122'),(76,'programs','0003_auto_20151120_1613','2016-01-22 20:04:46.494468'),(77,'rss_proxy','0001_initial','2016-01-22 20:04:46.572259'),(78,'self_paced','0001_initial','2016-01-22 20:04:47.142336'),(79,'sessions','0001_initial','2016-01-22 20:04:47.246566'),(80,'student','0001_initial','2016-01-22 20:05:04.804775'),(81,'shoppingcart','0001_initial','2016-01-22 20:05:20.059855'),(82,'shoppingcart','0002_auto_20151208_1034','2016-01-22 20:05:21.457216'),(83,'shoppingcart','0003_auto_20151217_0958','2016-01-22 20:05:22.922754'),(84,'splash','0001_initial','2016-01-22 20:05:23.712228'),(85,'static_replace','0001_initial','2016-01-22 20:05:24.489567'),(86,'status','0001_initial','2016-01-22 20:05:26.131108'),(87,'student','0002_auto_20151208_1034','2016-01-22 20:05:27.780831'),(88,'submissions','0001_initial','2016-01-22 20:05:29.846153'),(89,'submissions','0002_auto_20151119_0913','2016-01-22 20:05:30.098711'),(90,'survey','0001_initial','2016-01-22 20:05:31.137171'),(91,'teams','0001_initial','2016-01-22 20:05:33.436431'),(92,'third_party_auth','0001_initial','2016-01-22 20:05:38.010830'),(93,'track','0001_initial','2016-01-22 20:05:38.097314'),(94,'user_api','0001_initial','2016-01-22 20:05:44.556502'),(95,'util','0001_initial','2016-01-22 20:05:45.512595'),(96,'util','0002_data__default_rate_limit_config','2016-01-22 20:05:45.583331'),(97,'verify_student','0001_initial','2016-01-22 20:05:56.456343'),(98,'verify_student','0002_auto_20151124_1024','2016-01-22 20:05:58.780629'),(99,'verify_student','0003_auto_20151113_1443','2016-01-22 20:05:59.622634'),(100,'wiki','0001_initial','2016-01-22 20:06:30.273480'),(101,'wiki','0002_remove_article_subscription','2016-01-22 20:06:30.358472'),(102,'workflow','0001_initial','2016-01-22 20:06:30.741041'),(103,'xblock_django','0001_initial','2016-01-22 20:06:31.645852'),(104,'contentstore','0001_initial','2016-01-22 20:07:00.042399'),(105,'course_creators','0001_initial','2016-01-22 20:07:00.153769'),(106,'xblock_config','0001_initial','2016-01-22 20:07:00.552942');
/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2016-01-22 20:07:05
common/test/db_cache/bok_choy_migrations_data_default.sql 0 → 100644
-- MySQL dump 10.13 Distrib 5.6.14, for debian-linux-gnu (x86_64)
--
-- Host: localhost Database: edxtest
-- ------------------------------------------------------
-- Server version 5.6.14-1+debphp.org~precise+1
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Dumping data for table `django_migrations`
--
LOCK TABLES `django_migrations` WRITE;
/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-02-29 17:58:04.910265'),(2,'auth','0001_initial','2016-02-29 17:58:05.155297'),(3,'admin','0001_initial','2016-02-29 17:58:05.261098'),(4,'assessment','0001_initial','2016-02-29 17:58:08.458980'),(5,'assessment','0002_staffworkflow','2016-02-29 17:58:08.659081'),(6,'contenttypes','0002_remove_content_type_name','2016-02-29 17:58:08.818265'),(7,'auth','0002_alter_permission_name_max_length','2016-02-29 17:58:08.855835'),(8,'auth','0003_alter_user_email_max_length','2016-02-29 17:58:08.894682'),(9,'auth','0004_alter_user_username_opts','2016-02-29 17:58:08.918567'),(10,'auth','0005_alter_user_last_login_null','2016-02-29 17:58:08.971615'),(11,'auth','0006_require_contenttypes_0002','2016-02-29 17:58:08.976069'),(12,'bookmarks','0001_initial','2016-02-29 17:58:09.243246'),(13,'branding','0001_initial','2016-02-29 17:58:09.364920'),(14,'bulk_email','0001_initial','2016-02-29 17:58:09.630882'),(15,'bulk_email','0002_data__load_course_email_template','2016-02-29 17:58:09.715512'),(16,'instructor_task','0001_initial','2016-02-29 17:58:09.872596'),(17,'certificates','0001_initial','2016-02-29 17:58:10.722743'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-02-29 17:58:10.736641'),(19,'certificates','0003_data__default_modes','2016-02-29 17:58:10.775334'),(20,'certificates','0004_certificategenerationhistory','2016-02-29 17:58:10.911498'),(21,'certificates','0005_auto_20151208_0801','2016-02-29 17:58:11.002303'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-02-29 17:58:11.050549'),(23,'certificates','0007_certificateinvalidation','2016-02-29 17:58:11.166297'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-02-29 17:58:11.198023'),(25,'commerce','0002_commerceconfiguration','2016-02-29 17:58:11.291907'),(26,'contentserver','0001_initial','2016-02-29 17:58:11.375676'),(27,'cors_csrf','0001_initial','2016-02-29 17:58:11.462189'),(28,'course_action_state','0001_initial','2016-02-29 17:58:11.783744'),(29,'course_groups','0001_initial','2016-02-29 17:58:12.790006'),(30,'course_modes','0001_initial','2016-02-29 17:58:12.902796'),(31,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-02-29 17:58:12.957209'),(32,'course_modes','0003_auto_20151113_1443','2016-02-29 17:58:12.979234'),(33,'course_modes','0004_auto_20151113_1457','2016-02-29 17:58:13.095320'),(34,'course_modes','0005_auto_20151217_0958','2016-02-29 17:58:13.116644'),(35,'course_modes','0006_auto_20160208_1407','2016-02-29 17:58:13.208193'),(36,'course_overviews','0001_initial','2016-02-29 17:58:13.283088'),(37,'course_overviews','0002_add_course_catalog_fields','2016-02-29 17:58:13.499090'),(38,'course_overviews','0003_courseoverviewgeneratedhistory','2016-02-29 17:58:13.525866'),(39,'course_overviews','0004_courseoverview_org','2016-02-29 17:58:13.567577'),(40,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-02-29 17:58:13.585458'),(41,'course_overviews','0006_courseoverviewimageset','2016-02-29 17:58:13.636894'),(42,'course_overviews','0007_courseoverviewimageconfig','2016-02-29 17:58:13.764616'),(43,'course_overviews','0008_remove_courseoverview_facebook_url','2016-02-29 17:58:13.815867'),(44,'course_overviews','0009_readd_facebook_url','2016-02-29 17:58:13.872971'),(45,'course_structures','0001_initial','2016-02-29 17:58:13.899706'),(46,'courseware','0001_initial','2016-02-29 17:58:16.765503'),(47,'coursewarehistoryextended','0001_initial','2016-02-29 17:58:16.912454'),(48,'credentials','0001_initial','2016-02-29 17:58:17.072454'),(49,'credit','0001_initial','2016-02-29 17:58:18.478297'),(50,'dark_lang','0001_initial','2016-02-29 17:58:18.635638'),(51,'dark_lang','0002_data__enable_on_install','2016-02-29 17:58:18.654315'),(52,'default','0001_initial','2016-02-29 17:58:19.121037'),(53,'default','0002_add_related_name','2016-02-29 17:58:19.287646'),(54,'default','0003_alter_email_max_length','2016-02-29 17:58:19.355367'),(55,'django_comment_common','0001_initial','2016-02-29 17:58:19.826406'),(56,'django_notify','0001_initial','2016-02-29 17:58:20.691115'),(57,'django_openid_auth','0001_initial','2016-02-29 17:58:20.948196'),(58,'edx_proctoring','0001_initial','2016-02-29 17:58:25.028310'),(59,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-02-29 17:58:25.270722'),(60,'edx_proctoring','0003_auto_20160101_0525','2016-02-29 17:58:25.651659'),(61,'edx_proctoring','0004_auto_20160201_0523','2016-02-29 17:58:25.881209'),(62,'edxval','0001_initial','2016-02-29 17:58:26.490038'),(63,'edxval','0002_data__default_profiles','2016-02-29 17:58:26.519947'),(64,'embargo','0001_initial','2016-02-29 17:58:27.388295'),(65,'embargo','0002_data__add_countries','2016-02-29 17:58:27.716022'),(66,'external_auth','0001_initial','2016-02-29 17:58:28.339793'),(67,'lms_xblock','0001_initial','2016-02-29 17:58:28.600358'),(68,'sites','0001_initial','2016-02-29 17:58:28.639305'),(69,'microsite_configuration','0001_initial','2016-02-29 17:58:30.470563'),(70,'microsite_configuration','0002_auto_20160202_0228','2016-02-29 17:58:31.089884'),(71,'milestones','0001_initial','2016-02-29 17:58:32.206321'),(72,'milestones','0002_data__seed_relationship_types','2016-02-29 17:58:32.242905'),(73,'milestones','0003_coursecontentmilestone_requirements','2016-02-29 17:58:33.317573'),(74,'milestones','0004_auto_20151221_1445','2016-02-29 17:58:33.593210'),(75,'mobile_api','0001_initial','2016-02-29 17:58:33.849089'),(76,'notes','0001_initial','2016-02-29 17:58:34.185220'),(77,'oauth2','0001_initial','2016-02-29 17:58:35.892898'),(78,'oauth2_provider','0001_initial','2016-02-29 17:58:36.227236'),(79,'oauth_provider','0001_initial','2016-02-29 17:58:37.052991'),(80,'organizations','0001_initial','2016-02-29 17:58:37.290331'),(81,'programs','0001_initial','2016-02-29 17:58:37.667708'),(82,'programs','0002_programsapiconfig_cache_ttl','2016-02-29 17:58:38.067119'),(83,'programs','0003_auto_20151120_1613','2016-02-29 17:58:39.752602'),(84,'programs','0004_programsapiconfig_enable_certification','2016-02-29 17:58:40.179508'),(85,'programs','0005_programsapiconfig_max_retries','2016-02-29 17:58:40.596412'),(86,'rss_proxy','0001_initial','2016-02-29 17:58:40.641620'),(87,'self_paced','0001_initial','2016-02-29 17:58:41.078194'),(88,'sessions','0001_initial','2016-02-29 17:58:41.133704'),(89,'student','0001_initial','2016-02-29 17:58:53.980867'),(90,'shoppingcart','0001_initial','2016-02-29 17:59:06.233175'),(91,'shoppingcart','0002_auto_20151208_1034','2016-02-29 17:59:07.059000'),(92,'shoppingcart','0003_auto_20151217_0958','2016-02-29 17:59:07.907660'),(93,'splash','0001_initial','2016-02-29 17:59:08.351083'),(94,'static_replace','0001_initial','2016-02-29 17:59:08.818049'),(95,'static_replace','0002_assetexcludedextensionsconfig','2016-02-29 17:59:09.319193'),(96,'status','0001_initial','2016-02-29 17:59:10.500113'),(97,'student','0002_auto_20151208_1034','2016-02-29 17:59:11.650009'),(98,'submissions','0001_initial','2016-02-29 17:59:12.491480'),(99,'submissions','0002_auto_20151119_0913','2016-02-29 17:59:12.683289'),(100,'submissions','0003_submission_status','2016-02-29 17:59:12.811034'),(101,'survey','0001_initial','2016-02-29 17:59:13.767659'),(102,'teams','0001_initial','2016-02-29 17:59:16.603866'),(103,'third_party_auth','0001_initial','2016-02-29 17:59:19.924001'),(104,'track','0001_initial','2016-02-29 17:59:19.974203'),(105,'user_api','0001_initial','2016-02-29 17:59:24.197607'),(106,'util','0001_initial','2016-02-29 17:59:24.978364'),(107,'util','0002_data__default_rate_limit_config','2016-02-29 17:59:25.017205'),(108,'verify_student','0001_initial','2016-02-29 17:59:33.329519'),(109,'verify_student','0002_auto_20151124_1024','2016-02-29 17:59:34.238392'),(110,'verify_student','0003_auto_20151113_1443','2016-02-29 17:59:35.001393'),(111,'wiki','0001_initial','2016-02-29 17:59:56.418517'),(112,'wiki','0002_remove_article_subscription','2016-02-29 17:59:56.476734'),(113,'workflow','0001_initial','2016-02-29 17:59:56.760505'),(114,'xblock_django','0001_initial','2016-02-29 17:59:57.572216'),(115,'xblock_django','0002_auto_20160204_0809','2016-02-29 17:59:58.380438'),(116,'contentstore','0001_initial','2016-02-29 18:00:19.534458'),(117,'course_creators','0001_initial','2016-02-29 18:00:19.597219'),(118,'xblock_config','0001_initial','2016-02-29 18:00:19.855097');
/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2016-02-29 18:00:23
common/test/db_cache/bok_choy_migrations_data_student_module_history.sql 0 → 100644
-- MySQL dump 10.13 Distrib 5.6.14, for debian-linux-gnu (x86_64)
--
-- Host: localhost Database: student_module_history_test
-- ------------------------------------------------------
-- Server version 5.6.14-1+debphp.org~precise+1
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Dumping data for table `django_migrations`
--
LOCK TABLES `django_migrations` WRITE;
/*!40000 ALTER TABLE `django_migrations` DISABLE KEYS */;
INSERT INTO `django_migrations` VALUES (1,'contenttypes','0001_initial','2016-02-29 18:01:39.511327'),(2,'auth','0001_initial','2016-02-29 18:01:39.547129'),(3,'admin','0001_initial','2016-02-29 18:01:39.568351'),(4,'assessment','0001_initial','2016-02-29 18:01:40.149754'),(5,'assessment','0002_staffworkflow','2016-02-29 18:01:40.160082'),(6,'contenttypes','0002_remove_content_type_name','2016-02-29 18:01:40.231907'),(7,'auth','0002_alter_permission_name_max_length','2016-02-29 18:01:40.251926'),(8,'auth','0003_alter_user_email_max_length','2016-02-29 18:01:40.276419'),(9,'auth','0004_alter_user_username_opts','2016-02-29 18:01:40.297224'),(10,'auth','0005_alter_user_last_login_null','2016-02-29 18:01:40.317098'),(11,'auth','0006_require_contenttypes_0002','2016-02-29 18:01:40.319374'),(12,'bookmarks','0001_initial','2016-02-29 18:01:40.396366'),(13,'branding','0001_initial','2016-02-29 18:01:40.453783'),(14,'bulk_email','0001_initial','2016-02-29 18:01:40.565081'),(15,'bulk_email','0002_data__load_course_email_template','2016-02-29 18:01:40.574000'),(16,'instructor_task','0001_initial','2016-02-29 18:01:40.613011'),(17,'certificates','0001_initial','2016-02-29 18:01:40.980762'),(18,'certificates','0002_data__certificatehtmlviewconfiguration_data','2016-02-29 18:01:40.991190'),(19,'certificates','0003_data__default_modes','2016-02-29 18:01:41.003742'),(20,'certificates','0004_certificategenerationhistory','2016-02-29 18:01:41.061308'),(21,'certificates','0005_auto_20151208_0801','2016-02-29 18:01:41.113837'),(22,'certificates','0006_certificatetemplateasset_asset_slug','2016-02-29 18:01:41.131141'),(23,'certificates','0007_certificateinvalidation','2016-02-29 18:01:41.189247'),(24,'commerce','0001_data__add_ecommerce_service_user','2016-02-29 18:01:41.200385'),(25,'commerce','0002_commerceconfiguration','2016-02-29 18:01:41.260287'),(26,'contentserver','0001_initial','2016-02-29 18:01:41.323580'),(27,'cors_csrf','0001_initial','2016-02-29 18:01:41.388714'),(28,'course_action_state','0001_initial','2016-02-29 18:01:41.514544'),(29,'course_groups','0001_initial','2016-02-29 18:01:42.036647'),(30,'course_modes','0001_initial','2016-02-29 18:01:42.068946'),(31,'course_modes','0002_coursemode_expiration_datetime_is_explicit','2016-02-29 18:01:42.084746'),(32,'course_modes','0003_auto_20151113_1443','2016-02-29 18:01:42.100516'),(33,'course_modes','0004_auto_20151113_1457','2016-02-29 18:01:42.187139'),(34,'course_modes','0005_auto_20151217_0958','2016-02-29 18:01:42.205556'),(35,'course_modes','0006_auto_20160208_1407','2016-02-29 18:01:42.289731'),(36,'course_overviews','0001_initial','2016-02-29 18:01:42.319879'),(37,'course_overviews','0002_add_course_catalog_fields','2016-02-29 18:01:42.402948'),(38,'course_overviews','0003_courseoverviewgeneratedhistory','2016-02-29 18:01:42.417563'),(39,'course_overviews','0004_courseoverview_org','2016-02-29 18:01:42.435373'),(40,'course_overviews','0005_delete_courseoverviewgeneratedhistory','2016-02-29 18:01:42.448847'),(41,'course_overviews','0006_courseoverviewimageset','2016-02-29 18:01:42.472337'),(42,'course_overviews','0007_courseoverviewimageconfig','2016-02-29 18:01:42.561497'),(43,'course_overviews','0008_remove_courseoverview_facebook_url','2016-02-29 18:01:42.582078'),(44,'course_overviews','0009_readd_facebook_url','2016-02-29 18:01:42.608691'),(45,'course_structures','0001_initial','2016-02-29 18:01:42.625296'),(46,'courseware','0001_initial','2016-02-29 18:01:44.499738'),(47,'coursewarehistoryextended','0001_initial','2016-02-29 18:01:44.691094'),(48,'credentials','0001_initial','2016-02-29 18:01:44.788912'),(49,'credit','0001_initial','2016-02-29 18:01:45.621214'),(50,'dark_lang','0001_initial','2016-02-29 18:01:45.752882'),(51,'dark_lang','0002_data__enable_on_install','2016-02-29 18:01:45.765638'),(52,'default','0001_initial','2016-02-29 18:01:46.107319'),(53,'default','0002_add_related_name','2016-02-29 18:01:46.247368'),(54,'default','0003_alter_email_max_length','2016-02-29 18:01:46.265241'),(55,'django_comment_common','0001_initial','2016-02-29 18:01:46.569556'),(56,'django_notify','0001_initial','2016-02-29 18:01:47.200935'),(57,'django_openid_auth','0001_initial','2016-02-29 18:01:47.399851'),(58,'edx_proctoring','0001_initial','2016-02-29 18:01:50.659784'),(59,'edx_proctoring','0002_proctoredexamstudentattempt_is_status_acknowledged','2016-02-29 18:01:50.845795'),(60,'edx_proctoring','0003_auto_20160101_0525','2016-02-29 18:01:51.213992'),(61,'edx_proctoring','0004_auto_20160201_0523','2016-02-29 18:01:51.397904'),(62,'edxval','0001_initial','2016-02-29 18:01:51.631665'),(63,'edxval','0002_data__default_profiles','2016-02-29 18:01:51.651781'),(64,'embargo','0001_initial','2016-02-29 18:01:52.230500'),(65,'embargo','0002_data__add_countries','2016-02-29 18:01:52.401900'),(66,'external_auth','0001_initial','2016-02-29 18:01:52.851531'),(67,'lms_xblock','0001_initial','2016-02-29 18:01:53.081369'),(68,'sites','0001_initial','2016-02-29 18:01:53.104608'),(69,'microsite_configuration','0001_initial','2016-02-29 18:01:54.497782'),(70,'microsite_configuration','0002_auto_20160202_0228','2016-02-29 18:01:55.048275'),(71,'milestones','0001_initial','2016-02-29 18:01:55.459781'),(72,'milestones','0002_data__seed_relationship_types','2016-02-29 18:01:55.481439'),(73,'milestones','0003_coursecontentmilestone_requirements','2016-02-29 18:01:55.522487'),(74,'milestones','0004_auto_20151221_1445','2016-02-29 18:01:55.688330'),(75,'mobile_api','0001_initial','2016-02-29 18:01:56.844055'),(76,'notes','0001_initial','2016-02-29 18:01:57.066857'),(77,'oauth2','0001_initial','2016-02-29 18:01:58.412080'),(78,'oauth2_provider','0001_initial','2016-02-29 18:01:58.699157'),(79,'oauth_provider','0001_initial','2016-02-29 18:01:59.448662'),(80,'organizations','0001_initial','2016-02-29 18:01:59.538595'),(81,'programs','0001_initial','2016-02-29 18:01:59.922304'),(82,'programs','0002_programsapiconfig_cache_ttl','2016-02-29 18:02:00.317781'),(83,'programs','0003_auto_20151120_1613','2016-02-29 18:02:01.977886'),(84,'programs','0004_programsapiconfig_enable_certification','2016-02-29 18:02:02.413721'),(85,'programs','0005_programsapiconfig_max_retries','2016-02-29 18:02:02.799700'),(86,'rss_proxy','0001_initial','2016-02-29 18:02:02.828648'),(87,'self_paced','0001_initial','2016-02-29 18:02:03.260889'),(88,'sessions','0001_initial','2016-02-29 18:02:03.285722'),(89,'student','0001_initial','2016-02-29 18:02:14.964735'),(90,'shoppingcart','0001_initial','2016-02-29 18:02:26.167709'),(91,'shoppingcart','0002_auto_20151208_1034','2016-02-29 18:02:26.959607'),(92,'shoppingcart','0003_auto_20151217_0958','2016-02-29 18:02:27.772373'),(93,'splash','0001_initial','2016-02-29 18:02:28.180056'),(94,'static_replace','0001_initial','2016-02-29 18:02:28.600901'),(95,'static_replace','0002_assetexcludedextensionsconfig','2016-02-29 18:02:29.072670'),(96,'status','0001_initial','2016-02-29 18:02:30.149381'),(97,'student','0002_auto_20151208_1034','2016-02-29 18:02:31.383085'),(98,'submissions','0001_initial','2016-02-29 18:02:31.704655'),(99,'submissions','0002_auto_20151119_0913','2016-02-29 18:02:31.818741'),(100,'submissions','0003_submission_status','2016-02-29 18:02:31.875924'),(101,'survey','0001_initial','2016-02-29 18:02:32.613867'),(102,'teams','0001_initial','2016-02-29 18:02:35.208964'),(103,'third_party_auth','0001_initial','2016-02-29 18:02:38.385039'),(104,'track','0001_initial','2016-02-29 18:02:38.420093'),(105,'user_api','0001_initial','2016-02-29 18:02:42.631297'),(106,'util','0001_initial','2016-02-29 18:02:43.444988'),(107,'util','0002_data__default_rate_limit_config','2016-02-29 18:02:43.472520'),(108,'verify_student','0001_initial','2016-02-29 18:02:51.517974'),(109,'verify_student','0002_auto_20151124_1024','2016-02-29 18:02:52.305832'),(110,'verify_student','0003_auto_20151113_1443','2016-02-29 18:02:53.133517'),(111,'wiki','0001_initial','2016-02-29 18:03:14.972792'),(112,'wiki','0002_remove_article_subscription','2016-02-29 18:03:15.007100'),(113,'workflow','0001_initial','2016-02-29 18:03:15.147297'),(114,'xblock_django','0001_initial','2016-02-29 18:03:15.981916'),(115,'xblock_django','0002_auto_20160204_0809','2016-02-29 18:03:16.844433'),(116,'contentstore','0001_initial','2016-02-29 18:03:37.119213'),(117,'course_creators','0001_initial','2016-02-29 18:03:37.149132'),(118,'xblock_config','0001_initial','2016-02-29 18:03:37.381904');
/*!40000 ALTER TABLE `django_migrations` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2016-02-29 18:03:40
common/test/db_cache/bok_choy_schema_student_module_history.sql 0 → 100644
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
DROP TABLE IF EXISTS `coursewarehistoryextended_studentmodulehistoryextended`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `coursewarehistoryextended_studentmodulehistoryextended` (
`version` varchar(255) DEFAULT NULL,
`created` datetime(6) NOT NULL,
`state` longtext,
`grade` double DEFAULT NULL,
`max_grade` double DEFAULT NULL,
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`student_module_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `coursewarehistoryextended_studentmodulehistoryextended_2af72f10` (`version`),
KEY `coursewarehistoryextended_studentmodulehistoryextended_e2fa5388` (`created`)
) ENGINE=InnoDB AUTO_INCREMENT=10000 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `django_migrations`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `django_migrations` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`app` varchar(255) NOT NULL,
`name` varchar(255) NOT NULL,
`applied` datetime(6) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=119 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
lms/djangoapps/ccx/tests/test_field_override_performance.py
......@@ -46,6 +46,8 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
providers.
"""
__test__ = False
# Tell Django to clean out all databases, not just default
multi_db = True
# TEST_DATA must be overridden by subclasses
TEST_DATA = None
......@@ -151,7 +153,10 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
"""
return check_sum_of_calls(XBlock, ['__init__'], instantiations, instantiations, include_arguments=False)
def instrument_course_progress_render(self, course_width, enable_ccx, view_as_ccx, queries, reads, xblocks):
def instrument_course_progress_render(
self, course_width, enable_ccx, view_as_ccx,
default_queries, history_queries, reads, xblocks
):
"""
Renders the progress page, instrumenting Mongo reads and SQL queries.
"""
......@@ -173,10 +178,11 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
# can actually take affect.
OverrideFieldData.provider_classes = None
with self.assertNumQueries(queries):
with self.assertMongoCallCount(reads):
with self.assertXBlockInstantiations(xblocks):
self.grade_course(self.course, view_as_ccx)
with self.assertNumQueries(default_queries, using='default'):
with self.assertNumQueries(history_queries, using='student_module_history'):
with self.assertMongoCallCount(reads):
with self.assertXBlockInstantiations(xblocks):
self.grade_course(self.course, view_as_ccx)
@ddt.data(*itertools.product(('no_overrides', 'ccx'), range(1, 4), (True, False), (True, False)))
@ddt.unpack
......@@ -201,8 +207,12 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
raise SkipTest("Can't use a MongoModulestore test as a CCX course")
with self.settings(FIELD_OVERRIDE_PROVIDERS=providers[overrides]):
queries, reads, xblocks = self.TEST_DATA[(overrides, course_width, enable_ccx, view_as_ccx)]
self.instrument_course_progress_render(course_width, enable_ccx, view_as_ccx, queries, reads, xblocks)
default_queries, history_queries, reads, xblocks = self.TEST_DATA[
(overrides, course_width, enable_ccx, view_as_ccx)
]
self.instrument_course_progress_render(
course_width, enable_ccx, view_as_ccx, default_queries, history_queries, reads, xblocks
)
class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
......@@ -213,25 +223,30 @@ class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
__test__ = True
TEST_DATA = {
# (providers, course_width, enable_ccx, view_as_ccx): # of sql queries, # of mongo queries, # of xblocks
('no_overrides', 1, True, False): (48, 6, 13),
('no_overrides', 2, True, False): (135, 6, 84),
('no_overrides', 3, True, False): (480, 6, 335),
('ccx', 1, True, False): (48, 6, 13),
('ccx', 2, True, False): (135, 6, 84),
('ccx', 3, True, False): (480, 6, 335),
('ccx', 1, True, True): (48, 6, 13),
('ccx', 2, True, True): (135, 6, 84),
('ccx', 3, True, True): (480, 6, 335),
('no_overrides', 1, False, False): (48, 6, 13),
('no_overrides', 2, False, False): (135, 6, 84),
('no_overrides', 3, False, False): (480, 6, 335),
('ccx', 1, False, False): (48, 6, 13),
('ccx', 2, False, False): (135, 6, 84),
('ccx', 3, False, False): (480, 6, 335),
('ccx', 1, False, True): (48, 6, 13),
('ccx', 2, False, True): (135, 6, 84),
('ccx', 3, False, True): (480, 6, 335),
# (providers, course_width, enable_ccx, view_as_ccx): (
# # of sql queries to default,
# # sql queries to student_module_history,
# # of mongo queries,
# # of xblocks
# )
('no_overrides', 1, True, False): (47, 1, 6, 13),
('no_overrides', 2, True, False): (119, 16, 6, 84),
('no_overrides', 3, True, False): (399, 81, 6, 335),
('ccx', 1, True, False): (47, 1, 6, 13),
('ccx', 2, True, False): (119, 16, 6, 84),
('ccx', 3, True, False): (399, 81, 6, 335),
('ccx', 1, True, True): (47, 1, 6, 13),
('ccx', 2, True, True): (119, 16, 6, 84),
('ccx', 3, True, True): (399, 81, 6, 335),
('no_overrides', 1, False, False): (47, 1, 6, 13),
('no_overrides', 2, False, False): (119, 16, 6, 84),
('no_overrides', 3, False, False): (399, 81, 6, 335),
('ccx', 1, False, False): (47, 1, 6, 13),
('ccx', 2, False, False): (119, 16, 6, 84),
('ccx', 3, False, False): (399, 81, 6, 335),
('ccx', 1, False, True): (47, 1, 6, 13),
('ccx', 2, False, True): (119, 16, 6, 84),
('ccx', 3, False, True): (399, 81, 6, 335),
}
......@@ -243,22 +258,22 @@ class TestFieldOverrideSplitPerformance(FieldOverridePerformanceTestCase):
__test__ = True
TEST_DATA = {
('no_overrides', 1, True, False): (48, 4, 9),
('no_overrides', 2, True, False): (135, 19, 54),
('no_overrides', 3, True, False): (480, 84, 215),
('ccx', 1, True, False): (48, 4, 9),
('ccx', 2, True, False): (135, 19, 54),
('ccx', 3, True, False): (480, 84, 215),
('ccx', 1, True, True): (50, 4, 13),
('ccx', 2, True, True): (137, 19, 84),
('ccx', 3, True, True): (482, 84, 335),
('no_overrides', 1, False, False): (48, 4, 9),
('no_overrides', 2, False, False): (135, 19, 54),
('no_overrides', 3, False, False): (480, 84, 215),
('ccx', 1, False, False): (48, 4, 9),
('ccx', 2, False, False): (135, 19, 54),
('ccx', 3, False, False): (480, 84, 215),
('ccx', 1, False, True): (48, 4, 9),
('ccx', 2, False, True): (135, 19, 54),
('ccx', 3, False, True): (480, 84, 215),
('no_overrides', 1, True, False): (47, 1, 4, 9),
('no_overrides', 2, True, False): (119, 16, 19, 54),
('no_overrides', 3, True, False): (399, 81, 84, 215),
('ccx', 1, True, False): (47, 1, 4, 9),
('ccx', 2, True, False): (119, 16, 19, 54),
('ccx', 3, True, False): (399, 81, 84, 215),
('ccx', 1, True, True): (49, 1, 4, 13),
('ccx', 2, True, True): (121, 16, 19, 84),
('ccx', 3, True, True): (401, 81, 84, 335),
('no_overrides', 1, False, False): (47, 1, 4, 9),
('no_overrides', 2, False, False): (119, 16, 19, 54),
('no_overrides', 3, False, False): (399, 81, 84, 215),
('ccx', 1, False, False): (47, 1, 4, 9),
('ccx', 2, False, False): (119, 16, 19, 54),
('ccx', 3, False, False): (399, 81, 84, 215),
('ccx', 1, False, True): (47, 1, 4, 9),
('ccx', 2, False, True): (119, 16, 19, 54),
('ccx', 3, False, True): (399, 81, 84, 215),
}
lms/djangoapps/courseware/management/commands/clean_history.py deleted 100644 → 0
"""A command to clean the StudentModuleHistory table.
When we added XBlock storage, each field modification wrote a new history row
to the db. Now that we have bulk saves to avoid that database hammering, we
need to clean out the unnecessary rows from the database.
This command that does that.
"""
import datetime
import json
import logging
import optparse
import time
import traceback
from django.core.management.base import NoArgsCommand
from django.db import transaction
from django.db.models import Max
from courseware.models import StudentModuleHistory
class Command(NoArgsCommand):
"""The actual clean_history command to clean history rows."""
help = "Deletes unneeded rows from the StudentModuleHistory table."
option_list = NoArgsCommand.option_list + (
optparse.make_option(
'--batch',
type='int',
default=100,
help="Batch size, number of module_ids to examine in a transaction.",
),
optparse.make_option(
'--dry-run',
action='store_true',
default=False,
help="Don't change the database, just show what would be done.",
),
optparse.make_option(
'--sleep',
type='float',
default=0,
help="Seconds to sleep between batches.",
),
)
def handle_noargs(self, **options):
# We don't want to see the SQL output from the db layer.
logging.getLogger("django.db.backends").setLevel(logging.INFO)
smhc = StudentModuleHistoryCleaner(
dry_run=options["dry_run"],
)
smhc.main(batch_size=options["batch"], sleep=options["sleep"])
class StudentModuleHistoryCleaner(object):
"""Logic to clean rows from the StudentModuleHistory table."""
DELETE_GAP_SECS = 0.5 # Rows this close can be discarded.
STATE_FILE = "clean_history.json"
BATCH_SIZE = 100
def __init__(self, dry_run=False):
self.dry_run = dry_run
self.next_student_module_id = 0
self.last_student_module_id = 0
def main(self, batch_size=None, sleep=0):
"""Invoked from the management command to do all the work."""
batch_size = batch_size or self.BATCH_SIZE
self.last_student_module_id = self.get_last_student_module_id()
self.load_state()
while self.next_student_module_id <= self.last_student_module_id:
with transaction.atomic():
for smid in self.module_ids_to_check(batch_size):
try:
self.clean_one_student_module(smid)
except Exception: # pylint: disable=broad-except
trace = traceback.format_exc()
self.say("Couldn't clean student_module_id {}:\n{}".format(smid, trace))
if self.dry_run:
transaction.set_rollback(True)
else:
self.say("Committing")
self.save_state()
if sleep:
time.sleep(sleep)
def say(self, message):
"""
Display a message to the user.
The message will have a trailing newline added to it.
"""
print message
def load_state(self):
"""
Load the latest state from disk.
"""
try:
state_file = open(self.STATE_FILE)
except IOError:
self.say("No stored state")
self.next_student_module_id = 0
else:
with state_file:
state = json.load(state_file)
self.say(
"Loaded stored state: {}".format(
json.dumps(state, sort_keys=True)
)
)
self.next_student_module_id = state['next_student_module_id']
def save_state(self):
"""
Save the state to disk.
"""
state = {
'next_student_module_id': self.next_student_module_id,
}
with open(self.STATE_FILE, "w") as state_file:
json.dump(state, state_file)
self.say("Saved state: {}".format(json.dumps(state, sort_keys=True)))
def get_last_student_module_id(self):
"""
Return the id of the last student_module.
"""
last = StudentModuleHistory.objects.all() \
.aggregate(Max('student_module'))['student_module__max']
self.say("Last student_module_id is {}".format(last))
return last
def module_ids_to_check(self, batch_size):
"""Produce a sequence of student module ids to check.
`batch_size` is how many module ids to produce, max.
The sequence starts with `next_student_module_id`, and goes up to
and including `last_student_module_id`.
`next_student_module_id` is updated as each id is yielded.
"""
start = self.next_student_module_id
for smid in range(start, start + batch_size):
if smid > self.last_student_module_id:
break
yield smid
self.next_student_module_id = smid + 1
def get_history_for_student_modules(self, student_module_id):
"""
Get the history rows for a student module.
```student_module_id```: the id of the student module we're
interested in.
Return a list: [(id, created), ...], all the rows of history.
"""
history = StudentModuleHistory.objects \
.filter(student_module=student_module_id) \
.order_by('created', 'id')
return [(row.id, row.created) for row in history]
def delete_history(self, ids_to_delete):
"""
Delete history rows.
```ids_to_delete```: a non-empty list (or set...) of history row ids to delete.
"""
assert ids_to_delete
StudentModuleHistory.objects.filter(id__in=ids_to_delete).delete()
def clean_one_student_module(self, student_module_id):
"""Clean one StudentModule's-worth of history.
`student_module_id`: the id of the StudentModule to process.
"""
delete_gap = datetime.timedelta(seconds=self.DELETE_GAP_SECS)
history = self.get_history_for_student_modules(student_module_id)
if not history:
self.say("No history for student_module_id {}".format(student_module_id))
return
ids_to_delete = []
next_created = None
for history_id, created in reversed(history):
if next_created is not None:
# Compare this timestamp with the next one.
if (next_created - created) < delete_gap:
# This row is followed closely by another, we can discard
# this one.
ids_to_delete.append(history_id)
next_created = created
verb = "Would have deleted" if self.dry_run else "Deleting"
self.say("{verb} {to_delete} rows of {total} for student_module_id {id}".format(
verb=verb,
to_delete=len(ids_to_delete),
total=len(history),
id=student_module_id,
))
if ids_to_delete and not self.dry_run:
self.delete_history(ids_to_delete)
lms/djangoapps/courseware/models.py
......@@ -24,9 +24,9 @@ from django.dispatch import receiver, Signal
from model_utils.models import TimeStampedModel
from student.models import user_by_anonymous_id
from submissions.models import score_set, score_reset
import coursewarehistoryextended
from xmodule_django.models import CourseKeyField, LocationKeyField, BlockTypeKeyField
log = logging.getLogger(__name__)
log = logging.getLogger("edx.courseware")
......@@ -149,18 +149,15 @@ class StudentModule(models.Model):
return unicode(repr(self))
class StudentModuleHistory(models.Model):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size."""
class BaseStudentModuleHistory(models.Model):
"""Abstract class containing most fields used by any class
storing Student Module History"""
objects = ChunkingManager()
HISTORY_SAVING_TYPES = {'problem'}
class Meta(object):
app_label = "courseware"
get_latest_by = "created"
abstract = True
student_module = models.ForeignKey(StudentModule, db_index=True)
version = models.CharField(max_length=255, null=True, blank=True, db_index=True)
# This should be populated from the modified field in StudentModule
......@@ -169,11 +166,59 @@ class StudentModuleHistory(models.Model):
grade = models.FloatField(null=True, blank=True)
max_grade = models.FloatField(null=True, blank=True)
@receiver(post_save, sender=StudentModule)
@property
def csm(self):
"""
Finds the StudentModule object for this history record, even if our data is split
across multiple data stores. Django does not handle this correctly with the built-in
student_module property.
"""
return StudentModule.objects.get(pk=self.student_module_id)
@staticmethod
def get_history(student_modules):
"""
Find history objects across multiple backend stores for a given StudentModule
"""
history_entries = []
if settings.FEATURES.get('ENABLE_CSMH_EXTENDED'):
history_entries += coursewarehistoryextended.models.StudentModuleHistoryExtended.objects.filter(
# Django will sometimes try to join to courseware_studentmodule
# so just do an in query
student_module__in=[module.id for module in student_modules]
).order_by('-id')
# If we turn off reading from multiple history tables, then we don't want to read from
# StudentModuleHistory anymore, we believe that all history is in the Extended table.
if settings.FEATURES.get('ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES'):
# we want to save later SQL queries on the model which allows us to prefetch
history_entries += StudentModuleHistory.objects.prefetch_related('student_module').filter(
student_module__in=student_modules
).order_by('-id')
return history_entries
class StudentModuleHistory(BaseStudentModuleHistory):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size."""
class Meta(object):
app_label = "courseware"
get_latest_by = "created"
student_module = models.ForeignKey(StudentModule, db_index=True)
def __unicode__(self):
return unicode(repr(self))
def save_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Checks the instance's module_type, and creates & saves a
StudentModuleHistory entry if the module_type is one that
StudentModuleHistoryExtended entry if the module_type is one that
we save.
"""
if instance.module_type in StudentModuleHistory.HISTORY_SAVING_TYPES:
......@@ -185,8 +230,11 @@ class StudentModuleHistory(models.Model):
max_grade=instance.max_grade)
history_entry.save()
def __unicode__(self):
return unicode(repr(self))
# When the extended studentmodulehistory table exists, don't save
# duplicate history into courseware_studentmodulehistory, just retain
# data for reading.
if not settings.FEATURES.get('ENABLE_CSMH_EXTENDED'):
post_save.connect(save_history, sender=StudentModule)
class XBlockFieldBase(models.Model):
......
lms/djangoapps/courseware/tests/test_access.py
......@@ -520,6 +520,7 @@ class UserRoleTestCase(TestCase):
"""
Tests for user roles.
"""
def setUp(self):
super(UserRoleTestCase, self).setUp()
self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
......
lms/djangoapps/courseware/tests/test_grades.py
......@@ -240,6 +240,9 @@ class TestProgressSummary(TestCase):
(2/5) (3/5) (0/1) - (1/3) - (3/10)
"""
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self):
super(TestProgressSummary, self).setUp()
self.course_key = CourseLocator(
......
lms/djangoapps/courseware/tests/test_i18n.py
......@@ -20,6 +20,7 @@ class BaseI18nTestCase(TestCase):
"""
Base utilities for i18n test classes to derive from
"""
def assert_tag_has_attr(self, content, tag, attname, value):
"""Assert that a tag in `content` has a certain value in a certain attribute."""
regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d\- ]+)['"][^>]*>""".format(tag=tag, attname=attname)
......
lms/djangoapps/courseware/tests/test_model_data.py
......@@ -103,6 +103,8 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
"""Tests for user_state storage via StudentModule"""
other_key_factory = partial(DjangoKeyValueStore.Key, Scope.user_state, 2, location('usage_id')) # user_id=2, not 1
existing_field_name = "a_field"
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self):
super(TestStudentModuleStorage, self).setUp()
......@@ -137,8 +139,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
# to discover if something other than the DjangoXBlockUserStateClient
# has written to the StudentModule (such as UserStateCache setting the score
# on the StudentModule).
with self.assertNumQueries(3):
self.kvs.set(user_state_key('a_field'), 'new_value')
with self.assertNumQueries(2, using='default'):
with self.assertNumQueries(1, using='student_module_history'):
self.kvs.set(user_state_key('a_field'), 'new_value')
self.assertEquals(1, StudentModule.objects.all().count())
self.assertEquals({'b_field': 'b_value', 'a_field': 'new_value'}, json.loads(StudentModule.objects.all()[0].state))
......@@ -149,8 +152,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
# to discover if something other than the DjangoXBlockUserStateClient
# has written to the StudentModule (such as UserStateCache setting the score
# on the StudentModule).
with self.assertNumQueries(3):
self.kvs.set(user_state_key('not_a_field'), 'new_value')
with self.assertNumQueries(2, using='default'):
with self.assertNumQueries(1, using='student_module_history'):
self.kvs.set(user_state_key('not_a_field'), 'new_value')
self.assertEquals(1, StudentModule.objects.all().count())
self.assertEquals({'b_field': 'b_value', 'a_field': 'a_value', 'not_a_field': 'new_value'}, json.loads(StudentModule.objects.all()[0].state))
......@@ -161,8 +165,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
# to discover if something other than the DjangoXBlockUserStateClient
# has written to the StudentModule (such as UserStateCache setting the score
# on the StudentModule).
with self.assertNumQueries(3):
self.kvs.delete(user_state_key('a_field'))
with self.assertNumQueries(2, using='default'):
with self.assertNumQueries(1, using='student_module_history'):
self.kvs.delete(user_state_key('a_field'))
self.assertEquals(1, StudentModule.objects.all().count())
self.assertRaises(KeyError, self.kvs.get, user_state_key('not_a_field'))
......@@ -201,8 +206,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
# We also need to read the database to discover if something other than the
# DjangoXBlockUserStateClient has written to the StudentModule (such as
# UserStateCache setting the score on the StudentModule).
with self.assertNumQueries(3):
self.kvs.set_many(kv_dict)
with self.assertNumQueries(2, using="default"):
with self.assertNumQueries(1, using="student_module_history"):
self.kvs.set_many(kv_dict)
for key in kv_dict:
self.assertEquals(self.kvs.get(key), kv_dict[key])
......@@ -223,6 +229,9 @@ class TestStudentModuleStorage(OtherUserFailureTestMixin, TestCase):
@attr('shard_1')
class TestMissingStudentModule(TestCase):
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self):
super(TestMissingStudentModule, self).setUp()
......@@ -244,13 +253,15 @@ class TestMissingStudentModule(TestCase):
self.assertEquals(0, len(self.field_data_cache))
self.assertEquals(0, StudentModule.objects.all().count())
# We are updating a problem, so we write to courseware_studentmodulehistory
# We are updating a problem, so we write to courseware_studentmodulehistoryextended
# as well as courseware_studentmodule. We also need to read the database
# to discover if something other than the DjangoXBlockUserStateClient
# has written to the StudentModule (such as UserStateCache setting the score
# on the StudentModule).
with self.assertNumQueries(5):
self.kvs.set(user_state_key('a_field'), 'a_value')
# Django 1.8 also has a number of other BEGIN and SAVESTATE queries.
with self.assertNumQueries(4, using='default'):
with self.assertNumQueries(1, using='student_module_history'):
self.kvs.set(user_state_key('a_field'), 'a_value')
self.assertEquals(1, sum(len(cache) for cache in self.field_data_cache.cache.values()))
self.assertEquals(1, StudentModule.objects.all().count())
......
lms/djangoapps/courseware/tests/test_submitting_problems.py
......@@ -19,7 +19,7 @@ from capa.tests.response_xml_factory import (
CodeResponseXMLFactory,
)
from courseware import grades
from courseware.models import StudentModule, StudentModuleHistory
from courseware.models import StudentModule, BaseStudentModuleHistory
from courseware.tests.helpers import LoginEnrollmentTestCase
from lms.djangoapps.lms_xblock.runtime import quote_slashes
from student.tests.factories import UserFactory
......@@ -121,6 +121,8 @@ class TestSubmittingProblems(ModuleStoreTestCase, LoginEnrollmentTestCase, Probl
Check that a course gets graded properly.
"""
# Tell Django to clean out all databases, not just default
multi_db = True
# arbitrary constant
COURSE_SLUG = "100"
COURSE_NAME = "test_course"
......@@ -319,6 +321,9 @@ class TestCourseGrader(TestSubmittingProblems):
"""
Suite of tests for the course grader.
"""
# Tell Django to clean out all databases, not just default
multi_db = True
def basic_setup(self, late=False, reset=False, showanswer=False):
"""
Set up a simple course for testing basic grading functionality.
......@@ -451,26 +456,20 @@ class TestCourseGrader(TestSubmittingProblems):
self.submit_question_answer('p1', {'2_1': u'Correct'})
# Now fetch the state entry for that problem.
student_module = StudentModule.objects.get(
student_module = StudentModule.objects.filter(
course_id=self.course.id,
student=self.student_user
)
# count how many state history entries there are
baseline = StudentModuleHistory.objects.filter(
student_module=student_module
)
baseline_count = baseline.count()
self.assertEqual(baseline_count, 3)
baseline = BaseStudentModuleHistory.get_history(student_module)
self.assertEqual(len(baseline), 3)
# now click "show answer"
self.show_question_answer('p1')
# check that we don't have more state history entries
csmh = StudentModuleHistory.objects.filter(
student_module=student_module
)
current_count = csmh.count()
self.assertEqual(current_count, 3)
csmh = BaseStudentModuleHistory.get_history(student_module)
self.assertEqual(len(csmh), 3)
def test_grade_with_max_score_cache(self):
"""
......@@ -713,6 +712,8 @@ class TestCourseGrader(TestSubmittingProblems):
@attr('shard_1')
class ProblemWithUploadedFilesTest(TestSubmittingProblems):
"""Tests of problems with uploaded files."""
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self):
super(ProblemWithUploadedFilesTest, self).setUp()
......@@ -768,6 +769,8 @@ class TestPythonGradedResponse(TestSubmittingProblems):
"""
Check that we can submit a schematic and custom response, and it answers properly.
"""
# Tell Django to clean out all databases, not just default
multi_db = True
SCHEMATIC_SCRIPT = dedent("""
# for a schematic response, submission[i] is the json representation
......
lms/djangoapps/courseware/tests/test_user_state_client.py
......@@ -18,6 +18,8 @@ class TestDjangoUserStateClient(UserStateClientTestBase, TestCase):
Tests of the DjangoUserStateClient backend.
"""
__test__ = True
# Tell Django to clean out all databases, not just default
multi_db = True
def _user(self, user_idx):
return self.users[user_idx].username
......
lms/djangoapps/courseware/user_state_client.py
......@@ -15,7 +15,7 @@ except ImportError:
import dogstats_wrapper as dog_stats_api
from django.contrib.auth.models import User
from xblock.fields import Scope, ScopeBase
from courseware.models import StudentModule, StudentModuleHistory
from courseware.models import StudentModule, BaseStudentModuleHistory
from edx_user_state_client.interface import XBlockUserStateClient, XBlockUserState
......@@ -312,9 +312,7 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
if len(student_modules) == 0:
raise self.DoesNotExist()
history_entries = StudentModuleHistory.objects.prefetch_related('student_module').filter(
student_module__in=student_modules
).order_by('-id')
history_entries = BaseStudentModuleHistory.get_history(student_modules)
# If no history records exist, raise an error
if not history_entries:
......@@ -332,9 +330,9 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
if state == {}:
state = None
block_key = history_entry.student_module.module_state_key
block_key = history_entry.csm.module_state_key
block_key = block_key.map_into_course(
history_entry.student_module.course_id
history_entry.csm.course_id
)
yield XBlockUserState(username, block_key, state, history_entry.created, scope)
......
lms/djangoapps/courseware/views.py
......@@ -59,7 +59,7 @@ from courseware.courses import (
)
from courseware.masquerade import setup_masquerade
from courseware.model_data import FieldDataCache, ScoresClient
from courseware.models import StudentModuleHistory
from courseware.models import StudentModule, BaseStudentModuleHistory
from courseware.url_helpers import get_redirect_url
from courseware.user_state_client import DjangoXBlockUserStateClient
from edxmako.shortcuts import render_to_response, render_to_string, marketing_link
......@@ -1173,11 +1173,12 @@ def submission_history(request, course_id, student_username, location):
# This is ugly, but until we have a proper submissions API that we can use to provide
# the scores instead, it will have to do.
scores = list(StudentModuleHistory.objects.filter(
student_module__module_state_key=usage_key,
student_module__student__username=student_username,
student_module__course_id=course_key
).order_by('-id'))
csm = StudentModule.objects.filter(
module_state_key=usage_key,
student__username=student_username,
course_id=course_key)
scores = BaseStudentModuleHistory.get_history(csm)
if len(scores) != len(history_entries):
log.warning(
......
lms/djangoapps/coursewarehistoryextended/fields.py 0 → 100644
"""
Custom fields for use in the coursewarehistoryextended django app.
"""
from django.db.models.fields import AutoField
class UnsignedBigIntAutoField(AutoField):
"""
An unsigned 8-byte integer for auto-incrementing primary keys.
"""
def db_type(self, connection):
if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
return "bigint UNSIGNED AUTO_INCREMENT"
elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
# Sqlite will only auto-increment the ROWID column. Any INTEGER PRIMARY KEY column
# is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
# isn't an alias for ROWID, so we have to give up on the unsigned part.
return "integer"
elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
# Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and
# goes 1-9.2x10^18
return "BIGSERIAL"
else:
return None
lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
import coursewarehistoryextended.fields
from django.conf import settings
def bump_pk_start(apps, schema_editor):
if not schema_editor.connection.alias == 'student_module_history':
return
StudentModuleHistory = apps.get_model("courseware", "StudentModuleHistory")
biggest_id = StudentModuleHistory.objects.all().order_by('-id').first()
initial_id = settings.STUDENTMODULEHISTORYEXTENDED_OFFSET
if biggest_id is not None:
initial_id += biggest_id.id
if schema_editor.connection.vendor == 'mysql':
schema_editor.execute('ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended AUTO_INCREMENT=%s', [initial_id])
elif schema_editor.connection.vendor == 'sqlite3':
# This is a hack to force sqlite to add new rows after the earlier rows we
# want to migrate.
StudentModuleHistory(
id=initial_id,
course_key=None,
usage_key=None,
username="",
version="",
created=datetime.datetime.now(),
).save()
elif schema_editor.connection.vendor == 'postgresql':
schema_editor.execute("SELECT setval('coursewarehistoryextended_studentmodulehistoryextended_seq', %s)", [initial_id])
class Migration(migrations.Migration):
dependencies = [
('courseware', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='StudentModuleHistoryExtended',
fields=[
('version', models.CharField(db_index=True, max_length=255, null=True, blank=True)),
('created', models.DateTimeField(db_index=True)),
('state', models.TextField(null=True, blank=True)),
('grade', models.FloatField(null=True, blank=True)),
('max_grade', models.FloatField(null=True, blank=True)),
('id', coursewarehistoryextended.fields.UnsignedBigIntAutoField(serialize=False, primary_key=True)),
('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)),
],
options={
'get_latest_by': 'created',
},
),
migrations.RunPython(bump_pk_start, reverse_code=migrations.RunPython.noop),
]
lms/djangoapps/coursewarehistoryextended/models.py 0 → 100644
"""
WE'RE USING MIGRATIONS!
If you make changes to this model, be sure to create an appropriate migration
file and check it in at the same time as your model changes. To do that,
1. Go to the edx-platform dir
2. ./manage.py schemamigration courseware --auto description_of_your_change
3. Add the migration file created in edx-platform/lms/djangoapps/coursewarehistoryextended/migrations/
ASSUMPTIONS: modules have unique IDs, even across different module_types
"""
from django.db import models
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from coursewarehistoryextended.fields import UnsignedBigIntAutoField
from courseware.models import StudentModule, BaseStudentModuleHistory
class StudentModuleHistoryExtended(BaseStudentModuleHistory):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size.
This new extended CSMH has a larger primary key that won't run out of space
so quickly."""
class Meta(object):
app_label = 'coursewarehistoryextended'
get_latest_by = "created"
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False, on_delete=models.DO_NOTHING)
@receiver(post_save, sender=StudentModule)
def save_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Checks the instance's module_type, and creates & saves a
StudentModuleHistoryExtended entry if the module_type is one that
we save.
"""
if instance.module_type in StudentModuleHistoryExtended.HISTORY_SAVING_TYPES:
history_entry = StudentModuleHistoryExtended(student_module=instance,
version=None,
created=instance.modified,
state=instance.state,
grade=instance.grade,
max_grade=instance.max_grade)
history_entry.save()
@receiver(post_delete, sender=StudentModule)
def delete_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Django can't cascade delete across databases, so we tell it at the model level to
on_delete=DO_NOTHING and then listen for post_delete so we can clean up the CSMHE rows.
"""
StudentModuleHistoryExtended.objects.filter(student_module=instance).all().delete()
def __unicode__(self):
return unicode(repr(self))
lms/djangoapps/coursewarehistoryextended/tests.py 0 → 100644
"""
Tests for coursewarehistoryextended
Many aspects of this app are covered by the courseware tests,
but these are specific to the new storage model with multiple
backend tables.
"""
import json
from mock import patch
from django.test import TestCase
from django.conf import settings
from unittest import skipUnless
from nose.plugins.attrib import attr
from courseware.models import BaseStudentModuleHistory, StudentModuleHistory, StudentModule
from courseware.tests.factories import StudentModuleFactory, location, course_id
@attr('shard_1')
@skipUnless(settings.FEATURES["ENABLE_CSMH_EXTENDED"], "CSMH Extended needs to be enabled")
class TestStudentModuleHistoryBackends(TestCase):
""" Tests of data in CSMH and CSMHE """
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self):
super(TestStudentModuleHistoryBackends, self).setUp()
for record in (1, 2, 3):
# This will store into CSMHE via the post_save signal
csm = StudentModuleFactory.create(module_state_key=location('usage_id'),
course_id=course_id,
state=json.dumps({'type': 'csmhe', 'order': record}))
# This manually gets us a CSMH record to compare
csmh = StudentModuleHistory(student_module=csm,
version=None,
created=csm.modified,
state=json.dumps({'type': 'csmh', 'order': record}),
grade=csm.grade,
max_grade=csm.max_grade)
csmh.save()
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": True})
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": True})
def test_get_history_true_true(self):
student_module = StudentModule.objects.all()
history = BaseStudentModuleHistory.get_history(student_module)
self.assertEquals(len(history), 6)
self.assertEquals({'type': 'csmhe', 'order': 3}, json.loads(history[0].state))
self.assertEquals({'type': 'csmhe', 'order': 2}, json.loads(history[1].state))
self.assertEquals({'type': 'csmhe', 'order': 1}, json.loads(history[2].state))
self.assertEquals({'type': 'csmh', 'order': 3}, json.loads(history[3].state))
self.assertEquals({'type': 'csmh', 'order': 2}, json.loads(history[4].state))
self.assertEquals({'type': 'csmh', 'order': 1}, json.loads(history[5].state))
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": True})
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": False})
def test_get_history_true_false(self):
student_module = StudentModule.objects.all()
history = BaseStudentModuleHistory.get_history(student_module)
self.assertEquals(len(history), 3)
self.assertEquals({'type': 'csmhe', 'order': 3}, json.loads(history[0].state))
self.assertEquals({'type': 'csmhe', 'order': 2}, json.loads(history[1].state))
self.assertEquals({'type': 'csmhe', 'order': 1}, json.loads(history[2].state))
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": False})
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": True})
def test_get_history_false_true(self):
student_module = StudentModule.objects.all()
history = BaseStudentModuleHistory.get_history(student_module)
self.assertEquals(len(history), 3)
self.assertEquals({'type': 'csmh', 'order': 3}, json.loads(history[0].state))
self.assertEquals({'type': 'csmh', 'order': 2}, json.loads(history[1].state))
self.assertEquals({'type': 'csmh', 'order': 1}, json.loads(history[2].state))
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_CSMH_EXTENDED": False})
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES": False})
def test_get_history_false_false(self):
student_module = StudentModule.objects.all()
history = BaseStudentModuleHistory.get_history(student_module)
self.assertEquals(len(history), 0)
lms/envs/acceptance.py
......@@ -72,6 +72,14 @@ DATABASES = {
'timeout': 30,
},
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'TEST_NAME': TEST_ROOT / "db" / "test_student_module_history.db",
'OPTIONS': {
'timeout': 30,
},
}
}
......@@ -145,7 +153,9 @@ LETTUCE_APPS = ('courseware', 'instructor')
# This causes some pretty cryptic errors as lettuce tries
# to parse files in `instructor_task` as features.
# As a quick workaround, explicitly exclude the `instructor_task` app.
LETTUCE_AVOID_APPS = ('instructor_task',)
# The coursewarehistoryextended app also falls prey to this fuzzy
# for the courseware app.
LETTUCE_AVOID_APPS = ('instructor_task', 'coursewarehistoryextended')
LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome')
......
lms/envs/aws.py
......@@ -759,6 +759,11 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get(
# Course Content Bookmarks Settings
MAX_BOOKMARKS_PER_COURSE = ENV_TOKENS.get('MAX_BOOKMARKS_PER_COURSE', MAX_BOOKMARKS_PER_COURSE)
# Offset for pk of courseware.StudentModuleHistoryExtended
STUDENTMODULEHISTORYEXTENDED_OFFSET = ENV_TOKENS.get(
'STUDENTMODULEHISTORYEXTENDED_OFFSET', STUDENTMODULEHISTORYEXTENDED_OFFSET
)
# Cutoff date for granting audit certificates
if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None):
AUDIT_CERT_CUTOFF_DATE = dateutil.parser.parse(ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE'))
......@@ -766,3 +771,7 @@ if ENV_TOKENS.get('AUDIT_CERT_CUTOFF_DATE', None):
################################ Settings for Credentials Service ################################
CREDENTIALS_GENERATION_ROUTING_KEY = HIGH_PRIORITY_QUEUE
# The extended StudentModule history table
if FEATURES.get('ENABLE_CSMH_EXTENDED'):
INSTALLED_APPS += ('coursewarehistoryextended',)
lms/envs/aws_migrate.py
......@@ -13,18 +13,25 @@ from .aws import *
import os
from django.core.exceptions import ImproperlyConfigured
DB_OVERRIDES = dict(
PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
USER=os.environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
)
if DB_OVERRIDES['PASSWORD'] is None:
raise ImproperlyConfigured("No database password was provided for running "
"migrations. This is fatal.")
def get_db_overrides(db_name):
"""
Now that we have multiple databases, we want to look up from the environment
for both databases.
"""
db_overrides = dict(
PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES[db_name]['ENGINE']),
USER=os.environ.get('DB_MIGRATION_USER', DATABASES[db_name]['USER']),
NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES[db_name]['NAME']),
HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES[db_name]['HOST']),
PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES[db_name]['PORT']),
)
for override, value in DB_OVERRIDES.iteritems():
DATABASES['default'][override] = value
if db_overrides['PASSWORD'] is None:
raise ImproperlyConfigured("No database password was provided for running "
"migrations. This is fatal.")
return db_overrides
for db in ['default', 'student_module_history']:
DATABASES[db].update(get_db_overrides(db))
lms/envs/bok_choy.auth.json
......@@ -39,6 +39,14 @@
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
},
"student_module_history": {
"ENGINE": "django.db.backends.mysql",
"HOST": "localhost",
"NAME": "student_module_history_test",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
}
},
"DOC_STORE_CONFIG": {
......
lms/envs/bok_choy.py
......@@ -178,6 +178,11 @@ PROFILE_IMAGE_BACKEND = {
'base_url': os.path.join(MEDIA_URL, 'profile-images/'),
},
}
# Make sure we test with the extended history table
FEATURES['ENABLE_CSMH_EXTENDED'] = True
INSTALLED_APPS += ('coursewarehistoryextended',)
#####################################################################
# Lastly, see if the developer has any local overrides.
try:
......
lms/envs/common.py
......@@ -363,6 +363,18 @@ FEATURES = {
# Show Language selector.
'SHOW_LANGUAGE_SELECTOR': False,
# Write new CSM history to the extended table.
# This will eventually default to True and may be
# removed since all installs should have the separate
# extended history table.
'ENABLE_CSMH_EXTENDED': False,
# Read from both the CSMH and CSMHE history tables.
# This is the default, but can be disabled if all history
# lives in the Extended table, saving the frontend from
# making multiple queries.
'ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES': True
}
# Ignore static asset files on import which match this pattern
......@@ -413,6 +425,12 @@ GEOIPV6_PATH = REPO_ROOT / "common/static/data/geoip/GeoIPv6.dat"
# Where to look for a status message
STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json"
############################ Global Database Configuration #####################
DATABASE_ROUTERS = [
'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter',
]
############################ OpenID Provider ##################################
OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net']
......@@ -2759,6 +2777,13 @@ MOBILE_APP_USER_AGENT_REGEXES = [
r'edX/org.edx.mobile',
]
# Offset for courseware.StudentModuleHistoryExtended which is used to
# calculate the starting primary key for the underlying table. This gap
# should be large enough that you do not generate more than N courseware.StudentModuleHistory
# records before you have deployed the app to write to coursewarehistoryextended.StudentModuleHistoryExtended
# if you want to avoid an overlap in ids while searching for history across the two tables.
STUDENTMODULEHISTORYEXTENDED_OFFSET = 10000
# Deprecated xblock types
DEPRECATED_ADVANCED_COMPONENT_TYPES = []
......
lms/envs/dev.py
......@@ -48,6 +48,11 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "edx.db",
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "student_module_history.db",
'ATOMIC_REQUESTS': True,
}
}
......
lms/envs/devplus.py
......@@ -31,6 +31,15 @@ DATABASES = {
'HOST': '127.0.0.1',
'PORT': '3306',
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'student_module_history',
'USER': 'root',
'PASSWORD': '',
'HOST': '127.0.0.1',
'PORT': '3306',
'ATOMIC_REQUESTS': True,
}
}
......
lms/envs/static.py
......@@ -27,6 +27,11 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "edx.db",
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "student_module_history.db",
'ATOMIC_REQUESTS': True,
}
}
......
lms/envs/test.py
......@@ -186,7 +186,10 @@ DATABASES = {
'NAME': TEST_ROOT / 'db' / 'edx.db',
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': TEST_ROOT / 'db' / 'student_module_history.db'
},
}
if os.environ.get('DISABLE_MIGRATIONS'):
......@@ -194,6 +197,10 @@ if os.environ.get('DISABLE_MIGRATIONS'):
# to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations.
MIGRATION_MODULES = NoOpMigrationModules()
# Make sure we test with the extended history table
FEATURES['ENABLE_CSMH_EXTENDED'] = True
INSTALLED_APPS += ('coursewarehistoryextended',)
CACHES = {
# This is the cache used for most things.
# In staging/prod envs, the sessions also live here.
......
lms/envs/test_static_optimized.py
......@@ -19,7 +19,9 @@ DATABASES = {
'ENGINE': 'django.db.backends.sqlite3',
'ATOMIC_REQUESTS': True,
},
'student_module_history': {
'ENGINE': 'django.db.backends.sqlite3',
},
}
# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up
......
openedx/core/lib/django_courseware_routers.py 0 → 100644
"""
Database Routers for use with the coursewarehistoryextended django app.
"""
class StudentModuleHistoryExtendedRouter(object):
"""
A Database Router that separates StudentModuleHistoryExtended into its own database.
"""
DATABASE_NAME = 'student_module_history'
def _is_csmh(self, model):
"""
Return True if ``model`` is courseware.StudentModuleHistoryExtended.
"""
return (
model._meta.app_label == 'coursewarehistoryextended' and # pylint: disable=protected-access
model.__name__ == 'StudentModuleHistoryExtended'
)
def db_for_read(self, model, **hints): # pylint: disable=unused-argument
"""
Use the StudentModuleHistoryExtendedRouter.DATABASE_NAME if the model is StudentModuleHistoryExtended.
"""
if self._is_csmh(model):
return self.DATABASE_NAME
else:
return None
def db_for_write(self, model, **hints): # pylint: disable=unused-argument
"""
Use the StudentModuleHistoryExtendedRouter.DATABASE_NAME if the model is StudentModuleHistoryExtended.
"""
if self._is_csmh(model):
return self.DATABASE_NAME
else:
return None
def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument
"""
Disable relations if the model is StudentModuleHistoryExtended.
"""
if self._is_csmh(obj1) or self._is_csmh(obj2):
return False
return None
def allow_migrate(self, db, model): # pylint: disable=unused-argument
"""
Only sync StudentModuleHistoryExtended to StudentModuleHistoryExtendedRouter.DATABASE_Name
"""
if self._is_csmh(model):
return db == self.DATABASE_NAME
elif db == self.DATABASE_NAME:
return False
return None
pavelib/utils/test/suites/acceptance_suite.py
......@@ -69,8 +69,14 @@ class AcceptanceTestSuite(TestSuite):
def __init__(self, *args, **kwargs):
super(AcceptanceTestSuite, self).__init__(*args, **kwargs)
self.root = 'acceptance'
self.db = Env.REPO_ROOT / 'test_root/db/test_edx.db'
self.db_cache = Env.REPO_ROOT / 'common/test/db_cache/lettuce.db'
self.dbs = {
'default': Env.REPO_ROOT / 'test_root/db/test_edx.db',
'student_module_history': Env.REPO_ROOT / 'test_root/db/test_student_module_history.db'
}
self.db_caches = {
'default': Env.REPO_ROOT / 'common/test/db_cache/lettuce.db',
'student_module_history': Env.REPO_ROOT / 'common/test/db_cache/lettuce_student_module_history.db'
}
self.fasttest = kwargs.get('fasttest', False)
if kwargs.get('system'):
......@@ -114,24 +120,30 @@ class AcceptanceTestSuite(TestSuite):
definitions to sync and migrate.
"""
if self.db.isfile():
# Since we are using SQLLite, we can reset the database by deleting it on disk.
self.db.remove()
for db in self.dbs.keys():
if self.dbs[db].isfile():
# Since we are using SQLLite, we can reset the database by deleting it on disk.
self.dbs[db].remove()
if self.db_cache.isfile():
if all(self.db_caches[cache].isfile() for cache in self.db_caches.keys()):
# To speed up migrations, we check for a cached database file and start from that.
# The cached database file should be checked into the repo
# Copy the cached database to the test root directory
sh("cp {db_cache} {db}".format(db_cache=self.db_cache, db=self.db))
for db_alias in self.dbs.keys():
sh("cp {db_cache} {db}".format(db_cache=self.db_caches[db_alias], db=self.dbs[db_alias]))
# Run migrations to update the db, starting from its cached state
sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial")
sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial")
for db_alias in sorted(self.dbs.keys()):
# pylint: disable=line-too-long
sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
else:
# If no cached database exists, syncdb before migrating, then create the cache
sh("./manage.py lms --settings acceptance migrate --traceback --noinput")
sh("./manage.py cms --settings acceptance migrate --traceback --noinput")
for db_alias in sorted(self.dbs.keys()):
sh("./manage.py lms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
sh("./manage.py cms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
# Create the cache if it doesn't already exist
sh("cp {db} {db_cache}".format(db_cache=self.db_cache, db=self.db))
for db_alias in self.dbs.keys():
sh("cp {db} {db_cache}".format(db_cache=self.db_caches[db_alias], db=self.dbs[db_alias]))
scripts/reset-test-db.sh
......@@ -24,35 +24,55 @@
DB_CACHE_DIR="common/test/db_cache"
declare -A databases
declare -a database_order
databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
database_order=("default" "student_module_history")
# Ensure the test database exists.
echo "CREATE DATABASE IF NOT EXISTS edxtest;" | mysql -u root
for db in "${database_order[@]}"; do
echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root
# Clear out the test database
#
# We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
# "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
# or a jenkins worker environment) that already ran tests on another commit that had
# different migrations that created, dropped, or altered tables.
echo "Issuing a reset_db command to the bok_choy MySQL database."
./manage.py lms --settings bok_choy reset_db --traceback --noinput
# Clear out the test database
#
# We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
# "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
# or a jenkins worker environment) that already ran tests on another commit that had
# different migrations that created, dropped, or altered tables.
echo "Issuing a reset_db command to the $db bok_choy MySQL database."
./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db
# If there are cached database schemas/data, load them
if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then
echo "Missing $DB_CACHE_DIR/bok_choy_schema_$db.sql or $DB_CACHE_DIR/bok_choy_data_$db.json, or $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql rebuilding cache"
REBUILD_CACHE=true
fi
done
# If there are cached database schemas/data, load them
if [[ -f $DB_CACHE_DIR/bok_choy_schema.sql && -f $DB_CACHE_DIR/bok_choy_migrations_data.sql && -f $DB_CACHE_DIR/bok_choy_data.json ]]; then
if [[ -z $REBUILD_CACHE ]]; then
echo "Found the bok_choy DB cache files. Loading them into the database..."
# Load the schema, then the data (including the migration history)
echo "Loading the schema from the filesystem into the MySQL DB."
mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_schema.sql
echo "Loading the migration data from the filesystem into the MySQL DB."
mysql -u root edxtest < $DB_CACHE_DIR/bok_choy_migrations_data.sql
echo "Loading the fixture data from the filesystem into the MySQL DB."
./manage.py lms --settings bok_choy loaddata $DB_CACHE_DIR/bok_choy_data.json
# Re-run migrations to ensure we are up-to-date
echo "Running the lms migrations on the bok_choy DB."
./manage.py lms --settings bok_choy migrate --traceback --noinput
echo "Running the cms migrations on the bok_choy DB."
./manage.py cms --settings bok_choy migrate --traceback --noinput
for db in "${database_order[@]}"; do
# Load the schema, then the data (including the migration history)
echo "Loading the schema from the filesystem into the $db MySQL DB."
mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
echo "Loading the fixture data from the filesystem into the $db MySQL DB."
./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
# Migrations are stored in the default database
echo "Loading the migration data from the filesystem into the $db MySQL DB."
mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
# Re-run migrations to ensure we are up-to-date
echo "Running the lms migrations on the $db bok_choy DB."
./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
echo "Running the cms migrations on the $db bok_choy DB."
./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
done
# Otherwise, update the test database and update the cache
else
......@@ -60,19 +80,21 @@ else
# Clean the cache directory
mkdir -p $DB_CACHE_DIR && rm -f $DB_CACHE_DIR/bok_choy*
# Re-run migrations on the test database
echo "Issuing a migrate command to the bok_choy MySQL database for the lms django apps."
./manage.py lms --settings bok_choy migrate --traceback --noinput
echo "Issuing a migrate command to the bok_choy MySQL database for the cms django apps."
./manage.py cms --settings bok_choy migrate --traceback --noinput
# Dump the schema and data to the cache
echo "Using the dumpdata command to save the fixture data to the filesystem."
./manage.py lms --settings bok_choy dumpdata > $DB_CACHE_DIR/bok_choy_data.json
# dump_data does not dump the django_migrations table so we do it separately.
echo "Saving the django_migrations table of the bok_choy DB to the filesystem."
mysqldump -u root --no-create-info edxtest django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data.sql
echo "Saving the schema of the bok_choy DB to the filesystem."
mysqldump -u root --no-data --skip-comments --skip-dump-date edxtest > $DB_CACHE_DIR/bok_choy_schema.sql
fi
for db in "${database_order[@]}"; do
# Re-run migrations on the test database
echo "Issuing a migrate command to the $db bok_choy MySQL database for the lms django apps."
./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
echo "Issuing a migrate command to the $db bok_choy MySQL database for the cms django apps."
./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
# Dump the schema and data to the cache
echo "Using the dumpdata command to save the $db fixture data to the filesystem."
./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json
echo "Saving the schema of the $dh bok_choy DB to the filesystem."
mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
# dump_data does not dump the django_migrations table so we do it separately.
echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem."
mysqldump -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
done
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment