Cache databases to speed up acceptance tests

rakelib/acceptance_test.rake

-ACCEPTANCE_DB = 'test_root/db/test_edx.db'
+ACCEPTANCE_DB = File.join(REPO_ROOT, 'test_root/db/test_edx.db')
+ACCEPTANCE_DB_CACHE = File.join(REPO_ROOT, 'common/test/db_cache/lettuce.db')
 ACCEPTANCE_REPORT_DIR = report_dir_path('acceptance')
 directory ACCEPTANCE_REPORT_DIR
 
@@ -21,12 +22,31 @@ task :setup_acceptance_db do
     # migrations from the upgrade tables in the DB.
     # But for now for either system (lms or cms), use the lms
     # definitions to sync and migrate.
+
+    # Since we are using SQLLite, we can reset the database by deleting it on disk.
     if File.exists?(ACCEPTANCE_DB)
         File.delete(ACCEPTANCE_DB)
     end
 
-    sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
-    sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
+    # To speed up migrations, we check for a cached database file and start from that.
+    # The cached database file should be checked into the repo
+    if File.exists?(ACCEPTANCE_DB_CACHE)
+
+        # Copy the cached database to the test root directory
+        sh("cp #{ACCEPTANCE_DB_CACHE} #{ACCEPTANCE_DB}")
+
+        # Run migrations to update the db, starting from its cached state
+        sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
+
+    # If no cached database exists, syncdb before migrating, then create the cache
+    else
+        sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
+        sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
+
+        # Create the cache if it doesn't already exist
+        sh("cp #{ACCEPTANCE_DB} #{ACCEPTANCE_DB_CACHE}")
+    end
+
 end
 
 task :prep_for_acceptance_tests => [

rakelib/bok_choy.rake

@@ -150,17 +150,8 @@ namespace :'test:bok_choy' do
         # Invalidate the cache
         BOK_CHOY_CACHE.flush()
 
-        # HACK: Since the CMS depends on the existence of some database tables
-        # that are now in common but used to be in LMS (Role/Permissions for Forums)
-        # we need to create/migrate the database tables defined in the LMS.
-        # We might be able to address this by moving out the migrations from
-        # lms/django_comment_client, but then we'd have to repair all the existing
-        # migrations from the upgrade tables in the DB.
-        # But for now for either system (lms or cms), use the lms
-        # definitions to sync and migrate.
-        sh(django_admin('lms', 'bok_choy', 'reset_db', '--noinput'))
-        sh(django_admin('lms', 'bok_choy', 'syncdb', '--noinput'))
-        sh(django_admin('lms', 'bok_choy', 'migrate', '--noinput'))
+        # Reset the database
+        sh("#{REPO_ROOT}/scripts/reset-test-db.sh")
 
         # Collect static assets
         Rake::Task["gather_assets"].invoke('lms', 'bok_choy')

scripts/reset-test-db.sh

+#!/usr/bin/env bash
+
+############################################################################
+#
+#   reset-test-db.sh
+#
+#   Resets the MySQL test database for the bok-choy acceptance tests.
+#
+#   If it finds a cached schema and migration history, it will start
+#   from the cached version to speed up migrations.
+#
+#   If no cached database exists, it will create one.  This can be
+#   checked into the repo to speed up future tests.
+#
+#   Note that we do NOT want to re-use the cache between test runs!
+#   A newer commit could introduce migrations that do not exist
+#   in other commits, which could cause migrations to fail in the other
+#   commits.
+#
+#   For this reason, we always use a cache that was committed to master
+#   at the time the branch was created.
+#
+############################################################################
+
+DB_CACHE_DIR="common/test/db_cache"
+
+# Clear out the test database
+./manage.py lms --settings bok_choy reset_db --traceback --noinput
+
+# If there are cached database schemas/data, load them
+if [[ -f $DB_CACHE_DIR/bok_choy_schema.sql && -f $DB_CACHE_DIR/bok_choy_data.json ]]; then
+
+    # Load the schema, then the data (including the migration history)
+    mysql -u root test < $DB_CACHE_DIR/bok_choy_schema.sql
+    ./manage.py lms --settings bok_choy loaddata $DB_CACHE_DIR/bok_choy_data.json
+
+    # Re-run migrations to ensure we are up-to-date
+    ./manage.py lms --settings bok_choy migrate --traceback --noinput
+
+# Otherwise, update the test database and update the cache
+else
+
+    # Clean the cache directory
+    rm -rf $DB_CACHE_DIR && mkdir -p $DB_CACHE_DIR
+
+    # Re-run migrations on the test database
+    ./manage.py lms --settings bok_choy syncdb --traceback --noinput
+    ./manage.py lms --settings bok_choy migrate --traceback --noinput
+
+    # Dump the schema and data to the cache
+    ./manage.py lms --settings bok_choy dumpdata > $DB_CACHE_DIR/bok_choy_data.json
+    mysqldump -u root --no-data --skip-comments --skip-dump-date test > $DB_CACHE_DIR/bok_choy_schema.sql
+fi
+