Merge pull request #16851 from edx/estute/paver-fingerprint-migrations

TE-2350 paver function to fingerprint bokchoy db files

pavelib/database.py

@@ -3,6 +3,7 @@ tasks for controlling the databases used in tests
 """
 from __future__ import print_function
 import os
+import hashlib
 
 from paver.easy import sh, needs
 
@@ -18,7 +19,9 @@ def update_bokchoy_db_cache():
     """
     Update and cache the MYSQL database for bokchoy testing. This command
     will remove any previously cached database files and apply migrations
-    on a fresh db.
+    on a fresh db. Additionally, the collective sha1 checksum for all of
+    these files will be written to file, for future comparisons/checking
+    for updates.
 
     You can commit the resulting files in common/test/db_cache into
     git to speed up test runs
@@ -43,3 +46,59 @@ def update_bokchoy_db_cache():
             continue
 
     sh('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT))
+
+    # Write the fingerprint of the database files to disk for use in future
+    # comparisons
+    fingerprint = fingerprint_bokchoy_db_files()
+    with open('common/test/db_cache/bokchoy_migrations.sha1', 'w') as fingerprint_file:
+        fingerprint_file.write(fingerprint)
+
+
+def compare_bokchoy_db_fingerprints():
+    """
+    Determine if the current state of the bokchoy databases and related files
+    have changed since the last time they were updated in the repository by
+    comparing their fingerprint to the fingerprint saved in the repo
+    """
+    try:
+        fingerprint_filepath = '{}/common/test/db_cache/bokchoy_migrations.sha1'.format(Env.REPO_ROOT)
+        with open(fingerprint_filepath, 'r') as fingerprint_file:
+            cached_fingerprint = fingerprint_file.read().strip()
+    except IOError:
+        return False
+    current_fingerprint = fingerprint_bokchoy_db_files()
+    return current_fingerprint == cached_fingerprint
+
+
+def fingerprint_bokchoy_db_files():
+    """
+    Generate a sha1 checksum for files used to configure the bokchoy databases.
+    This checksum will represent the current 'state' of the databases,
+    including schema, migrations to be run and data. It can be used to determine
+    if the databases need to be updated.
+    """
+    # Run the calculate-bokchoy-migrations script, which will generate two
+    # yml files. These tell whether or not we need to run migrations
+    sh('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT))
+    db_files = [
+        # Bokchoy db schema and data fixtures
+        'bok_choy_data_default.json',
+        'bok_choy_data_student_module_history.json',
+        'bok_choy_migrations_data_default.sql',
+        'bok_choy_migrations_data_student_module_history.sql',
+        'bok_choy_schema_default.sql',
+        'bok_choy_schema_student_module_history.sql',
+        # Output files from scripts/calculate-bokchoy-migrations.sh
+        'bok_choy_default_migrations.yaml',
+        'bok_choy_student_module_history_migrations.yaml'
+    ]
+    hasher = hashlib.sha1()
+    file_paths = [
+        os.path.join('common/test/db_cache', db_file) for db_file in db_files
+    ]
+    for file_path in file_paths:
+        with open(file_path, 'rb') as file_handle:
+            hasher.update(file_handle.read())
+    fingerprint = hasher.hexdigest()
+    print("Computed fingerprint for bokchoy db files: {}".format(fingerprint))
+    return fingerprint

scripts/calculate-bokchoy-migrations.sh

+#!/usr/bin/env bash
+
+############################################################################
+#
+#   Output all migrations that would be applied to an
+#   empty database for the bok-choy acceptance tests.
+#
+############################################################################
+
+# Fail fast
+set -e
+
+if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then
+    MYSQL_HOST=""
+    SETTINGS="bok_choy"
+else
+    MYSQL_HOST="--host=edx.devstack.mysql"
+    SETTINGS="bok_choy_docker"
+fi
+
+declare -A databases
+declare -a database_order
+databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
+database_order=("default" "student_module_history")
+
+for db in "${database_order[@]}"; do
+    echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root
+
+    # Clear out the test database using the reset_db command which uses "DROP DATABASE" and
+    # "CREATE DATABASE". This will result in an empty database.
+    echo "Clearing out the $db bok_choy MySQL database."
+    ./manage.py lms --settings $SETTINGS reset_db --traceback --router $db
+    # Now output all the migrations in the platform to a file.
+    echo "Calculating migrations."
+
+    output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml"
+    ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file
+
+done