Merge pull request #7913 from edx/clytwynec/unique-dirs-for-test-results

Clytwynec/unique dirs for test results

pavelib/paver_tests/test_paver_bok_choy_cmds.py

@@ -14,19 +14,21 @@ class TestPaverBokChoyCmd(unittest.TestCase):
     def _expected_command(self, expected_text_append, expected_default_store=None):
         if expected_text_append:
             expected_text_append = "/" + expected_text_append
+        shard = os.environ.get('SHARD')
         expected_statement = (
             "DEFAULT_STORE={default_store} "
-            "SCREENSHOT_DIR='{repo_dir}/test_root/log' "
-            "BOK_CHOY_HAR_DIR='{repo_dir}/test_root/log/hars' "
-            "SELENIUM_DRIVER_LOG_DIR='{repo_dir}/test_root/log' "
+            "SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' "
+            "BOK_CHOY_HAR_DIR='{repo_dir}/test_root/log{shard_str}/hars' "
+            "SELENIUM_DRIVER_LOG_DIR='{repo_dir}/test_root/log{shard_str}' "
             "nosetests {repo_dir}/common/test/acceptance/tests{exp_text} "
             "--with-xunit "
-            "--xunit-file={repo_dir}/reports/bok_choy/xunit.xml "
-            "--verbosity=2"
+            "--xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml "
+            "--verbosity=2 "
         ).format(
             default_store=expected_default_store,
             repo_dir=REPO_DIR,
             exp_text=expected_text_append,
+            shard_str='/shard_' + shard if shard else '',
         )
         return expected_statement.strip()
 

pavelib/tests.py

@@ -3,7 +3,7 @@ Unit test tasks
 """
 import os
 import sys
-from paver.easy import sh, task, cmdopts, needs, call_task
+from paver.easy import sh, task, cmdopts, needs, call_task, no_help
 from pavelib.utils.test import suites
 from pavelib.utils.envs import Env
 from optparse import make_option
@@ -212,6 +212,28 @@ def coverage(options):
     call_task('diff_coverage', options=dict(options))
 
 
+@no_help
+@task
+@needs('pavelib.prereqs.install_prereqs')
+def combine_jenkins_coverage():
+    """
+    Combine coverage reports from jenkins build flow.
+    """
+    coveragerc = Env.REPO_ROOT / 'test_root' / '.jenkins-coveragerc'
+
+    for directory in Env.LIB_TEST_DIRS + ['cms', 'lms']:
+        report_dir = Env.REPORT_DIR / directory
+
+        # Only try to combine the coverage if we've run the tests.
+        if report_dir.isdir():
+            sh(
+                "cd {} && coverage combine --rcfile={}".format(
+                    report_dir,
+                    coveragerc,
+                )
+            )
+
+
 @task
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([

pavelib/utils/envs.py

@@ -28,6 +28,15 @@ class Env(object):
     BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy"
     BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc"
 
+    # If set, put reports for run in "unique" directories.
+    # The main purpose of this is to ensure that the reports can be 'slurped'
+    # in the main jenkins flow job without overwriting the reports from other
+    # build steps. For local development/testing, this shouldn't be needed.
+    if os.environ.get("SHARD", None):
+        shard_str = "shard_{}".format(os.environ.get("SHARD"))
+        BOK_CHOY_REPORT_DIR = BOK_CHOY_REPORT_DIR / shard_str
+        BOK_CHOY_LOG_DIR = BOK_CHOY_LOG_DIR / shard_str
+
     # For the time being, stubs are used by both the bok-choy and lettuce acceptance tests
     # For this reason, the stubs package is currently located in the Django app called "terrain"
     # where other lettuce configuration is stored.

pavelib/utils/test/suites/nose_suite.py

@@ -21,6 +21,15 @@ class NoseTestSuite(TestSuite):
         self.fail_fast = kwargs.get('fail_fast', False)
         self.run_under_coverage = kwargs.get('with_coverage', True)
         self.report_dir = Env.REPORT_DIR / self.root
+
+        # If set, put reports for run in "unique" directories.
+        # The main purpose of this is to ensure that the reports can be 'slurped'
+        # in the main jenkins flow job without overwriting the reports from other
+        # build steps. For local development/testing, this shouldn't be needed.
+        if os.environ.get("SHARD", None):
+            shard_str = "shard_{}".format(os.environ.get("SHARD"))
+            self.report_dir = self.report_dir / shard_str
+
         self.test_id_dir = Env.TEST_DIR / self.root
         self.test_ids = self.test_id_dir / 'noseids'
         self.extra_args = kwargs.get('extra_args', '')
@@ -112,12 +121,14 @@ class SystemTestSuite(NoseTestSuite):
     def cmd(self):
         cmd = (
             './manage.py {system} test --verbosity={verbosity} '
-            '{test_id} {test_opts} --traceback --settings=test {extra}'.format(
+            '{test_id} {test_opts} --traceback --settings=test {extra} '
+            '--with-xunit --xunit-file={xunit_report}'.format(
                 system=self.root,
                 verbosity=self.verbosity,
                 test_id=self.test_id,
                 test_opts=self.test_options_flags,
                 extra=self.extra_args,
+                xunit_report=self.report_dir / "nosetests.xml",
             )
         )
 

scripts/all-tests.sh

@@ -98,14 +98,12 @@ END
     "unit")
         case "$SHARD" in
             "lms")
-                paver test_system -s lms --extra_args="--with-flaky" || { EXIT=1; }
-                paver coverage
+                SHARD=1 paver test_system -s lms --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
                 ;;
             "cms-js-commonlib")
-                paver test_system -s cms --extra_args="--with-flaky" || { EXIT=1; }
-                paver test_js --coverage --skip_clean || { EXIT=1; }
-                paver test_lib --skip_clean --extra_args="--with-flaky" || { EXIT=1; }
-                paver coverage
+                SHARD=1 paver test_system -s cms --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
+                SHARD=1 paver test_js --coverage --skip_clean || { EXIT=1; }
+                SHARD=1 paver test_lib --skip_clean --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
                 ;;
             *)
                 paver test --extra_args="--with-flaky"
@@ -210,15 +208,4 @@ END
 END
                 ;;
         esac
-
-        # Move the reports to a directory that is unique to the shard
-        # so that when they are 'slurped' to the main flow job, they
-        # do not conflict with and overwrite reports from other shards.
-        mv reports/ reports_tmp/
-        mkdir -p reports/${TEST_SUITE}/${SHARD}
-        mv reports_tmp/* reports/${TEST_SUITE}/${SHARD}
-        rm -r reports_tmp/
-        exit $EXIT
-        ;;
-
 esac

scripts/jenkins-report.sh

 #!/usr/bin/env bash
 source scripts/jenkins-common.sh
 
-# Run coverage again to get the diff coverage report
-paver diff_coverage
+# Combine the data files that were generated using -p
+paver combine_jenkins_coverage
+
+# Get the diff coverage and html reports for unit tests
+paver coverage
 
 # JUnit test reporter will fail the build
 # if it thinks test results are old

test_root/.jenkins-coveragerc

+[paths]
+source =
+    /home/jenkins/workspace/edx-platform-unit-coverage
+    /home/jenkins/workspace/edx-platform-test-subset