all-tests.sh 7.69 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#!/usr/bin/env bash
set -e

###############################################################################
#
#   edx-all-tests.sh
#
#   Execute all tests for edx-platform.
#
#   This script can be called from a Jenkins
#   multiconfiguration job that defines these environment
#   variables:
#
#   `TEST_SUITE` defines which kind of test to run.
#   Possible values are:
#
#       - "quality": Run the quality (pep8/pylint) checks
#       - "unit": Run the JavaScript and Python unit tests
#            (also tests building the Sphinx documentation,
#             because we couldn't think of a better place to put it)
#       - "lms-acceptance": Run the acceptance (Selenium) tests for the LMS
#       - "cms-acceptance": Run the acceptance (Selenium) tests for Studio
#       - "bok-choy": Run acceptance tests that use the bok-choy framework
#
#   `SHARD` is a number (1, 2, or 3) indicating which subset of the tests
26
#       to build.  Currently, "lms-acceptance" and "bok-choy" each have two
27 28 29 30 31
#       shards (1 and 2), "cms-acceptance" has three shards (1, 2, and 3),
#       and all the other test suites have one shard.
#
#       For the "bok-choy", the tests are put into shard groups using the nose
#       'attr' decorator (e.g. "@attr('shard_1')").  Currently, anything with
32 33
#       the 'shard_1' attribute will run in the first shard.  All other bok-choy
#       tests will run in shard 2.
34 35
#
#       For the lettuce acceptance tests, ("lms-" and "cms-acceptance") they
36 37 38
#       are decorated with "@shard_{}" (e.g. @shard_1 for the first shard).
#       The lettuce tests must have a shard specified to be run in jenkins,
#       as there is no shard that runs unspecified tests.
39
#
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
#
#   Jenkins configuration:
#
#   - The edx-platform git repository is checked out by the Jenkins git plugin.
#
#   - Jenkins logs in as user "jenkins"
#
#   - The Jenkins file system root is "/home/jenkins"
#
#   - An init script creates a virtualenv at "/home/jenkins/edx-venv"
#     with some requirements pre-installed (such as scipy)
#
#  Jenkins worker setup:
#  See the edx/configuration repo for Jenkins worker provisioning scripts.
#  The provisioning scripts install requirements that this script depends on!
#
###############################################################################

# Clean up previous builds
git clean -qxfd

61
source scripts/jenkins-common.sh
62

63
# Violations thresholds for failing the build
64
PYLINT_THRESHOLD=5500
65

66 67 68 69 70 71 72
# If the environment variable 'SHARD' is not set, default to 'all'.
# This could happen if you are trying to use this script from
# jenkins and do not define 'SHARD' in your multi-config project.
# Note that you will still need to pass a value for 'TEST_SUITE'
# or else no tests will be executed.
SHARD=${SHARD:="all"}

73 74 75
case "$TEST_SUITE" in

    "quality")
76
        echo "Finding fixme's and storing report..."
Christine Lytwynec committed
77
        paver find_fixme > fixme.log || { cat fixme.log; EXIT=1; }
78
        echo "Finding pep8 violations and storing report..."
79
        paver run_pep8 > pep8.log || { cat pep8.log; EXIT=1; }
80
        echo "Finding pylint violations and storing in report..."
81
        paver run_pylint -l $PYLINT_THRESHOLD > pylint.log || { cat pylint.log; EXIT=1; }
82 83
        # Run quality task. Pass in the 'fail-under' percentage to diff-quality
        paver run_quality -p 100
84

85 86
        mkdir -p reports
        paver run_complexity > reports/code_complexity.log || echo "Unable to calculate code complexity. Ignoring error."
87 88 89 90 91 92 93 94
        # Need to create an empty test result so the post-build
        # action doesn't fail the build.
        cat > reports/quality.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="quality" tests="1" errors="0" failures="0" skip="0">
<testcase classname="quality" name="quality" time="0.604"></testcase>
</testsuite>
END
95
        exit $EXIT
96 97 98
        ;;

    "unit")
99
        case "$SHARD" in
100
            "lms")
101
                paver test_system -s lms --extra_args="--with-flaky" || { EXIT=1; }
102 103
                paver coverage
                ;;
104
            "cms-js-commonlib")
105 106 107
                paver test_system -s cms --extra_args="--with-flaky" || { EXIT=1; }
                paver test_js --coverage --skip_clean || { EXIT=1; }
                paver test_lib --skip_clean --extra_args="--with-flaky" || { EXIT=1; }
108 109 110
                paver coverage
                ;;
            *)
111
                paver test --extra_args="--with-flaky"
112 113 114
                paver coverage
                ;;
        esac
115 116

        exit $EXIT
117 118 119
        ;;

    "lms-acceptance")
120 121 122 123 124 125
        case "$SHARD" in

            "all")
                paver test_acceptance -s lms --extra_args="-v 3"
                ;;

126 127 128 129 130 131 132 133 134 135
            "2")
                mkdir -p reports
                mkdir -p reports/acceptance
                cat > reports/acceptance/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="lettuce.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
                ;;
136
            *)
137
                paver test_acceptance -s lms --extra_args="-v 3"
138 139
                ;;
        esac
140 141 142
        ;;

    "cms-acceptance")
143 144
        case "$SHARD" in

145
            "all"|"1")
146 147 148
                paver test_acceptance -s cms --extra_args="-v 3"
                ;;

149 150 151 152 153 154 155 156
            "2"|"3")
                mkdir -p reports/acceptance
                cat > reports/acceptance/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="lettuce.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
157
                ;;
158

159
        esac
160 161 162
        ;;

    "bok-choy")
163
        case "$SHARD" in
164 165

            "all")
166
                paver test_bokchoy || { EXIT=1; }
167 168
                ;;

169
            "1")
170
                paver test_bokchoy --extra_args="-a shard_1 --with-flaky" || { EXIT=1; }
171 172 173
                ;;

            "2")
174
                paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky" || { EXIT=1; }
175 176 177
                ;;

            "3")
178
                paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky" || { EXIT=1; }
179 180 181
                ;;

            "4")
182
                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False --with-flaky" || { EXIT=1; }
183 184 185
                ;;

            # Default case because if we later define another bok-choy shard on Jenkins
186
            # (e.g. Shard 5) in the multi-config project and expand this file
187 188 189 190 191 192 193 194 195
            # with an additional case condition, old branches without that commit
            # would not execute any tests on the worker assigned to that shard
            # and thus their build would fail.
            # This way they will just report 1 test executed and passed.
            *)
                # Need to create an empty test result so the post-build
                # action doesn't fail the build.
                # May be unnecessary if we changed the "Skip if there are no test files"
                # option to True in the jenkins job definitions.
196
                mkdir -p reports/bok_choy
197 198 199 200 201 202 203
                cat > reports/bok_choy/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="acceptance.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
                ;;
204
        esac
205 206 207 208 209 210 211 212 213

        # Move the reports to a directory that is unique to the shard
        # so that when they are 'slurped' to the main flow job, they
        # do not conflict with and overwrite reports from other shards.
        mv reports/ reports_tmp/
        mkdir -p reports/${TEST_SUITE}/${SHARD}
        mv reports_tmp/* reports/${TEST_SUITE}/${SHARD}
        rm -r reports_tmp/
        exit $EXIT
214 215 216
        ;;

esac