PLAT-1676 Use pytest for bok-choy tests

common/test/acceptance/.coveragerc

@@ -10,21 +10,32 @@ source =
 omit =
     lms/envs/*
     cms/envs/*
+    cms/manage.py
+    cms/djangoapps/contentstore/views/dev.py
     common/djangoapps/terrain/*
     common/djangoapps/*/migrations/*
+    openedx/core/djangoapps/debug/*
     openedx/core/djangoapps/*/migrations/*
     */test*
     */management/*
     */urls*
     */wsgi*
+    lms/debug/*
+    lms/djangoapps/*/features/*
     lms/djangoapps/*/migrations/*
+    cms/djangoapps/*/features/*
     cms/djangoapps/*/migrations/*
 
+concurrency = multiprocessing
 parallel = True
 
 [report]
 ignore_errors = True
 
+exclude_lines =
+   pragma: no cover
+   raise NotImplementedError
+
 [html]
 title = Bok Choy Test Coverage Report
 directory = reports/bok_choy/cover

common/test/acceptance/tests/discussion/test_discussion.py

@@ -1226,6 +1226,7 @@ class DiscussionUserProfileTest(UniqueCourseTest):
         self.profiled_user_id = self.setup_user(username=self.PROFILED_USERNAME)
         # now create a second user who will view the profile.
         self.user_id = self.setup_user()
+        UserProfileViewFixture([]).push()
 
     def setup_course(self):
         """

docs/testing.rst

@@ -437,14 +437,14 @@ To test only a certain feature, specify the file and the testcase class.
 
 ::
 
-    paver test_bokchoy -t studio/test_studio_bad_data.py:BadComponentTest
+    paver test_bokchoy -t studio/test_studio_bad_data.py::BadComponentTest
 
 To execute only a certain test case, specify the file name, class, and
 test case method.
 
 ::
 
-    paver test_bokchoy -t lms/test_lms.py:RegistrationTest.test_register
+    paver test_bokchoy -t lms/test_lms.py::RegistrationTest::test_register
 
 During acceptance test execution, log files and also screenshots of
 failed tests are captured in test\_root/log.
@@ -454,7 +454,7 @@ If you check this in, your tests will hang on jenkins.
 
 ::
 
-    from nose.tools import set_trace; set_trace()
+    import pdb; pdb.set_trace()
 
 By default, all bokchoy tests are run with the 'split' ModuleStore. To
 override the modulestore that is used, use the default\_store option.
@@ -506,7 +506,7 @@ relative to the ``common/test/acceptance/tests`` directory. This is an example f
 
 ::
 
-    paver test_a11y -t lms/test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y
+    paver test_a11y -t lms/test_lms_dashboard.py::LmsDashboardA11yTest::test_dashboard_course_listings_a11y
 
 **Coverage**:
 
@@ -644,7 +644,7 @@ Running Tests on Paver Scripts
 
 To run tests on the scripts that power the various Paver commands, use the following command::
 
-  nosetests paver
+  nosetests pavelib
 
 
 Testing internationalization with dummy translations

pavelib/paver_tests/test_paver_bok_choy_cmds.py

@@ -11,6 +11,7 @@ import ddt
 from mock import Mock, call, patch
 from paver.easy import BuildFailure, call_task, environment
 
+from pavelib.utils.envs import Env
 from pavelib.utils.test.suites import BokChoyTestSuite, Pa11yCrawler
 from pavelib.utils.test.suites.bokchoy_suite import DEMO_COURSE_IMPORT_DIR, DEMO_COURSE_TAR_GZ
 
@@ -40,10 +41,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
             ),
             "SELENIUM_DRIVER_LOG_DIR='{}/test_root/log{}'".format(REPO_DIR, shard_str),
             "VERIFY_XSS='{}'".format(verify_xss),
-            "nosetests",
+            "coverage",
+            "run",
+            "--rcfile={}".format(Env.BOK_CHOY_COVERAGERC),
+            "-m",
+            "pytest",
             "{}/common/test/acceptance/{}".format(REPO_DIR, name),
-            "--xunit-file={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
-            "--verbosity=2",
+            "--durations=20",
+            "--junitxml={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
+            "--verbose",
         ]
         return expected_statement
 
@@ -122,11 +128,11 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         Using 1 process means paver should ask for the traditional xunit plugin for plugin results
         """
         expected_verbosity_command = [
-            "--xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                 repo_dir=REPO_DIR,
                 shard_str='/shard_' + self.shard if self.shard else ''
             ),
-            "--verbosity=2",
+            "--verbose",
         ]
         suite = BokChoyTestSuite('', num_processes=1)
         self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -138,13 +144,13 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         """
         process_count = 2
         expected_verbosity_command = [
-            "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                 repo_dir=REPO_DIR,
                 shard_str='/shard_' + self.shard if self.shard else '',
             ),
-            "--processes={}".format(process_count),
-            "--no-color",
-            "--process-timeout=1200",
+            "-n {}".format(process_count),
+            "--color=no",
+            "--verbose",
         ]
         suite = BokChoyTestSuite('', num_processes=process_count)
         self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -155,27 +161,17 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         """
         process_count = 3
         expected_verbosity_command = [
-            "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                 repo_dir=REPO_DIR,
                 shard_str='/shard_' + self.shard if self.shard else '',
             ),
-            "--processes={}".format(process_count),
-            "--no-color",
-            "--process-timeout=1200",
+            "-n {}".format(process_count),
+            "--color=no",
+            "--verbose",
         ]
         suite = BokChoyTestSuite('', num_processes=process_count)
         self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
 
-    def test_invalid_verbosity_and_processes(self):
-        """
-        If an invalid combination of verbosity and number of processors is passed in, a
-        BuildFailure should be raised
-        """
-        suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
-        with self.assertRaises(BuildFailure):
-            # pylint: disable=pointless-statement
-            suite.verbosity_processes_command
-
 
 @ddt.ddt
 class TestPaverPa11yCrawlerCmd(unittest.TestCase):

pavelib/paver_tests/test_utils.py

@@ -6,9 +6,11 @@ import unittest
 
 from mock import patch
 
+from pavelib.utils.envs import Env
 from pavelib.utils.test.utils import MINIMUM_FIREFOX_VERSION, check_firefox_version
 
 
+@unittest.skipIf(Env.USING_DOCKER, 'Firefox version check works differently under Docker Devstack')
 class TestUtils(unittest.TestCase):
     """
     Test utils.py under pavelib/utils/test

pavelib/utils/test/bokchoy_options.py

@@ -19,8 +19,16 @@ BOKCHOY_DEFAULT_STORE_DEPR = make_option(
     default=os.environ.get('DEFAULT_STORE', 'split'),
     help='deprecated in favor of default-store'
 )
-BOKCHOY_FASTTEST = make_option('-a', '--fasttest', action='store_true', help='Skip some setup')
-BOKCHOY_COVERAGERC = make_option('--coveragerc', help='coveragerc file to use during this test')
+BOKCHOY_EVAL_ATTR = make_option(
+    "-a", "--eval-attr",
+    dest="eval_attr", help="Only run tests matching given attribute expression."
+)
+BOKCHOY_FASTTEST = make_option('--fasttest', action='store_true', help='Skip some setup')
+BOKCHOY_COVERAGERC = make_option(
+    '--coveragerc',
+    default=Env.BOK_CHOY_COVERAGERC,
+    help='coveragerc file to use during this test'
+)
 
 BOKCHOY_OPTS = [
     ('test-spec=', 't', 'Specific test to run'),
@@ -28,7 +36,9 @@ BOKCHOY_OPTS = [
     ('skip-clean', 'C', 'Skip cleaning repository before running tests'),
     make_option('-r', '--serversonly', action='store_true', help='Prepare suite and leave servers running'),
     make_option('-o', '--testsonly', action='store_true', help='Assume servers are running and execute tests only'),
+    BOKCHOY_COVERAGERC,
     BOKCHOY_DEFAULT_STORE,
+    BOKCHOY_EVAL_ATTR,
     make_option(
         '-d', '--test-dir',
         default='tests',

pavelib/utils/test/suites/bokchoy_suite.py

@@ -179,10 +179,11 @@ class BokChoyTestSuite(TestSuite):
       testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
       test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
       default_store - modulestore to use when running tests (split or draft)
+      eval_attr - only run tests matching given attribute expression
       num_processes - number of processes or threads to use in tests. Recommendation is that this
       is less than or equal to the number of available processors.
       verify_xss - when set, check for XSS vulnerabilities in the page HTML.
-      See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
+      See pytest documentation: https://docs.pytest.org/en/latest/
     """
     def __init__(self, *args, **kwargs):
         super(BokChoyTestSuite, self).__init__(*args, **kwargs)
@@ -196,6 +197,7 @@ class BokChoyTestSuite(TestSuite):
         self.testsonly = kwargs.get('testsonly', False)
         self.test_spec = kwargs.get('test_spec', None)
         self.default_store = kwargs.get('default_store', None)
+        self.eval_attr = kwargs.get('eval_attr', None)
         self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
         self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
         self.verify_xss = kwargs.get('verify_xss', os.environ.get('VERIFY_XSS', True))
@@ -203,7 +205,7 @@ class BokChoyTestSuite(TestSuite):
         self.har_dir = self.log_dir / 'hars'
         self.a11y_file = Env.BOK_CHOY_A11Y_CUSTOM_RULES_FILE
         self.imports_dir = kwargs.get('imports_dir', None)
-        self.coveragerc = kwargs.get('coveragerc', None)
+        self.coveragerc = kwargs.get('coveragerc', Env.BOK_CHOY_COVERAGERC)
         self.save_screenshots = kwargs.get('save_screenshots', False)
 
     def __enter__(self):
@@ -269,29 +271,22 @@ class BokChoyTestSuite(TestSuite):
     @property
     def verbosity_processes_command(self):
         """
-        Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
-        the proper combination for use with nosetests.
+        Construct the proper combination of multiprocessing, XUnit XML file, color, and verbosity for use with pytest.
         """
-        command = []
-
-        if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
-            msg = 'Cannot pass in both num_processors and verbosity. Quitting'
-            raise BuildFailure(msg)
+        command = ["--junitxml={}".format(self.xunit_report)]
 
         if self.num_processes != 1:
-            # Construct "multiprocess" nosetest command
-            command = [
-                "--xunitmp-file={}".format(self.xunit_report),
-                "--processes={}".format(self.num_processes),
-                "--no-color",
-                "--process-timeout=1200",
-            ]
-
-        else:
-            command = [
-                "--xunit-file={}".format(self.xunit_report),
-                "--verbosity={}".format(self.verbosity),
+            # Construct "multiprocess" pytest command
+            command += [
+                "-n {}".format(self.num_processes),
+                "--color=no",
             ]
+        if self.verbosity < 1:
+            command.append("--quiet")
+        elif self.verbosity > 1:
+            command.append("--verbose")
+        if self.eval_attr:
+            command.append("-a '{}'".format(self.eval_attr))
 
         return command
 
@@ -300,7 +295,7 @@ class BokChoyTestSuite(TestSuite):
         Infinite loop. Servers will continue to run in the current session unless interrupted.
         """
         print 'Bok-choy servers running. Press Ctrl-C to exit...\n'
-        print 'Note: pressing Ctrl-C multiple times can corrupt noseid files and system state. Just press it once.\n'
+        print 'Note: pressing Ctrl-C multiple times can corrupt system state. Just press it once.\n'
 
         while True:
             try:
@@ -312,7 +307,7 @@ class BokChoyTestSuite(TestSuite):
     @property
     def cmd(self):
         """
-        This method composes the nosetests command to send to the terminal. If nosetests aren't being run,
+        This method composes the pytest command to send to the terminal. If pytest isn't being run,
          the command returns None.
         """
         # Default to running all tests if no specific test is specified
@@ -321,12 +316,12 @@ class BokChoyTestSuite(TestSuite):
         else:
             test_spec = self.test_dir / self.test_spec
 
-        # Skip any additional commands (such as nosetests) if running in
+        # Skip any additional commands (such as pytest) if running in
         # servers only mode
         if self.serversonly:
             return None
 
-        # Construct the nosetests command, specifying where to save
+        # Construct the pytest command, specifying where to save
         # screenshots and XUnit XML reports
         cmd = [
             "DEFAULT_STORE={}".format(self.default_store),
@@ -335,11 +330,21 @@ class BokChoyTestSuite(TestSuite):
             "BOKCHOY_A11Y_CUSTOM_RULES_FILE='{}'".format(self.a11y_file),
             "SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
             "VERIFY_XSS='{}'".format(self.verify_xss),
-            "nosetests",
+        ]
+        if self.save_screenshots:
+            cmd.append("NEEDLE_SAVE_BASELINE=True")
+        cmd += [
+            "coverage",
+            "run",
+        ]
+        if self.coveragerc:
+            cmd.append("--rcfile={}".format(self.coveragerc))
+        cmd += [
+            "-m",
+            "pytest",
             test_spec,
+            "--durations=20",
         ] + self.verbosity_processes_command
-        if self.save_screenshots:
-            cmd.append("--with-save-baseline")
         if self.extra_args:
             cmd.append(self.extra_args)
         cmd.extend(self.passthrough_options)

requirements/edx/testing.txt

@@ -9,4 +9,11 @@
 #   * @edx/testeng - to discuss it's impact on test infrastructure
 #   * @edx/devops - to check system requirements
 
+execnet==1.4.1
+py==1.4.34
 pysqlite==2.8.3
+pytest==3.1.3
+pytest-attrib==0.1.3
+pytest-catchlog==1.2.2
+pytest-django==3.1.2
+pytest-xdist==1.18.1

scripts/accessibility-tests.sh

@@ -6,7 +6,7 @@ echo "Setting up for accessibility tests..."
 source scripts/jenkins-common.sh
 
 echo "Running explicit accessibility tests..."
-SELENIUM_BROWSER=phantomjs paver test_a11y --with-xunitmp
+SELENIUM_BROWSER=phantomjs paver test_a11y
 
 echo "Generating coverage report..."
 paver a11y_coverage

scripts/generic-ci-tests.sh

@@ -162,7 +162,7 @@ case "$TEST_SUITE" in
 
     "bok-choy")
 
-        PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS --with-flaky --with-xunit"
+        PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS"
 
         case "$SHARD" in
 
@@ -171,11 +171,11 @@ case "$TEST_SUITE" in
                 ;;
 
             [1-9]|10)
-                paver test_bokchoy --attr="shard=$SHARD" $PAVER_ARGS
+                paver test_bokchoy --eval-attr="shard==$SHARD" $PAVER_ARGS
                 ;;
 
             11|"noshard")
-                paver test_bokchoy --attr='!shard,a11y=False' $PAVER_ARGS
+                paver test_bokchoy --eval-attr='not shard and not a11y' $PAVER_ARGS
                 ;;
 
             # Default case because if we later define another bok-choy shard on Jenkins
@@ -190,7 +190,7 @@ case "$TEST_SUITE" in
                 # May be unnecessary if we changed the "Skip if there are no test files"
                 # option to True in the jenkins job definitions.
                 mkdir -p reports/bok_choy
-                emptyxunit "bok_choy/nosetests"
+                emptyxunit "bok_choy/xunit"
                 ;;
         esac
         ;;

setup.cfg

@@ -14,6 +14,9 @@ process-timeout=300
 #nocapture=1
 #pdb=1
 
+[tool:pytest]
+norecursedirs = .git conf node_modules test_root cms/envs lms/envs
+
 [pep8]
 # error codes: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
 # E501: line too long