Merge pull request #10464 from edx/clytwynec/add-paver-test-a11y

Clytwynec/add paver test a11y

common/test/acceptance/.a11ycoveragerc

+[run]
+data_file = reports/a11y/.coverage
+source =
+    lms
+    cms
+    common/djangoapps
+    common/lib
+    openedx/core/djangoapps
+
+omit =
+    lms/envs/*
+    cms/envs/*
+    common/djangoapps/terrain/*
+    common/djangoapps/*/migrations/*
+    openedx/core/djangoapps/*/migrations/*
+    */test*
+    */management/*
+    */urls*
+    */wsgi*
+    lms/djangoapps/*/migrations/*
+    cms/djangoapps/*/migrations/*
+
+parallel = True
+
+[report]
+ignore_errors = True
+include =
+    **/views/*.py
+    **/views.py
+
+[html]
+title = Bok Choy A11y Test Coverage Report
+directory = reports/a11y/cover
+
+[xml]
+output = reports/a11y/coverage.xml

common/test/acceptance/.coveragerc

 [run]
 data_file = reports/bok_choy/.coverage
-source = lms, cms, common/djangoapps, common/lib
-omit = lms/envs/*, cms/envs/*, common/djangoapps/terrain/*, common/djangoapps/*/migrations/*, openedx/core/djangoapps/*/migrations/*, */test*, */management/*, */urls*, */wsgi*, lms/djangoapps/*/migrations/*, cms/djangoapps/*/migrations/*
+source =
+    lms
+    cms
+    common/djangoapps
+    common/lib
+    openedx/core/djangoapps
+
+omit =
+    lms/envs/*
+    cms/envs/*
+    common/djangoapps/terrain/*
+    common/djangoapps/*/migrations/*
+    openedx/core/djangoapps/*/migrations/*
+    */test*
+    */management/*
+    */urls*
+    */wsgi*
+    lms/djangoapps/*/migrations/*
+    cms/djangoapps/*/migrations/*
+
 parallel = True
 
 [report]

docs/en_us/internal/testing.rst

@@ -456,14 +456,26 @@ To run all the bok choy accessibility tests use this command.
 
 ::
 
-    paver test_bokchoy --extra_args="-a 'a11y'"
+    paver test_a11y
 
 To run specific tests, use the ``-t`` flag to specify a nose-style test spec
 relative to the ``common/test/acceptance/tests`` directory. This is an example for it.
 
 ::
 
-    paver test_bokchoy --extra_args="-a 'a11y'" -t test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y
+    paver test_a11y -t test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y
+
+**Coverage**:
+
+To generate the coverage report for the views run during accessibility tests::
+
+    paver a11y_coverage
+
+Note that this coverage report is just a guideline to find areas that
+are missing tests.  If the view isn't 'covered', there definitely
+isn't a test for it.  If it is 'covered', we are loading that page
+during the tests but not necessarily calling ``page.a11y_audit.check_for_accessibility_errors`` on it.
+
 
 Options for Faster Development Cycles in Bok-Choy Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pavelib/bok_choy.py

@@ -16,10 +16,7 @@ except ImportError:
 
 __test__ = False  # do not collect
 
-
-@task
-@needs('pavelib.prereqs.install_prereqs')
-@cmdopts([
+BOKCHOY_OPTS = [
     ('test_spec=', 't', 'Specific test to run'),
     ('fasttest', 'a', 'Skip some setup'),
     ('serversonly', 'r', 'Prepare suite and leave servers running'),
@@ -33,7 +30,32 @@ __test__ = False  # do not collect
     make_option("-v", "--verbosity", action="count", dest="verbosity"),
     make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"),
     make_option("--skip_firefox_version_validation", action='store_false', dest="validate_firefox_version")
-])
+]
+
+
+def parse_bokchoy_opts(options):
+    """
+    Parses bok choy options.
+
+    Returns: dict of options.
+    """
+    return {
+        'test_spec': getattr(options, 'test_spec', None),
+        'fasttest': getattr(options, 'fasttest', False),
+        'num_processes': int(getattr(options, 'num_processes', 1)),
+        'serversonly': getattr(options, 'serversonly', False),
+        'testsonly': getattr(options, 'testsonly', False),
+        'default_store': getattr(options, 'default_store', os.environ.get('DEFAULT_STORE', 'split')),
+        'verbosity': getattr(options, 'verbosity', 2),
+        'extra_args': getattr(options, 'extra_args', ''),
+        'pdb': getattr(options, 'pdb', False),
+        'test_dir': getattr(options, 'test_dir', 'tests'),
+    }
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts(BOKCHOY_OPTS)
 def test_bokchoy(options):
     """
     Run acceptance tests that use the bok-choy framework.
@@ -57,18 +79,33 @@ def test_bokchoy(options):
     if validate_firefox:
         check_firefox_version()
 
-    opts = {
-        'test_spec': getattr(options, 'test_spec', None),
-        'num_processes': int(getattr(options, 'num_processes', 1)),
-        'fasttest': getattr(options, 'fasttest', False),
-        'serversonly': getattr(options, 'serversonly', False),
-        'testsonly': getattr(options, 'testsonly', False),
-        'default_store': getattr(options, 'default_store', os.environ.get('DEFAULT_STORE', 'split')),
-        'verbosity': getattr(options, 'verbosity', 2),
-        'extra_args': getattr(options, 'extra_args', ''),
-        'pdb': getattr(options, 'pdb', False),
-        'test_dir': getattr(options, 'test_dir', 'tests'),
-    }
+    opts = parse_bokchoy_opts(options)
+    run_bokchoy(**opts)
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts(BOKCHOY_OPTS)
+def test_a11y(options):
+    """
+    Run accessibility tests that use the bok-choy framework.
+    Skips some static asset steps if `fasttest` is True.
+    Using 'serversonly' will prepare and run servers, leaving a process running in the terminal. At
+        the same time, a user can open a separate terminal and use 'testsonly' for executing tests against
+        those running servers.
+
+    `test_spec` is a nose-style test specifier relative to the test directory
+    Examples:
+    - path/to/test.py
+    - path/to/test.py:TestFoo
+    - path/to/test.py:TestFoo.test_bar
+    It can also be left blank to run all tests in the suite that are tagged
+    with `@attr("a11y")`.
+    """
+    opts = parse_bokchoy_opts(options)
+    opts['report_dir'] = Env.BOK_CHOY_A11Y_REPORT_DIR
+    opts['coveragerc'] = Env.BOK_CHOY_A11Y_COVERAGERC
+    opts['extra_args'] = opts['extra_args'] + ' -a "a11y" '
     run_bokchoy(**opts)
 
 
@@ -113,13 +150,11 @@ def run_bokchoy(**opts):
     test_suite.run()
 
 
-@task
-def bokchoy_coverage():
+def parse_coverage(report_dir, coveragerc):
     """
-    Generate coverage reports for bok-choy tests
+    Generate coverage reports for bok-choy or a11y tests
     """
-    Env.BOK_CHOY_REPORT_DIR.makedirs_p()
-    coveragerc = Env.BOK_CHOY_COVERAGERC
+    report_dir.makedirs_p()
 
     msg = colorize('green', "Combining coverage reports")
     print msg
@@ -132,3 +167,29 @@ def bokchoy_coverage():
     sh("coverage html --rcfile={}".format(coveragerc))
     sh("coverage xml --rcfile={}".format(coveragerc))
     sh("coverage report --rcfile={}".format(coveragerc))
+
+
+@task
+def bokchoy_coverage():
+    """
+    Generate coverage reports for bok-choy tests
+    """
+    parse_coverage(
+        Env.BOK_CHOY_REPORT_DIR,
+        Env.BOK_CHOY_COVERAGERC
+    )
+
+
+@task
+def a11y_coverage():
+    """
+    Generate coverage reports for a11y tests. Note that this coverage report
+    is just a guideline to find areas that are missing tests.  If the view
+    isn't 'covered', there definitely isn't a test for it.  If it is
+    'covered', we are loading that page during the tests but not necessarily
+    calling ``page.a11y_audit.check_for_accessibility_errors`` on it.
+    """
+    parse_coverage(
+        Env.BOK_CHOY_A11Y_REPORT_DIR,
+        Env.BOK_CHOY_A11Y_COVERAGERC
+    )

pavelib/utils/envs.py

@@ -29,7 +29,9 @@ class Env(object):
     BOK_CHOY_DIR = REPO_ROOT / "common" / "test" / "acceptance"
     BOK_CHOY_LOG_DIR = REPO_ROOT / "test_root" / "log"
     BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy"
+    BOK_CHOY_A11Y_REPORT_DIR = REPORT_DIR / "a11y"
     BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc"
+    BOK_CHOY_A11Y_COVERAGERC = BOK_CHOY_DIR / ".a11ycoveragerc"
 
     # If set, put reports for run in "unique" directories.
     # The main purpose of this is to ensure that the reports can be 'slurped'

pavelib/utils/test/bokchoy_utils.py

@@ -18,10 +18,11 @@ except ImportError:
 __test__ = False  # do not collect
 
 
-def start_servers(default_store):
+def start_servers(default_store, coveragerc=None):
     """
     Start the servers we will run tests on, returns PIDs for servers.
     """
+    coveragerc = coveragerc or Env.BOK_CHOY_COVERAGERC
 
     def start_server(cmd, logfile, cwd=None):
         """
@@ -38,7 +39,7 @@ def start_servers(default_store):
             "manage {service} --settings bok_choy runserver "
             "{address} --traceback --noreload".format(
                 default_store=default_store,
-                coveragerc=Env.BOK_CHOY_COVERAGERC,
+                coveragerc=coveragerc,
                 service=service,
                 address=address,
             )

pavelib/utils/test/suites/bokchoy_suite.py

@@ -43,7 +43,7 @@ class BokChoyTestSuite(TestSuite):
         super(BokChoyTestSuite, self).__init__(*args, **kwargs)
         self.test_dir = Env.BOK_CHOY_DIR / kwargs.get('test_dir', 'tests')
         self.log_dir = Env.BOK_CHOY_LOG_DIR
-        self.report_dir = Env.BOK_CHOY_REPORT_DIR
+        self.report_dir = kwargs.get('report_dir', Env.BOK_CHOY_REPORT_DIR)
         self.xunit_report = self.report_dir / "xunit.xml"
         self.cache = Env.BOK_CHOY_CACHE
         self.fasttest = kwargs.get('fasttest', False)
@@ -56,6 +56,7 @@ class BokChoyTestSuite(TestSuite):
         self.extra_args = kwargs.get('extra_args', '')
         self.har_dir = self.log_dir / 'hars'
         self.imports_dir = kwargs.get('imports_dir', None)
+        self.coveragerc = kwargs.get('coveragerc', None)
 
     def __enter__(self):
         super(BokChoyTestSuite, self).__enter__()
@@ -165,7 +166,7 @@ class BokChoyTestSuite(TestSuite):
         # Ensure the test servers are available
         msg = colorize('green', "Confirming servers are running...")
         print msg
-        bokchoy_utils.start_servers(self.default_store)
+        bokchoy_utils.start_servers(self.default_store, self.coveragerc)
 
     def run_servers_continuously(self):
         """

scripts/accessibility-tests.sh

@@ -3,4 +3,7 @@ echo "Setting up for accessibility tests..."
 source scripts/jenkins-common.sh
 
 echo "Running explicit accessibility tests..."
-SELENIUM_BROWSER=phantomjs paver test_bokchoy --extra_args="-a 'a11y'"
+SELENIUM_BROWSER=phantomjs paver test_a11y
+
+echo "Generating coverage report..."
+paver a11y_coverage