Add pa11ycrawler command

.gitignore

@@ -58,6 +58,7 @@ jscover.log
 jscover.log.*
 .tddium*
 common/test/data/test_unicode/static/
+test_root/courses/
 django-pyfs
 
 ### Installation artifacts

common/test/acceptance/.pa11ycrawlercoveragerc

+[run]
+data_file = reports/pa11ycrawler/.coverage
+
+source =
+    lms
+    cms
+    common/djangoapps
+    common/lib
+    openedx/core/djangoapps
+    **/mako_lms/
+    **/mako_cms/
+
+omit =
+    lms/envs/*
+    cms/envs/*
+    common/djangoapps/terrain/*
+    common/djangoapps/*/migrations/*
+    openedx/core/djangoapps/*/migrations/*
+    */test*
+    */management/*
+    */urls*
+    */wsgi*
+    lms/djangoapps/*/migrations/*
+    cms/djangoapps/*/migrations/*
+
+parallel = True
+
+[report]
+ignore_errors = True
+include =
+    **/views/*.py
+    **/views.py
+
+
+[html]
+title = pa11ycrawler Coverage Report
+directory = reports/pa11ycrawler/cover
+
+[xml]
+output = reports/pa11ycrawler/coverage.xml

package.json

@@ -13,6 +13,8 @@
   "devDependencies": {
     "jshint": "^2.7.0",
     "edx-custom-a11y-rules": "edx/edx-custom-a11y-rules",
+    "pa11y": "3.6.0",
+    "pa11y-reporter-1.0-json": "1.0.2",
     "plato": "1.2.2"
   }
 }

pavelib/bok_choy.py

@@ -3,7 +3,7 @@ Run acceptance tests that use the bok-choy framework
 http://bok-choy.readthedocs.org/en/latest/
 """
 from paver.easy import task, needs, cmdopts, sh
-from pavelib.utils.test.suites.bokchoy_suite import BokChoyTestSuite
+from pavelib.utils.test.suites.bokchoy_suite import BokChoyTestSuite, A11yCrawler
 from pavelib.utils.envs import Env
 from pavelib.utils.test.utils import check_firefox_version
 from optparse import make_option
@@ -115,30 +115,44 @@ def test_a11y(options):
 
 @task
 @needs('pavelib.prereqs.install_prereqs')
-@cmdopts([
-    ('test_spec=', 't', 'Specific test to run'),
-    ('fasttest', 'a', 'Skip some setup'),
+@cmdopts(BOKCHOY_OPTS + [
     ('imports_dir=', 'd', 'Directory containing (un-archived) courses to be imported'),
-    ('default_store=', 's', 'Default modulestore'),
-    make_option("--verbose", action="store_const", const=2, dest="verbosity"),
-    make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
-    make_option("-v", "--verbosity", action="count", dest="verbosity"),
 ])
 def perf_report_bokchoy(options):
     """
     Generates a har file for with page performance info.
     """
-    opts = {
-        'test_spec': getattr(options, 'test_spec', None),
-        'fasttest': getattr(options, 'fasttest', False),
-        'default_store': getattr(options, 'default_store', os.environ.get('DEFAULT_STORE', 'split')),
-        'imports_dir': getattr(options, 'imports_dir', None),
-        'verbosity': getattr(options, 'verbosity', 2),
-        'test_dir': 'performance',
-    }
+    opts = parse_bokchoy_opts(options)
+    opts['imports_dir'] = getattr(options, 'imports_dir', None)
+    opts['test_dir'] = 'performance'
+
     run_bokchoy(**opts)
 
 
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts(BOKCHOY_OPTS + [
+    ('with-html', 'w', 'Include html reports'),
+])
+def pa11ycrawler(options):
+    """
+    Runs pa11ycrawler against the demo-test-course to generates accessibility
+    reports. (See https://github.com/edx/demo-test-course)
+
+    Note: Like the bok-choy tests, this can be used with the `serversonly`
+    flag to get an environment running. The setup for this is the same as
+    for bok-choy tests, only test course is imported as well.
+    """
+    opts = parse_bokchoy_opts(options)
+    opts['report_dir'] = Env.PA11YCRAWLER_REPORT_DIR
+    opts['coveragerc'] = Env.PA11YCRAWLER_COVERAGERC
+    test_suite = A11yCrawler('a11y_crawler', **opts)
+    test_suite.run()
+
+    if getattr(options, 'with_html', False):
+        test_suite.generate_html_reports()
+
+
 def run_bokchoy(**opts):
     """
     Runs BokChoyTestSuite with the given options.
@@ -197,3 +211,14 @@ def a11y_coverage():
         Env.BOK_CHOY_A11Y_REPORT_DIR,
         Env.BOK_CHOY_A11Y_COVERAGERC
     )
+
+
+@task
+def pa11ycrawler_coverage():
+    """
+    Generate coverage reports for bok-choy tests
+    """
+    parse_coverage(
+        Env.PA11YCRAWLER_REPORT_DIR,
+        Env.PA11YCRAWLER_COVERAGERC
+    )

pavelib/paver_tests/test_paver_bok_choy_cmds.py

@@ -4,9 +4,11 @@ Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_bok_ch
 """
 import os
 import unittest
+
+from mock import patch, call
 from test.test_support import EnvironmentVarGuard
 from paver.easy import BuildFailure
-from pavelib.utils.test.suites import BokChoyTestSuite
+from pavelib.utils.test.suites import BokChoyTestSuite, A11yCrawler
 
 REPO_DIR = os.getcwd()
 
@@ -167,3 +169,72 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
         with self.assertRaises(BuildFailure):
             BokChoyTestSuite.verbosity_processes_string(suite)
+
+
+class TestPaverPa11ycrawlerCmd(unittest.TestCase):
+
+    """
+    Paver pa11ycrawler command test cases.  Most of the functionality is
+    inherited from BokChoyTestSuite, so those tests aren't duplicated.
+    """
+
+    def setUp(self):
+        super(TestPaverPa11ycrawlerCmd, self).setUp()
+
+        # Mock shell commands
+        mock_sh = patch('pavelib.utils.test.suites.bokchoy_suite.sh')
+        self._mock_sh = mock_sh.start()
+
+        # Cleanup mocks
+        self.addCleanup(mock_sh.stop)
+
+    def _expected_command(self, report_dir):
+        """
+        Returns the expected command to run pa11ycrawler.
+        """
+        cms_start_url = (
+            "http://localhost:8031/auto_auth?redirect=true&course_id=course-v1"
+            "%3AedX%2BTest101%2Bcourse&staff=true"
+        )
+
+        lms_start_url = (
+            "http://localhost:8003/auto_auth?redirect=true&course_id=course-v1"
+            "%3AedX%2BTest101%2Bcourse&staff=true"
+        )
+
+        expected_statement = (
+            'pa11ycrawler run "{cms_start_url}" "{lms_start_url}" '
+            '--pa11ycrawler-allowed-domains=localhost '
+            '--pa11ycrawler-reports-dir={report_dir} '
+            '--pa11ycrawler-deny-url-matcher=logout '
+            '--pa11y-reporter="1.0-json" '
+            '--depth-limit=6 '
+        ).format(
+            cms_start_url=cms_start_url,
+            lms_start_url=lms_start_url,
+            report_dir=report_dir,
+        )
+        return expected_statement
+
+    def test_default(self):
+        suite = A11yCrawler('')
+        self.assertEqual(
+            suite.cmd, self._expected_command(suite.pa11y_report_dir))
+
+    def test_get_test_course(self):
+        suite = A11yCrawler('')
+        suite.get_test_course()
+        self._mock_sh.assert_has_calls([
+            call(
+                'wget {targz} -O {dir}demo_course.tar.gz'.format(targz=suite.tar_gz_file, dir=suite.imports_dir)),
+            call(
+                'tar zxf {dir}demo_course.tar.gz -C {dir}'.format(dir=suite.imports_dir)),
+        ])
+
+    def test_generate_html_reports(self):
+        suite = A11yCrawler('')
+        suite.generate_html_reports()
+        self._mock_sh.assert_has_calls([
+            call(
+                'pa11ycrawler json-to-html --pa11ycrawler-reports-dir={}'.format(suite.pa11y_report_dir)),
+        ])

pavelib/utils/envs.py

@@ -37,6 +37,9 @@ class Env(object):
         "lib" / "custom_a11y_rules.js"
     )
 
+    PA11YCRAWLER_REPORT_DIR = REPORT_DIR / "pa11ycrawler"
+    PA11YCRAWLER_COVERAGERC = BOK_CHOY_DIR / ".pa11ycrawlercoveragerc"
+
     # If set, put reports for run in "unique" directories.
     # The main purpose of this is to ensure that the reports can be 'slurped'
     # in the main jenkins flow job without overwriting the reports from other

pavelib/utils/test/suites/__init__.py

@@ -6,4 +6,4 @@ from .nose_suite import NoseTestSuite, SystemTestSuite, LibTestSuite
 from .python_suite import PythonTestSuite
 from .js_suite import JsTestSuite
 from .acceptance_suite import AcceptanceTestSuite
-from .bokchoy_suite import BokChoyTestSuite
+from .bokchoy_suite import BokChoyTestSuite, A11yCrawler

pavelib/utils/test/suites/bokchoy_suite.py

@@ -2,9 +2,11 @@
 Class used for defining and running Bok Choy acceptance test suite
 """
 from time import sleep
+from urllib import urlencode
 
 from common.test.acceptance.fixtures.course import CourseFixture, FixtureError
 
+from path import Path as path
 from paver.easy import sh, BuildFailure
 from pavelib.utils.test.suites.suite import TestSuite
 from pavelib.utils.envs import Env
@@ -162,6 +164,26 @@ class BokChoyTestSuite(TestSuite):
         # load data in db_fixtures
         self.load_data()
 
+        # load courses if self.imports_dir is set
+        self.load_courses()
+
+        # Ensure the test servers are available
+        msg = colorize('green', "Confirming servers are running...")
+        print msg
+        bokchoy_utils.start_servers(self.default_store, self.coveragerc)
+
+    def load_courses(self):
+        """
+        Loads courses from self.imports_dir.
+
+        Note: self.imports_dir is the directory that contains the directories
+        that have courses in them. For example, if the course is located in
+        `test_root/courses/test-example-course/`, self.imports_dir should be
+        `test_root/courses/`.
+        """
+        msg = colorize('green', "Importing courses from {}...".format(self.imports_dir))
+        print msg
+
         if self.imports_dir:
             sh(
                 "DEFAULT_STORE={default_store}"
@@ -171,11 +193,6 @@ class BokChoyTestSuite(TestSuite):
                 )
             )
 
-        # Ensure the test servers are available
-        msg = colorize('green', "Confirming servers are running...")
-        print msg
-        bokchoy_utils.start_servers(self.default_store, self.coveragerc)
-
     def load_data(self):
         """
         Loads data into database from db_fixtures
@@ -241,3 +258,88 @@ class BokChoyTestSuite(TestSuite):
 
         cmd = (" ").join(cmd)
         return cmd
+
+
+class A11yCrawler(BokChoyTestSuite):
+    """
+    Sets up test environment with mega-course loaded, and runs pa11ycralwer
+    against it.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(A11yCrawler, self).__init__(*args, **kwargs)
+
+        self.pa11y_report_dir = os.path.join(self.report_dir, 'pa11ycrawler_reports')
+        self.imports_dir = path('test_root/courses/')
+        self.tar_gz_file = "https://github.com/edx/demo-test-course/archive/master.tar.gz"
+
+    def __enter__(self):
+        self.get_test_course()
+        super(A11yCrawler, self).__enter__()
+
+    def get_test_course(self):
+        """
+        Fetches the test course.
+        """
+        self.imports_dir.makedirs_p()
+        zipped_course = self.imports_dir + 'demo_course.tar.gz'
+
+        msg = colorize('green', "Fetching the test course from github...")
+        print msg
+
+        sh(
+            'wget {tar_gz_file} -O {zipped_course}'.format(
+                tar_gz_file=self.tar_gz_file,
+                zipped_course=zipped_course,
+            )
+        )
+
+        msg = colorize('green', "Uncompressing the test course...")
+        print msg
+
+        sh(
+            'tar zxf {zipped_course} -C {courses_dir}'.format(
+                zipped_course=zipped_course,
+                courses_dir=self.imports_dir,
+            )
+        )
+
+    def generate_html_reports(self):
+        """
+        Runs pa11ycrawler json-to-html
+        """
+        cmd_str = (
+            'pa11ycrawler json-to-html --pa11ycrawler-reports-dir={report_dir}'
+        ).format(report_dir=self.pa11y_report_dir)
+
+        sh(cmd_str)
+
+    @property
+    def cmd(self):
+        """
+        Runs pa11ycrawler as staff user against the test course.
+        """
+        params = urlencode({
+            "redirect": 'true',
+            "staff": 'true',
+            "course_id": "course-v1:edX+Test101+course",
+        })
+        cms_start_url = 'http://localhost:8031/auto_auth?{}'.format(params)
+        lms_start_url = 'http://localhost:8003/auto_auth?{}'.format(params)
+        cmd_str = (
+            'pa11ycrawler run "{cms_start_url}" "{lms_start_url}" '
+            '--pa11ycrawler-allowed-domains={allowed_domains} '
+            '--pa11ycrawler-reports-dir={report_dir} '
+            '--pa11ycrawler-deny-url-matcher={dont_go_here} '
+            '--pa11y-reporter="{reporter}" '
+            '--depth-limit={depth} '
+        ).format(
+            cms_start_url=cms_start_url,
+            lms_start_url=lms_start_url,
+            allowed_domains='localhost',
+            report_dir=self.pa11y_report_dir,
+            reporter="1.0-json",
+            dont_go_here="logout",
+            depth="6",
+        )
+        return cmd_str

requirements/edx/github.txt

@@ -67,6 +67,7 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
 
 # Used for testing
 git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
+git+https://github.com/edx/pa11ycrawler.git@0.0.1#egg=pa11ycrawler
 
 # Our libraries:
 git+https://github.com/edx/XBlock.git@xblock-0.4.7#egg=XBlock==0.4.7

scripts/accessibility-crawler.sh

+#!/usr/bin/env bash
+set -e
+
+echo "Setting up for accessibility tests..."
+source scripts/jenkins-common.sh
+
+echo "Running pa11ycrawler against test course..."
+paver pa11ycrawler
+
+echo "Generating coverage report..."
+paver pa11ycrawler_coverage