Merge pull request #12391 from edx/robrap/jenkins-rule-thresholds

TNL-4509: Set Jenkins thresholds per rule.

cms/templates/404.html

@@ -19,9 +19,9 @@ from openedx.core.djangolib.markup import Text, HTML
       ${_('The page that you were looking for was not found.')}
       ${Text(_('Go back to the {homepage} or let us know about any pages that may have been moved at {email}.')).format(
         homepage=HTML('<a href="/">homepage</a>'),
-        email=HTML('<a href="mailto:{address}">{address}</a>'.format(
+        email=HTML('<a href="mailto:{address}">{address}</a>').format(
           address=Text(settings.TECH_SUPPORT_EMAIL)
-        ))
+        )
       )}
       </p>
     </article>

cms/templates/500.html

@@ -30,9 +30,9 @@ ${Text(_("{studio_name} Server Error")).format(
         )}
         ${_("We've logged the error and our staff is currently working to resolve this error as soon as possible.")}
         ${Text(_(u'If the problem persists, please email us at {email_link}.')).format(
-          email_link=HTML(u'<a href="mailto:{email_address}">{email_address}</a>'.format(
-              email_address=Text(settings.TECH_SUPPORT_EMAIL),
-          ))
+          email_link=HTML(u'<a href="mailto:{email_address}">{email_address}</a>').format(
+            email_address=Text(settings.TECH_SUPPORT_EMAIL),
+          )
         )}
       </p>
     </article>

cms/templates/course_outline.html

@@ -121,7 +121,7 @@ from openedx.core.djangolib.markup import Text, HTML
             <h3 class="sr">${_("Page Actions")}</h3>
             <ul>
                 <li class="nav-item">
-                    <a href="#" class="button button-new" data-category="chapter" data-parent="${context_course.location | h}" data-default-name="${_('Section')}" title="${_('Click to add a new section')}">
+                    <a href="#" class="button button-new" data-category="chapter" data-parent="${context_course.location}" data-default-name="${_('Section')}" title="${_('Click to add a new section')}">
                         <i class="icon fa fa-plus"></i>${_('New Section')}
                     </a>
                 </li>

cms/templates/group_configurations.html

@@ -103,7 +103,7 @@ from openedx.core.djangolib.markup import Text, HTML
       <h3 class="title-3">${_("Other Course Settings")}</h3>
         <nav class="nav-related" aria-label="${_('Other Course Settings')}">
           <ul>
-            <li class="nav-item"><a href="${details_url}">${_("Details &amp; Schedule")}</a></li>
+            <li class="nav-item"><a href="${details_url}">${_("Details & Schedule")}</a></li>
             <li class="nav-item"><a href="${grading_url}">${_("Grading")}</a></li>
             <li class="nav-item"><a href="${course_team_url}">${_("Course Team")}</a></li>
             <li class="nav-item"><a href="${advanced_settings_url}">${_("Advanced Settings")}</a></li>

cms/templates/widgets/header.html

-<%page expression_filter="h"/>
+<%page expression_filter="h" args="online_help_token"/>
 <%namespace name='static' file='../static_content.html'/>
 <%!
   from django.conf import settings
@@ -6,7 +6,6 @@
   from django.utils.translation import ugettext as _
   from contentstore.context_processors import doc_url
 %>
-<%page args="online_help_token"/>
 <div class="wrapper-header wrapper" id="view-top">
   <header class="primary" role="banner">
 
@@ -61,7 +60,7 @@
                     <a href="${tabs_url}">${_("Pages")}</a>
                   </li>
                   <li class="nav-item nav-course-courseware-uploads">
-                    <a href="${assets_url}">${_("Files &amp; Uploads")}</a>
+                    <a href="${assets_url}">${_("Files & Uploads")}</a>
                   </li>
                   <li class="nav-item nav-course-courseware-textbooks">
                     <a href="${textbooks_url}">${_("Textbooks")}</a>
@@ -83,7 +82,7 @@
               <div class="nav-sub">
                 <ul>
                   <li class="nav-item nav-course-settings-schedule">
-                    <a href="${settings_url}">${_("Schedule &amp; Details")}</a>
+                    <a href="${settings_url}">${_("Schedule & Details")}</a>
                   </li>
                   <li class="nav-item nav-course-settings-grading">
                     <a href="${grading_url}">${_("Grading")}</a>

lms/templates/instructor/instructor_dashboard_2/instructor_analytics.html

@@ -14,7 +14,7 @@ from openedx.core.djangolib.markup import HTML, Text
                 link_start=HTML('<a href="{dashboard_url}" target="_blank">').format(
                     dashboard_url=escape_uri_path('{base_url}/courses/{course_id}'.format(
                         base_url=settings.ANALYTICS_DASHBOARD_URL,
-                        course_id=Text(section_data['course_id'])
+                        course_id=section_data['course_id'],
                     ))
                 ),
                 analytics_dashboard_name=settings.ANALYTICS_DASHBOARD_NAME,

lms/templates/navigation.html

@@ -9,7 +9,7 @@ from django.utils.translation import ugettext as _
 from lms.djangoapps.ccx.overrides import get_current_ccx
 from microsite_configuration import microsite
 from microsite_configuration.templatetags.microsite import platform_name
-from openedx.core.djangolib.markup import Text, HTML
+from openedx.core.djangolib.markup import HTML, Text
 
 # App that handles subdomain specific branding
 from branding import api as branding_api
@@ -194,15 +194,12 @@ site_status_msg = get_site_status_msg(course_id)
 </header>
 % if course:
 <!--[if lte IE 9]>
-<div class="ie-banner" aria-hidden="true">
-${Text(_('{begin_strong}Warning:{end_strong} Your browser is not fully supported. We strongly recommend using {chrome_link} or {ff_link}.')).format(
+<div class="ie-banner" aria-hidden="true">${Text(_('{begin_strong}Warning:{end_strong} Your browser is not fully supported. We strongly recommend using {chrome_link} or {ff_link}.')).format(
     begin_strong=HTML('<strong>'),
     end_strong=HTML('</strong>'),
     chrome_link=HTML('<a href="https://www.google.com/chrome" target="_blank">Chrome</a>'),
     ff_link=HTML('<a href="http://www.mozilla.org/firefox" target="_blank">Firefox</a>'),
-  )
-}
-</div>
+)}</div>
 <![endif]-->
 % endif
 

pavelib/paver_tests/test_paver_quality.py

@@ -4,6 +4,7 @@ Tests for paver quality tasks
 import os
 from path import Path as path
 import tempfile
+import textwrap
 import unittest
 from mock import patch, MagicMock, mock_open
 from ddt import ddt, file_data
@@ -132,6 +133,64 @@ class TestPaverReportViolationsCounts(unittest.TestCase):
         actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo")  # pylint: disable=protected-access
         self.assertEqual(actual_count, None)
 
+    def test_get_safelint_counts_happy(self):
+        report = textwrap.dedent("""
+            test.html: 30:53: javascript-jquery-append:  $('#test').append(print_tos);
+
+            javascript-concat-html: 310 violations
+            javascript-escape:      7 violations
+
+            2608 violations total
+        """)
+        with open(self.f.name, 'w') as f:
+            f.write(report)
+        counts = pavelib.quality._get_safelint_counts(self.f.name)  # pylint: disable=protected-access
+        self.assertDictEqual(counts, {
+            'rules': {
+                'javascript-concat-html': 310,
+                'javascript-escape': 7,
+            },
+            'total': 2608,
+        })
+
+    def test_get_safelint_counts_bad_counts(self):
+        report = textwrap.dedent("""
+            javascript-concat-html: violations
+        """)
+        with open(self.f.name, 'w') as f:
+            f.write(report)
+        counts = pavelib.quality._get_safelint_counts(self.f.name)  # pylint: disable=protected-access
+        self.assertDictEqual(counts, {
+            'rules': {},
+            'total': None,
+        })
+
+    def test_get_safecommit_count_happy(self):
+        report = textwrap.dedent("""
+            Linting lms/templates/navigation.html:
+
+            2 violations total
+
+            Linting scripts/tests/templates/test.underscore:
+
+            3 violations total
+        """)
+        with open(self.f.name, 'w') as f:
+            f.write(report)
+        count = pavelib.quality._get_safecommit_count(self.f.name)  # pylint: disable=protected-access
+
+        self.assertEqual(count, 5)
+
+    def test_get_safecommit_count_bad_counts(self):
+        report = textwrap.dedent("""
+            Linting lms/templates/navigation.html:
+        """)
+        with open(self.f.name, 'w') as f:
+            f.write(report)
+        count = pavelib.quality._get_safecommit_count(self.f.name)  # pylint: disable=protected-access
+
+        self.assertIsNone(count)
+
 
 class TestPrepareReportDir(unittest.TestCase):
     """

pavelib/paver_tests/test_safecommit.py

+"""
+Tests for paver safecommit quality tasks
+"""
+from mock import patch
+
+import pavelib.quality
+from paver.easy import call_task
+
+from .utils import PaverTestCase
+
+
+class PaverSafeCommitTest(PaverTestCase):
+    """
+    Test run_safecommit_report with a mocked environment in order to pass in
+    opts.
+
+    """
+
+    def setUp(self):
+        super(PaverSafeCommitTest, self).setUp()
+        self.reset_task_messages()
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safecommit_count')
+    def test_safecommit_violation_number_not_found(self, _mock_count, _mock_report_dir, _mock_write_metric):
+        """
+        run_safecommit_report encounters an error parsing the safecommit output
+        log.
+
+        """
+        _mock_count.return_value = None
+        with self.assertRaises(SystemExit):
+            call_task('pavelib.quality.run_safecommit_report')
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safecommit_count')
+    def test_safecommit_vanilla(self, _mock_count, _mock_report_dir, _mock_write_metric):
+        """
+        run_safecommit_report finds violations.
+        """
+        _mock_count.return_value = 0
+        call_task('pavelib.quality.run_safecommit_report')

pavelib/paver_tests/test_safelint.py

 """
-Tests for paver quality tasks
+Tests for paver safelint quality tasks
 """
 from mock import patch
 
@@ -20,43 +20,99 @@ class PaverSafeLintTest(PaverTestCase):
 
     @patch.object(pavelib.quality, '_write_metric')
     @patch.object(pavelib.quality, '_prepare_report_dir')
-    @patch.object(pavelib.quality, '_get_count_from_last_line')
-    def test_safelint_violation_number_not_found(self, _mock_count, _mock_report_dir, _mock_write_metric):
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric):
         """
         run_safelint encounters an error parsing the safelint output log
         """
-        _mock_count.return_value = None
+        _mock_counts.return_value = {}
         with self.assertRaises(SystemExit):
             call_task('pavelib.quality.run_safelint')
 
     @patch.object(pavelib.quality, '_write_metric')
     @patch.object(pavelib.quality, '_prepare_report_dir')
-    @patch.object(pavelib.quality, '_get_count_from_last_line')
-    def test_safelint_vanilla(self, _mock_count, _mock_report_dir, _mock_write_metric):
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_vanilla(self, _mock_counts, _mock_report_dir, _mock_write_metric):
         """
         run_safelint finds violations, but a limit was not set
         """
-        _mock_count.return_value = 1
+        _mock_counts.return_value = {'total': 0}
         call_task('pavelib.quality.run_safelint')
 
     @patch.object(pavelib.quality, '_write_metric')
     @patch.object(pavelib.quality, '_prepare_report_dir')
-    @patch.object(pavelib.quality, '_get_count_from_last_line')
-    def test_safelint_too_many_violations(self, _mock_count, _mock_report_dir, _mock_write_metric):
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_invalid_thresholds_option(self, _mock_counts, _mock_report_dir, _mock_write_metric):
+        """
+        run_safelint fails when thresholds option is poorly formatted
+        """
+        _mock_counts.return_value = {'total': 0}
+        with self.assertRaises(SystemExit):
+            call_task('pavelib.quality.run_safelint', options={"thresholds": "invalid"})
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_invalid_thresholds_option_key(self, _mock_counts, _mock_report_dir, _mock_write_metric):
+        """
+        run_safelint fails when thresholds option is poorly formatted
+        """
+        _mock_counts.return_value = {'total': 0}
+        with self.assertRaises(SystemExit):
+            call_task('pavelib.quality.run_safelint', options={"thresholds": '{"invalid": 3}'})
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_too_many_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric):
         """
         run_safelint finds more violations than are allowed
         """
-        _mock_count.return_value = 4
+        _mock_counts.return_value = {'total': 4}
         with self.assertRaises(SystemExit):
-            call_task('pavelib.quality.run_safelint', options={"limit": "3"})
+            call_task('pavelib.quality.run_safelint', options={"thresholds": '{"total": 3}'})
 
     @patch.object(pavelib.quality, '_write_metric')
     @patch.object(pavelib.quality, '_prepare_report_dir')
-    @patch.object(pavelib.quality, '_get_count_from_last_line')
-    def test_safelint_under_limit(self, _mock_count, _mock_report_dir, _mock_write_metric):
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_under_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric):
         """
         run_safelint finds fewer violations than are allowed
         """
-        _mock_count.return_value = 4
+        _mock_counts.return_value = {'total': 4}
+        # No System Exit is expected
+        call_task('pavelib.quality.run_safelint', options={"thresholds": '{"total": 5}'})
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_rule_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric):
+        """
+        run_safelint encounters an error parsing the safelint output log for a
+        given rule threshold that was set.
+        """
+        _mock_counts.return_value = {'total': 4}
+        with self.assertRaises(SystemExit):
+            call_task('pavelib.quality.run_safelint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'})
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_too_many_rule_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric):
+        """
+        run_safelint finds more rule violations than are allowed
+        """
+        _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}}
+        with self.assertRaises(SystemExit):
+            call_task('pavelib.quality.run_safelint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'})
+
+    @patch.object(pavelib.quality, '_write_metric')
+    @patch.object(pavelib.quality, '_prepare_report_dir')
+    @patch.object(pavelib.quality, '_get_safelint_counts')
+    def test_safelint_under_rule_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric):
+        """
+        run_safelint finds fewer rule violations than are allowed
+        """
+        _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}}
         # No System Exit is expected
-        call_task('pavelib.quality.run_safelint', options={"limit": "5"})
+        call_task('pavelib.quality.run_safelint', options={"thresholds": '{"rules": {"javascript-escape": 5}}'})

scripts/all-tests.sh

@@ -13,7 +13,7 @@ set -e
 # Violations thresholds for failing the build
 export PYLINT_THRESHOLD=4175
 export JSHINT_THRESHOLD=7550
-export SAFELINT_THRESHOLD=2700
+source scripts/safelint_thresholds.sh
 
 doCheckVars() {
     if [ -n "$CIRCLECI" ] ; then

scripts/generic-ci-tests.sh

@@ -85,7 +85,9 @@ case "$TEST_SUITE" in
         echo "Running code complexity report (python)."
         paver run_complexity > reports/code_complexity.log || echo "Unable to calculate code complexity. Ignoring error."
         echo "Running safe template linter report."
-        paver run_safelint -l $SAFELINT_THRESHOLD > safelint.log || { cat safelint.log; EXIT=1; }
+        paver run_safelint -t $SAFELINT_THRESHOLDS > safelint.log || { cat safelint.log; EXIT=1; }
+        echo "Running safe commit linter report."
+        paver run_safecommit_report > safecommit.log || { cat safecommit.log; EXIT=1; }
         # Run quality task. Pass in the 'fail-under' percentage to diff-quality
         echo "Running diff quality."
         paver run_quality -p 100 || EXIT=1

scripts/safe-commit-linter.sh

@@ -15,11 +15,17 @@ show_help() {
     echo "Runs the Safe Template Linter against all files in a git commit."
     echo ""
     echo "Mandatory arguments to long options are mandatory for short options too."
+    echo "  -h, --help                  Output this help."
     echo "  -m, --main-branch=COMMIT    Run against files changed between the"
     echo "                              current branch and this commit."
     echo "                              Defaults to origin/master."
     echo ""
-    echo "For additional help:"
+    echo "This scripts does not give a grand total.  Be sure to check for"
+    echo "0 violations on each file."
+    echo ""
+    echo "For more help using the safe template linter, including details on how"
+    echo "to understand and fix any violations, read the docs here:"
+    echo ""
     echo "  http://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/safe_templates.html#safe-template-linter"
 
 }

scripts/safelint_thresholds.sh

+#!/usr/bin/env bash
+set -e
+
+###############################################################################
+#
+#   safelint_thresholds.sh
+#
+#   The thresholds used for paver run_safelint when used with various CI
+#   systems.
+#
+###############################################################################
+
+# Violations thresholds for failing the build
+export SAFELINT_THRESHOLDS='
+    {
+        "rules": {
+            "javascript-concat-html": 313,
+            "javascript-escape": 7,
+            "javascript-interpolate": 71,
+            "javascript-jquery-append": 120,
+            "javascript-jquery-html": 313,
+            "javascript-jquery-insert-into-target": 26,
+            "javascript-jquery-insertion": 30,
+            "javascript-jquery-prepend": 12,
+            "mako-html-entities": 0,
+            "mako-invalid-html-filter": 33,
+            "mako-invalid-js-filter": 249,
+            "mako-js-html-string": 0,
+            "mako-js-missing-quotes": 0,
+            "mako-missing-default": 248,
+            "mako-multiple-page-tags": 0,
+            "mako-unknown-context": 0,
+            "mako-unparseable-expression": 0,
+            "mako-unwanted-html-filter": 0,
+            "python-close-before-format": 0,
+            "python-concat-html": 28,
+            "python-custom-escape": 13,
+            "python-deprecated-display-name": 53,
+            "python-interpolate-html": 68,
+            "python-parse-error": 0,
+            "python-requires-html-or-text": 0,
+            "python-wrap-html": 289,
+            "underscore-not-escaped": 709
+        },
+        "total": 2565
+    }'
+export SAFELINT_THRESHOLDS=${SAFELINT_THRESHOLDS//[[:space:]]/}

scripts/tests/templates/test.underscore

-<%= invalid %>
+<%=
+    'multi-line invalid'
+%>

scripts/tests/test_safe_template_linter.py

@@ -10,8 +10,8 @@ import textwrap
 from unittest import TestCase
 
 from ..safe_template_linter import (
-    _process_os_walk, FileResults, JavaScriptLinter, MakoTemplateLinter, ParseString,
-    StringLines, PythonLinter, UnderscoreTemplateLinter, Rules
+    _lint, FileResults, JavaScriptLinter, MakoTemplateLinter, ParseString,
+    StringLines, PythonLinter, SummaryResults, UnderscoreTemplateLinter, Rules
 )
 
 
@@ -83,6 +83,15 @@ class TestSafeTemplateLinter(TestCase):
     Test some top-level linter functions
     """
 
+    def setUp(self):
+        """
+        Setup patches on linters for testing.
+        """
+        self.patch_is_valid_directory(MakoTemplateLinter)
+        self.patch_is_valid_directory(JavaScriptLinter)
+        self.patch_is_valid_directory(UnderscoreTemplateLinter)
+        self.patch_is_valid_directory(PythonLinter)
+
     def patch_is_valid_directory(self, linter_class):
         """
         Creates a mock patch for _is_valid_directory on a Linter to always
@@ -96,37 +105,137 @@ class TestSafeTemplateLinter(TestCase):
         self.addCleanup(patcher.stop)
         return patch_start
 
-    def test_process_os_walk(self):
+    def test_lint_defaults(self):
         """
-        Tests the top-level processing of template files, including Mako
-        includes.
+        Tests the top-level linting with default options.
         """
         out = StringIO()
-
-        options = {
-            'list_files': False,
-            'verbose': False,
-        }
-
-        template_linters = [MakoTemplateLinter(), JavaScriptLinter(), UnderscoreTemplateLinter(), PythonLinter()]
-
-        self.patch_is_valid_directory(MakoTemplateLinter)
-        self.patch_is_valid_directory(JavaScriptLinter)
-        self.patch_is_valid_directory(UnderscoreTemplateLinter)
-        self.patch_is_valid_directory(PythonLinter)
-
-        num_violations = _process_os_walk('scripts/tests/templates', template_linters, options, out)
+        summary_results = SummaryResults()
+
+        _lint(
+            'scripts/tests/templates',
+            template_linters=[MakoTemplateLinter(), UnderscoreTemplateLinter(), JavaScriptLinter(), PythonLinter()],
+            options={
+                'list_files': False,
+                'verbose': False,
+                'rule_totals': False,
+            },
+            summary_results=summary_results,
+            out=out,
+        )
 
         output = out.getvalue()
-        self.assertEqual(num_violations, 7)
+        # Assert violation details are displayed.
         self.assertIsNotNone(re.search('test\.html.*{}'.format(Rules.mako_missing_default.rule_id), output))
         self.assertIsNotNone(re.search('test\.coffee.*{}'.format(Rules.javascript_concat_html.rule_id), output))
         self.assertIsNotNone(re.search('test\.coffee.*{}'.format(Rules.underscore_not_escaped.rule_id), output))
         self.assertIsNotNone(re.search('test\.js.*{}'.format(Rules.javascript_concat_html.rule_id), output))
         self.assertIsNotNone(re.search('test\.js.*{}'.format(Rules.underscore_not_escaped.rule_id), output))
-        self.assertIsNotNone(re.search('test\.underscore.*{}'.format(Rules.underscore_not_escaped.rule_id), output))
+        lines_with_rule = 0
+        lines_without_rule = 0  # Output with verbose setting only.
+        for underscore_match in re.finditer('test\.underscore:.*\n', output):
+            if re.search(Rules.underscore_not_escaped.rule_id, underscore_match.group()) is not None:
+                lines_with_rule += 1
+            else:
+                lines_without_rule += 1
+        self.assertGreaterEqual(lines_with_rule, 1)
+        self.assertEquals(lines_without_rule, 0)
         self.assertIsNone(re.search('test\.py.*{}'.format(Rules.python_parse_error.rule_id), output))
         self.assertIsNotNone(re.search('test\.py.*{}'.format(Rules.python_wrap_html.rule_id), output))
+        # Assert no rule totals.
+        self.assertIsNone(re.search('{}:\s*{} violations'.format(Rules.python_parse_error.rule_id, 0), output))
+        # Assert final total
+        self.assertIsNotNone(re.search('{} violations total'.format(7), output))
+
+    def test_lint_with_verbose(self):
+        """
+        Tests the top-level linting with verbose option.
+        """
+        out = StringIO()
+        summary_results = SummaryResults()
+
+        _lint(
+            'scripts/tests/templates',
+            template_linters=[MakoTemplateLinter(), UnderscoreTemplateLinter(), JavaScriptLinter(), PythonLinter()],
+            options={
+                'list_files': False,
+                'verbose': True,
+                'rule_totals': False,
+            },
+            summary_results=summary_results,
+            out=out,
+        )
+
+        output = out.getvalue()
+        lines_with_rule = 0
+        lines_without_rule = 0  # Output with verbose setting only.
+        for underscore_match in re.finditer('test\.underscore:.*\n', output):
+            if re.search(Rules.underscore_not_escaped.rule_id, underscore_match.group()) is not None:
+                lines_with_rule += 1
+            else:
+                lines_without_rule += 1
+        self.assertGreaterEqual(lines_with_rule, 1)
+        self.assertGreaterEqual(lines_without_rule, 1)
+        # Assert no rule totals.
+        self.assertIsNone(re.search('{}:\s*{} violations'.format(Rules.python_parse_error.rule_id, 0), output))
+        # Assert final total
+        self.assertIsNotNone(re.search('{} violations total'.format(7), output))
+
+    def test_lint_with_rule_totals(self):
+        """
+        Tests the top-level linting with rule totals option.
+        """
+        out = StringIO()
+        summary_results = SummaryResults()
+
+        _lint(
+            'scripts/tests/templates',
+            template_linters=[MakoTemplateLinter(), UnderscoreTemplateLinter(), JavaScriptLinter(), PythonLinter()],
+            options={
+                'list_files': False,
+                'verbose': False,
+                'rule_totals': True,
+            },
+            summary_results=summary_results,
+            out=out,
+        )
+
+        output = out.getvalue()
+        self.assertIsNotNone(re.search('test\.py.*{}'.format(Rules.python_wrap_html.rule_id), output))
+
+        # Assert totals output.
+        self.assertIsNotNone(re.search('{}:\s*{} violations'.format(Rules.python_parse_error.rule_id, 0), output))
+        self.assertIsNotNone(re.search('{}:\s*{} violations'.format(Rules.python_wrap_html.rule_id, 1), output))
+        self.assertIsNotNone(re.search('{} violations total'.format(7), output))
+
+    def test_lint_with_list_files(self):
+        """
+        Tests the top-level linting with list files option.
+        """
+        out = StringIO()
+        summary_results = SummaryResults()
+
+        _lint(
+            'scripts/tests/templates',
+            template_linters=[MakoTemplateLinter(), UnderscoreTemplateLinter(), JavaScriptLinter(), PythonLinter()],
+            options={
+                'list_files': True,
+                'verbose': False,
+                'rule_totals': False,
+            },
+            summary_results=summary_results,
+            out=out,
+        )
+
+        output = out.getvalue()
+        # Assert file with rule is not output.
+        self.assertIsNone(re.search('test\.py.*{}'.format(Rules.python_wrap_html.rule_id), output))
+        # Assert file is output.
+        self.assertIsNotNone(re.search('test\.py', output))
+
+        # Assert no totals.
+        self.assertIsNone(re.search('{}:\s*{} violations'.format(Rules.python_parse_error.rule_id, 0), output))
+        self.assertIsNone(re.search('{} violations total'.format(7), output))
 
 
 @ddt

themes/edx.org/lms/templates/header.html

@@ -6,6 +6,7 @@
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 
+from openedx.core.djangolib.markup import HTML, Text
 # App that handles subdomain specific branding
 import branding
 # app that handles site status messages
@@ -59,9 +60,9 @@ site_status_msg = get_site_status_msg(course_id)
 
     % if course and not disable_courseware_header:
     <h2 class="course-header">
-      <span class="provider">${course.display_org_with_default | h}:</span>
-      <span class="course-number">${course.display_number_with_default | h}</span>
-      <span class="course-name">${course.display_name_with_default_escaped}</span>
+      <span class="provider">${course.display_org_with_default}:</span>
+      <span class="course-number">${course.display_number_with_default}</span>
+      <span class="course-name">${course.display_name_with_default}</span>
     </h2>
     % endif
 
@@ -143,7 +144,12 @@ site_status_msg = get_site_status_msg(course_id)
 </header>
 % if course:
 <!--[if lte IE 9]>
-<div class="ie-banner" aria-hidden="true">${_('<strong>Warning:</strong> Your browser is not fully supported. We strongly recommend using {chrome_link} or {ff_link}.').format(chrome_link='<a href="https://www.google.com/chrome" target="_blank">Chrome</a>', ff_link='<a href="http://www.mozilla.org/firefox" target="_blank">Firefox</a>')}</div>
+<div class="ie-banner" aria-hidden="true">${Text(_('{begin_strong}Warning:{end_strong} Your browser is not fully supported. We strongly recommend using {chrome_link} or {ff_link}.')).format(
+    begin_strong=HTML('<strong>'),
+    end_strong=HTML('</strong>'),
+    chrome_link=HTML('<a href="https://www.google.com/chrome" target="_blank">Chrome</a>'),
+    ff_link=HTML('<a href="http://www.mozilla.org/firefox" target="_blank">Firefox</a>'),
+)}</div>
 <![endif]-->
 % endif