test_paver_quality.py 15 KB
Newer Older
1 2 3
"""
Tests for paver quality tasks
"""
4
import os
5
from path import Path as path
6
import tempfile
7
import textwrap
8
import unittest
9
from mock import patch, MagicMock, mock_open
10 11
from ddt import ddt, file_data

12 13
import pavelib.quality
import paver.easy
14
import paver.tasks
15
from paver.easy import BuildFailure
16

17

18 19
@ddt
class TestPaverQualityViolations(unittest.TestCase):
20 21 22
    """
    For testing the paver violations-counting tasks
    """
23
    def setUp(self):
24
        super(TestPaverQualityViolations, self).setUp()
25 26
        self.f = tempfile.NamedTemporaryFile(delete=False)
        self.f.close()
27
        self.addCleanup(os.remove, self.f.name)
28 29 30 31

    def test_pylint_parser_other_string(self):
        with open(self.f.name, 'w') as f:
            f.write("hello")
32
        num = pavelib.quality._count_pylint_violations(f.name)  # pylint: disable=protected-access
33 34 35 36 37 38
        self.assertEqual(num, 0)

    def test_pylint_parser_pep8(self):
        # Pep8 violations should be ignored.
        with open(self.f.name, 'w') as f:
            f.write("foo/hello/test.py:304:15: E203 whitespace before ':'")
39
        num = pavelib.quality._count_pylint_violations(f.name)  # pylint: disable=protected-access
40 41 42 43
        self.assertEqual(num, 0)

    @file_data('pylint_test_list.json')
    def test_pylint_parser_count_violations(self, value):
44 45 46 47 48
        """
        Tests:
        - Different types of violations
        - One violation covering multiple lines
        """
49 50
        with open(self.f.name, 'w') as f:
            f.write(value)
51
        num = pavelib.quality._count_pylint_violations(f.name)  # pylint: disable=protected-access
52 53 54 55 56
        self.assertEqual(num, 1)

    def test_pep8_parser(self):
        with open(self.f.name, 'w') as f:
            f.write("hello\nhithere")
57
        num, _violations = pavelib.quality._pep8_violations(f.name)  # pylint: disable=protected-access
58 59
        self.assertEqual(num, 2)

60

61
class TestPaverReportViolationsCounts(unittest.TestCase):
62
    """
63 64
    For testing utility functions for getting counts from reports for
    run_jshint, run_complexity, run_safelint, and run_safecommit_report.
65 66 67
    """

    def setUp(self):
68
        super(TestPaverReportViolationsCounts, self).setUp()
69 70 71 72 73 74 75 76 77 78 79 80 81

        # Mock the paver @needs decorator
        self._mock_paver_needs = patch.object(pavelib.quality.run_quality, 'needs').start()
        self._mock_paver_needs.return_value = 0

        # Temporary file infrastructure
        self.f = tempfile.NamedTemporaryFile(delete=False)
        self.f.close()

        # Cleanup various mocks and tempfiles
        self.addCleanup(self._mock_paver_needs.stop)
        self.addCleanup(os.remove, self.f.name)

82
    def test_get_jshint_violations_count(self):
83 84
        with open(self.f.name, 'w') as f:
            f.write("3000 violations found")
85
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "jshint")  # pylint: disable=protected-access
86 87 88 89 90
        self.assertEqual(actual_count, 3000)

    def test_get_violations_no_number_found(self):
        with open(self.f.name, 'w') as f:
            f.write("Not expected string regex")
91
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "jshint")  # pylint: disable=protected-access
92 93 94 95 96 97 98 99
        self.assertEqual(actual_count, None)

    def test_get_violations_count_truncated_report(self):
        """
        A truncated report (i.e. last line is just a violation)
        """
        with open(self.f.name, 'w') as f:
            f.write("foo/bar/js/fizzbuzz.js: line 45, col 59, Missing semicolon.")
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "jshint")  # pylint: disable=protected-access
        self.assertEqual(actual_count, None)

    def test_complexity_value(self):
        with open(self.f.name, 'w') as f:
            f.write("Average complexity: A (1.93953443446)")
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "python_complexity")  # pylint: disable=protected-access
        self.assertEqual(actual_count, 1.93953443446)

    def test_truncated_complexity_report(self):
        with open(self.f.name, 'w') as f:
            f.write("M 110:4 FooBar.default - A")
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "python_complexity")  # pylint: disable=protected-access
        self.assertEqual(actual_count, None)

    def test_no_complexity_report(self):
        with self.assertRaises(BuildFailure):
            pavelib.quality._get_count_from_last_line("non-existent-file", "python_complexity")  # pylint: disable=protected-access

    def test_generic_value(self):
        """
        Default behavior is to look for an integer appearing at head of line
        """
        with open(self.f.name, 'w') as f:
            f.write("5.777 good to see you")
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo")  # pylint: disable=protected-access
        self.assertEqual(actual_count, 5)

    def test_generic_value_none_found(self):
        """
        Default behavior is to look for an integer appearing at head of line
        """
        with open(self.f.name, 'w') as f:
            f.write("hello 5.777 good to see you")
        actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo")  # pylint: disable=protected-access
135 136
        self.assertEqual(actual_count, None)

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
    def test_get_safelint_counts_happy(self):
        """
        Test happy path getting violation counts from safelint report.
        """
        report = textwrap.dedent("""
            test.html: 30:53: javascript-jquery-append:  $('#test').append(print_tos);

            javascript-concat-html: 310 violations
            javascript-escape:      7 violations

            2608 violations total
        """)
        with open(self.f.name, 'w') as f:
            f.write(report)
        counts = pavelib.quality._get_safelint_counts(self.f.name)  # pylint: disable=protected-access
        self.assertDictEqual(counts, {
            'rules': {
                'javascript-concat-html': 310,
                'javascript-escape': 7,
            },
            'total': 2608,
        })

    def test_get_safelint_counts_bad_counts(self):
        """
        Test getting violation counts from truncated and malformed safelint
        report.
        """
        report = textwrap.dedent("""
            javascript-concat-html: violations
        """)
        with open(self.f.name, 'w') as f:
            f.write(report)
        counts = pavelib.quality._get_safelint_counts(self.f.name)  # pylint: disable=protected-access
        self.assertDictEqual(counts, {
            'rules': {},
            'total': None,
        })

    def test_get_safecommit_count_happy(self):
        """
        Test happy path getting violation count from safecommit report.
        """
        report = textwrap.dedent("""
            Linting lms/templates/navigation.html:

            2 violations total

            Linting scripts/tests/templates/test.underscore:

            3 violations total
        """)
        with open(self.f.name, 'w') as f:
            f.write(report)
        count = pavelib.quality._get_safecommit_count(self.f.name)  # pylint: disable=protected-access

        self.assertEqual(count, 5)

    def test_get_safecommit_count_bad_counts(self):
        """
        Test getting violation count from truncated safecommit report.
        """
        report = textwrap.dedent("""
            Linting lms/templates/navigation.html:
        """)
        with open(self.f.name, 'w') as f:
            f.write(report)
        count = pavelib.quality._get_safecommit_count(self.f.name)  # pylint: disable=protected-access

        self.assertIsNone(count)

    def test_get_safecommit_count_no_files(self):
        """
        Test getting violation count from safecommit report where no files were
        linted.
        """
        report = textwrap.dedent("""
            No files linted.
        """)
        with open(self.f.name, 'w') as f:
            f.write(report)
        count = pavelib.quality._get_safecommit_count(self.f.name)  # pylint: disable=protected-access

        self.assertEqual(count, 0)

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244

class TestPrepareReportDir(unittest.TestCase):
    """
    Tests the report directory preparation
    """

    def setUp(self):
        super(TestPrepareReportDir, self).setUp()
        self.test_dir = tempfile.mkdtemp()
        self.test_file = tempfile.NamedTemporaryFile(delete=False, dir=self.test_dir)
        self.addCleanup(os.removedirs, self.test_dir)

    def test_report_dir_with_files(self):
        self.assertTrue(os.path.exists(self.test_file.name))
        pavelib.quality._prepare_report_dir(path(self.test_dir))  # pylint: disable=protected-access
        self.assertFalse(os.path.exists(self.test_file.name))

    def test_report_dir_without_files(self):
        os.remove(self.test_file.name)
        pavelib.quality._prepare_report_dir(path(self.test_dir))  # pylint: disable=protected-access
        self.assertEqual(os.listdir(path(self.test_dir)), [])


245 246 247 248 249 250
class TestPaverRunQuality(unittest.TestCase):
    """
    For testing the paver run_quality task
    """

    def setUp(self):
251
        super(TestPaverRunQuality, self).setUp()
252

253 254 255 256 257 258 259 260 261 262 263 264 265
        # test_no_diff_quality_failures seems to alter the way that paver
        # executes these lines is subsequent tests.
        # https://github.com/paver/paver/blob/master/paver/tasks.py#L175-L180
        #
        # The other tests don't appear to have the same impact. This was
        # causing a test order dependency. This line resets that state
        # of environment._task_in_progress so that the paver commands in the
        # tests will be considered top level tasks by paver, and we can predict
        # which path it will chose in the above code block.
        #
        # TODO: Figure out why one test is altering the state to begin with.
        paver.tasks.environment = paver.tasks.Environment()

266 267 268
        # mock the @needs decorator to skip it
        self._mock_paver_needs = patch.object(pavelib.quality.run_quality, 'needs').start()
        self._mock_paver_needs.return_value = 0
269 270 271 272
        patcher = patch('pavelib.quality.sh')
        self._mock_paver_sh = patcher.start()
        self.addCleanup(patcher.stop)
        self.addCleanup(self._mock_paver_needs.stop)
273

274
    @patch('__builtin__.open', mock_open())
275 276
    def test_failure_on_diffquality_pep8(self):
        """
277
        If pep8 finds errors, pylint and jshint should still be run
278
        """
279 280 281 282 283 284 285 286
        # Mock _get_pep8_violations to return a violation
        _mock_pep8_violations = MagicMock(
            return_value=(1, ['lms/envs/common.py:32:2: E225 missing whitespace around operator'])
        )
        with patch('pavelib.quality._get_pep8_violations', _mock_pep8_violations):
            with self.assertRaises(SystemExit):
                pavelib.quality.run_quality("")

287 288
        # Test that pep8, pylint, and jshint were called by counting the calls to
        # _get_pep8_violations (for pep8) and sh (for diff-quality pylint & jshint)
289
        self.assertEqual(_mock_pep8_violations.call_count, 1)
290
        self.assertEqual(self._mock_paver_sh.call_count, 2)
291

292
    @patch('__builtin__.open', mock_open())
293 294 295 296 297 298 299
    def test_failure_on_diffquality_pylint(self):
        """
        If diff-quality fails on pylint, the paver task should also fail
        """

        # Underlying sh call must fail when it is running the pylint diff-quality task
        self._mock_paver_sh.side_effect = CustomShMock().fail_on_pylint
300 301 302 303
        _mock_pep8_violations = MagicMock(return_value=(0, []))
        with patch('pavelib.quality._get_pep8_violations', _mock_pep8_violations):
            with self.assertRaises(SystemExit):
                pavelib.quality.run_quality("")
304

305
        # Test that both pep8 and pylint were called by counting the calls
306 307
        # Assert that _get_pep8_violations (which calls "pep8") is called once
        self.assertEqual(_mock_pep8_violations.call_count, 1)
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
        # And assert that sh was called twice (for the calls to pylint & jshint). This means that even in
        # the event of a diff-quality pylint failure, jshint is still called.
        self.assertEqual(self._mock_paver_sh.call_count, 2)

    @patch('__builtin__.open', mock_open())
    def test_failure_on_diffquality_jshint(self):
        """
        If diff-quality fails on jshint, the paver task should also fail
        """

        # Underlying sh call must fail when it is running the jshint diff-quality task
        self._mock_paver_sh.side_effect = CustomShMock().fail_on_jshint
        _mock_pep8_violations = MagicMock(return_value=(0, []))
        with patch('pavelib.quality._get_pep8_violations', _mock_pep8_violations):
            with self.assertRaises(SystemExit):
                pavelib.quality.run_quality("")
                self.assertRaises(BuildFailure)
        # Test that both pep8 and pylint were called by counting the calls
        # Assert that _get_pep8_violations (which calls "pep8") is called once
        self.assertEqual(_mock_pep8_violations.call_count, 1)
328
        # And assert that sh was called twice (for the calls to pep8 and pylint)
329
        self.assertEqual(self._mock_paver_sh.call_count, 2)
330

331
    @patch('__builtin__.open', mock_open())
332 333 334 335 336
    def test_other_exception(self):
        """
        If diff-quality fails for an unknown reason on the first run (pep8), then
        pylint should not be run
        """
337
        self._mock_paver_sh.side_effect = [Exception('unrecognized failure!'), 0]
338
        with self.assertRaises(SystemExit):
339
            pavelib.quality.run_quality("")
340
            self.assertRaises(Exception)
341
        # Test that pylint is NOT called by counting calls
342
        self.assertEqual(self._mock_paver_sh.call_count, 1)
343

344
    @patch('__builtin__.open', mock_open())
345 346
    def test_no_diff_quality_failures(self):
        # Assert nothing is raised
347 348 349 350 351
        _mock_pep8_violations = MagicMock(return_value=(0, []))
        with patch('pavelib.quality._get_pep8_violations', _mock_pep8_violations):
            pavelib.quality.run_quality("")
        # Assert that _get_pep8_violations (which calls "pep8") is called once
        self.assertEqual(_mock_pep8_violations.call_count, 1)
352
        # And assert that sh was called twice (for the call to "pylint" & "jshint")
353
        self.assertEqual(self._mock_paver_sh.call_count, 2)
354 355 356 357 358 359 360 361 362 363 364 365 366


class CustomShMock(object):
    """
    Diff-quality makes a number of sh calls. None of those calls should be made during tests; however, some
    of them need to have certain responses.
    """

    def fail_on_pylint(self, arg):
        """
        For our tests, we need the call for diff-quality running pep8 reports to fail, since that is what
        is going to fail when we pass in a percentage ("p") requirement.
        """
367
        if "pylint" in arg:
368 369 370 371
            # Essentially mock diff-quality exiting with 1
            paver.easy.sh("exit 1")
        else:
            return
372 373 374 375 376 377 378 379 380 381 382

    def fail_on_jshint(self, arg):
        """
        For our tests, we need the call for diff-quality running pep8 reports to fail, since that is what
        is going to fail when we pass in a percentage ("p") requirement.
        """
        if "jshint" in arg:
            # Essentially mock diff-quality exiting with 1
            paver.easy.sh("exit 1")
        else:
            return