Commit 594bd287 by Calen Pennington Committed by GitHub

Merge pull request #53 from edx/cale/lint-amnesty

Cale/lint amnesty
parents 8f7eeb5b f7213763
......@@ -11,6 +11,7 @@ matrix:
sudo: false
install:
- pip install --upgrade pip setuptools
- pip install tox-travis
script:
......
......@@ -29,6 +29,26 @@ The file written contains a hash of its contents, to detect subsequent editing.
is detected, the edited file will be moved aside so it can be compared to the
newly written file.
Using lint-amnesty
------------------
The ``lint-amnesty`` command can be used to squash all existing pylint errors
in a codebase, so that from then the repository can maintain pylint-cleanliness.
Install the package using ``pip``::
$ pip install edx-lint
The ``lint-amnesty`` command expects pylint errors in the ``--output-format=parseable``
format::
$ pylint my.python.package --output-format=parseable | lint-amnesty
This will add comments for every existing pylint violation that look like::
# pylint: disable=some-error # lint-amnesty
It will also remove any existing suppressions that pylint flags as being ``useless-suppressions``.
Customizing edx_lint
--------------------
You can customize the resulting pylintrc file by creating a pylintrc_tweaks file in the
......
"""
Commandline scripts to give one-time amnesty for specify linting error conditions.
"""
from __future__ import print_function
from collections import namedtuple, defaultdict
import logging
import re
import sys
import click
import click_log
LOG = logging.getLogger(__name__)
PYLINT_PARSEABLE_REGEX = re.compile(
r"""^(?P<filename>[^:]+):(?P<linenum>\d+): """
r"""\[(?P<error_code>[^(]+)\((?P<error_name>[^)]+)\), (?P<function>[^\]]*)\] """
r"""(?P<error_msg>.*)"""
)
PYLINT_EXCEPTION_REGEX = re.compile(r"""\s*#\s*pylint:\s+disable=(?P<disables>[^#$]+?)(?=\s*(#|$))""")
PylintError = namedtuple('PylintError', ['filename', 'linenum', 'error_code', 'error_name', 'function', 'error_msg'])
def parse_pylint_output(pylint_output):
"""
Parse the pylint output-format=parseable lines into PylintError tuples.
"""
for line in pylint_output:
if not line.strip():
continue
if line[0:5] in ("-"*5, "*"*5):
continue
parsed = PYLINT_PARSEABLE_REGEX.search(line)
if parsed is None:
LOG.warning(
"Unable to parse %r. If this is a lint failure, please re-run pylint with the "
"--output-format=parseable option, otherwise, you can ignore this message.",
line
)
continue
parsed_dict = parsed.groupdict()
parsed_dict['linenum'] = int(parsed_dict['linenum'])
yield PylintError(**parsed_dict)
def format_pylint_disables(error_names, tag=True):
"""
Format a list of error_names into a 'pylint: disable=' line.
"""
tag_str = "lint-amnesty, " if tag else ""
if error_names:
return " # {tag}pylint: disable={disabled}".format(
disabled=", ".join(sorted(error_names)),
tag=tag_str,
)
else:
return ""
def fix_pylint(line, errors):
"""
Yield any modified versions of ``line`` needed to address the errors in ``errors``.
"""
if not errors:
yield line
return
current = PYLINT_EXCEPTION_REGEX.search(line)
if current:
original_errors = {disable.strip() for disable in current.group('disables').split(',')}
else:
original_errors = set()
disabled_errors = set(original_errors)
for error in errors:
if error.error_name == 'useless-suppression':
parsed = re.search("""Useless suppression of '(?P<error_name>[^']+)'""", error.error_msg)
disabled_errors.discard(parsed.group('error_name'))
elif error.error_name == 'missing-docstring' and error.error_msg == 'Missing module docstring':
yield format_pylint_disables({error.error_name}).strip() + '\n'
else:
disabled_errors.add(error.error_name)
disable_string = format_pylint_disables(disabled_errors, not disabled_errors <= original_errors)
if current:
yield PYLINT_EXCEPTION_REGEX.sub(disable_string, line)
else:
yield re.sub(r'($\s*)', disable_string + r'\1', line)
@click.command()
@click.option(
'--pylint-output', default=sys.stdin, type=click.File(),
help="An input file containing pylint --output-format=parseable errors. Defaults to stdin."
)
@click_log.simple_verbosity_option(default=u'INFO')
@click_log.init()
def pylint_amnesty(pylint_output):
"""
Add ``# pylint: disable`` clauses to add exceptions to all existing pylint errors in a codebase.
"""
errors = defaultdict(lambda: defaultdict(set))
for pylint_error in parse_pylint_output(pylint_output):
errors[pylint_error.filename][pylint_error.linenum].add(pylint_error)
for file_with_errors in sorted(errors):
try:
opened_file = open(file_with_errors)
except IOError:
LOG.warning("Unable to open %s for edits", file_with_errors, exc_info=True)
else:
with opened_file as input_file:
output_lines = []
for line_num, line in enumerate(input_file, start=1):
output_lines.extend(
fix_pylint(
line,
errors[file_with_errors][line_num]
)
)
with open(file_with_errors, 'w') as output_file:
output_file.writelines(output_lines)
......@@ -106,7 +106,7 @@ def write_main(argv):
# pkg_resources always reads binary data (in both python2 and python3).
# ConfigParser.read_string only exists in python3, so we have to wrap the string
# from pkg_resources in a cStringIO so that we can pass it into ConfigParser.readfp.
cfg.readfp(cStringIO(resource_string), resource_name) # pylint: disable=deprecated-method
cfg.readfp(cStringIO(resource_string), resource_name) # pylint: disable=deprecated-method
if os.path.exists(tweaks_name):
print("Applying local tweaks from %s" % tweaks_name)
......
......@@ -24,7 +24,227 @@ load-plugins=edx_lint.pylint,pylint_django,pylint_celery
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time. See also the "--disable" option for examples.
#enable=
enable=
# These are controlled by explicit choices in the pylintrc files
blacklisted-name,
line-too-long,
# These affect the correctness of the code
syntax-error,
init-is-generator,
return-in-init,
function-redefined,
not-in-loop,
return-outside-function,
yield-outside-function,
return-arg-in-generator,
nonexistent-operator,
duplicate-argument-name,
abstract-class-instantiated,
bad-reversed-sequence,
continue-in-finally,
method-hidden,
access-member-before-definition,
no-method-argument,
no-self-argument,
invalid-slots-object,
assigning-non-slot,
invalid-slots,
inherit-non-class,
inconsistent-mro,
duplicate-bases,
non-iterator-returned,
unexpected-special-method-signature,
invalid-length-returned,
import-error,
used-before-assignment,
undefined-variable,
undefined-all-variable,
invalid-all-object,
no-name-in-module,
unbalance-tuple-unpacking,
unpacking-non-sequence,
bad-except-order,
raising-bad-type,
misplaced-bare-raise,
raising-non-exception,
nonimplemented-raised,
catching-non-exception,
slots-on-old-class,
super-on-old-class,
bad-super-call,
missing-super-argument,
no-member,
not-callable,
assignment-from-no-return,
no-value-for-parameter,
too-many-function-args,
unexpected-keyword-arg,
redundant-keyword-arg,
invalid-sequence-index,
invalid-slice-index,
assignment-from-none,
not-context-manager,
invalid-unary-operand-type,
unsupported-binary-operation,
repeated-keyword,
not-an-iterable,
not-a-mapping,
unsupported-membership-test,
unsubscriptable-object,
logging-unsupported-format,
logging-too-many-args,
logging-too-few-args,
bad-format-character,
truncated-format-string,
mixed-fomat-string,
format-needs-mapping,
missing-format-string-key,
too-many-format-args,
too-few-format-args,
bad-str-strip-call,
model-unicode-not-callable,
super-method-not-called,
non-parent-method-called,
test-inherits-tests,
translation-of-non-string,
redefined-variable-type,
cyclical-import,
unreachable,
dangerous-default-value,
pointless-statement,
pointless-string-statement,
expression-not-assigned,
duplicate-key,
confusing-with-statement,
using-constant-test,
lost-exception,
assert-on-tuple,
attribute-defined-outside-init,
bad-staticmethod-argument,
arguments-differ,
signature-differs,
abstract-method,
super-init-not-called,
relative-import,
import-self,
misplaced-future,
invalid-encoded-data,
global-variable-undefined,
redefined-outer-name,
redefined-builtin,
redefined-in-handler,
undefined-loop-variable,
cell-var-from-loop,
duplicate-except,
nonstandard-exception,
binary-op-exception,
property-on-old-class,
bad-format-string-key,
unused-format-string-key,
bad-format-string,
missing-format-argument-key,
unused-format-string-argument,
format-combined-specification,
missing-format-attribute,
invalid-format-index,
anomalous-backslash-in-string,
anomalous-unicode-escape-in-string,
bad-open-mode,
boolean-datetime,
# Checking failed for some reason
fatal,
astroid-error,
parse-error,
method-check-failed,
django-not-available,
raw-checker-failed,
django-not-available-placeholder,
# Documentation is important
empty-docstring,
invalid-characters-in-docstring,
missing-docstring,
wrong-spelling-in-comment,
wrong-spelling-in-docstring,
# Unused code should be deleted
unused-import,
unused-variable,
unused-argument,
# These are dangerous!
exec-used,
eval-used,
# These represent idiomatic python. Not adhering to them
# will raise red flags with future readers.
bad-classmethod-argument,
bad-mcs-classmethod-argument,
bad-mcs-method-argument,
bad-whitespace,
consider-iterating-dictionary,
consider-using-enumerate,
literal-used-as-attribute,
multiple-imports,
multiple-statements,
old-style-class,
simplifiable-range,
singleton-comparison,
superfluous-parens,
unidiomatic-typecheck,
unneeded-not,
wrong-assert-type,
simplifiable-if-statement,
no-classmethod-decorator,
no-staticmethod-decorator,
unnecessary-pass,
unnecessary-lambda,
useless-else-on-loop,
unnecessary-semicolon,
reimported,
global-variable-not-assigned,
global-at-module-level,
bare-except,
broad-except,
logging-not-lazy,
redundant-unittest-assert,
model-missing-unicode,
model-has-unicode,
model-no-explicit-unicode,
protected-access,
# Don't use things that are deprecated
deprecated-module,
deprecated-method,
# These help manage code complexity
too-many-nested-blocks,
too-many-statements,
too-many-boolean-expressions,
# Consistent import order makes finding where code is
# imported from easier
ungrouped-imports,
wrong-import-order,
wrong-import-position,
wildcard-import,
# These should be auto-fixed by any competent editor
missing-final-newline,
mixed-line-endings,
trailing-newlines,
trailing-whitespace,
unexpected-line-ending-format,
mixed-indentation,
# These attempt to limit pylint line-noise
bad-option-value,
unrecognized-inline-option,
useless-suppression,
bad-inline-option,
deprecated-pragma,
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
......@@ -36,28 +256,83 @@ load-plugins=edx_lint.pylint,pylint_django,pylint_celery
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=
# These should be left to the discretion of the reviewer
bad-continuation,
invalid-name,
misplaced-comparison-constant,
file-ignored,
bad-indentation,
lowercase-l-suffix,
unused-wildcard-import,
global-statement
# These are disabled by pylint by default
apply-builtin,
backtick,
basestring-builtin,
buffer-builtin,
cmp-builtin,
cmp-method,
coerce-builtin,
coerce-method,
delslice-method,
dict-iter-method,
dict-view-method,
duplicate-code,
execfile-builtin,
file-builtin,
filter-builtin-not-iterating,
fixme,
getslice-method,
hex-method,
import-star-module-level,
indexing-exception,
input-builtin,
intern-builtin,
locally-disabled,
locally-enabled,
too-few-public-methods,
bad-builtin,
star-args,
abstract-class-not-used,
abstract-class-little-used,
no-init,
fixme,
logging-format-interpolation,
too-many-lines,
long-builtin,
long-suffix,
map-builtin-not-iterating,
metaclass-assignment,
next-method-called,
no-absolute-import,
no-init,
no-self-use,
nonzero-method,
oct-method,
old-division,
old-ne-operator,
old-octal-literal,
old-raise-syntax,
parameter-unpacking,
print-statement,
raising-string,
range-builtin-not-iterating,
raw_input-builtin,
reduce-builtin,
reload-builtin,
round-builtin,
setslice-method,
standarderror-builtin,
suppressed-message,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-instance-attributes,
too-few-public-methods,
too-many-lines,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
too-many-branches,
too-many-arguments,
too-many-locals,
unused-wildcard-import,
duplicate-code
unichr-builtin,
unicode-builtin,
unpacking-in-except,
using-cmp-argument,
xrange-builtin,
zip-builtin-not-iterating,
[REPORTS]
......@@ -139,7 +414,7 @@ no-docstring-rgx=__.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Met
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
docstring-min-length=5
[FORMAT]
......
"""Pylint plugin: test classes derived from test classes."""
import astroid
from astroid.scoped_nodes import get_locals # not sure this is the right import
from pylint.checkers import BaseChecker, utils
from pylint.interfaces import IAstroidChecker
......@@ -24,7 +23,7 @@ def is_test_case_class(node):
if not node.is_subtype_of('unittest.case.TestCase'):
return False
dunder_test = get_locals(node).get("__test__")
dunder_test = node.locals.get("__test__")
if dunder_test:
if isinstance(dunder_test[0], astroid.AssName):
value = list(dunder_test[0].assigned_stmts())
......
......@@ -55,32 +55,291 @@
[MASTER]
ignore = ,input
persistent = yes
load-plugins = edx_lint.pylint,pylint_django,pylint_celery
load-plugins = edx_lint.pylint
[MESSAGES CONTROL]
enable =
blacklisted-name,
line-too-long,
syntax-error,
init-is-generator,
return-in-init,
function-redefined,
not-in-loop,
return-outside-function,
yield-outside-function,
return-arg-in-generator,
nonexistent-operator,
duplicate-argument-name,
abstract-class-instantiated,
bad-reversed-sequence,
continue-in-finally,
method-hidden,
access-member-before-definition,
no-method-argument,
no-self-argument,
invalid-slots-object,
assigning-non-slot,
invalid-slots,
inherit-non-class,
inconsistent-mro,
duplicate-bases,
non-iterator-returned,
unexpected-special-method-signature,
invalid-length-returned,
import-error,
used-before-assignment,
undefined-variable,
undefined-all-variable,
invalid-all-object,
no-name-in-module,
unbalance-tuple-unpacking,
unpacking-non-sequence,
bad-except-order,
raising-bad-type,
misplaced-bare-raise,
raising-non-exception,
nonimplemented-raised,
catching-non-exception,
slots-on-old-class,
super-on-old-class,
bad-super-call,
missing-super-argument,
no-member,
not-callable,
assignment-from-no-return,
no-value-for-parameter,
too-many-function-args,
unexpected-keyword-arg,
redundant-keyword-arg,
invalid-sequence-index,
invalid-slice-index,
assignment-from-none,
not-context-manager,
invalid-unary-operand-type,
unsupported-binary-operation,
repeated-keyword,
not-an-iterable,
not-a-mapping,
unsupported-membership-test,
unsubscriptable-object,
logging-unsupported-format,
logging-too-many-args,
logging-too-few-args,
bad-format-character,
truncated-format-string,
mixed-fomat-string,
format-needs-mapping,
missing-format-string-key,
too-many-format-args,
too-few-format-args,
bad-str-strip-call,
model-unicode-not-callable,
super-method-not-called,
non-parent-method-called,
test-inherits-tests,
translation-of-non-string,
redefined-variable-type,
cyclical-import,
unreachable,
dangerous-default-value,
pointless-statement,
pointless-string-statement,
expression-not-assigned,
duplicate-key,
confusing-with-statement,
using-constant-test,
lost-exception,
assert-on-tuple,
attribute-defined-outside-init,
bad-staticmethod-argument,
arguments-differ,
signature-differs,
abstract-method,
super-init-not-called,
relative-import,
import-self,
misplaced-future,
invalid-encoded-data,
global-variable-undefined,
redefined-outer-name,
redefined-builtin,
redefined-in-handler,
undefined-loop-variable,
cell-var-from-loop,
duplicate-except,
nonstandard-exception,
binary-op-exception,
property-on-old-class,
bad-format-string-key,
unused-format-string-key,
bad-format-string,
missing-format-argument-key,
unused-format-string-argument,
format-combined-specification,
missing-format-attribute,
invalid-format-index,
anomalous-backslash-in-string,
anomalous-unicode-escape-in-string,
bad-open-mode,
boolean-datetime,
fatal,
astroid-error,
parse-error,
method-check-failed,
django-not-available,
raw-checker-failed,
django-not-available-placeholder,
empty-docstring,
invalid-characters-in-docstring,
missing-docstring,
wrong-spelling-in-comment,
wrong-spelling-in-docstring,
unused-import,
unused-variable,
unused-argument,
exec-used,
eval-used,
bad-classmethod-argument,
bad-mcs-classmethod-argument,
bad-mcs-method-argument,
bad-whitespace,
consider-iterating-dictionary,
consider-using-enumerate,
literal-used-as-attribute,
multiple-imports,
multiple-statements,
old-style-class,
simplifiable-range,
singleton-comparison,
superfluous-parens,
unidiomatic-typecheck,
unneeded-not,
wrong-assert-type,
simplifiable-if-statement,
no-classmethod-decorator,
no-staticmethod-decorator,
unnecessary-pass,
unnecessary-lambda,
useless-else-on-loop,
unnecessary-semicolon,
reimported,
global-variable-not-assigned,
global-at-module-level,
bare-except,
broad-except,
logging-not-lazy,
redundant-unittest-assert,
model-missing-unicode,
model-has-unicode,
model-no-explicit-unicode,
protected-access,
deprecated-module,
deprecated-method,
too-many-nested-blocks,
too-many-statements,
too-many-boolean-expressions,
ungrouped-imports,
wrong-import-order,
wrong-import-position,
wildcard-import,
missing-final-newline,
mixed-line-endings,
trailing-newlines,
trailing-whitespace,
unexpected-line-ending-format,
mixed-indentation,
bad-option-value,
unrecognized-inline-option,
useless-suppression,
bad-inline-option,
deprecated-pragma,
disable =
bad-continuation,
invalid-name,
misplaced-comparison-constant,
file-ignored,
bad-indentation,
lowercase-l-suffix,
unused-wildcard-import,
global-statement
apply-builtin,
backtick,
basestring-builtin,
buffer-builtin,
cmp-builtin,
cmp-method,
coerce-builtin,
coerce-method,
delslice-method,
dict-iter-method,
dict-view-method,
duplicate-code,
execfile-builtin,
file-builtin,
filter-builtin-not-iterating,
fixme,
getslice-method,
hex-method,
import-star-module-level,
indexing-exception,
input-builtin,
intern-builtin,
locally-disabled,
locally-enabled,
too-few-public-methods,
bad-builtin,
star-args,
abstract-class-not-used,
abstract-class-little-used,
no-init,
fixme,
logging-format-interpolation,
too-many-lines,
long-builtin,
long-suffix,
map-builtin-not-iterating,
metaclass-assignment,
next-method-called,
no-absolute-import,
no-init,
no-self-use,
nonzero-method,
oct-method,
old-division,
old-ne-operator,
old-octal-literal,
old-raise-syntax,
parameter-unpacking,
print-statement,
raising-string,
range-builtin-not-iterating,
raw_input-builtin,
reduce-builtin,
reload-builtin,
round-builtin,
setslice-method,
standarderror-builtin,
suppressed-message,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-instance-attributes,
too-few-public-methods,
too-many-lines,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
too-many-branches,
too-many-arguments,
too-many-locals,
unused-wildcard-import,
duplicate-code
unichr-builtin,
unicode-builtin,
unpacking-in-except,
using-cmp-argument,
xrange-builtin,
zip-builtin-not-iterating,
[REPORTS]
output-format = text
......@@ -103,7 +362,7 @@ inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$
good-names = f,i,j,k,db,ex,Run,_,__
bad-names = foo,bar,baz,toto,tutu,tata
no-docstring-rgx = __.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Meta$
docstring-min-length = -1
docstring-min-length = 5
[FORMAT]
max-line-length = 120
......@@ -180,4 +439,4 @@ int-import-graph =
[EXCEPTIONS]
overgeneral-exceptions = Exception
# 6dce538dd71d2e031194c436e3c0a4f25fa4b139
# 6eb8ad6e6d8cfc8d0aaa08b832cbd7b957662448
# Local tweaks to pylintrc for edx_lint's own code.
[MASTER]
ignore+ = ,input
load-plugins = edx_lint.pylint
git+https://github.com/cpennington/astroid.git@edx#egg=astroid
git+https://github.com/PyCQA/pylint.git@master#egg=pylint
pylint-django>=0.7.2,<1.0.0
pylint-celery==0.3
six>=1.10.0,<2.0.0
click>=6.0
click-log==0.1.8
[metadata]
name = edx-lint
description = edX-authored pylint checkers
url = https://github.com/edx/edx-lint
author = edX
author_email = oscm@edx.org
license = Apache
classifiers =
Environment :: Console
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Operating System :: OS Independent
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3.5
Topic :: Software Development :: Quality Assurance
[files]
packages =
edx_lint
edx_lint.cmd
edx_lint.pylint
package-data =
edx_lint =
files/*
[entry_points]
console_scripts =
edx_lint = edx_lint.cmd.main:main
lint-amnesty = edx_lint.cmd.amnesty:pylint_amnesty
......@@ -14,46 +14,6 @@ A collection of code quality tools:
from setuptools import setup
setup(
name='edx-lint',
version='0.5.3',
description='edX-authored pylint checkers',
url='https://github.com/edx/edx-lint',
author='edX',
author_email='oscm@edx.org',
license='Apache',
classifiers=[
"Environment :: Console",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.5",
"Topic :: Software Development :: Quality Assurance",
],
packages=[
'edx_lint',
'edx_lint.cmd',
'edx_lint.pylint',
],
package_data={
'edx_lint': [
'files/*',
],
},
entry_points={
'console_scripts': [
'edx_lint = edx_lint.cmd.main:main',
],
},
install_requires=[
'pylint==1.6.5',
'pylint-django>=0.7.2,<1.0.0',
'pylint-celery==0.3',
'six>=1.10.0,<2.0.0',
],
setup_requires=['pbr>=1.9', 'setuptools>=17.1'],
pbr=True,
)
......@@ -6,7 +6,7 @@ deps =
coverage
commands =
coverage run -p -m unittest discover -b
coverage run -p -m unittest {posargs:discover -b}
[testenv:coverage]
commands =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment