run pylint and pep8 quality tasks

.pep8

 [pep8]
 max-line-length = 120
-exclude = dependencies, watchdog.py, veda_deliver_xuetang.py, scripts
+exclude = dependencies, watchdog.py, veda_deliver_xuetang.py, scripts, settings, migrations

Makefile

 PACKAGES = VEDA VEDA_OS01 control frontend youtube_callback scripts
+PEP8_THRESHOLD=50
+PYLINT_THRESHOLD=855
 
 production-requirements:
 	pip install -r requirements.txt
@@ -12,7 +14,7 @@ migrate:
 static:
 	python manage.py collectstatic --noinput
 
-validate: test ## Run tests and quality checks
+validate: test quality
 
 test: clean
 	coverage run -m pytest --durations=10
@@ -22,6 +24,10 @@ test: clean
 clean:
 	coverage erase
 
-quality:
-	pep8 --config=.pep8 $(PACKAGES) *.py
-	pylint --rcfile=pylintrc $(PACKAGES) *.py
+quality: quality_pep8 quality_pylint
+
+quality_pep8:
+	paver run_pep8 -l ${PEP8_THRESHOLD}
+
+quality_pylint:
+	paver run_pylint -l ${PYLINT_THRESHOLD}

VEDA/tests/test_utils.py

@@ -64,7 +64,7 @@ class UtilTests(TestCase):
             'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
         },
         {
-            'urls': ('http://api.cielo24','add/job'),
+            'urls': ('http://api.cielo24', 'add/job'),
             'params': {'p': 100},
             'expected_url': 'http://api.cielo24/add/job?p=100'
         },

control/tests/test_file_ingest.py

@@ -7,7 +7,7 @@ import unittest
 from django.test import TestCase
 import requests
 
-from veda_file_ingest import VedaIngest, VideoProto
+from control.veda_file_ingest import VedaIngest, VideoProto
 
 """
 This is an API connection test

control/tests/test_reporting.py

@@ -12,7 +12,7 @@ A basic unittest for the "Course Addition Tool"
 sys.path.append(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
-from veda_utils import Report
+from control.veda_utils import Report
 
 
 class TestReporting(TestCase):

control/tests/test_val.py

@@ -10,8 +10,8 @@ import requests
 import responses
 
 from control.veda_val import VALAPICall
-from veda_file_ingest import VideoProto
 from VEDA import utils
+from control.veda_file_ingest import VideoProto
 
 
 requests.packages.urllib3.disable_warnings()

control/veda_deliver.py

@@ -33,7 +33,6 @@ except ImportError:
 LOGGER = logging.getLogger(__name__)
 
 
-
 try:
     boto.config.add_section('Boto')
 except:

control/veda_utils.py

@@ -15,6 +15,7 @@ from control.control_env import *
 from control.veda_encode import VedaEncode
 from VEDA.utils import get_config
 
+
 class EmailAlert():
 
     def __init__(self, **kwargs):

frontend/tests/test_abvid_reporting.py

@@ -15,6 +15,7 @@ sys.path.append(
 import abvid_reporting
 from frontend.course_validate import VEDACat
 
+
 class TestVariables(TestCase):
 
     def setUp(self):

pavelib/__init__.py

+"""
+paver commands
+"""
+from . import quality

pavelib/quality.py

+"""
+Check code quality
+"""
+import json
+import os
+import re
+from string import join
+from paver.easy import BuildFailure, call_task, cmdopts, needs, sh, task
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+REPORTS_DIR = os.path.join(ROOT_DIR, 'reports')
+
+PACKAGES = [
+    'VEDA',
+    'VEDA_OS01',
+    'control',
+    'frontend',
+    'youtube_callback',
+    'scripts',
+    'bin',
+]
+
+
+@task
+@cmdopts([
+    ("limit=", "l", "limit for number of acceptable violations"),
+])
+def run_pep8(options):  # pylint: disable=unused-argument
+    """
+    Run pep8 on system code.
+    Fail the task if any violations are found.
+    """
+    violations_limit = int(getattr(options, 'limit', -1))
+
+    sh('pep8 . | tee {report_dir}/pep8.report'.format(report_dir=REPORTS_DIR))
+
+    num_violations, __ = _count_pep8_violations(
+        '{report_dir}/pep8.report'.format(report_dir=REPORTS_DIR)
+    )
+
+    violations_message = '{violations_base_message}{violations_limit_message}'.format(
+        violations_base_message='Too many pep8 violations. Number of pep8 violations: {}. '.format(num_violations),
+        violations_limit_message='The limit is {violations_limit}. '.format(violations_limit=violations_limit),
+    )
+    print violations_message
+
+    # Fail if number of violations is greater than the limit
+    if num_violations > violations_limit > -1:
+        raise BuildFailure(violations_message)
+
+
+def _count_pep8_violations(report_file):
+    """
+    Returns a tuple of (num_violations, violations_list) for all
+    pep8 violations in the given report_file.
+    """
+    with open(report_file) as f:
+        violations_list = f.readlines()
+
+    num_lines = len(violations_list)
+    return num_lines, violations_list
+
+
+@task
+@cmdopts([
+    ('errors', 'e', 'Check for errors only'),
+    ('limit=', 'l', 'limit for number of acceptable violations'),
+])
+def run_pylint(options):
+    """
+    Run pylint on system code. When violations limit is passed in,
+    fail the task if too many violations are found.
+    """
+    num_violations = 0
+    violations_limit = int(getattr(options, 'limit', -1))
+    errors = getattr(options, 'errors', False)
+
+    flags = []
+    if errors:
+        flags.append('--errors-only')
+
+    sh(
+        'PYTHONPATH={python_path} pylint {packages} {flags} --msg-template={msg_template} | '
+        'tee {report_dir}/pylint.report'.format(
+            python_path=ROOT_DIR,
+            packages=' '.join(PACKAGES),
+            flags=' '.join(flags),
+            msg_template='"{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"',
+            report_dir=REPORTS_DIR
+        )
+    )
+
+    num_violations = _count_pylint_violations(
+        '{report_dir}/pylint.report'.format(report_dir=REPORTS_DIR)
+    )
+
+    violations_message = '{violations_base_message}{violations_limit_message}'.format(
+        violations_base_message='Too many pylint violations. Number of pylint violations: {}. '.format(num_violations),
+        violations_limit_message='The limit is {violations_limit}.'.format(violations_limit=violations_limit)
+    )
+    print violations_message
+
+    # Fail if number of violations is greater than the limit
+    if num_violations > violations_limit > -1:
+        raise BuildFailure(violations_message)
+
+
+def _count_pylint_violations(report_file):
+    """
+    Parses a pylint report line-by-line and determines the number of violations reported
+    """
+    num_violations_report = 0
+    # An example string:
+    # scripts/reencode_crawler.py:57: [C0303(trailing-whitespace), ] Trailing whitespace
+    pylint_pattern = re.compile(r'.(\d+):\ \[(\D\d+.+\]).')
+
+    for line in open(report_file):
+        violation_list_for_line = pylint_pattern.split(line)
+        # If the string is parsed into four parts, then we've found a violation. Example of split parts:
+        # test file, line number, violation name, violation details
+        if len(violation_list_for_line) == 4:
+            num_violations_report += 1
+
+    return num_violations_report

pavement.py

+import os
+import sys
+
+# Ensure that we can import pavelib
+sys.path.insert(0, os.path.dirname(__file__))
+
+from pavelib import *

pylintrc

+# ***************************
+# ** DO NOT EDIT THIS FILE **
+# ***************************
+#
+# This file was generated by edx-lint: http://github.com/edx/edx-lint
+#
+# If you want to change this file, you have two choices, depending on whether
+# you want to make a local change that applies only to this repo, or whether
+# you want to make a central change that applies to all repos using edx-lint.
+#
+# LOCAL CHANGE:
+#
+# 1. Edit the local pylintrc_tweaks file to add changes just to this
+#    repo's file.
+#
+# 2. Run:
+#
+#       $ edx_lint write pylintrc
+#
+# 3. This will modify the local file.  Submit a pull request to get it
+#    checked in so that others will benefit.
+#
+#
+# CENTRAL CHANGE:
+#
+# 1. Edit the pylintrc file in the edx-lint repo at
+#    https://github.com/edx/edx-lint/blob/master/edx_lint/files/pylintrc
+#
+# 2. Make a new version of edx_lint, which involves the usual steps of
+#    incrementing the version number, submitting and reviewing a pull
+#    request, and updating the edx-lint version reference in this repo.
+#
+# 3. Install the newer version of edx-lint.
+#
+# 4. Run:
+#
+#       $ edx_lint write pylintrc
+#
+# 5. This will modify the local file.  Submit a pull request to get it
+#    checked in so that others will benefit.
+#
+#
+#
+#
+#
+# STAY AWAY FROM THIS FILE!
+#
+#
+#
+#
+#
+# SERIOUSLY.
+#
+# ------------------------------
+[MASTER]
+ignore = 
+persistent = yes
+load-plugins = edx_lint.pylint,pylint_django,pylint_celery
+
+[MESSAGES CONTROL]
+enable = 
+	# These are controlled by explicit choices in the pylintrc files
+	blacklisted-name,
+	line-too-long,
+	# These affect the correctness of the code
+	syntax-error,
+	init-is-generator,
+	return-in-init,
+	function-redefined,
+	not-in-loop,
+	return-outside-function,
+	yield-outside-function,
+	return-arg-in-generator,
+	nonexistent-operator,
+	duplicate-argument-name,
+	abstract-class-instantiated,
+	bad-reversed-sequence,
+	continue-in-finally,
+	method-hidden,
+	access-member-before-definition,
+	no-method-argument,
+	no-self-argument,
+	invalid-slots-object,
+	assigning-non-slot,
+	invalid-slots,
+	inherit-non-class,
+	inconsistent-mro,
+	duplicate-bases,
+	non-iterator-returned,
+	unexpected-special-method-signature,
+	invalid-length-returned,
+	import-error,
+	used-before-assignment,
+	undefined-variable,
+	undefined-all-variable,
+	invalid-all-object,
+	no-name-in-module,
+	unbalance-tuple-unpacking,
+	unpacking-non-sequence,
+	bad-except-order,
+	raising-bad-type,
+	misplaced-bare-raise,
+	raising-non-exception,
+	nonimplemented-raised,
+	catching-non-exception,
+	slots-on-old-class,
+	super-on-old-class,
+	bad-super-call,
+	missing-super-argument,
+	no-member,
+	not-callable,
+	assignment-from-no-return,
+	no-value-for-parameter,
+	too-many-function-args,
+	unexpected-keyword-arg,
+	redundant-keyword-arg,
+	invalid-sequence-index,
+	invalid-slice-index,
+	assignment-from-none,
+	not-context-manager,
+	invalid-unary-operand-type,
+	unsupported-binary-operation,
+	repeated-keyword,
+	not-an-iterable,
+	not-a-mapping,
+	unsupported-membership-test,
+	unsubscriptable-object,
+	logging-unsupported-format,
+	logging-too-many-args,
+	logging-too-few-args,
+	bad-format-character,
+	truncated-format-string,
+	mixed-fomat-string,
+	format-needs-mapping,
+	missing-format-string-key,
+	too-many-format-args,
+	too-few-format-args,
+	bad-str-strip-call,
+	model-unicode-not-callable,
+	super-method-not-called,
+	non-parent-method-called,
+	test-inherits-tests,
+	translation-of-non-string,
+	redefined-variable-type,
+	cyclical-import,
+	unreachable,
+	dangerous-default-value,
+	pointless-statement,
+	pointless-string-statement,
+	expression-not-assigned,
+	duplicate-key,
+	confusing-with-statement,
+	using-constant-test,
+	lost-exception,
+	assert-on-tuple,
+	attribute-defined-outside-init,
+	bad-staticmethod-argument,
+	arguments-differ,
+	signature-differs,
+	abstract-method,
+	super-init-not-called,
+	relative-import,
+	import-self,
+	misplaced-future,
+	invalid-encoded-data,
+	global-variable-undefined,
+	redefined-outer-name,
+	redefined-builtin,
+	redefined-in-handler,
+	undefined-loop-variable,
+	cell-var-from-loop,
+	duplicate-except,
+	nonstandard-exception,
+	binary-op-exception,
+	property-on-old-class,
+	bad-format-string-key,
+	unused-format-string-key,
+	bad-format-string,
+	missing-format-argument-key,
+	unused-format-string-argument,
+	format-combined-specification,
+	missing-format-attribute,
+	invalid-format-index,
+	anomalous-backslash-in-string,
+	anomalous-unicode-escape-in-string,
+	bad-open-mode,
+	boolean-datetime,
+	# Checking failed for some reason
+	fatal,
+	astroid-error,
+	parse-error,
+	method-check-failed,
+	django-not-available,
+	raw-checker-failed,
+	django-not-available-placeholder,
+	# Documentation is important
+	empty-docstring,
+	invalid-characters-in-docstring,
+	missing-docstring,
+	wrong-spelling-in-comment,
+	wrong-spelling-in-docstring,
+	# Unused code should be deleted
+	unused-import,
+	unused-variable,
+	unused-argument,
+	# These are dangerous!
+	exec-used,
+	eval-used,
+	# These represent idiomatic python. Not adhering to them
+	# will raise red flags with future readers.
+	bad-classmethod-argument,
+	bad-mcs-classmethod-argument,
+	bad-mcs-method-argument,
+	bad-whitespace,
+	consider-iterating-dictionary,
+	consider-using-enumerate,
+	literal-used-as-attribute,
+	multiple-imports,
+	multiple-statements,
+	old-style-class,
+	simplifiable-range,
+	singleton-comparison,
+	superfluous-parens,
+	unidiomatic-typecheck,
+	unneeded-not,
+	wrong-assert-type,
+	simplifiable-if-statement,
+	no-classmethod-decorator,
+	no-staticmethod-decorator,
+	unnecessary-pass,
+	unnecessary-lambda,
+	useless-else-on-loop,
+	unnecessary-semicolon,
+	reimported,
+	global-variable-not-assigned,
+	global-at-module-level,
+	bare-except,
+	broad-except,
+	logging-not-lazy,
+	redundant-unittest-assert,
+	model-missing-unicode,
+	model-has-unicode,
+	model-no-explicit-unicode,
+	protected-access,
+	# Don't use things that are deprecated
+	deprecated-module,
+	deprecated-method,
+	# These help manage code complexity
+	too-many-nested-blocks,
+	too-many-statements,
+	too-many-boolean-expressions,
+	# Consistent import order makes finding where code is
+	# imported from easier
+	ungrouped-imports,
+	wrong-import-order,
+	wrong-import-position,
+	wildcard-import,
+	# These should be auto-fixed by any competent editor
+	missing-final-newline,
+	mixed-line-endings,
+	trailing-newlines,
+	trailing-whitespace,
+	unexpected-line-ending-format,
+	mixed-indentation,
+	# These attempt to limit pylint line-noise
+	bad-option-value,
+	unrecognized-inline-option,
+	useless-suppression,
+	bad-inline-option,
+	deprecated-pragma,
+disable = 
+	# These should be left to the discretion of the reviewer
+	bad-continuation,
+	invalid-name,
+	misplaced-comparison-constant,
+	file-ignored,
+	bad-indentation,
+	lowercase-l-suffix,
+	unused-wildcard-import,
+	global-statement,
+	no-else-return,
+	# These are disabled by pylint by default
+	apply-builtin,
+	backtick,
+	basestring-builtin,
+	buffer-builtin,
+	cmp-builtin,
+	cmp-method,
+	coerce-builtin,
+	coerce-method,
+	delslice-method,
+	dict-iter-method,
+	dict-view-method,
+	duplicate-code,
+	execfile-builtin,
+	file-builtin,
+	filter-builtin-not-iterating,
+	fixme,
+	getslice-method,
+	hex-method,
+	import-star-module-level,
+	indexing-exception,
+	input-builtin,
+	intern-builtin,
+	locally-disabled,
+	locally-enabled,
+	logging-format-interpolation,
+	long-builtin,
+	long-suffix,
+	map-builtin-not-iterating,
+	metaclass-assignment,
+	next-method-called,
+	no-absolute-import,
+	no-init,
+	no-self-use,
+	nonzero-method,
+	oct-method,
+	old-division,
+	old-ne-operator,
+	old-octal-literal,
+	old-raise-syntax,
+	parameter-unpacking,
+	print-statement,
+	raising-string,
+	range-builtin-not-iterating,
+	raw_input-builtin,
+	reduce-builtin,
+	reload-builtin,
+	round-builtin,
+	setslice-method,
+	standarderror-builtin,
+	suppressed-message,
+	too-few-public-methods,
+	too-many-ancestors,
+	too-many-arguments,
+	too-many-branches,
+	too-many-instance-attributes,
+	too-many-lines,
+	too-many-locals,
+	too-many-public-methods,
+	too-many-return-statements,
+	unichr-builtin,
+	unicode-builtin,
+	unpacking-in-except,
+	using-cmp-argument,
+	xrange-builtin,
+	zip-builtin-not-iterating,
+
+[REPORTS]
+output-format = text
+files-output = no
+reports = no
+evaluation = 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+[BASIC]
+bad-functions = map,filter,apply,input
+module-rgx = (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+const-rgx = (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$
+class-rgx = [A-Z_][a-zA-Z0-9]+$
+function-rgx = ([a-z_][a-z0-9_]{2,40}|test_[a-z0-9_]+)$
+method-rgx = ([a-z_][a-z0-9_]{2,40}|setUp|set[Uu]pClass|tearDown|tear[Dd]ownClass|assert[A-Z]\w*|maxDiff|test_[a-z0-9_]+)$
+attr-rgx = [a-z_][a-z0-9_]{2,30}$
+argument-rgx = [a-z_][a-z0-9_]{2,30}$
+variable-rgx = [a-z_][a-z0-9_]{2,30}$
+class-attribute-rgx = ([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$
+good-names = f,i,j,k,db,ex,Run,_,__
+bad-names = foo,bar,baz,toto,tutu,tata
+no-docstring-rgx = __.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Meta$
+docstring-min-length = 5
+
+[FORMAT]
+max-line-length = 120
+ignore-long-lines = ^\s*(# )?<?https?://\S+>?$
+single-line-if-stmt = no
+no-space-check = trailing-comma,dict-separator
+max-module-lines = 1000
+indent-string = '    '
+
+[MISCELLANEOUS]
+notes = FIXME,XXX,TODO
+
+[SIMILARITIES]
+min-similarity-lines = 4
+ignore-comments = yes
+ignore-docstrings = yes
+ignore-imports = no
+
+[TYPECHECK]
+ignore-mixin-members = yes
+ignored-classes = SQLObject
+unsafe-load-any-extension = yes
+generated-members = 
+	REQUEST,
+	acl_users,
+	aq_parent,
+	objects,
+	DoesNotExist,
+	can_read,
+	can_write,
+	get_url,
+	size,
+	content,
+	status_code,
+	create,
+	build,
+	fields,
+	tag,
+	org,
+	course,
+	category,
+	name,
+	revision,
+	_meta,
+
+[VARIABLES]
+init-import = no
+dummy-variables-rgx = _|dummy|unused|.*_unused
+additional-builtins = 
+
+[CLASSES]
+defining-attr-methods = __init__,__new__,setUp
+valid-classmethod-first-arg = cls
+valid-metaclass-classmethod-first-arg = mcs
+
+[DESIGN]
+max-args = 5
+ignored-argument-names = _.*
+max-locals = 15
+max-returns = 6
+max-branches = 12
+max-statements = 50
+max-parents = 7
+max-attributes = 7
+min-public-methods = 2
+max-public-methods = 20
+
+[IMPORTS]
+deprecated-modules = regsub,TERMIOS,Bastion,rexec
+import-graph = 
+ext-import-graph = 
+int-import-graph = 
+
+[EXCEPTIONS]
+overgeneral-exceptions = Exception
+
+# 2303568fe30e9fdd336b485a1a52e3b0aa0449b7

test_requirements.txt

@@ -8,4 +8,9 @@ moto==1.0.1
 responses==0.6.1
 pytest==3.0.6
 pytest-django==3.1.2
-pytest-django-ordering==1.0.1
\ No newline at end of file
+pytest-django-ordering==1.0.1
+Paver==1.2.4
+pylint==1.7.1
+pylint-celery==0.3
+pylint-django==0.7.2
+edx-lint==0.5.4