Codecov, travis, quality improvements

This adds codecov, quality improvements, bug fixes, import fixes.

.coveragerc

+# .coveragerc for edx-video-pipeline
+[run]
+data_file = reports/.coverage
+source =
+    VEDA_OS01
+    control
+    frontend
+    youtube_callback
+    scripts
+
+omit =
+    templates/*
+    frontend/tests/*
+    dependencies/*
+    control/*
+    VEDA/tests/*
+
+concurrency=multiprocessing
+
+[report]
+ignore_errors = True
+
+exclude_lines =
+   pragma: no cover
+   raise NotImplementedError
+
+[html]
+title = edx-video-worker Python Test Coverage Report
+directory = reports/cover
+
+[xml]
+output = reports/coverage.xml

.gitignore

@@ -6,3 +6,7 @@
 static/admin/
 
 sandbox.db
+
+.coverage
+
+coverage/

.pep8

 [pep8]
 max-line-length = 120
-ignore = E402
 exclude = dependencies, watchdog.py, veda_deliver_xuetang.py, scripts

.travis.yml

 language: python
+
 python:
   - "2.7"
+
 sudo: required
 
-install: pip install -r requirements.txt
-install: pip install pep8
+install:
+  - pip install -r requirements.txt
 
 # build tests
 script:
- - python VEDA/tests/test_build.py
- - pep8
- - cd control/tests nosetests
+    - make validate
 
+after_success:
+  - pip install -U codecov
+  - codecov

Makefile

+PACKAGES = VEDA VEDA_OS01 control frontend youtube_callback scripts
+
+validate: test ## Run tests and quality checks
+
+test: clean
+	 nosetests --with-coverage --cover-inclusive --cover-branches \
+		--cover-html --cover-html-dir=build/coverage/html/ \
+		--cover-xml --cover-xml-file=build/coverage/coverage.xml --verbosity=2 \
+		$(foreach package,$(PACKAGES),--cover-package=$(package)) \
+		$(PACKAGES)
+
+clean:
+	coverage erase
+
+quality:
+	pep8 --config=.pep8 $(PACKAGES) *.py
+	pylint --rcfile=pylintrc $(PACKAGES) *.py

VEDA/django_secrets.py

@@ -19,7 +19,7 @@ with open(read_yaml, 'r') as stream:
         return_dict = None
 
 
-DJANGO_SECRET_KEY = return_dict['django_secret_key']
+DJANGO_SECRET_KEY = return_dict['django_secret_key'] or 'test_secret_key'
 DJANGO_ADMIN = ('', '')
 DEBUG = True
 DATABASES = return_dict['DATABASES']

VEDA/urls.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 sys.path.append(os.path.abspath(__file__))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
 
 from django.conf import settings
 from rest_framework import routers

VEDA_OS01/admin.py

 from django.contrib import admin
-from VEDA_OS01.models import Course
-from VEDA_OS01.models import Video
-from VEDA_OS01.models import Encode
-from VEDA_OS01.models import URL
-from VEDA_OS01.models import Destination
-from VEDA_OS01.models import Institution
-from VEDA_OS01.models import VedaUpload
+from VEDA_OS01.models import Course, Video, Encode, URL, Destination, Institution, VedaUpload
 
 
 class CourseAdmin(admin.ModelAdmin):

VEDA_OS01/api.py

@@ -9,13 +9,12 @@ will get the token id from a url view, auth it, then push forward with a success
 
 import os
 import sys
-import oauth2_provider
 
-from oauth2_provider import models
+from oauth2_provider.models import AccessToken
+
 from rest_framework.authtoken.models import Token
 from django.contrib.auth.models import User
 
-from pipeline_env import *
 
 primary_directory = os.path.dirname(__file__)
 sys.path.append(primary_directory)
@@ -23,7 +22,7 @@ sys.path.append(primary_directory)
 
 def token_finisher(token_id):
     try:
-        d = oauth2_provider.models.AccessToken.objects.get(token=token_id)
+        d = AccessToken.objects.get(token=token_id)
     except:
         return False
 
@@ -34,7 +33,3 @@ def token_finisher(token_id):
     except:
         token = Token.objects.get(user=d.user)
     return token.key
-
-
-if __name__ == '__main__':
-    pass

VEDA_OS01/models.py

-
+"""
+Models for Video Pipeline
+"""
 import uuid
 from django.db import models
 
@@ -12,7 +14,7 @@ class Institution (models.Model):
     institution_name = models.CharField(max_length=50)
 
     def __unicode__(self):
-        return '%s %s' % (self.institution_name, self.institution_code) or u''
+        return u'%s %s'.format(self.institution_name, self.institution_code) or u''
 
 
 class Course (models.Model):
@@ -38,9 +40,7 @@ class Course (models.Model):
     review_proc = models.BooleanField('Producer Review', default=False)
     last_vid_number = models.IntegerField('Last Video ID', default=0)
 
-    """
-    Youtube
-    """
+    # Youtube
     yt_proc = models.BooleanField('Process for Youtube', default=True)
     yt_logon = models.CharField(
         'Youtube SFTP U/N',
@@ -53,9 +53,7 @@ class Course (models.Model):
         null=True, blank=True
     )
 
-    """
-    3Play Media (Transcription)
-    """
+    # 3Play Media (Transcription)
     tp_proc = models.BooleanField('Process for 3Play', default=False)
     tp_username = models.CharField(
         '3Play Username',
@@ -91,9 +89,7 @@ class Course (models.Model):
         null=True, blank=True
     )
 
-    """
-    Cielo24
-    """
+    # Cielo24
     c24_proc = models.BooleanField('Process for Cielo24', default=False)
     c24_username = models.CharField(
         'Cielo24 Username',
@@ -177,7 +173,7 @@ class Course (models.Model):
     )
 
     def __unicode__(self):
-        return '%s %s %s' % (
+        return u'%s %s %s'.format(
             self.institution,
             self.edx_classid,
             self.course_name
@@ -199,9 +195,7 @@ class Video (models.Model):
         max_length=100,
         null=True, blank=True
     )
-    """
-    Master File Properties
-    """
+    # Master File Properties
     video_orig_filesize = models.BigIntegerField(
         'Master Filesize',
         null=True, blank=True
@@ -226,9 +220,7 @@ class Video (models.Model):
         max_length=50,
         null=True, blank=True
     )
-    """
-    Status
-    """
+    # Status
     video_trans_start = models.DateTimeField('Process Start', null=True, blank=True)
     video_trans_end = models.DateTimeField('Process Complete', null=True, blank=True)
 
@@ -298,7 +290,7 @@ class Video (models.Model):
         get_latest_by = 'video_trans_start'
 
     def __unicode__(self):
-        return '%s' % (self.edx_id) or u''
+        return u'%s'.format(self.edx_id) or u''
 
 
 class Destination (models.Model):
@@ -307,7 +299,7 @@ class Destination (models.Model):
     destination_nick = models.CharField('Nickname (3 Char.)', max_length=3, null=True, blank=True)
 
     def __unicode__(self):
-        return '%s' % (self.destination_name) or u''
+        return u'%s'.format(self.destination_name) or u''
 
 
 class Encode (models.Model):
@@ -355,7 +347,7 @@ class Encode (models.Model):
     xuetang_proc = models.BooleanField('Submit to XuetangX', default=False)
 
     def __unicode__(self):
-        return '%s' % (self.encode_name)
+        return u'%s'.format(self.encode_name)
 
 
 class URL (models.Model):
@@ -378,7 +370,7 @@ class URL (models.Model):
         get_latest_by = 'url_date'
 
     def __unicode__(self):
-        return '%s : %s : %s' % (self.videoID, self.encode_profile.encode_name, self.url_date) or u''
+        return u'%s : %s : %s'.format(self.videoID, self.encode_profile.encode_name, self.url_date) or u''
 
 
 class VedaUpload (models.Model):
@@ -430,7 +422,7 @@ class VedaUpload (models.Model):
         get_latest_by = 'upload_date'
 
     def __unicode__(self):
-        return '%s %s %s %s' % (
+        return u'%s %s %s %s'.format(
             self.client_information,
             self.upload_filename,
             self.status_email,

VEDA_OS01/views.py

@@ -115,7 +115,3 @@ def user_login(request):
         return HttpResponseRedirect(request.path)
     else:
         return HttpResponseRedirect('../admin')  # settings.LOGIN_REDIRECT_URL)
-
-
-if __name__ == "__main__":
-    pass

codecov.yml

+comment: off
+coverage:
+  status:
+    patch:
+      default:
+        target: 85
+    project:
+      default:
+        target: 85

control/celeryapp.py

@@ -29,10 +29,17 @@ with open(auth_yaml, 'r') as stream:
     except yaml.YAMLError as exc:
         auth_dict = None
 
-CEL_BROKER = 'amqp://' + auth_dict['rabbitmq_user'] + ':' + auth_dict['rabbitmq_pass'] + '@' \
-             + auth_dict['rabbitmq_broker'] + ':5672//'
-CEL_BACKEND = 'amqp://' + auth_dict['rabbitmq_user'] + ':' + auth_dict['rabbitmq_pass'] + \
-              '@' + auth_dict['rabbitmq_broker'] + ':5672//'
+CEL_BROKER = 'amqp://{rabbitmq_user}:{rabbitmq_pass}@{rabbitmq_broker}:5672//'.format(
+    rabbitmq_user=auth_dict['rabbitmq_user'],
+    rabbitmq_pass=auth_dict['rabbitmq_pass'],
+    rabbitmq_broker=auth_dict['rabbitmq_broker']
+)
+
+CEL_BACKEND = 'amqp://{rabbitmq_user}:{rabbitmq_pass}@{rabbitmq_broker}:5672//'.format(
+    rabbitmq_user=auth_dict['rabbitmq_user'],
+    rabbitmq_pass=auth_dict['rabbitmq_pass'],
+    rabbitmq_broker=auth_dict['rabbitmq_broker']
+)
 
 app = Celery(auth_dict['celery_app_name'], broker=CEL_BROKER, backend=CEL_BACKEND, include=[])
 

control/tests/test_encode.py

@@ -11,8 +11,9 @@ Test encode profiler
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
-from veda_env import *
-from veda_encode import VedaEncode
+
+from control.veda_encode import VedaEncode
+from VEDA_OS01.models import Course, URL, Video, Encode
 
 
 class TestEncode(unittest.TestCase):

control/tests/test_file_discovery.py

@@ -9,7 +9,7 @@ Test VEDA API
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
-from veda_file_discovery import FileDiscovery
+from control.veda_file_discovery import FileDiscovery
 
 
 class TestValidation(unittest.TestCase):

control/tests/test_heal.py

@@ -10,8 +10,8 @@ Test heal processor
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
-from veda_env import *
-from veda_heal import VedaHeal
+from control.veda_heal import VedaHeal
+from VEDA_OS01.models import URL, Video, Encode
 
 
 class TestEncode(unittest.TestCase):

control/tests/test_hotstore.py

@@ -11,9 +11,8 @@ sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
 
-from veda_hotstore import Hotstore
+from control.veda_hotstore import Hotstore
 from veda_file_ingest import VideoProto
-from veda_env import *
 
 
 class TestHotstore(unittest.TestCase):
@@ -29,7 +28,8 @@ class TestHotstore(unittest.TestCase):
 
         self.H1 = Hotstore(
             video_object=VP,
-            upload_filepath=self.upload_filepath
+            upload_filepath=self.upload_filepath,
+            video_proto=VP
         )
 
     def test_single_upload(self):

control/tests/test_video_validation.py

@@ -9,10 +9,13 @@ Test VEDA API
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
-from veda_video_validation import Validation
+from control.veda_video_validation import Validation
 
 
 class TestValidation(unittest.TestCase):
+    """
+    Test class for Validation
+    """
 
     def setUp(self):
 

frontend/tests/test_abvid_reporting.py

@@ -12,7 +12,7 @@ sys.path.append(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 import abvid_reporting
-
+from frontend.course_validate import VEDACat
 
 class TestVariables(unittest.TestCase):
 

reports/test

+Ignore everything in this directory.

test_requirements.txt

+## NOTE:  Test requirements.
+codecov==2.2.0
+pep8==1.7.0
+coverage==3.7.1