Skip to content

E
edx-video-worker
Commit 62d6530f by Gregory Martin
Options

Change Encode Indentation

parent a94870ee
Showing with 34 additions and 458 deletions
.travis.yml
...@@ -18,8 +18,6 @@ install: ...@@ -18,8 +18,6 @@ install:
script: video_worker script: video_worker
script: script:
- python tests/test_api_connect.py
- python tests/test_build.py
- make validate - make validate
after_success: after_success:
......
instance_config.yaml
--- ---
#--
# Global settings
heal_start: 1
heal_end: 144
global_timeout: 40
# --- # ---
# Database information # Database information
# --- # ---
...@@ -88,9 +81,9 @@ celery_receiver_queue: encode_worker,large_encode_worker ...@@ -88,9 +81,9 @@ celery_receiver_queue: encode_worker,large_encode_worker
celery_threads: celery_threads:
## CLOUD ## CLOUD
rabbitmq_broker: rabbitmq_broker: ""
rabbitmq_pass: rabbitmq_pass: ""
rabbitmq_user: rabbitmq_user: ""
# --- # ---
# Shotgun Variables (internal mediateam) # Shotgun Variables (internal mediateam)
...@@ -138,7 +131,6 @@ val_profile_dict: ...@@ -138,7 +131,6 @@ val_profile_dict:
override: override:
- desktop_mp4 - desktop_mp4
- mobile_low - mobile_low
- mobile_high
audio_mp3: audio_mp3:
- audio_mp3 - audio_mp3
...@@ -149,4 +141,13 @@ val_profile_dict: ...@@ -149,4 +141,13 @@ val_profile_dict:
hls: hls:
- hls - hls
#--
# Global settings
heal_start: 3
heal_end: 144
global_timeout: 40
ffmpeg_compiled: 'ffmpeg'
ffprobe_compiled: 'ffprobe'
... ...
tests/test_api_connect.py deleted 100644 → 0
import os
import sys
import unittest
import requests
# for Travis
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from video_worker.config import WorkerSetup
from video_worker.generate_apitoken import val_tokengen, veda_tokengen
"""
This is an API connection test
set to pass if instance_config.yaml is missing
"""
# Disable warning
requests.packages.urllib3.disable_warnings()
class TestAPIConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_val_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'val_api_url',
'val_client_id',
'val_password',
'val_secret_key',
'val_username',
'val_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_veda_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'veda_api_url',
'veda_auth_url',
'veda_client_id',
'veda_secret_key',
'veda_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_val_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
val_token = val_tokengen()
self.assertFalse(val_token is None)
headers = {
'Authorization': 'Bearer ' + val_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['val_api_url'], headers=headers, timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def test_veda_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
veda_token = veda_tokengen()
self.assertFalse(veda_token is None)
headers = {
'Authorization': 'Token ' + veda_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['veda_api_url'] + '/', headers=headers, timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
tests/test_asset_connect.py deleted 100644 → 0
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
import os
import unittest
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from video_worker.config import WorkerSetup
"""
Test for deliverable connection
set to pass if instance_config.yaml is missing
"""
class TestAssetConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_storage_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection()
try:
bucket = conn.get_bucket(self.settings['veda_s3_hotstore_bucket'])
self.assertTrue(True)
except S3ResponseError:
self.assertFalse(True)
def test_delivery_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection()
try:
bucket = conn.get_bucket(self.settings['veda_deliverable_bucket'])
self.assertTrue(True)
except S3ResponseError:
self.assertFalse(True)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
tests/test_encode.py deleted 100644 → 0
import os
import sys
import unittest
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from video_worker.abstractions import Video, Encode
from video_worker.config import WorkerSetup
from video_worker.generate_encode import CommandGenerate
"""
test Encode Abstraction and Command Gen
"""
class Test_Encode_Command(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
"""
Gen abstractions
"""
# Video
self.VideoObject = Video(
veda_id='XXXXXXXX2016-V00TEST',
)
self.VideoObject.activate()
# Encode
self.E = Encode(
VideoObject=self.VideoObject,
profile_name=self.encode_profile
)
self.E.pull_data()
self.ffcommand = None
def test_generate_command(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
"""
Generate the (shell) command / Encode Object
"""
self.assertTrue(self.VideoObject.valid is True)
"""
Generate the Encode Object
"""
self.E.pull_data()
self.assertFalse(self.E.filetype is None)
"""
Generate the (shell) command
"""
self.ffcommand = CommandGenerate(
VideoObject=self.VideoObject,
EncodeObject=self.E
).generate()
self.assertFalse(self.ffcommand is None)
"""
TODO: More sophisticated encode tests
"""
# TODO: pillarbox, letterbox commands
# TODO: scalar commands
# TODO: destination file, etc.
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
tests/test_intake.py deleted 100644 → 0
import os
import sys
import unittest
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from video_worker.__init__ import VideoWorker
from video_worker.abstractions import Video, Encode
from video_worker.config import WorkerSetup
"""
file intake test
"""
class TestIntake(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VideoWorker(
veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid
)
def test_intake(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
# copied from __init__
self.VW.VideoObject = Video(
veda_id=self.VW.veda_id
)
self.VW.VideoObject.activate()
self.assertTrue(self.VW.VideoObject.valid)
self.VW.settings = self.settings
self.VW._engine_intake()
print self.VW.VideoObject
self.assertTrue(self.VW.VideoObject.valid)
self.assertTrue(
os.path.exists(
os.path.join(
self.VW.workdir,
self.VW.source_file
)
)
)
self.assertTrue(self.VW.VideoObject.valid)
@unittest.skip("not implemented")
def tearDown(self):
if self.jobid is not None:
shutil.rmtree(self.VW.workdir)
else:
os.remove(
os.path.join(
self.VW.workdir,
self.VW.output_file
)
)
os.remove(
os.path.join(
self.VW.workdir,
self.VW.source_file
)
)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
tests/test_video_abstraction.py deleted 100644 → 0
import os
import sys
import unittest
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from video_worker.abstractions import Video
"""
init end-to-end function tests
"""
class TestVideoAbstraction(unittest.TestCase):
def setUp(self):
self.VideoObject = Video()
self.VideoObject.activate()
def test_video_object(self):
self.assertTrue(os.path.exists(self.VideoObject.mezz_filepath))
def test_video_validate(self):
self.assertTrue(self.VideoObject.valid)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
video_worker/__init__.py
...@@ -92,9 +92,6 @@ class VideoWorker(object): ...@@ -92,9 +92,6 @@ class VideoWorker(object):
WS = WorkerSetup( WS = WorkerSetup(
instance_yaml=self.instance_yaml instance_yaml=self.instance_yaml
) )
if self.setup:
WS.setup = True
WS.run() WS.run()
self.settings = WS.settings_dict self.settings = WS.settings_dict
......
video_worker/abstractions.py
...@@ -200,9 +200,9 @@ class Encode(object): ...@@ -200,9 +200,9 @@ class Encode(object):
self.encode_suffix = e['encode_suffix'] self.encode_suffix = e['encode_suffix']
self.encode_pk = e['id'] self.encode_pk = e['id']
if self.encode_suffix is None: if self.encode_suffix is None:
# In the case of an API Error # In the case of an API Error
self._default_encodes() self._default_encodes()
def _default_encodes(self): def _default_encodes(self):
""" """
......
video_worker/config.py
...@@ -6,6 +6,7 @@ based on variables within ...@@ -6,6 +6,7 @@ based on variables within
import logging import logging
import os import os
import sys
import yaml import yaml
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -24,9 +25,6 @@ class WorkerSetup: ...@@ -24,9 +25,6 @@ class WorkerSetup:
self.settings_dict = {} self.settings_dict = {}
def run(self): def run(self):
"""
Read Extant Settings or Generate New Ones
"""
if not os.path.exists(self.instance_yaml): if not os.path.exists(self.instance_yaml):
logger.error('Not Configured') logger.error('Not Configured')
return return
...@@ -34,6 +32,6 @@ class WorkerSetup: ...@@ -34,6 +32,6 @@ class WorkerSetup:
with open(self.instance_yaml, 'r') as stream: with open(self.instance_yaml, 'r') as stream:
try: try:
self.settings_dict = yaml.load(stream) self.settings_dict = yaml.load(stream)
except yaml.YAMLError as exc: except yaml.YAMLError:
logger.error('Config YAML read error') logger.error('Config YAML read error')
return return
video_worker/tests/test_config.py
...@@ -2,14 +2,13 @@ ...@@ -2,14 +2,13 @@
This file tests configuration settings for worker setup. This file tests configuration settings for worker setup.
""" """
import os
import unittest import unittest
import yaml import yaml
from ddt import ddt, data, unpack from ddt import ddt, data, unpack
from mock import Mock, patch, mock_open from mock import patch
from utils import create_worker_setup, TEST_INSTANCE_YAML, DUMMY_INSTANCE_YAML from utils import create_worker_setup, TEST_INSTANCE_YAML
@ddt @ddt
...@@ -17,45 +16,21 @@ class ConfigTest(unittest.TestCase): ...@@ -17,45 +16,21 @@ class ConfigTest(unittest.TestCase):
""" """
Config test class. Config test class.
""" """
@data(
(True, TEST_INSTANCE_YAML),
(False, TEST_INSTANCE_YAML),
(False, '/dummy-instance-config-path'),
(True, '/dummy-instance-config-path')
)
@unpack
@patch('video_worker.config.WorkerSetup._READ_SETTINGS')
@patch('video_worker.config.WorkerSetup._CONFIGURE')
def test_run(self, setup, instance_yaml, mock_configure, mock_read_settings):
"""
Tests that correct method is called based on worker setup configuration
"""
WS = create_worker_setup({
'instance_yaml': instance_yaml,
'setup': setup
})
WS.run()
instance_yaml_exists = os.path.exists(WS.instance_yaml)
self.assertEqual(mock_configure.called, setup or not instance_yaml_exists)
self.assertEqual(mock_read_settings.called, not setup and instance_yaml_exists)
@data( @data(
('/dummy-instance-config-path', 'Not Configured'), ('/dummy-instance-config-path', 'Not Configured'),
(TEST_INSTANCE_YAML, '') (TEST_INSTANCE_YAML, '')
) )
@unpack @unpack
@patch('video_worker.config.logger') @patch('video_worker.config.logger')
def test_read_settings(self, instance_yaml, error_message, mock_logger): def test_run(self, instance_yaml, error_message, mock_logger):
""" """
Tests that `_READ_SETTINGS` method works correctly. Tests that `_read_settings` method works correctly.
""" """
WS = create_worker_setup({ WS = create_worker_setup({
'instance_yaml': instance_yaml 'instance_yaml': instance_yaml
}) })
self.assertEqual(WS.settings_dict, {}) self.assertEqual(WS.settings_dict, {})
WS._READ_SETTINGS() WS.run()
if error_message: if error_message:
self.assertFalse(len(WS.settings_dict)) self.assertFalse(len(WS.settings_dict))
...@@ -64,55 +39,16 @@ class ConfigTest(unittest.TestCase): ...@@ -64,55 +39,16 @@ class ConfigTest(unittest.TestCase):
self.assertNotEqual(WS.settings_dict, {}) self.assertNotEqual(WS.settings_dict, {})
self.assertTrue(len(WS.settings_dict)) self.assertTrue(len(WS.settings_dict))
@patch('video_worker.config.raw_input', create=True)
@patch('video_worker.config.yaml.load')
def test_configure(self, yaml_load, mock_raw_input):
"""
Tests that `_CONFIGURE` method works correctly.
"""
with open(DUMMY_INSTANCE_YAML, 'w+') as file:
mock_raw_input.return_value = 'dummy-value'
yaml_load.return_value = {
'dummy_key': 'dummy-value',
'empty_value_key': ''
}
WS = create_worker_setup({
'instance_yaml': DUMMY_INSTANCE_YAML
})
self.assertEqual(WS.settings_dict, {})
WS._CONFIGURE()
self.assertNotEqual(WS.settings_dict, {})
self.assertTrue(len(WS.settings_dict))
# Close and delete the file after used
file.close()
os.remove(DUMMY_INSTANCE_YAML)
@patch('video_worker.config.logger') @patch('video_worker.config.logger')
@patch('video_worker.config.yaml.load') @patch('video_worker.config.yaml.load')
def test_read_settings_load_error(self, yaml_load, mock_logger): def test_run_load_error(self, yaml_load, mock_logger):
""" """
Tests that `_READ_SETTINGS` method raises correct log in case of yaml load error. Tests that `_read_settings` method raises correct log in case of yaml load error.
""" """
error_message = 'Config YAML read error' error_message = 'Config YAML read error'
yaml_load.side_effect = yaml.YAMLError(error_message) yaml_load.side_effect = yaml.YAMLError(error_message)
WS = create_worker_setup() WS = create_worker_setup()
self.assertEqual(WS.settings_dict, {}) self.assertEqual(WS.settings_dict, {})
WS._READ_SETTINGS() WS.run()
mock_logger.error.assert_called_with(error_message)
self.assertEqual(WS.settings_dict, {})
@patch('video_worker.config.logger')
@patch('video_worker.config.yaml.load')
def test_configure_load_error(self, yaml_load, mock_logger):
"""
Tests that `_CONFIGURE` method raises correct log in case of yaml load error.
"""
error_message = 'default YAML read error'
yaml_load.side_effect = yaml.YAMLError(error_message)
WS = create_worker_setup()
self.assertEqual(WS.settings_dict, {})
WS._CONFIGURE()
mock_logger.error.assert_called_with(error_message) mock_logger.error.assert_called_with(error_message)
self.assertEqual(WS.settings_dict, {}) self.assertEqual(WS.settings_dict, {})
video_worker/tests/test_instance_config.yaml
...@@ -27,8 +27,14 @@ veda_s3_upload_bucket: 'dummy-upload-bucket' ...@@ -27,8 +27,14 @@ veda_s3_upload_bucket: 'dummy-upload-bucket'
veda_s3_hotstore_bucket: 'dummy-hotstore-bucket' veda_s3_hotstore_bucket: 'dummy-hotstore-bucket'
veda_deliverable_bucket: 'dummy-deliverable-bucket' veda_deliverable_bucket: 'dummy-deliverable-bucket'
ffmpeg_compiled: 'ffmpeg'
val_client_id: None val_client_id: None
val_video_images_url: 'https://www.testimg.com/update/images' val_video_images_url: 'https://www.testimg.com/update/images'
aws_video_images_prefix: 'video-images/' aws_video_images_prefix: 'video-images/'
aws_video_images_bucket: 'imagesbucket' aws_video_images_bucket: 'imagesbucket'
# Celery
celery_deliver_queue: ""
#--
# Global settings
global_timeout: 40
video_worker/tests/test_video_worker.py
...@@ -82,17 +82,6 @@ class VideoWorkerTest(unittest.TestCase): ...@@ -82,17 +82,6 @@ class VideoWorkerTest(unittest.TestCase):
) )
self.assertEqual(VW.workdir, expected_workdir) self.assertEqual(VW.workdir, expected_workdir)
@data(True, False)
@patch('nose.run')
def test_video_worker_test(self, nose_result, mock_nose):
"""
Test `test` method works correctly.
"""
# Mock nose.run
mock_nose.return_value = nose_result
result = self.VW.test()
self.assertEqual(result, nose_result)
@data( @data(
( (
{ {
...@@ -116,7 +105,7 @@ class VideoWorkerTest(unittest.TestCase): ...@@ -116,7 +105,7 @@ class VideoWorkerTest(unittest.TestCase):
( (
{ {
'encode_profile': 'static-pipeline', 'encode_profile': 'static-pipeline',
'path_exists': False 'exists_side_effects': [True, False]
} }
), ),
# Success # Success
...@@ -162,7 +151,7 @@ class VideoWorkerTest(unittest.TestCase): ...@@ -162,7 +151,7 @@ class VideoWorkerTest(unittest.TestCase):
# First 2 calls to os.path.exists will be called in WS.run() so we need to make sure our TEST_INSTANCE_YAML # First 2 calls to os.path.exists will be called in WS.run() so we need to make sure our TEST_INSTANCE_YAML
# file path is check correctly. # file path is check correctly.
mock_exists.side_effect = [True, True, path_exists] mock_exists.side_effect = mock_data.get('exists_side_effects', [True, True, True])
video_images_setup_mock.return_value = WS.settings_dict video_images_setup_mock.return_value = WS.settings_dict
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment