Skip to content

E
edx-video-worker
Commit 36824e03 by George Hatzis-Schoch
Options

Edited logging format and fixed initialization errors

parent 9a22f718
Showing with 57 additions and 57 deletions
video_worker/__init__.py
......@@ -35,9 +35,9 @@ try:
except:
pass
server_name = os.envion['SERVER_NAME']
server_name = os.environ['SERVER_NAME']
boto.config.set('Boto', 'http_socket_timeout', BOTO_TIMEOUT)
logging.basicConfig(format="(%s) %(message)s" % server_name, level=logging.INFO)
logging.basicConfig(format="[ENCODE_WORKER] %s " % server_name, level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
logger = logging.getLogger(__name__)
......@@ -80,7 +80,7 @@ class VideoWorker(object):
self.settings = get_config()
if self.encode_profile is None:
logger.error('[ENCODE_WORKER] No Encode Profile Specified')
logger.error('No Encode Profile Specified')
return
self.VideoObject = Video(
......@@ -95,13 +95,13 @@ class VideoWorker(object):
self.VideoObject.activate()
if not self.VideoObject.valid:
logger.error('[ENCODE_WORKER] {id} : Invalid Video Data'.format(id=self.VideoObject.val_id))
logger.error('{id} : Invalid Video Data'.format(id=self.VideoObject.val_id))
return
if not os.path.exists(self.workdir):
os.mkdir(self.workdir)
logger.info('[ENCODE_WORKER] {id} | {encoding} : Ready for Encode'.format(
logger.info('{id} | {encoding} : Ready for Encode'.format(
id=self.VideoObject.val_id,
encoding=self.encode_profile
))
......@@ -119,7 +119,7 @@ class VideoWorker(object):
self._engine_intake()
if not self.VideoObject.valid:
logger.error('[ENCODE_WORKER] Invalid Video / Local')
logger.error('Invalid Video / Local')
return
if self.VideoObject.val_id is not None:
......@@ -141,7 +141,7 @@ class VideoWorker(object):
else:
self._static_pipeline()
logger.info('[ENCODE_WORKER] {id} | {encoding} : Encode Complete'.format(
logger.info('{id} | {encoding} : Encode Complete'.format(
id=self.VideoObject.val_id,
encoding=self.encode_profile
))
......@@ -153,7 +153,7 @@ class VideoWorker(object):
(veda_id, encode_profile),
queue=self.settings['celery_deliver_queue']
)
logger.info('[ENCODE_WORKER] {id} | {encoding} : encoded file queued for delivery'.format(
logger.info('{id} | {encoding} : encoded file queued for delivery'.format(
id=self.VideoObject.val_id,
encoding=self.encode_profile
))
......@@ -183,7 +183,7 @@ class VideoWorker(object):
Activate HLS, use hls lib to upload
"""
if not os.path.exists(os.path.join(self.workdir, self.source_file)):
logger.error('[ENCODE_WORKER] : {id} | {encoding} Local raw video file not found'.format(
logger.error(': {id} | {encoding} Local raw video file not found'.format(
id=self.VideoObject.val_id,
encoding=self.encode_profile
))
......@@ -214,7 +214,7 @@ class VideoWorker(object):
Copy file down from AWS S3 storage bucket
"""
if not self.VideoObject.valid:
logger.error('[ENCODE_WORKER] : {id} Invalid Video'.format(
logger.error(': {id} Invalid Video'.format(
id=self.VideoObject.val_id,
))
return
......@@ -230,7 +230,7 @@ class VideoWorker(object):
try:
bucket = conn.get_bucket(self.settings['veda_s3_hotstore_bucket'])
except S3ResponseError:
logger.error('[ENCODE_WORKER] Invalid hotstore S3 bucket')
logger.error('Invalid hotstore S3 bucket')
return
if self.VideoObject.mezz_extension is not None and len(self.VideoObject.mezz_extension) > 0:
......@@ -243,7 +243,7 @@ class VideoWorker(object):
source_key = bucket.get_key(self.source_file)
if source_key is None:
logger.error('[ENCODE_WORKER] : {id} S3 Intake object not found'.format(
logger.error(': {id} S3 Intake object not found'.format(
id=self.VideoObject.val_id
))
return
......@@ -253,7 +253,7 @@ class VideoWorker(object):
)
if not os.path.exists(os.path.join(self.workdir, self.source_file)):
logger.error('[ENCODE_WORKER] : {id} engine intake download error'.format(
logger.error(': {id} engine intake download error'.format(
id=self.VideoObject.val_id
))
return
......@@ -296,7 +296,7 @@ class VideoWorker(object):
--no need to move the source--
"""
if not os.path.exists(os.path.join(self.workdir, self.source_file)):
logger.error('[ENCODE_WORKER] : {id} Encode input file not found'.format(
logger.error(': {id} Encode input file not found'.format(
id=self.VideoObject.val_id
))
return
......@@ -312,7 +312,7 @@ class VideoWorker(object):
self.output_file = self.ffcommand.split('/')[-1]
if not os.path.exists(os.path.join(self.workdir, self.output_file)):
logger.error('[ENCODE_WORKER] : {id} Encode output file not found'.format(
logger.error(': {id} Encode output file not found'.format(
id=self.VideoObject.val_id
))
......
video_worker/abstractions.py
......@@ -53,7 +53,7 @@ class Video(object):
test case
"""
if self.veda_id is not None and len(settings['veda_api_url']) == 0:
logger.error('[ENCODE_WORKER] VEDA API Config Incorrect')
logger.error('VEDA API Config Incorrect')
return
if self.veda_id is None and self.mezz_filepath is None:
......@@ -160,7 +160,7 @@ class Encode(object):
veda_token = generate_apitoken.veda_tokengen()
if veda_token is None:
logger.error(
'[ENCODE_WORKER] : {id} {encode} VEDA Token Generate'.format(
': {id} {encode} VEDA Token Generate'.format(
id=self.VideoObject.veda_id,
encode=self.profile_name
)
......@@ -186,7 +186,7 @@ class Encode(object):
if len(enc_dict['results']) == 0:
logger.error(
'[ENCODE_WORKER] : {id} {encode} VEDA API Encode Mismatch: No Data'.format(
': {id} {encode} VEDA API Encode Mismatch: No Data'.format(
id=self.VideoObject.veda_id,
encode=self.profile_name
)
......
video_worker/api_communicate.py
......@@ -51,7 +51,7 @@ class UpdateAPIStatus:
self.veda_token = generate_apitoken.veda_tokengen()
if self.veda_token is None:
logger.error('[ENCODE_WORKER] VEDA API connection: Invalid Setup/Method')
logger.error('VEDA API connection: Invalid Setup/Method')
return None
self.veda_headers = {
......@@ -99,7 +99,7 @@ class UpdateAPIStatus:
)
if y.status_code != 200:
logger.error('[ENCODE_WORKER] VEDA API: API config')
logger.error('VEDA API: API config')
return None
return json.loads(y.text)
......@@ -129,7 +129,7 @@ class UpdateAPIStatus:
)
if w.status_code != 200:
logger.error('[ENCODE_WORKER] VEDA API: GET Failure, no objects')
logger.error('VEDA API: GET Failure, no objects')
def send_val_data(self):
"""
......@@ -177,7 +177,7 @@ class UpdateAPIStatus:
if r1.status_code != 200 and r1.status_code != 404:
# Total API Failure
logger.error('[ENCODE_WORKER] VAL Communication')
logger.error('VAL Communication')
return None
if r1.status_code == 404:
......@@ -195,7 +195,7 @@ class UpdateAPIStatus:
)
if r2.status_code > 299:
logger.error('[ENCODE_WORKER] VAL POST/PUT')
logger.error('VAL POST/PUT')
return None
elif r1.status_code == 200:
......@@ -231,5 +231,5 @@ class UpdateAPIStatus:
)
if r2.status_code > 299:
logger.error('[ENCODE_WORKER] VAL POST/PUT')
logger.error('VAL POST/PUT')
return None
video_worker/generate_apitoken.py
......@@ -37,7 +37,7 @@ def veda_tokengen():
)
if veda_token_response.status_code != 200:
logger.error('[ENCODE_WORKER] VEDA token generation')
logger.error('VEDA token generation')
return
veda_token = ast.literal_eval(veda_token_response.text)['access_token']
......@@ -58,7 +58,7 @@ def veda_tokengen():
)
if veda_auth_response.status_code != 200:
logger.error('[ENCODE_WORKER] VEDA token authorization')
logger.error('VEDA token authorization')
return
return veda_auth_response.text.strip()
......@@ -79,7 +79,7 @@ def val_tokengen():
response = requests.post(settings['val_token_url'] + '/', data=payload, timeout=settings['global_timeout'])
if response.status_code != 200:
logger.error('[ENCODE_WORKER] VAL token generation')
logger.error('VAL token generation')
return
return ast.literal_eval(response.text)['access_token']
video_worker/generate_delivery.py
......@@ -139,7 +139,7 @@ class Deliverable(object):
if b is None:
logger.error(
'[ENCODE_WORKER] : {file} Deliverable Fail: s3 Bucket Connection Error'.format(
': {file} Deliverable Fail: s3 Bucket Connection Error'.format(
file=self.output_file
)
)
......
video_worker/generate_encode.py
......@@ -40,11 +40,11 @@ class CommandGenerate:
Generate command for ffmpeg lib
"""
if self.VideoObject is None:
logger.error('[ENCODE_WORKER] Command generation: No Video object')
logger.error('Command generation: No Video object')
return
if self.EncodeObject is None:
logger.error('[ENCODE_WORKER] Command generation: No Encode object')
logger.error('Command generation: No Encode object')
return
if self.workdir is None:
......
video_worker/tests/test_api_communicate.py
......@@ -103,7 +103,7 @@ class ApiCommunicateTest(unittest.TestCase):
if not veda_token:
self.assertFalse(response)
self.assertFalse(mock_send_veda_status.called)
mock_logger.error.assert_called_with('[ENCODE_WORKER] VEDA API connection: Invalid Setup/Method')
mock_logger.error.assert_called_with('VEDA API connection: Invalid Setup/Method')
elif not veda_video_status:
self.assertFalse(response)
self.assertFalse(mock_send_veda_status.called)
......@@ -159,7 +159,7 @@ class ApiCommunicateTest(unittest.TestCase):
self.assertFalse(response)
elif response_status_code != 200:
self.assertTrue(mock_patch.called)
mock_logger.error.assert_called_with('[ENCODE_WORKER] VEDA API: GET Failure, no objects')
mock_logger.error.assert_called_with('VEDA API: GET Failure, no objects')
@data(
# Check empty values
......@@ -192,7 +192,7 @@ class ApiCommunicateTest(unittest.TestCase):
'course_url': ['dummy-course-id'],
'get_status_code': 400,
'post_status_code': 200,
'error_message': '[ENCODE_WORKER] VAL Communication'
'error_message': 'VAL Communication'
},
),
(
......@@ -202,7 +202,7 @@ class ApiCommunicateTest(unittest.TestCase):
'course_url': ['dummy-course-id'],
'get_status_code': 404,
'post_status_code': 300,
'error_message': '[ENCODE_WORKER] VAL POST/PUT'
'error_message': 'VAL POST/PUT'
},
),
(
......@@ -212,7 +212,7 @@ class ApiCommunicateTest(unittest.TestCase):
'course_url': ['dummy-course-id'],
'get_status_code': 200,
'post_status_code': 300,
'error_message': '[ENCODE_WORKER] VAL POST/PUT'
'error_message': 'VAL POST/PUT'
},
),
# Success
......@@ -278,7 +278,7 @@ class ApiCommunicateTest(unittest.TestCase):
@data(
(False, 400, ''),
(True, 400, '[ENCODE_WORKER] VEDA API: API config'),
(True, 400, 'VEDA API: API config'),
(True, 200, '')
)
@unpack
......
video_worker/tests/test_generate_encode.py
......@@ -40,13 +40,13 @@ class CommandGenerateTest(unittest.TestCase):
(
{
'video_object': None,
'error_message': '[ENCODE_WORKER] Command generation: No Video object'
'error_message': 'Command generation: No Video object'
}
),
(
{
'encode_object': None,
'error_message': '[ENCODE_WORKER] Command generation: No Encode object'
'error_message': 'Command generation: No Encode object'
}
),
(
......
video_worker/tests/test_generate_token.py
......@@ -32,7 +32,7 @@ class GenerateApiTokenTest(unittest.TestCase):
mock_post.return_value = Mock(status_code=400, text='', content={})
response = veda_tokengen()
self.assertFalse(response)
mock_logger.error.assert_called_with('[ENCODE_WORKER] VEDA token generation')
mock_logger.error.assert_called_with('VEDA token generation')
@patch('video_worker.generate_apitoken.requests.post')
@patch('video_worker.generate_apitoken.logger')
......@@ -51,7 +51,7 @@ class GenerateApiTokenTest(unittest.TestCase):
]
response = veda_tokengen()
self.assertFalse(response)
mock_logger.error.assert_called_with('[ENCODE_WORKER] VEDA token authorization')
mock_logger.error.assert_called_with('VEDA token authorization')
@patch('video_worker.generate_apitoken.requests.post')
def test_veda_tokengen(self, mock_post):
......@@ -76,7 +76,7 @@ class GenerateApiTokenTest(unittest.TestCase):
mock_post.return_value = Mock(status_code=400, text='', content={})
response = val_tokengen()
self.assertFalse(response)
mock_logger.error.assert_called_with('[ENCODE_WORKER] VAL token generation')
mock_logger.error.assert_called_with('VAL token generation')
@patch('video_worker.generate_apitoken.requests.post')
def test_val_tokengen(self, mock_post):
......
video_worker/tests/test_video_worker.py
......@@ -69,21 +69,21 @@ class VideoWorkerTest(unittest.TestCase):
@data(
(
{
'error_message': '[ENCODE_WORKER] No Encode Profile Specified'
'error_message': 'No Encode Profile Specified'
}
),
(
{
'is_valid': False,
'encode_profile': 'static-pipeline',
'error_message': '[ENCODE_WORKER] dummy-val-id : Invalid Video Data'
'error_message': 'dummy-val-id : Invalid Video Data'
}
),
(
{
'is_valid_engine_intake': False,
'encode_profile': 'static-pipeline',
'error_message': '[ENCODE_WORKER] Invalid Video / Local'
'error_message': 'Invalid Video / Local'
}
),
(
......@@ -204,7 +204,7 @@ class VideoWorkerTest(unittest.TestCase):
self.VW._hls_pipeline()
self.assertIsNone(self.VW.endpoint_url)
mock_logger.assert_called_with('[ENCODE_WORKER] : None | None Local raw video file not found')
mock_logger.assert_called_with(': None | None Local raw video file not found')
@data(
(
......@@ -287,28 +287,28 @@ class VideoWorkerTest(unittest.TestCase):
(
{
'valid_video': False,
'error_message': '[ENCODE_WORKER] : None Invalid Video'
'error_message': ': None Invalid Video'
}
),
(
{
'source_file': None,
'mock_get_bucket': False,
'error_message': '[ENCODE_WORKER] Invalid hotstore S3 bucket'
'error_message': 'Invalid hotstore S3 bucket'
}
),
(
{
'source_file': None,
'mock_get_bucket_key': True,
'error_message': '[ENCODE_WORKER] : None S3 Intake object not found'
'error_message': ': None S3 Intake object not found'
}
),
(
{
'source_file': None,
'path_exists': False,
'error_message': '[ENCODE_WORKER] : None engine intake download error'
'error_message': ': None engine intake download error'
}
),
# Success
......@@ -414,13 +414,13 @@ class VideoWorkerTest(unittest.TestCase):
@data(
(
{
'error_message': '[ENCODE_WORKER] : None Encode input file not found'
'error_message': ': None Encode input file not found'
}
),
(
{
'path_exists': [True, False],
'error_message': '[ENCODE_WORKER] : None Encode output file not found'
'error_message': ': None Encode output file not found'
}
),
(
......
video_worker/validate.py
......@@ -44,7 +44,7 @@ class ValidateVideo:
"""
if not os.path.exists(self.filepath):
logger.error(
'[ENCODE_WORKER] : {filepath} File QA fail: File is not found'.format(
': {filepath} File QA fail: File is not found'.format(
filepath=self.filepath
)
)
......@@ -52,7 +52,7 @@ class ValidateVideo:
if os.stat(self.filepath).st_size == 0:
logger.error(
'[ENCODE_WORKER] : {filepath} File QA fail: Filesize is 0'.format(
': {filepath} File QA fail: Filesize is 0'.format(
filepath=self.filepath
)
)
......@@ -119,7 +119,7 @@ class ValidateVideo:
"""
if not os.path.exists(self.filepath):
logger.error(
'[ENCODE_WORKER] : {filepath} File QA fail: Filesize is 0'.format(
': {filepath} File QA fail: Filesize is 0'.format(
filepath=self.filepath
)
)
......
video_worker/video_images.py
......@@ -81,7 +81,7 @@ class VideoImages(object):
"""
if not self.video_object:
logger.error(
'[ENCODE_WORKER] : {id} Video Image generation failed: No Video Object'.format(
': {id} Video Image generation failed: No Video Object'.format(
)
)
return
......@@ -133,7 +133,7 @@ class VideoImages(object):
try:
bucket = s3_connection.get_bucket(self.settings['aws_video_images_bucket'])
except S3ResponseError:
logger.error('[ENCODE_WORKER] : Invalid Storage Bucket for Video Images')
logger.error(': Invalid Storage Bucket for Video Images')
return
image_keys = []
......@@ -182,7 +182,7 @@ class VideoImages(object):
)
if not response.ok:
logger.error('[ENCODE_WORKER] : {id} {message}'.format(
logger.error(': {id} {message}'.format(
id=self.video_object.val_id,
message=response.content
))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment