Unverified Commit dc99e5c7 by Gregory Martin Committed by GitHub

Merge pull request #67 from edx/yro/refine_heal_add_logging

Refine HEAL process, make logging louder
parents 89d4920e 71e238b5
...@@ -61,7 +61,7 @@ class DeliverCli: ...@@ -61,7 +61,7 @@ class DeliverCli:
'worker', 'worker',
'--loglevel=info', '--loglevel=info',
'--concurrency=' + str(auth_dict['celery_threads']), '--concurrency=' + str(auth_dict['celery_threads']),
'-Q ' + auth_dict['celery_deliver_queue'], '-Q ' + auth_dict['celery_deliver_queue'] + ',' + auth_dict['celery_heal_queue'],
'-n deliver.%h' '-n deliver.%h'
)) ))
) )
......
...@@ -34,7 +34,7 @@ class HealCli: ...@@ -34,7 +34,7 @@ class HealCli:
.replace(hour=0, minute=0, second=0, microsecond=0) \ .replace(hour=0, minute=0, second=0, microsecond=0) \
.astimezone(pytz.utc) + timedelta(days=1) .astimezone(pytz.utc) + timedelta(days=1)
maintainer_healer.apply_async((self.binscript,), queue='transcode_stat', eta=go_time) maintainer_healer.apply_async((self.binscript,), queue=auth_dict['celery_heal_queue'], eta=go_time)
def main(): def main():
......
...@@ -8,6 +8,7 @@ import datetime ...@@ -8,6 +8,7 @@ import datetime
import logging import logging
import shutil import shutil
from os.path import expanduser from os.path import expanduser
import sys
import boto import boto
import boto.s3 import boto.s3
...@@ -35,6 +36,9 @@ except ImportError: ...@@ -35,6 +36,9 @@ except ImportError:
from veda_deliver_3play import ThreePlayMediaClient from veda_deliver_3play import ThreePlayMediaClient
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
try: try:
boto.config.add_section('Boto') boto.config.add_section('Boto')
except: except:
...@@ -66,6 +70,7 @@ class VedaDelivery: ...@@ -66,6 +70,7 @@ class VedaDelivery:
Check the destination, route via available methods, Check the destination, route via available methods,
throw error if method is not extant throw error if method is not extant
""" """
LOGGER.info('[VIDEO_DELIVER] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile))
if self.encode_profile == 'hls': if self.encode_profile == 'hls':
# HLS encodes are a pass through # HLS encodes are a pass through
self.hls_run() self.hls_run()
......
...@@ -9,6 +9,7 @@ Roll through videos, check for completion ...@@ -9,6 +9,7 @@ Roll through videos, check for completion
""" """
import datetime import datetime
from datetime import timedelta from datetime import timedelta
import logging
import os import os
import sys import sys
import uuid import uuid
...@@ -27,6 +28,10 @@ from VEDA.utils import get_config ...@@ -27,6 +28,10 @@ from VEDA.utils import get_config
time_safetygap = datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=1) time_safetygap = datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=1)
LOGGER = logging.getLogger(__name__)
# TODO: Remove this temporary logging to stdout
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
class VedaHeal(object): class VedaHeal(object):
""" """
...@@ -87,6 +92,7 @@ class VedaHeal(object): ...@@ -87,6 +92,7 @@ class VedaHeal(object):
""" """
Determine expected and completed encodes Determine expected and completed encodes
""" """
LOGGER.info('[HEAL] : {id}'.format(id=video_object.edx_id))
if self.freezing_bug is True: if self.freezing_bug is True:
if video_object.video_trans_status == 'Corrupt File': if video_object.video_trans_status == 'Corrupt File':
return [] return []
...@@ -116,8 +122,11 @@ class VedaHeal(object): ...@@ -116,8 +122,11 @@ class VedaHeal(object):
except KeyError: except KeyError:
pass pass
# list comparison requeued_encodes = self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object)
return self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object) LOGGER.info(
'[HEAL] : Status: {status}, Encodes: {encodes}'.format(status=self.val_status, encodes=requeued_encodes)
)
return requeued_encodes
def differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object): def differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
""" """
......
...@@ -105,7 +105,7 @@ val_username: ...@@ -105,7 +105,7 @@ val_username:
val_transcript_create_url: val_transcript_create_url:
val_video_transcript_status_url: val_video_transcript_status_url:
# Celery Worker Config Information
rabbitmq_broker: rabbitmq_broker:
rabbitmq_pass: rabbitmq_pass:
rabbitmq_user: rabbitmq_user:
......
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
# This configuration should only have static settings. # This configuration should only have static settings.
# Celery Info # Celery Info
onsite_worker: False
celery_threads: 1
celery_app_name: veda_production celery_app_name: veda_production
# can do multiple queues like so: foo,bar,baz celery_worker_queue: encode_worker,large_encode_worker
celery_worker_queue: encode_worker
celery_deliver_queue: deliver_worker celery_deliver_queue: deliver_worker
celery_threads: 1 celery_heal_queue: heal_queue
# S3 upload settings # S3 upload settings
multi_upload_barrier: 2000000000 multi_upload_barrier: 2000000000
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment