Merge pull request #50 from edx/ammar/logging-and-cleanup

logging and cleanup

VEDA/settings/base.py

@@ -3,6 +3,7 @@ Base settings
 
 """
 import os
+from VEDA.settings.utils import get_logger_config
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname((__file__))))
 
@@ -164,26 +165,4 @@ SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer'
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        }
-    },
-    'loggers': {
-        'django.request': {
-            'handlers': ['mail_admins'],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-    }
-}
+LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG')

VEDA/settings/local.py

 from VEDA.settings.base import *
+from VEDA.settings.utils import get_logger_config
 
 DEBUG = True
 ALLOWED_HOSTS = ['*']
@@ -24,6 +25,8 @@ JWT_AUTH.update({
     'JWT_VERIFY_AUDIENCE': False
 })
 
+LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG')
+
 # See if the developer has any local overrides.
 if os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'private.py')):
     from .private import *  # pylint: disable=import-error, wildcard-import

VEDA/settings/production.py

@@ -2,12 +2,15 @@ import yaml
 from os import environ
 from VEDA.settings.base import *
 from VEDA.utils import get_config
+from VEDA.settings.utils import get_logger_config
 
 DEBUG = False
 TEMPLATE_DEBUG = DEBUG
 
 ALLOWED_HOSTS = ['*']
 
+LOGGING = get_logger_config()
+
 # Keep track of the names of settings that represent dicts. Instead of overriding the values in base.py,
 # the values read from disk should UPDATE the pre-configured dicts.
 DICT_UPDATE_KEYS = ('DATABASES', 'SECRET_KEY', 'STATIC_ROOT_PATH', 'JWT_AUTH')

VEDA/settings/test.py

@@ -3,6 +3,7 @@ Test settings
 
 """
 from VEDA.settings.base import *
+from VEDA.settings.utils import get_logger_config
 
 DATABASES = {
     'default': {
@@ -10,3 +11,5 @@ DATABASES = {
         'NAME': 'pipeline.db',
     }
 }
+
+LOGGING = get_logger_config(debug=False, dev_env=True, local_loglevel='DEBUG')

VEDA/settings/utils.py

+"""
+Common utils for settings.
+"""
+import os
+import platform
+import sys
+from logging.handlers import SysLogHandler
+
+
+def get_logger_config(log_dir='/var/tmp',
+                      logging_env="no_env",
+                      edx_filename="edx.log",
+                      dev_env=False,
+                      debug=False,
+                      local_loglevel='INFO',
+                      service_variant='video-pipeline'):
+    """
+    Return the appropriate logging config dictionary. You should assign the
+    result of this to the LOGGING var in your settings.
+
+    If dev_env is set to true logging will not be done via local rsyslogd,
+    instead, application logs will be dropped in log_dir.
+
+    "edx_filename" is ignored unless dev_env is set to true since otherwise logging is handled by rsyslogd.
+    """
+
+    # Revert to INFO if an invalid string is passed in
+    if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
+        local_loglevel = 'INFO'
+
+    hostname = platform.node().split(".")[0]
+    syslog_format = (
+        "[service_variant={service_variant}]"
+        "[%(name)s][env:{logging_env}] %(levelname)s "
+        "[{hostname}  %(process)d] [%(filename)s:%(lineno)d] "
+        "- %(message)s"
+    ).format(
+        service_variant=service_variant,
+        logging_env=logging_env, hostname=hostname
+    )
+
+    if debug:
+        handlers = ['console']
+    else:
+        handlers = ['local']
+
+    logger_config = {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s %(levelname)s %(process)d '
+                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
+            },
+            'syslog_format': {'format': syslog_format},
+            'raw': {'format': '%(message)s'},
+        },
+        'filters': {
+            'require_debug_false': {
+                '()': 'django.utils.log.RequireDebugFalse'
+            }
+        },
+        'handlers': {
+            'console': {
+                'level': 'DEBUG' if debug else 'INFO',
+                'class': 'logging.StreamHandler',
+                'formatter': 'standard',
+                'stream': sys.stdout,
+            },
+            'mail_admins': {
+                'level': 'ERROR',
+                'filters': ['require_debug_false'],
+                'class': 'django.utils.log.AdminEmailHandler'
+            }
+        },
+        'loggers': {
+            'django': {
+                'handlers': handlers,
+                'propagate': True,
+                'level': 'INFO'
+            },
+            'requests': {
+                'handlers': handlers,
+                'propagate': True,
+                'level': 'WARNING'
+            },
+            'factory': {
+                'handlers': handlers,
+                'propagate': True,
+                'level': 'WARNING'
+            },
+            'django.request': {
+                'handlers': handlers + [] if debug else ['mail_admins'],
+                'propagate': True,
+                'level': 'WARNING'
+            },
+            '': {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False
+            },
+        }
+    }
+
+    if dev_env:
+        edx_file_loc = os.path.join(log_dir, edx_filename)
+        logger_config['handlers'].update({
+            'local': {
+                'class': 'logging.handlers.RotatingFileHandler',
+                'level': local_loglevel,
+                'formatter': 'standard',
+                'filename': edx_file_loc,
+                'maxBytes': 1024 * 1024 * 2,
+                'backupCount': 5,
+            },
+        })
+    else:
+        logger_config['handlers'].update({
+            'local': {
+                'level': local_loglevel,
+                'class': 'logging.handlers.SysLogHandler',
+                # Use a different address for Mac OS X
+                'address': '/var/run/syslog' if sys.platform == "darwin" else '/dev/log',
+                'formatter': 'syslog_format',
+                'facility': SysLogHandler.LOG_LOCAL0,
+            },
+        })
+
+    return logger_config

VEDA/utils.py

@@ -2,9 +2,9 @@
 Common utils.
 """
 import os
-import yaml
 import urllib
 
+import yaml
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 

VEDA_OS01/transcripts.py

@@ -26,7 +26,6 @@ from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 
-logging.basicConfig()
 LOGGER = logging.getLogger(__name__)
 
 # 3PlayMedia possible send-along statuses for a transcription callback.