Merge pull request #1910 from edx/max/newrelic-for-workers

Add newrelic monitoring for edxapp workers

playbooks/roles/edxapp/defaults/main.yml

@@ -341,6 +341,7 @@ EDXAPP_XML_S3_KEY: !!null
 
 EDXAPP_NEWRELIC_LMS_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-edxapp-lms"
 EDXAPP_NEWRELIC_CMS_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-edxapp-cms"
+EDXAPP_NEWRELIC_WORKERS_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-edxapp-workers"
 
 EDXAPP_AWS_STORAGE_BUCKET_NAME: 'edxuploads'
 
@@ -386,24 +387,31 @@ EDXAPP_CELERY_WORKERS:
   - queue: low
     service_variant: cms
     concurrency: 3
+    monitor: True
   - queue: default
     service_variant: cms
     concurrency: 4
+    monitor: True
   - queue: high
     service_variant: cms
     concurrency: 1
+    monitor: True
   - queue: low
     service_variant: lms
     concurrency: 1
+    monitor: True
   - queue: default
     service_variant: lms
     concurrency: 3
+    monitor: True
   - queue: high
     service_variant: lms
     concurrency: 4
+    monitor: True
   - queue: high_mem
     service_variant: lms
     concurrency: 2
+    monitor: False
 
 EDXAPP_DEFAULT_CACHE_VERSION: "1"
 EDXAPP_OAUTH_ENFORCE_SECURE: True

playbooks/roles/edxapp/templates/workers.conf.j2

 {% for w in edxapp_workers %}
 [program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
 
-environment=CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE=aws,PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }}
+environment={% if COMMON_ENABLE_NEWRELIC %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE=aws,PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }}
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edxapp_venv_bin}}/python {{ edxapp_code_dir }}/manage.py {{ w.service_variant }} --settings=aws celery worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }}
+command={{ edxapp_venv_dir + '/bin/newrelic-admin run-program ' if w.monitor and COMMON_ENABLE_NEWRELIC else ''}}{{ edxapp_venv_bin }}/python {{ edxapp_code_dir }}/manage.py {{ w.service_variant }} --settings=aws celery worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }}
 killasgroup=true
 stopasgroup=true
 stopwaitsecs=432000

util/jenkins/ansible-provision.sh

@@ -185,6 +185,7 @@ COMMON_ENABLE_DATADOG: $enable_monitoring
 FORUM_NEW_RELIC_ENABLE: $enable_monitoring
 EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
 EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
+EDXAPP_NEWRELIC_WORKERS_APPNAME: sandbox-${dns_name}-edxapp-workers
 XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
 FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
 SANDBOX_USERNAME: $github_username