Merge pull request #1322 from edx/jarv/bring-down-worker-count

bringing down the worker count to default to 2

playbooks/roles/analytics-api/defaults/main.yml

@@ -58,6 +58,7 @@ ANALYTICS_API_CONFIG:
       HOST: 'localhost'
       PORT: '3306'
 
+ANALYTICS_API_GUNICORN_WORKERS: "2"
 #
 # vars are namespace with the module name.
 #
@@ -79,7 +80,6 @@ analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
 analytics_api_conf_dir: "{{ analytics_api_home }}"
 analytics_api_gunicorn_host: "127.0.0.1"
 analytics_api_gunicorn_port: "8100"
-analytics_api_gunicorn_workers: "8"
 analytics_api_gunicorn_timeout: "300"
 
 analytics_api_django_settings: "production"

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2

@@ -15,4 +15,4 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
 
 source {{ analytics_api_app_dir }}/analytics_api_env
 
-{{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application
+{{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ ANALYTICS_API_GUNICORN_WORKERS }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/analytics-server/defaults/main.yml

@@ -18,7 +18,7 @@ AS_DB_ANALYTICS_HOST: 'localhost'
 AS_SERVER_PORT: '9000'
 AS_ENV_LANG: 'en_US.UTF-8'
 AS_LOG_LEVEL: 'INFO'
-AS_WORKERS: '4'
+AS_WORKERS: '2'
 # add public keys to enable the automator user
 # for running manage.py commands
 

playbooks/roles/analytics/defaults/main.yml

@@ -18,7 +18,7 @@ ANALYTICS_DB_ANALYTICS_HOST: 'localhost'
 ANALYTICS_SERVER_PORT: '9000'
 ANALYTICS_ENV_LANG: 'en_US.UTF-8'
 ANALYTICS_LOG_LEVEL: 'INFO'
-ANALYTICS_WORKERS: '4'
+ANALYTICS_WORKERS: '2'
 
 DATABASES:
   default: &databases_default

playbooks/roles/edxapp/defaults/main.yml

@@ -254,6 +254,12 @@ EDXAPP_CODE_JAIL_LIMITS:
 
 EDXAPP_VIRTUAL_UNIVERSITIES: []
 EDXAPP_SUBDOMAIN_BRANDING: {}
+# Set the number of workers explicitely for lms and cms
+# Should be set to
+# EDXAPP_WORKERS:
+#   lms: <num workers>
+#   cms: <num workers>
+EDXAPP_WORKERS: !!null
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings

playbooks/roles/edxapp/templates/cms.conf.j2

@@ -6,11 +6,16 @@
 {% set executable = edxapp_venv_dir + '/bin/gunicorn' %}
 {% endif %}
 
-{% if ansible_processor|length > 0 %}
+{% if EDXAPP_WORKERS -%}
+command={{ executable }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ EDXAPP_WORKERS.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
+{% else -%}
+{# This is for backwards compatibility, set workers explicitely using EDXAPP_WORKERS #}
+{% if ansible_processor|length > 0 -%}
 command={{ executable }}  --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
-{% else %}
+{% else -%}
 command={{ executable }} --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
-{% endif %}
+{% endif -%}
+{% endif -%}
 
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}

playbooks/roles/edxapp/templates/lms.conf.j2

@@ -6,11 +6,16 @@
 {% set executable = edxapp_venv_dir + '/bin/gunicorn' %}
 {% endif %}
 
-{% if ansible_processor|length > 0 %}
+{% if EDXAPP_WORKERS -%}
+command={{ executable }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ EDXAPP_WORKERS.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
+{% else -%}
+{# This is for backwards compatibility, set workers explicitely using EDXAPP_WORKERS #}
+{% if ansible_processor|length > 0 -%}
 command={{ executable }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
-{% else %}
+{% else -%}
 command={{ executable }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
 {% endif %}
+{% endif %}
 
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}

playbooks/roles/ora/defaults/main.yml

@@ -76,7 +76,7 @@ ORA_AWS_SECRET_ACCESS_KEY: ''
 # Default nginx listen port
 # These should be overrided if you want
 # to serve all content on port 80
-ora_gunicorn_workers: 4
+ora_gunicorn_workers: 2
 ora_gunicorn_port: 8060
 ora_gunicorn_host: 127.0.0.1
 

playbooks/roles/xqueue/defaults/main.yml

@@ -31,6 +31,9 @@ XQUEUE_MYSQL_PASSWORD: 'password'
 XQUEUE_MYSQL_HOST: 'localhost'
 XQUEUE_MYSQL_PORT: '3306'
 XQUEUE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-xqueue"
+# Set the number of workers explicitely for xqueue
+XQUEUE_WORKERS: !!null
+XQUEUE_WORKERS_PER_QUEUE: 12
 
 # Internal vars below this line
 #############################################
@@ -51,7 +54,7 @@ xqueue_gunicorn_host: 127.0.0.1
 
 xqueue_env_config:
   XQUEUES: $XQUEUE_QUEUES
-  XQUEUE_WORKERS_PER_QUEUE: 12
+  XQUEUE_WORKERS_PER_QUEUE: $XQUEUE_WORKERS_PER_QUEUE
   LOGGING_ENV : $XQUEUE_LOGGING_ENV
   SYSLOG_SERVER: $XQUEUE_SYSLOG_SERVER
   LOG_DIR : "{{ COMMON_DATA_DIR }}/logs/xqueue"

playbooks/roles/xqueue/templates/xqueue.conf.j2

@@ -6,11 +6,15 @@
 {% set executable = xqueue_venv_bin + '/gunicorn' %}
 {% endif %}
 
+{% if XQUEUE_WORKERS -%}
+command={{ executable }} --preload -b {{ xqueue_gunicorn_host }}:{{ xqueue_gunicorn_port }} -w {{ XQUEUE_WORKERS }} --timeout=300 --pythonpath={{ xqueue_code_dir }} xqueue.wsgi
+{% else -%}
 {% if ansible_processor|length > 0 %}
 command={{ executable }} --preload -b {{ xqueue_gunicorn_host }}:{{ xqueue_gunicorn_port }} -w {{ ansible_processor|length * 2 }} --timeout=300 --pythonpath={{ xqueue_code_dir }} xqueue.wsgi
-{% else %}
+{% else -%}
 command={{ executable }} --preload -b {{ xqueue_gunicorn_host }}:{{ xqueue_gunicorn_port }} -w 2 --timeout=300 --pythonpath={{ xqueue_code_dir }} xqueue.wsgi
-{% endif %}
+{% endif -%}
+{% endif -%}
 
 user={{ common_web_user }}
 directory={{ xqueue_code_dir }}