Conditionally install workers and start them.

playbooks/roles/edxapp/tasks/collect_static.yml

-# This check needs to be run to see if rake can be used but its failure should not stop the run.
-- name: check if rake gather_assets is available
-  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} rake -T | grep gather_assets
-  environment: "{{ deploy_environment }}"
-  register: grep_gather_assets
-  ignore_errors: yes
-  tags:
-  - lms
-  - lms-preview
-  - lms-xml
-  - cms
-  - deploy
-
-- name: check if django can collect lms static data
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws
-  register: check_lms_collect_static
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  ignore_errors: yes
-  tags:
-  - lms
-  - lms-preview
-  - lms-xml
-  - deploy
-
-- name: check if django can collect cms static data
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
-  register: check_cms_collect_static
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  ignore_errors: yes
-  tags:
-  - cms
-  - deploy
-
-- name: check if django can update cms templates
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py help update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
-  register: check_cms_update_templates
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  ignore_errors: yes
-  tags:
-  - cms
-  - deploy
-
 # Gather lms assets using rake if possible
 - name: gather lms static assets with rake
   shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws

playbooks/roles/edxapp/tasks/deploy.yml

@@ -8,6 +8,11 @@
   - cms
   - deploy
 
+- name: stop edx workers
+  service: name=edx-workers state=stopped
+  tags:
+  - deploy
+
 # Do A Checkout
 - name: git checkout edx-platform repo into $app_base_dir
   git: dest={{edx_platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
@@ -121,8 +126,56 @@
   - install
   - deploy
 
+# This check needs to be run to see if rake can be used but its failure should not stop the run.
+- name: check if rake gather_assets is available
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} rake -T | grep gather_assets
+  environment: "{{ deploy_environment }}"
+  register: grep_gather_assets
+  ignore_errors: yes
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - cms
+  - deploy
+
+- name: check if django can collect lms static data
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws
+  register: check_lms_collect_static
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  ignore_errors: yes
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+- name: check if django can collect cms static data
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
+  register: check_cms_collect_static
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  ignore_errors: yes
+  tags:
+  - cms
+  - deploy
+
+- name: check if django can update cms templates
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py help update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
+  register: check_cms_update_templates
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  ignore_errors: yes
+  tags:
+  - cms
+  - deploy
+
 - include: collect_static.yml
-  when: celery_workers is not defined
+  when: celery_worker is not defined
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for the local mysql5.5, not sure if this fix will ever get merged
@@ -144,9 +197,16 @@
 
 - name: restart edxapp
   service: name=edxapp state=restarted
+  when: celery_worker is not defined
   tags:
   - lms
   - lms-xml
   - lms-preview
   - cms
   - deploy
+
+- name: restart workers
+  service: name=edx-workers state=restarted
+  when: celery_worker is defined
+  tags:
+  - deploy

playbooks/roles/edxapp/tasks/main.yml

@@ -48,11 +48,19 @@
 - name: creating edxapp upstart script
   sudo: True
   template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root
+  when: "celery_worker is not defined"
   tags:
   - upstart
   - gunicorn
   - update
 
+- name: create edx-workers upstart script
+  template: src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf owner=root group=root
+  when: "celery_worker is defined"
+  tags:
+  - upstart
+  - update
+
 - include: npm.yml
 - include: ruby.yml
 - include: deploy.yml

playbooks/roles/edxapp/templates/edx-workers.conf.j2

+# edx workers
+# managed by puppet
+
+description "start edX app workers"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+pre-start script
+
+   start cms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=cms
+   start cms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=cms
+   start cms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=cms
+
+   start lms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=lms
+   start lms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=lms
+   start lms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=lms
+
+   start lms-xml QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=lms-xml
+   start lms-xml QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=lms-xml
+   start lms-xml QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=lms-xml
+
+end script