Merge pull request #164 from edx/feanil/ansible_workers_rebase

Feanil/ansible workers rebase

playbooks/edx-east/feanil_deploy.yml

@@ -8,6 +8,7 @@
     - common
     - nginx
     - edxapp
+    - { role: 'edxapp', celery_worker: True }
 
 - hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_xserver
   sudo: True

playbooks/edx_sandbox.yml

@@ -26,3 +26,5 @@
     - nginx
     - edxlocal
     - edxapp
+    - rabbitmq
+    - { role: 'edxapp', celery_worker: True }

playbooks/roles/edxapp/tasks/cms.yml

@@ -18,8 +18,10 @@
   - update
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms
+  when: celery_worker is not defined
   
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms-backend
+  when: celery_worker is not defined
 
 - name: Create CMS log target directory
   file: path={{log_base_dir}}/cms state=directory owner=syslog group=adm mode=2770
@@ -29,5 +31,9 @@
   - logging
   - update
 
-# Creates LMS upstart file
-- include: upstart.yml service_variant=cms
+# Creates CMS upstart file
+- include: upstart.yml basename=cms
+  when: celery_worker is not defined
+
+- include: upstart.yml basename=edx-worker-cms
+  when: celery_worker is defined

playbooks/roles/edxapp/tasks/collect_static.yml

+# Gather lms assets using rake if possible
+- name: gather lms static assets with rake
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
+  when: grep_gather_assets.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+# Gather lms assets using django if necessary(When rake doesn't know how)
+- name: gather lms static assets with django
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+
+# Gather cms assets using rake if possible
+- name: gather cms static assets with rake
+#  script: gather_assets.sh
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
+  when: grep_gather_assets.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+- name: gather cms static assets with django
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+- name: update cms templates
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
+  when: check_cms_update_templates.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+# Add failure checks for if no static assets were deployed.
+- name: lms asset static failure check
+  shell: /bin/false
+  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc != 0
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+- name: cms asset static failure check
+  shell: /bin/false
+  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc != 0
+  tags:
+  - cms
+  - deploy
+

playbooks/roles/edxapp/tasks/deploy.yml

 # Stop all services.
 - name: stop edxapp services
   service: name=edxapp state=stopped
+  when: celery_worker is not defined
   tags:
   - lms
   - lms-xml
@@ -8,6 +9,12 @@
   - cms
   - deploy
 
+- name: stop edx workers
+  service: name=edx-workers state=stopped
+  when: celery_worker is defined
+  tags:
+  - deploy
+
 # Do A Checkout
 - name: git checkout edx-platform repo into $app_base_dir
   git: dest={{edx_platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
@@ -169,91 +176,8 @@
   - cms
   - deploy
 
-# Gather lms assets using rake if possible
-- name: gather lms static assets with rake
-  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
-  when: grep_gather_assets.rc == 0
-  notify:
-  - restart edxapp
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  tags:
-  - lms
-  - lms-preview
-  - lms-xml
-  - deploy
-
-# Gather lms assets using django if necessary(When rake doesn't know how)
-- name: gather lms static assets with django
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
-  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc == 0
-  notify:
-  - restart edxapp
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  tags:
-  - lms
-  - lms-preview
-  - lms-xml
-  - deploy
-
-
-# Gather cms assets using rake if possible
-- name: gather cms static assets with rake
-#  script: gather_assets.sh
-  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
-  when: grep_gather_assets.rc == 0
-  notify:
-  - restart edxapp
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  tags:
-  - cms
-  - deploy
-
-- name: gather cms static assets with django
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
-  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc == 0
-  notify:
-  - restart edxapp
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  tags:
-  - cms
-  - deploy
-
-- name: update cms templates
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
-  when: check_cms_update_templates.rc == 0
-  notify:
-  - restart edxapp
-  sudo: yes
-  sudo_user: www-data
-  environment: "{{ deploy_environment }}"
-  tags:
-  - cms
-  - deploy
-
-# Add failure checks for if no static assets were deployed.
-- name: lms asset static failure check
-  shell: /bin/false
-  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc != 0
-  tags:
-  - lms
-  - lms-preview
-  - lms-xml
-  - deploy
-
-- name: cms asset static failure check
-  shell: /bin/false
-  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc != 0
-  tags:
-  - cms
-  - deploy
+- include: collect_static.yml
+  when: celery_worker is not defined
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for the local mysql5.5, not sure if this fix will ever get merged
@@ -275,9 +199,16 @@
 
 - name: restart edxapp
   service: name=edxapp state=restarted
+  when: celery_worker is not defined
   tags:
   - lms
   - lms-xml
   - lms-preview
   - cms
   - deploy
+
+- name: restart workers
+  service: name=edx-workers state=restarted
+  when: celery_worker is defined
+  tags:
+  - deploy

playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -24,8 +24,10 @@
   - update
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-preview
+  when: celery_worker is not defined
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-preview-backend
+  when: celery_worker is not defined
 
 # Creates LMS Preview upstart file
-- include: upstart.yml service_variant=lms-preview
+- include: upstart.yml basename=lms-preview

playbooks/roles/edxapp/tasks/lms-xml.yml

@@ -26,8 +26,14 @@
   - update
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml
+  when: celery_worker is not defined
   
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml-backend
+  when: celery_worker is not defined
 
 # Creates upstart file
-- include: upstart.yml service_variant=lms-xml
+- include: upstart.yml basename=lms-xml
+  when: celery_worker is not defined
+
+- include: upstart.yml basename=edx-worker-lms-xml
+  when: celery_worker is defined

playbooks/roles/edxapp/tasks/lms.yml

@@ -22,6 +22,14 @@
   - update
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
+  when: celery_worker is not defined
+
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
+  when: celery_worker is not defined
+
 # Creates LMS upstart file
-- include: upstart.yml service_variant=lms
+- include: upstart.yml basename=lms
+  when: celery_worker is not defined
+
+- include: upstart.yml basename=edx-worker-lms
+  when: celery_worker is defined

playbooks/roles/edxapp/tasks/main.yml

@@ -48,11 +48,19 @@
 - name: creating edxapp upstart script
   sudo: True
   template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root
+  when: "celery_worker is not defined"
   tags:
   - upstart
   - gunicorn
   - update
 
+- name: create edx-workers upstart script
+  template: src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf owner=root group=root
+  when: "celery_worker is defined"
+  tags:
+  - upstart
+  - update
+
 - include: npm.yml
 - include: ruby.yml
 - include: deploy.yml

playbooks/roles/edxapp/tasks/upstart.yml

 # write the gunicorn upstart script for {{ service_variant }}
-- name: writing {{ service_variant }} upstart script to /etc/init
+- name: writing {{ basename }} upstart script to /etc/init
   sudo: True
-  template: src={{ service_variant }}.conf.j2 dest=/etc/init/{{ service_variant }}.conf owner=root group=root
+  template: src={{ basename }}.conf.j2 dest=/etc/init/{{ basename }}.conf owner=root group=root
   tags:
   - upstart
   - gunicorn

playbooks/roles/edxapp/templates/cms.conf.j2

 # gunicorn
 # Templated and placed by ansible from jinja2 source
- 
-description "gunicorn server"
-author "Calen Pennington <cpennington@mitx.mit.edu>"
- 
+description "cms gunicorn"
+# CMS Upstart Script
 start on started edxapp
 stop on stopped edxapp
  

playbooks/roles/edxapp/templates/edx-worker-cms.conf.j2

+# gunicorn
+# Templated and placed by ansible from jinja2 source
+# CMS Celery Worker Upstart Script
+description "cms celery worker"
+
+stop on stopping edx-workers
+
+respawn
+
+instance edx.${SERVICE_VARIANT}.core.${QUEUE}
+
+#env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
+#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
+env CONCURRENCY=${CONCURRENCY}
+env LOGLEVEL=info
+env DJANGO_SETTINGS_MODULE=cms.envs.aws
+env PYTHONPATH={{edx_platform_code_dir}}
+env SERVICE_VARIANT=${SERVICE_VARIANT}
+
+setuid www-data
+
+chdir {{edx_platform_code_dir}}
+
+exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-worker-lms-xml.conf.j2

+# gunicorn
+# Templated and placed by ansible from jinja2 source
+# lms-xml Celery Worker Upstart Script
+description "cms celery worker"
+
+stop on stopping edx-workers
+
+respawn
+
+instance edx.${SERVICE_VARIANT}.core.${QUEUE}
+
+#env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
+#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
+env CONCURRENCY=${CONCURRENCY}
+env LOGLEVEL=info
+env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env PYTHONPATH={{edx_platform_code_dir}}
+env SERVICE_VARIANT=${SERVICE_VARIANT}
+
+setuid www-data
+
+chdir {{edx_platform_code_dir}}
+
+exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-worker-lms.conf.j2

+# gunicorn
+# Templated and placed by ansible from jinja2 source
+# CMS Celery Worker Upstart Script
+description "lms celery worker"
+
+stop on stopping edx-workers
+
+respawn
+
+instance edx.${SERVICE_VARIANT}.core.${QUEUE}
+
+#env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
+#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
+env CONCURRENCY=${CONCURRENCY}
+env LOGLEVEL=info
+env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env PYTHONPATH={{edx_platform_code_dir}}
+env SERVICE_VARIANT=${SERVICE_VARIANT}
+
+setuid www-data
+
+chdir {{edx_platform_code_dir}}
+
+exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-workers.conf.j2

+# edx workers
+# managed by puppet or ansible
+
+description "start edX app workers"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+pre-start script
+
+   {% if 'cms' in service_variants_enabled %}
+   start edx-worker-cms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=cms
+   start edx-worker-cms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=cms
+   start edx-worker-cms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=cms
+   {% endif %}
+
+   {% if 'lms' in service_variants_enabled %}
+   start edx-worker-lms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=lms
+   start edx-worker-lms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=lms
+   start edx-worker-lms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=lms
+   {% endif %}
+
+   {% if 'lms-xml' in service_variants_enabled %}
+   start edx-worker-lms-xml QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=lms-xml
+   start edx-worker-lms-xml QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=lms-xml
+   start edx-worker-lms-xml QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=lms-xml
+   {% endif %}
+
+end script

playbooks/roles/edxapp/templates/lms-xml.conf.j2

 # gunicorn
-
-description "gunicorn server"
-author "Calen Pennington <cpennington@mitx.mit.edu>"
+# Templated and placed by ansible from jinja2 source
+description "lms-xml gunicorn server"
 
 start on started edxapp
 stop on stopped edxapp
@@ -39,4 +38,3 @@ post-start script
     fi
   done
 end script
-

playbooks/roles/edxapp/templates/lms.conf.j2

 # gunicorn
-
-description "gunicorn server"
-author "Calen Pennington <cpennington@mitx.mit.edu>"
-
+# Templated and placed by ansible from jinja2 source
 start on started edxapp
 stop on stopped edxapp
 
@@ -39,4 +36,3 @@ post-start script
     fi
   done
 end script
-