switching edxapp and edxapp-workers over to supervisord

playbooks/edx-east/edx_continuous_integration.yml

@@ -12,20 +12,19 @@
       nginx_sites:
       - cms
       - lms
-      - lms-preview
-      - ora
-      - xqueue
-      - xserver
+#      - ora
+#      - xqueue
+#      - xserver
     - edxlocal
     - supervisor
-    - mongo
-    - edxapp
+#    - mongo
+#    - edxapp
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - { role: 'edxapp', celery_worker: True }
-    - oraclejdk
-    - elasticsearch
-    - forum
-    - { role: "xqueue", update_users: True }
-    - xserver
-    - ora
-    - discern
+#    - oraclejdk
+#    - elasticsearch
+#    - forum
+#    - { role: "xqueue", update_users: True }
+#    - xserver
+#    - ora
+#    - discern

playbooks/roles/discern/tasks/deploy.yml

@@ -69,8 +69,10 @@
 
 #Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
 #support virtualenvs as of this comment
-- name: discern | django syncdb, migrate, and collectstatic for discern
-  shell: "{{ discern_venv_dir }}/bin/python {{discern_code_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+- name: discern | django syncdb migrate and collectstatic for discern
+  shell: >
+    command="{{ discern_venv_dir }}/bin/python {{discern_code_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+      chdir={{ discern_code_dir }}
   sudo_user: "{{ discern_user }}"
   with_items:
     - syncdb
@@ -78,10 +80,11 @@
     - collectstatic
   tags:
     - deploy
-
 #Have this separate from the other three because it doesn't take the noinput flag
 - name: discern | django update_index for discern
-  shell: "{{ discern_venv_dir}}/bin/python {{discern_code_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+  shell: >
+    command="{{ discern_venv_dir}}/bin/python {{discern_code_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+      chdir={{ discern_code_dir }}
   sudo_user: "{{ discern_user }}"
   tags:
     - deploy

playbooks/roles/discern/tasks/main.yml

@@ -51,5 +51,5 @@
 - name: discern | create a symlink for venv python
   file: >
     src="{{ discern_venv_bin }}/python"
-    dest={{ cfg_dir }}/python.discern
+    dest={{ bin_dir }}/python.discern
     state=link

playbooks/roles/edxapp/defaults/main.yml

@@ -75,7 +75,7 @@ EDXAPP_CELERY_BROKER_HOSTNAME: ''
 EDXAPP_LOGGING_ENV:  'sandbox'
 
 EDXAPP_SYSLOG_SERVER: ''
-EDXAPP_RABBIT_HOSTNAME: 'rabbit.{{ENV_NAME}}.vpc.edx.org'
+EDXAPP_RABBIT_HOSTNAME: 'localhost'
 EDXAPP_XML_MAPPINGS: {}
 
 EDXAPP_LMS_NGINX_PORT: 80
@@ -110,6 +110,30 @@ edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles"
 edxapp_course_data_dir: "{{ edxapp_data_dir }}/data"
 edxapp_upload_dir: "{{ edxapp_data_dir }}/uploads"
 edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"
+edxapp_workers:
+  - queue: low
+    service_variant: cms
+    concurrency: 3
+  - queue: default
+    service_variant: cms
+    concurrency: 4
+  - queue: high
+    service_variant: cms
+    concurrency: 1
+  - queue: low
+    service_variant: lms
+    concurrency: 1
+  - queue: default
+    service_variant: lms
+    concurrency: 3
+  - queue: high
+    service_variant: lms
+    concurrency: 4
+
+
+
+
+
 
 # TODO: old style variable syntax is necessary
 # here until ansible 1.4
@@ -306,6 +330,7 @@ service_variants_enabled:
   - cms
 
 edxapp_lms_env: 'lms.envs.aws'
+edxapp_cms_env: 'cms.envs.aws'
 
 
 #Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores

playbooks/roles/edxapp/tasks/cms.yml

-# requires:
-#  - group_vars/all
-#  - common/tasks/main.yml
----
-- name: create cms application config
-  template: >
-    src=cms.env.json.j2
-    dest={{ edxapp_app_dir }}/cms.env.json
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
-
-- name: create cms auth file
-  template: >
-    src=cms.auth.json.j2
-    dest={{ edxapp_app_dir }}/cms.auth.json
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
-
-# Creates CMS upstart file
-- include: upstart.yml basename=cms
-  when: celery_worker is not defined
-
-- include: upstart.yml basename=edx-worker-cms
-  when: celery_worker is defined

playbooks/roles/edxapp/tasks/deploy.yml

 # Stop all services.
-- name: edxapp | stop edxapp services
-  service: name=edxapp state=stopped
+#
+
+- name: edxapp | stop the edxapp services (supervisor)
+  supervisorctl: >
+    name=edxapp
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
   when: celery_worker is not defined
   tags:
-  - lms
-  - lms-preview
-  - cms
   - deploy
 
 - name: edxapp | stop edx workers
@@ -14,15 +17,6 @@
   tags:
   - deploy
 
-# update json configs for the application
-
-- include: lms.yml
-  when: "'lms' in service_variants_enabled"
-- include: cms.yml
-  when: "'cms' in service_variants_enabled"
-- include: lms-preview.yml
-  when: "'lms-preview' in service_variants_enabled"
-
 
 # Do A Checkout
 - name: edxapp | checkout edx-platform repo into {{edxapp_code_dir}}
@@ -32,6 +26,7 @@
   tags:
   - deploy
 
+
 - name: edxapp | git clean after checking out edx-platform
   shell: cd {{edxapp_code_dir}} && git clean -xdf
   when: edx_platform_checkout.changed
@@ -168,54 +163,19 @@
   tags:
   - deploy
 
+# creates the supervisor jobs for the
+# service variants configured, runs
+# gather_assets and db migrations
+- include: service_variant_config.yml
 
-- name: edxapp | syncdb and migrate
-  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin}}/django-admin.py syncdb --migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-  - deploy
-
-- name: edxapp | db migrate
-  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
-  when: migrate_only is defined and migrate_only|lower == "yes"
-  tags:
-  - deploy
-
-# Gather lms assets using rake if possible
-- name: edxapp | gather lms static assets with rake
-  shell: SERVICE_VARIANT={{ edxapp_lms_variant }} rake lms:gather_assets:aws
-    executable=/bin/bash
-    chdir={{ edxapp_code_dir }}
-  notify:
-  - edxapp | restart edxapp
-  sudo_user: "{{ edxapp_user }}"
-  when: celery_worker is not defined
-  environment: "{{ edxapp_deploy_environment }}"
-  tags:
-  - deploy
-
-# Gather cms assets using rake if possible
-- name: edxapp | gather cms static assets with rake
-  shell: SERVICE_VARIANT={{ edxapp_cms_variant }} rake cms:gather_assets:aws
-    executable=/bin/bash
-    chdir={{ edxapp_code_dir }}
-  notify:
-  - edxapp | restart edxapp
-  sudo_user: "{{ edxapp_user }}"
-  when: celery_worker is not defined
-  environment: "{{ edxapp_deploy_environment }}"
-  tags:
-  - deploy
-
-
-- name: edxapp | restart edxapp
-  service: name=edxapp state=restarted
+- name: edxapp | restart the edxapp services (supervisor)
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} restart edxapp"
   when: celery_worker is not defined
   tags:
-  - deploy
+    - deploy
 
-- name: edxapp | restart workers
-  service: name=edx-workers state=restarted
+- name: edxapp | restart the celery worker services (supervisor)
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} restart edxapp_worker"
   when: celery_worker is defined
   tags:
-  - deploy
+    - deploy

playbooks/roles/edxapp/tasks/lms-preview.yml

-# requires:
-#  - group_vars/all
-#  - common/tasks/main.yml
----
-- name: create lms application config
-  template: >
-    src=lms-preview.env.json.j2
-    dest={{ edxapp_app_dir }}/lms-preview.env.json
-  tags:
-  - deploy
-
-- name: create lms auth file
-  template: >
-    src=lms-preview.auth.json.j2
-    dest={{ edxapp_app_dir }}/lms-preview.auth.json
-  tags:
-  - deploy
-
-# Creates LMS Preview upstart file
-- include: upstart.yml basename=lms-preview

playbooks/roles/edxapp/tasks/lms.yml

----
-- name: create lms application config
-  template: >
-    src=lms.env.json.j2
-    dest={{ edxapp_app_dir }}/lms.env.json
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
-
-- name: create lms auth file
-  template: >
-    src=lms.auth.json.j2
-    dest={{ edxapp_app_dir }}/lms.auth.json
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
-
-# Creates LMS upstart file
-- include: upstart.yml basename=lms
-  when: celery_worker is not defined
-
-- include: upstart.yml basename=edx-worker-lms
-  when: celery_worker is defined

playbooks/roles/edxapp/tasks/main.yml

@@ -46,33 +46,17 @@
 - name: edxapp | install system packages on which LMS and CMS rely
   apt: pkg={{','.join(edxapp_debian_pkgs)}} state=present
 
-- name: edxapp | creating edxapp upstart script
-  sudo: True
-  template: >
-    src=edxapp.conf.j2 dest=/etc/init/edxapp.conf
-    owner=root group=root
-  when: "celery_worker is not defined"
-
-- name: edxapp | create edx-workers upstart script
-  template: >
-    src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf
-    owner=root group=root
-  when: "celery_worker is defined"
-
 - name: edxapp | create log directories for service variants
   file: >
     path={{ edxapp_log_dir }}/{{ item }} state=directory
     owner={{ common_log_user }} group={{ common_log_user }}
     mode=0750
-  with_items:
-    - lms
-    - cms
-    - lms-preview
+  with_items: service_variants_enabled
 
 - include: deploy.yml
 
 - name: edxapp | create a symlink for venv python
   file: >
     src="{{ edxapp_venv_bin }}/python"
-    dest={{ cfg_dir }}/python.edxapp
+    dest={{ bin_dir }}/python.edxapp
     state=link

playbooks/roles/edxapp/tasks/service_variant_config.yml

+- name: edxapp | create edx-workers upstart script
+  template: >
+    src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf
+    owner=root group=root
+  when: celery_worker is defined
+
+
+- name: "create {{ item }} application config"
+  template: >
+    src={{ item }}.env.json.j2
+    dest={{ edxapp_app_dir }}/{{ item }}.env.json
+  sudo_user: "{{ edxapp_user }}"
+  with_items: service_variants_enabled
+  tags:
+  - deploy
+
+- name: "create {{ item }} auth file"
+  template: >
+    src={{ item }}.auth.json.j2
+    dest={{ edxapp_app_dir }}/{{ item }}.auth.json
+  sudo_user: "{{ edxapp_user }}"
+  with_items: service_variants_enabled
+  tags:
+  - deploy
+
+# write the supervisor script for {{ service_variant }}
+# for non-celery servers
+
+- name: "writing {{ item }} supervisor script"
+  template: >
+    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  with_items: service_variants_enabled
+  when: celery_worker is not defined
+  notify: supervisor | reload supervisor
+  tags:
+  - deploy
+
+- name: "writing edxapp supervisor script"
+  template: >
+    src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  when: celery_worker is not defined
+  tags:
+  - deploy
+
+
+# write the supervisor script for celery workers
+
+- name: writing celery worker supervisor script
+  template: >
+    src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  when: celery_worker is defined
+  tags:
+  - deploy
+
+- name: supervisor | reload supervisor
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} reload"
+
+# Gather assets using rake if possible
+
+- name: edxapp | gather {{ item }} static assets with rake
+  shell: >
+    command=SERVICE_VARIANT={{ item }} rake {{ item }}:gather_assets:aws
+    executable=/bin/bash
+    chdir={{ edxapp_code_dir }}
+  sudo_user: "{{ edxapp_user }}"
+  when: celery_worker is not defined
+  with_items: service_variants_enabled
+  environment: "{{ edxapp_deploy_environment }}"
+  tags:
+  - deploy
+
+
+- name: edxapp | syncdb and migrate
+  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin}}/django-admin.py syncdb --migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  when: migrate_db is defined and migrate_db|lower == "yes"
+  tags:
+  - deploy
+
+- name: edxapp | db migrate
+  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  when: migrate_only is defined and migrate_only|lower == "yes"
+  tags:
+  - deploy
+
+

playbooks/roles/edxapp/tasks/upstart.yml

-# write the gunicorn upstart script for {{ service_variant }}
-- name: writing {{ basename }} upstart script to /etc/init
-  sudo: True
-  template: src={{ basename }}.conf.j2 dest=/etc/init/{{ basename }}.conf owner=root group=root
-  tags:
-  - upstart
-  - gunicorn
-  - update
-  - deploy

playbooks/roles/edxapp/templates/cms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-description "cms gunicorn"
-# CMS Upstart Script
-start on started edxapp
-stop on stopped edxapp
- 
-respawn
-respawn limit 3 30
- 
-env PID=/var/tmp/cms.pid
+[program:cms]
 {% if ansible_processor|length > 0 %}
-env WORKERS={{ ansible_processor|length * worker_core_mult.cms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b 127.0.0.1:{{ edxapp_cms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
 {% else %}
-env WORKERS={{ worker_core_mult.cms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b 127.0.0.1:{{ edxapp_cms_gunicorn_port }} -w {{ worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
 {% endif %}
-env PORT={{edxapp_cms_gunicorn_port}}
-env ADDRESS={{edxapp_cms_gunicorn_host}}
-env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE=cms.envs.aws
-env SERVICE_VARIANT="cms"
- 
-chdir {{edxapp_code_dir}}
-setuid www-data
 
-exec {{edxapp_venv_dir}}/bin/gunicorn_django -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edxapp_code_dir}} --settings=cms.envs.aws
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+environment=PORT={{edxapp_cms_gunicorn_port}},ADDRESS={{edxapp_cms_gunicorn_host}},LANG=en_US.UTF-8,DJANGO_SETTINGS_MODULE={{ edxapp_cms_env }},SERVICE_VARIANT="cms"
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log

playbooks/roles/edxapp/templates/edxapp.conf.j2

-#/etc/init/edxapp.conf
-
-description "Starts and stops multiple edX services, e.g., lms, cms, etc., installed in a stacked configuration."
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-##
-## Each awaited service is responsible for ensuring that it is ready
-## for service when it returns.
-##
-pre-start script
-
-{% if 'lms' in service_variants_enabled %}
-if [ -e /etc/init/lms.conf ]; then
-  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-{% if 'lms-preview' in service_variants_enabled %}
-if [ -e /etc/init/lms-preview.conf ]; then
-  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-{% if 'cms' in service_variants_enabled %}
-if [ -e /etc/init/cms.conf ]; then
-  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-end script
-
-script
-  # Noop process for other edX components to take their
-  # marching orders from.  In the edxapp deployment, 
-  # lms, cms, etc. will listen for this process to start
-  # and stop and follow suit.
-  while true
-    do
-      logger -t $0 "edX App Shell Daemon is running..."
-      sleep 600
-    done
-end script
-
-pre-stop script
-
-{% if 'lms' in service_variants_enabled %}
-if [ -e /etc/init/lms.conf ]; then
-  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-{% if 'lms-preview' in service_variants_enabled %}
-if [ -e /etc/init/lms-preview.conf ]; then
-  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-{% if 'cms' in service_variants_enabled %}
-if [ -e /etc/init/cms.conf ]; then
-  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-end script
-
+[group:edxapp]
+programs={{ ",".join(service_variants_enabled) }}

playbooks/roles/edxapp/templates/lms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-start on started edxapp
-stop on stopped edxapp
-
-respawn
-respawn limit 3 30
-
-env PID=/var/tmp/lms.pid
+[program:lms]
 {% if ansible_processor|length > 0 %}
-env WORKERS={{ ansible_processor|length * worker_core_mult.lms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b 127.0.0.1:{{ edxapp_lms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
 {% else %}
-env WORKERS={{ worker_core_mult.lms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b 127.0.0.1:{{ edxapp_lms_gunicorn_port }} -w {{ worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
 {% endif %}
-env PORT={{edxapp_lms_gunicorn_port}}
-env ADDRESS={{edxapp_lms_gunicorn_host}}
-env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }}
-env SERVICE_VARIANT="lms"
-
-chdir {{edxapp_code_dir}}
-setuid www-data
-
-exec {{edxapp_venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edxapp_code_dir}} lms.wsgi
 
-post-start script
-  while true
-  do
-    if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then
-      break;
-    else
-      sleep 1;
-    fi
-  done
-end script
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+environment=PORT={{edxapp_lms_gunicorn_port}},ADDRESS={{edxapp_lms_gunicorn_host}},LANG=en_US.UTF-8,DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }},SERVICE_VARIANT="lms"
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log

playbooks/roles/edxapp/templates/workers.conf.j2

+{% for w in edxapp_workers %}
+[program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
+
+environment=CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE=aws,PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }}
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+
+command={{ edxapp_venv_bin}}/python {{ edxapp_code_dir }}/manage.py {{ w.service_variant }} --settings=aws celery worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.`hostname` --concurrency={{ w.concurrency }}
+
+{% endfor %}
+
+[group:edxapp_worker]
+programs={%- for w in edxapp_workers %}{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}{%- if not loop.last %},{%- endif %}{%- endfor %}

playbooks/roles/forum/tasks/deploy.yml

 ---
 
 - name: forum | stop the forum service
-  service: name=cs_comments_service state=stopped
+  supervisorctl: >
+    name=forum
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
   tags:
   - deploy
 
@@ -11,9 +15,8 @@
     dest={{ forum_supervisor_wrapper }}
     mode=0755
   sudo_user: "{{ forum_user }}"
-  notify:
-    - forum | restart the forum service
-
+  tags:
+  - deploy
 
 - name:  forum | git checkout forum repo into {{ forum_code_dir }}
   git: dest={{ forum_code_dir }} repo={{ forum_source_repo }} version={{ forum_version }}

playbooks/roles/forum/tasks/main.yml

@@ -43,6 +43,7 @@
   template: >
     src=forum.conf.j2 dest={{ supervisor_cfg_dir }}/forum.conf
     owner={{ common_web_user }} group={{ supervisor_user }}
+    mode=0644
   register: forum_supervisor
 
 - include: deploy.yml

playbooks/roles/nginx/defaults/main.yml

 # Variables for nginx role
 ---
+
+nginx_app_dir: "{{ app_dir }}/nginx"
+nginx_data_dir: "{{ data_dir }}/nginx"
+nginx_conf_dir: "{{ app_dir }}/conf.d"
+nginx_log_dir: "{{ log_dir }}/nginx"
+nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available"
+nginx_sites_enabled_dir: "{{ nginx_app_dir }}/sites-enabled"
+nginx_user: root
+
 pkgs:
   nginx:
     state: installed

playbooks/roles/nginx/tasks/main.yml

 # requires:
 #   - common/tasks/main.yml
 ---
+
+- name: nginx | create nginx app dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ nginx_user }}"
+    group="{{ common_web_group }}"
+  with_items:
+    - "{{ nginx_app_dir }}"
+    - "{{ nginx_sites_available_dir }}"
+    - "{{ nginx_sites_enabled_dir }}"
+  notify: nginx | restart nginx
+
+- name: nginx | create nginx data dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ common_web_user }}"
+    group="{{ nginx_user }}"
+  with_items:
+    - "{{ nginx_data_dir }}"
+    - "{{ nginx_log_dir }}"
+  notify: nginx | restart nginx
+
 - name: nginx | Install nginx
   apt: pkg=nginx state={{ pkgs.nginx.state }}
   notify: nginx | restart nginx
 
 - name: nginx | Server configuration file
-  copy: >
-    src={{secure_dir}}/files/nginx.conf dest=/etc/nginx/nginx.conf
+  template: >
+    src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
     owner=root group=root mode=0644
-  when: nginx_conf is defined
   notify: nginx | reload nginx
 
 - name: nginx | Creating common nginx configuration
   template: >
-    src=edx-release.j2 dest=/etc/nginx/sites-available/edx-release
+    src=edx-release.j2 dest={{ nginx_sites_available_dir }}/edx-release
     owner=root group=root mode=0600
   notify: nginx | reload nginx
 
 - name: nginx | Creating link for common nginx configuration
   file: >
-    src=/etc/nginx/sites-available/edx-release dest=/etc/nginx/sites-enabled/edx-release
+    src={{ nginx_sites_available_dir }}/edx-release
+    dest={{ nginx_sites_enabled_dir }}/edx-release
     state=link owner=root group=root
   notify: nginx | reload nginx
 
 - name: nginx | Copying nginx configs for {{ nginx_sites }}
   template: >
-    src={{ item }}.j2 dest=/etc/nginx/sites-available/{{ item }}
+    src={{ item }}.j2 dest={{ nginx_sites_available_dir }}/{{ item }}
     owner=root group=root mode=0600
   notify: nginx | reload nginx
   with_items: nginx_sites
@@ -34,23 +58,24 @@
 - name: nginx | Creating nginx config links for {{ nginx_sites }}
   file: >
     src=/etc/nginx/sites-available/{{ item  }}
-    dest=/etc/nginx/sites-enabled/{{ item }} state=link owner=root group=root
+    dest={{ nginx_sites_enabled_dir }}/{{ item }}
+    state=link owner=root group=root
   notify: nginx | reload nginx
   with_items: nginx_sites
 
 - name: nginx | Write out default htpasswd file
   copy: >
-    content={{ nginx_cfg.htpasswd }} dest=/etc/nginx/nginx.htpasswd
+    content={{ nginx_cfg.htpasswd }} dest={{ nginx_app_dir }}/nginx.htpasswd
     owner=www-data group=www-data mode=0600
 
 - name: nginx | Create nginx log file location (just in case)
   file: >
-    path={{log_dir}}/nginx state=directory
-    owner=syslog group=syslog mode=2770 recurse=yes
+    path={{ nginx_log_dir}} state=directory
+    owner={{ common_web_user }} group={{ common_web_user }}
 
 # removing default link
 - name: nginx | Removing default nginx config and restart (enabled)
-  file: path=/etc/nginx/sites-enabled/default state=absent
+  file: path={{ nginx_sites_enabled_dir }}/default state=absent
   notify: nginx | reload nginx
 
 # Note that nginx logs to /var/log until it reads its configuration, so /etc/logrotate.d/nginx is still good
@@ -65,10 +90,6 @@
     dest=/etc/logrotate.d/nginx-error src=edx_logrotate_nginx_error.j2
     owner=root group=root mode=644
 
-- name: nginx | Removing default nginx config (available)
-  file: path=/etc/nginx/sites-available/default state=absent
-  notify: nginx | reload nginx
-
 # If tasks that notify restart nginx don't change the state of the remote system
 # their corresponding notifications don't get run.  If nginx has been stopped for
 # any reason, this will ensure that it is started up again.

playbooks/roles/nginx/templates/basic-auth.j2

      auth_basic            "Restricted";
-     auth_basic_user_file  /etc/nginx/nginx.htpasswd;
+     auth_basic_user_file  {{ nginx_app_dir }}/nginx.htpasswd;
      index index.html
      proxy_set_header X-Forwarded-Proto https;

playbooks/roles/nginx/templates/cms.j2

@@ -11,8 +11,8 @@ server {
 
   server_name studio.*;
 
-  access_log {{log_dir}}/nginx/access.log;
-  error_log {{log_dir}}/nginx/error.log error;
+  access_log {{ nginx_log_dir }}/access.log;
+  error_log {{ nginx_log_dir }}/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots. 
   # CMS requires larger value for course assest, values provided 

playbooks/roles/nginx/templates/edx_logrotate_nginx_access.j2

 # Put in place by ansible
 
-{{log_dir}}/nginx/access.log {
+{{ nginx_log_dir }}/nginx/access.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/edx_logrotate_nginx_error.j2

 # Put in place by ansible
 
-{{log_dir}}/nginx/error.log {
+{{ nginx_log_dir }}/nginx/error.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/lms.j2

@@ -9,8 +9,8 @@ server {
 
   listen {{EDXAPP_LMS_NGINX_PORT}} default;
 
-  access_log {{log_dir}}/nginx/access.log;
-  error_log {{log_dir}}/nginx/error.log error;
+  access_log {{ nginx_log_dir }}/access.log;
+  error_log {{ nginx_log_dir }}/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots.
   # CMS requires larger value for course assest, values provided

playbooks/roles/nginx/templates/nginx.conf.j2

+user www-data;
+worker_processes 4;
+pid /var/run/nginx.pid;
+
+events {
+        worker_connections 768;
+        # multi_accept on;
+}
+
+http {
+
+        ##
+        # Basic Settings
+        ##
+
+        sendfile on;
+        tcp_nopush on;
+        tcp_nodelay on;
+        keepalive_timeout 65;
+        types_hash_max_size 2048;
+        # increase header buffer for for https://edx-wiki.atlassian.net/browse/LMS-467&gt
+        # see http://orensol.com/2009/01/18/nginx-and-weird-400-bad-request-responses/
+        large_client_header_buffers 4 16k;
+        # server_tokens off;
+
+        # server_names_hash_bucket_size 64;
+        # server_name_in_redirect off;
+
+        include /etc/nginx/mime.types;
+        default_type application/octet-stream;
+
+        ##
+        # Logging Settings
+        ##
+
+        log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user [$time_local]  '
+                            '"$request" $status $body_bytes_sent $request_time '
+                            '"$http_referer" "$http_user_agent"';
+        access_log {{ nginx_log_dir }}/access.log p_combined;
+        error_log {{ nginx_log_dir }}/error.log;
+
+        ##
+        # Gzip Settings
+        ##
+
+        gzip on;
+        gzip_disable "msie6";
+
+        gzip_vary on;
+        gzip_proxied any;
+        gzip_comp_level 6;
+        gzip_buffers 16 8k;
+        gzip_http_version 1.1;
+        gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+        ##
+        # Virtual Host Configs
+        ##
+
+        include {{ nginx_sites_enabled_dir }}/*.conf;
+        include {{ nginx_conf_dir }}/*;
+}
+

playbooks/roles/ora/tasks/main.yml

@@ -31,4 +31,4 @@
 - name: ora | create a symlink for venv python
   file: >
     src="{{ ora_venv_bin }}/python"
-    dest={{ cfg_dir }}/python.ora state=link
+    dest={{ bin_dir }}/python.ora state=link

playbooks/roles/supervisor/tasks/main.yml

@@ -84,7 +84,7 @@
 - name: supervisor | create a symlink for supervisor cfg
   file: >
     src={{ supervisor_cfg_dir }}
-    dest={{ cfg_dir }}/supervisor.{{ supervisor_cfg_dir|basename }}
+    dest={{ bin_dir }}/supervisor.{{ supervisor_cfg_dir|basename }}
     state=link
 
 

playbooks/roles/xqueue/tasks/main.yml

@@ -15,7 +15,7 @@
 
 - name: xqueue | create xqueue app and venv dir
   file: >
-    path="{{ xqueue_app_dir }}"
+    path="{{ item }}"
     state=directory
     owner="{{ xqueue_user }}"
     group="{{ common_web_group }}"
@@ -51,5 +51,5 @@
 - name: xqueue | create a symlink for venv python
   file: >
     src="{{ xqueue_venv_bin }}/python"
-    dest={{ cfg_dir }}/python.xqueue
+    dest={{ bin_dir }}/python.xqueue
     state=link

playbooks/roles/xserver/tasks/main.yml

@@ -68,6 +68,6 @@
 - name: xserver | create a symlink for venv python
   file: >
     src="{{ xserver_venv_bin }}/python"
-    dest={{ cfg_dir }}/python.xserver
+    dest={{ bin_dir }}/python.xserver
     state=link