Stanford dosa merge WIP/more install tuning

More tuning to recipes to get an integrated worker stack to install
properly under the new régime.

playbooks/edx-west/stage-worker.yml

@@ -9,18 +9,28 @@
     migrate_db: "no"
   vars_files:
     - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
-    - "{{ secure_dir }}/vars/users.yml"
-    - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
+    - "{{ secure_dir }}/vars/xqueue_stage_vars.yml"
+    #- "{{ secure_dir }}/vars/users.yml"
+    #- "{{ secure_dir }}/vars/edxapp_stage_users.yml"
     - "{{ secure_dir }}/vars/datadog_stage.yml"
   roles:
-    - common
-    - supervisor
+    - role: 'common'
+      tags: common
+    - role: 'supervisor'
+      tags: supervisor
     - role: 'edxapp'
       celery_worker: True 
       tags: celery
+    - role: nginx
+      nginx_sites:
+      - xqueue
+      - basic-auth
+      tags: nginx
     - role: xqueue
+      xqueue_create_db: "no"
       tags: xqueue
-    - datadog
+    - role: 'datadog'
+      tags: datadog
 
 - name: Deploy certs, notifier only to first util machine
   hosts: ~tag_Name_util1_stage
@@ -35,10 +45,10 @@
     - "{{ secure_dir }}/vars/notifier_stage_vars.yml"
   roles:
     - certs
-    - role: virtualenv
-      virtualenv_user: "notifier"
-      virtualenv_user_home: "/opt/wwc/notifier"
-      virtualenv_name: "notifier"
+#    - role: virtualenv
+#      virtualenv_user: "notifier"
+#      virtualenv_user_home: "/opt/wwc/notifier"
+#      virtualenv_name: "notifier"
     - notifier
 
 #- name: Deploy certs to first util machine

playbooks/roles/datadog/tasks/main.yml

@@ -119,7 +119,7 @@
 - name: datadog | update dogstreams
   lineinfile:
     dest="/etc/dd-agent/datadog.conf" 
-    "line=dogstreams:{{log_base_dir}}/nginx/access.log:nginx_log_http_response_counter:count"
+    "line=dogstreams:{{nginx_log_dir}}/access.log:nginx_log_http_response_counter:count"
     state=present 
     insertafter=EOF 
     "regexp=^\#?\s*dogstreams:.+$"

playbooks/roles/edxapp/handlers/main.yml

@@ -5,7 +5,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
-  when: not devstack
+  when: not devstack and not celery_worker
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
   tags: deploy

playbooks/roles/edxapp/tasks/deploy.yml

@@ -298,7 +298,7 @@
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
   sudo_user: "{{ supervisor_service_user }}"
-  when: not devstack
+  when: not devstack and not celery_worker
   with_items: service_variants_enabled
   tags: deploy
 

playbooks/roles/nginx/templates/edx_logrotate_nginx_access.j2

 # Put in place by ansible
 
-{{log_base_dir}}/nginx/access.log {
+{{ nginx_log_dir }}/access.log {
   create 0640 www-data syslog
   compress
   delaycompress

playbooks/roles/nginx/templates/ora.j2

@@ -7,9 +7,6 @@ upstream app_server {
 server {
   listen {{ ORA_NGINX_PORT }} default_server;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
-
   location / {
 
     {% if  ORA_BASIC_AUTH %}
@@ -46,13 +43,4 @@ server {
     proxy_redirect off;
     proxy_pass http://app_server;
   }
-
-# Monitoring support for datadog.
-  location /nginx_status {
-    stub_status on;
-    access_log off;
-    allow 127.0.0.1/32;
-    deny all;
-  }
-
 }

playbooks/roles/nginx/templates/xqueue.j2

@@ -7,10 +7,13 @@ upstream xqueue_app_server {
 server {
   listen {{ XQUEUE_NGINX_PORT }} default_server;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
+  access_log {{nginx_log_dir}}/access.log;
+  error_log {{nginx_log_dir}}/error.log error;
 
   location / {
+    {% if  XQUEUE_BASIC_AUTH %}
+        {% include "basic-auth.j2" %}
+    {% endif %}
     try_files $uri @proxy_to_app;
   }
 

playbooks/roles/xqueue/templates/xqueue.auth.json.j2

-{{ xqueue_auth_config | to_nice_json }}
+{# ansible template system bug causes this play to blow up if this template #}
+{# lacks whitespace. we don't know why, but this fixes it. #}
+ {{ xqueue_auth_config | to_nice_json }}