add forum notifier to prod and stage workers

removing git accounts for notifier repo, open now

add datadog api key Since this file will be the model for others
to use for their vars files, put in a placeholder DD api key, since
it's required to be *something* currently.

celerybeat wants to write its pidfile to ~notifier/src.  Not sure why
the PID or CELERYD_PID_FILE environment variables don't correctly point
it to /var/run/... but in the meantime, this should get us by.

notifier: remove git_ssh wrapper not needed now that repo is public

run notifier on first util machine only

remove references to forums_digest.sh. File was a vestige of prior
code when this was run from cron instead of celery beat.

playbooks/edx-west/prod-worker.yml

-# this gets all running prod webservers
+# For all util machines
 - hosts: tag_environment_prod:&tag_function_util
 # or we can get subsets of them by name
 #- hosts: ~tag_Name_util(1|2)_prod
@@ -17,3 +17,19 @@
   roles:
     - common
     - { role: 'edxapp', celery_worker: True }
+
+# run the notifier on the first util machine only
+- hosts: ~tag_Name_util1_prod
+  sudo: True
+  vars:
+    secure_dir: '../../../configuration-secure/ansible'
+    migrate_db: "no"
+  vars_files:
+    - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
+    - "{{ secure_dir }}/vars/notifier_prod_vars.yml"
+  roles:
+    - role: virtualenv
+      virtualenv_user: "notifier"
+      virtualenv_user_home: "/opt/wwc/notifier"
+      virtualenv_name: "notifier"
+    - notifier

playbooks/edx-west/stage-worker.yml

@@ -5,6 +5,8 @@
   sudo: True
   vars:
     secure_dir: ../../../edx-secret/ansible
+    # this indicates the path to site-specific (with precedence)
+    # things like nginx template files
     local_dir: ../../../edx-secret/ansible/local
     migrate_db: "no"
   vars_files:
@@ -14,3 +16,19 @@
   roles:
     - common
     - { role: 'edxapp', celery_worker: True }
+
+# run the notifier on the first util machine only
+- hosts: ~tag_Name_util1_stage
+  sudo: True
+  vars:
+    secure_dir: '../../../configuration-secure/ansible'
+    migrate_db: "no"
+  vars_files:
+    - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/vars/notifier_stage_vars.yml"
+  roles:
+    - role: virtualenv
+      virtualenv_user: "notifier"
+      virtualenv_user_home: "/opt/wwc/notifier"
+      virtualenv_name: "notifier"
+    - notifier

playbooks/roles/notifier/defaults/main.yml

@@ -41,6 +41,8 @@ notifier_supervisor_log_dest: "/mnt/logs/supervisor"
 
 notifer_requests_ca_bundle: "/etc/ssl/certs/ca-certificates.crt"
 
+notifier_dd_api_key: "NOT_USED"    # data dog
+
 notifier_debian_pkgs:
   - apparmor-utils
   - build-essential

playbooks/roles/notifier/files/git_ssh.sh

-#!/bin/sh
-exec /usr/bin/ssh -o StrictHostKeyChecking=no -i /etc/git-identity "$@"

playbooks/roles/notifier/files/opt/notifier/bin/forums_digest.sh

-#!/bin/bash
-
-. $HOME/.bashrc
-
-minutes=$1
-digest_date=`date --utc '+%Y-%m-%dT%H:%MZ'`
-
-cd /opt/wwc/notifier/src && /opt/wwc/notifier/virtualenvs/notifier/bin/python /opt/wwc/notifier/src/manage.py forums_digest --to_datetime=${digest_date} --minutes=${minutes}

playbooks/roles/notifier/tasks/deploy.yml

-#
-# TODO: Needed while this repo is private
-#
-- name: notifier | upload ssh script
-  copy: 
-    src=git_ssh.sh dest=/tmp/git_ssh.sh 
-    force=yes owner=root group=adm mode=750
-  notify:
-    - notifier | restart notifier
-  tags:
-  - notifier
-  - deploy
-  - install
-  - update
-
-#
-# TODO: Needed while this repo is private
-#
-- name: notifier | install read-only ssh key required for checkout
-  copy: 
-    src={{ notifier_git_identity_path }} dest=/etc/git-identity 
-    force=yes owner=ubuntu group=adm mode=60
-  tags:
-  - notifier
-  - deploy
-  - install
-  - update
-
+---
 - name: notifier | stop notifier-celery-beat
   supervisorctl: name=notifier-celery-beat state=restarted
+  ignore_errors: yes
 
 - name: notifier | stop notifier-celery-workers
   supervisorctl: name=notifier-celery-workers state=restarted
+  ignore_errors: yes
 
 - name: notifier | checkout code
   git: 
     dest={{ notifier_code_dir }} repo={{ notifier_source_repo }} 
     version={{ notifier_version }}
-  environment:
-    GIT_SSH: /tmp/git_ssh.sh
   notify:
     - notifier | restart notifier
   tags:
@@ -46,40 +19,14 @@
     - install
     - update
 
-#
-# TODO: Needed while this repo is private
-#
-- name: notifier | update src permissions
+- name: notifier | source repo group perms
   file: 
-    path={{ notifier_code_dir }} state=directory owner={{ notifier_user }} 
-    group={{ notifier_user }} mode=2750 recurse=yes
-  tags:
-  - notifier
-  - deploy
-  - install
-  - update
-
-#
-# TODO: Needed while this repo is private
-#
-- name: notifier | remove read-only ssh key for the content repo
-  file: path=/etc/git-identity state=absent
-  tags:
-  - notifier
-  - deploy
-  - install
-  - update
-
-#
-# TODO: Needed while this repo is private
-#
-- name: notifier | remove ssh script
-  file: path=/tmp/git_ssh.sh state=absent
+    path={{ notifier_source_repo }} mode=2775 state=directory
   tags:
-  - notifier
-  - deploy
-  - install
-  - update
+    - notifier
+    - deploy
+    - install
+    - update
 
 - name: notifier | install application requirements
   pip: 

playbooks/roles/notifier/tasks/main.yml

@@ -96,7 +96,6 @@
     owner={{ notifier_user }} 
     group={{ notifier_user }}
 
-
 - name: notifier | ensure .bashrc exists
   shell: touch {{ notifier_home }}/.bashrc
   sudo: true  
@@ -126,16 +125,17 @@
   - install
   - update
 
-- name: notifier | create notifier/bin directory
-  file: 
-    path="{{ notifier_home }}/bin" mode=2775 state=directory
+- name: notifier | create notifier/db directory
+  file:
+    path="{{ notifier_home }}/db" mode=2775 state=directory
   tags:
   - notifier
   - install
   - update
 
-- name: notifier | make the script executable
-  file: path={{ notifier_home }}/bin/forums_digest.sh state=file mode=2755
+- name: notifier | create notifier/bin directory
+  file: 
+    path="{{ notifier_home }}/bin" mode=2775 state=directory
   tags:
   - notifier
   - install
@@ -168,4 +168,4 @@
   - install
   - update
 
-- include: deploy.yml
+- include: deploy.yml
\ No newline at end of file