Skip to content

C
configuration
Commit b67dba4e by e0d
Options

Merge pull request #208 from edx/e0d/notifier-updates 

E0d/notifier updates
parents 46bcaa04 41a73fa3
Showing with 373 additions and 54 deletions
playbooks/edx-east/edx_notifier.yml 0 → 100644
- name: Configure stage instance(s)
hosts: notifier_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure loadtest instance(s)
hosts: notifier_loadtest
sudo: True
vars_files:
- "{{ secure_dir }}/vars/loadtest/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure stage edge instance(s)
hosts: notifier_edge_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure prod instance(s)
hosts: notifier_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure edge prod instance(s)
hosts: notifier_edge_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
vars:
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
playbooks/edx_notifier.yml 0 → 100644
- name: Configure stage instance(s)
hosts: notifier_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure loadtest instance(s)
hosts: notifier_loadtest
sudo: True
vars_files:
- "{{ secure_dir }}/vars/loadtest/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure stage edge instance(s)
hosts: notifier_edge_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure prod instance(s)
hosts: notifier_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure edge prod instance(s)
hosts: notifier_edge_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
vars:
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
playbooks/roles/common/tasks/main.yml
---
- include: create_users.yml
- include: create_github_users.yml
when: github_users is defined
#- include: create_github_users.yml
# when: github_users is defined
- name: common | Add user www-data
# This user should be created on the system by default
......
playbooks/roles/notifier/files/opt/notifier/bin/forums_digest.sh 0 → 100644
#!/bin/bash
. $HOME/.bashrc
minutes=$1
digest_date=`date --utc '+%Y-%m-%dT%H:%MZ'`
cd /opt/wwc/notifier/src && /opt/wwc/notifier/virtualenvs/notifier/bin/python /opt/wwc/notifier/src/manage.py forums_digest --to_datetime=${digest_date} --minutes=${minutes}
playbooks/roles/notifier/handlers/main.yml
......@@ -3,6 +3,21 @@
##
## for future compliance, when the API comes on line.
##
- name: notifier | restart the notifier service
service: name=notifier state=restarted
- name: notifier | install notifier-celery-beat
supervisorctl: name=notifier-celery-beat state=present
- name: notifier | install notifier-celery-workers
supervisorctl: name=notifier-celery-workers state=present
- name: notifier | restart notifier
supervisorctl: name=notifier-celery-beat state=restarted
notify:
- notifier | install notifier-celery-workers
- notifier | install notifier-celery-beat
- name: notifier | restart notifier-celery-beat
supervisorctl: name=notifier-celery-beat state=restarted
- name: notifier | restart notifier-celery-workers
supervisorctl: name=notifier-celery-workers state=restarted
playbooks/roles/notifier/tasks/deploy.yml
......@@ -5,8 +5,11 @@
copy:
src=git_ssh.sh dest=/tmp/git_ssh.sh
force=yes owner=root group=adm mode=750
notify:
- notifier | restart notifier
tags:
- notifier
- deploy
- install
- update
......@@ -18,17 +21,27 @@
src={{ notifier_git_identity_path }} dest=/etc/git-identity
force=yes owner=ubuntu group=adm mode=60
tags:
- notifier
- deploy
- install
- update
- name: notifier | stop notifier-celery-beat
supervisorctl: name=notifier-celery-beat state=restarted
- name: notifier | stop notifier-celery-workers
supervisorctl: name=notifier-celery-workers state=restarted
- name: notifier | checkout code
git:
dest={{ notifier_code_dir }} repo={{ notifier_source_repo }}
version={{ notifier_version }}
environment:
GIT_SSH: /tmp/git_ssh.sh
notify:
- notifier | restart notifier
tags:
- notifier
- deploy
- install
- update
......@@ -41,6 +54,7 @@
path={{ notifier_code_dir }} state=directory owner={{ notifier_user }}
group={{ notifier_user }} mode=2750 recurse=yes
tags:
- notifier
- deploy
- install
- update
......@@ -51,6 +65,7 @@
- name: notifier | remove read-only ssh key for the content repo
file: path=/etc/git-identity state=absent
tags:
- notifier
- deploy
- install
- update
......@@ -61,6 +76,7 @@
- name: notifier | remove ssh script
file: path=/tmp/git_ssh.sh state=absent
tags:
- notifier
- deploy
- install
- update
......@@ -71,7 +87,23 @@
virtualenv="{{ notifier_venv_dir }}" state=present
sudo: true
sudo_user: "{{ notifier_user }}"
notify:
- notifier | restart notifier
tags:
- notifier
- deploy
- install
- update
\ No newline at end of file
- update
- name: notifier | syncdb
shell: >
cd {{ notifier_code_dir }} && {{ notifier_venv_dir }}/bin/python manage.py syncdb
sudo: true
sudo_user: "{{ notifier_user }}"
notify:
- notifier | restart notifier
tags:
- notifier
- deploy
- install
- update
playbooks/roles/notifier/tasks/main.yml
......@@ -24,9 +24,19 @@
- install
- update
- name: notifier | check if incommon ca is installed
command: test -e /usr/share/ca-certificates/incommon/InCommonServerCA.crt
register: incommon_present
ignore_errors: yes
tags:
- notifier
- install
- update
- name: common | create incommon ca directory
file:
path="/usr/share/ca-certificates/incommon" mode=2775 state=directory
when: incommon_present|failed
tags:
- notifier
- install
......@@ -35,6 +45,7 @@
- name: common | retrieve incommon server CA
shell: curl https://www.incommon.org/cert/repository/InCommonServerCA.txt -o /usr/share/ca-certificates/incommon/InCommonServerCA.crt
when: incommon_present|failed
tags:
- notifier
- install
......@@ -115,4 +126,46 @@
- install
- update
- name: notifier | create notifier/bin directory
file:
path="{{ notifier_home }}/bin" mode=2775 state=directory
tags:
- notifier
- install
- update
- name: notifier | make the script executable
file: path={{ notifier_home }}/bin/forums_digest.sh state=file mode=2755
tags:
- notifier
- install
- update
- name: common | create supervisor log directoy
file:
path={{notifier_supervisor_log_dest }} mode=2750 state=directory
tags:
- notifier
- install
- update
- ubuntu
- name: notifier | supervisord config for celery workers
template:
src=etc/supervisor/conf.d/notifier-celery-workers.conf.j2 dest=/etc/supervisor/conf.d/notifier-celery-workers.conf
notify: notifier | restart notifier-celery-workers
tags:
- notifier
- install
- update
- name: notifier | supervisord config for celery beat
template:
src=etc/supervisor/conf.d/notifier-celery-beat.conf.j2 dest=/etc/supervisor/conf.d/notifier-celery-beat.conf
notify: notifier | restart notifier-celery-beat
tags:
- notifier
- install
- update
- include: deploy.yml
\ No newline at end of file
playbooks/roles/notifier/templates/etc/supervisor/conf.d/notifier-celery-beat.conf.j2 0 → 100644
;
; {{ ansible_managed }}
;
[program:notifier-celery-beat]
command={{ notifier_venv_dir }}/bin/python manage.py celery beat -l DEBUG
process_name=%(program_name)s
numprocs=1
directory={{ notifier_code_dir }}
umask=022
autostart=true
autorestart=true
startsecs=10
startretries=3
exitcodes=0,2
stopsignal=TERM
stopwaitsecs=10
user=notifier
redirect_stderr=false
stdout_logfile={{ notifier_supervisor_log_dest }}/notifier-celery-beat-stdout.log
stdout_logfile_maxbytes=1MB
stdout_logfile_backups=10
stdout_capture_maxbytes=1MB
stderr_logfile={{notifier_supervisor_log_dest }}/notifier-celery-beat-stderr.log
stderr_logfile_maxbytes=1MB
stderr_logfile_backups=10
stderr_capture_maxbytes=1MB
environment=PID='/var/tmp/notifier-celery-beat.pid',LANG=en_US.UTF-8,
{%- for name,value in notifier_env_vars.items() -%}
{{name}}="{{value}}"{%- if not loop.last -%},{%- endif -%}
{%- endfor -%}
\ No newline at end of file
playbooks/roles/notifier/templates/etc/supervisor/conf.d/notifier-celery-workers.conf.j2 0 → 100644
;
; {{ ansible_managed }}
;
[program:notifier-celery-workers]
command={{ notifier_venv_dir }}/bin/python manage.py celery worker -l DEBUG
process_name=%(program_name)s
numprocs=1
directory={{ notifier_code_dir }}
umask=022
autostart=true
autorestart=true
startsecs=10
startretries=3
exitcodes=0,2
stopsignal=TERM
stopwaitsecs=10
user=notifier
redirect_stderr=false
stdout_logfile={{notifier_supervisor_log_dest }}/notifier-celery-workers-stdout.log
stdout_logfile_maxbytes=1MB
stdout_logfile_backups=10
stdout_capture_maxbytes=1MB
stderr_logfile={{notifier_supervisor_log_dest }}/notifier-celery-workers-stderr.log
stderr_logfile_maxbytes=1MB
stderr_logfile_backups=10
stderr_capture_maxbytes=1MB
environment=PID='/var/tmp/notifier-celery-workers.pid',LANG=en_US.UTF-8,
{%- for name,value in notifier_env_vars.items() -%}
{{name}}="{{value}}"{%- if not loop.last -%},{%- endif -%}
{%- endfor -%}
\ No newline at end of file
playbooks/roles/notifier/templates/notifier.conf.j2 deleted 100644 → 0
description "Notifier Service"
start on runlevel [2345]
stop on runlevel [!2345]
setuid {{ notifier_web_user }}
env PID=/var/tmp/notifier_service.pid
chdir {{ notifier_code_dir }}
exec {{ notifier_home }}/src/manage.py runserver
\ No newline at end of file
playbooks/roles/notifier/templates/notifier_env.j2
# Application Environment
export NOTIFIER_ENV="{{ notifier_env }}"
# {{ ansible_managed }}
# email settings independent of transport
export EMAIL_BACKEND="{{ notifier_email_backend }}"
export EMAIL_HOST="{{ notifier_email_host }}"
export EMAIL_PORT="{{ notifer_email_port }}"
export EMAIL_HOST_USER="{{ notifer_email_user }}"
export EMAIL_HOST_PASSWORD="{{ notifier_email_pass }}"
export EMAIL_DOMAIN="{{ notifier_email_domain }}"
export EMAIL_REWRITE_RECIPIENT="{{ notifer_email_rewrite_recipient }}"
# LMS links, images, etc
export LMS_URL_BASE="{{ notifier_lms_url_base }}"
export SECRET_KEY="{{ notifier_lms_secret_key }}"
# Comments Service Endpoint, for digest pulls
export CS_URL_BASE="{{ notifier_comment_service_base }}"
export CS_API_KEY="{{ notifier_comment_service_api_key }}"
# User Service Endpoint, for notification prefs
export US_URL_BASE="{{ notifier_user_service_base }}"
export US_API_KEY="{{ notifier_user_service_api_key }}"
# celery
export BROKER_URL="{{ notifier_celery_broker_url }}"
# have requests use the OS ca certs
export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
# basic auth credentils
export US_HTTP_AUTH_USER="{{ notifier_user_service_http_auth_user }}"
export US_HTTP_AUTH_PASS="{{ notifier_user_service_http_auth_pass }}"
{% for name,value in notifier_env_vars.items() %}
{% if value %}
export {{ name }}="{{ value }}"
{% endif %}
{% endfor %}
\ No newline at end of file
playbooks/roles/notifier/vars/main.yml
......@@ -9,6 +9,9 @@ notifier_code_dir: "{{ notifier_home }}/src"
notifier_version: "rc/digests"
notifier_git_identity_path: "{{ secure_dir }}/files/git-identity"
notifier_requirements_file: "{{ notifier_code_dir }}/requirements.txt"
notifier_log_level: "INFO"
notifier_rsyslog_enabled: "yes"
notifier_digest_task_interval: "5"
notifier_env: "Development"
......@@ -19,7 +22,6 @@ notifier_email_user: ""
notifier_email_pass: ""
notifier_email_host: ""
notifier_email_domain: "notifications.edx.org"
notifier_email_rewrite_recipient: ""
......@@ -27,7 +29,7 @@ notifier_lms_url_base: "http://localhost:8000"
notifier_lms_secret_key: "PUT_YOUR_SECRET_KEY_HERE"
notifier_comment_service_base: "http://localhost:4567"
2notifier_comment_service_api_key: "PUT_YOUR_API_KEY_HERE"
notifier_comment_service_api_key: "PUT_YOUR_API_KEY_HERE"
notifier_user_service_base: "http://localhost:8000"
notifier_user_service_api_key: "PUT_YOUR_API_KEY_HERE"
......@@ -35,6 +37,10 @@ notifier_user_service_http_auth_user: "guido"
notifier_user_service_http_auth_pass: "vanrossum"
notifier_celery_broker_url: "django://"
notifier_supervisor_log_dest: "/mnt/logs/supervisor"
notifer_requests_ca_bundle: "/etc/ssl/certs/ca-certificates.crt"
notifier_debian_pkgs:
- apparmor-utils
- build-essential
......@@ -43,4 +49,31 @@ notifier_debian_pkgs:
- gcc
- ipython
- pkg-config
- rsyslog
\ No newline at end of file
- rsyslog
- supervisor
#
# This structure is iterated over in order to build both
# the environment file for the notifier which is sourced
# into the applications's environment and for building
# the env variable for the supervisor job definition.
#
notifier_env_vars:
NOTIFIER_ENV: "{{ notifier_env }}"
EMAIL_BACKEND: "{{ notifier_email_backend }}"
EMAIL_DOMAIN: "{{ notifier_email_domain }}"
EMAIL_REWRITE_RECIPIENT: "{{ notifier_email_rewrite_recipient }}"
LMS_URL_BASE: "{{ notifier_lms_url_base }}"
SECRET_KEY: "{{ notifier_lms_secret_key }}"
CS_URL_BASE: "{{ notifier_comment_service_base }}"
CS_API_KEY: "{{ notifier_comment_service_api_key }}"
US_URL_BASE: "{{ notifier_user_service_base }}"
US_API_KEY: "{{ notifier_user_service_api_key }}"
DATADOG_API_KEY: "{{ notifier_dd_api_key }}"
LOG_LEVEL: "{{ notifier_log_level }}"
RSYSLOG_ENABLED: "{{ notifier_rsyslog_enabled }}"
BROKER_URL: "{{ notifier_celery_broker_url }}"
REQUESTS_CA_BUNDLE: "{{ notifer_requests_ca_bundle }}"
US_HTTP_AUTH_USER: "{{ notifier_user_service_http_auth_user }}"
US_HTTP_AUTH_PASS: "{{ notifier_user_service_http_auth_pass }}"
FORUM_DIGEST_TASK_INTERVAL: "{{ notifier_digest_task_interval }}"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment