Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
configuration
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
OpenEdx
configuration
Commits
334e7cda
Commit
334e7cda
authored
Nov 14, 2013
by
Joe Blaylock
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #445 from edx/jrbl/datadog-loghandler
Stanford-oriented Datadog updates
parents
eea60f9f
a05b70db
Show whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
206 additions
and
7 deletions
+206
-7
playbooks/edx-west/prod-app.yml
+1
-1
playbooks/edx-west/prod-jumpbox.yml
+1
-1
playbooks/edx-west/prod-ora.yml
+1
-1
playbooks/edx-west/prod-worker.yml
+1
-1
playbooks/edx-west/prod-xqueue.yml
+2
-2
playbooks/roles/datadog/files/usr/share/datadog/agent/nginx_log_http_response_counter.py
+100
-0
playbooks/roles/datadog/handlers/main.yml
+0
-0
playbooks/roles/datadog/tasks/main.yml
+63
-0
playbooks/roles/datadog/templates/nginx.yaml.j2
+13
-0
playbooks/roles/nginx/tasks/main.yml
+1
-1
playbooks/roles/nginx/templates/ora.j2
+12
-0
playbooks/roles/nginx/templates/xqueue.j2
+11
-0
No files found.
playbooks/edx-west/prod-app.yml
View file @
334e7cda
...
...
@@ -24,7 +24,7 @@
-
"
{{
secure_dir
}}/vars/users.yml"
-
"
{{
secure_dir
}}/vars/edxapp_prod_users.yml"
-
"
{{
secure_dir
}}/vars/shib_prod_vars.yml"
-
"
{{
secure_dir
}}/vars/datadog_
stage
.yml"
-
"
{{
secure_dir
}}/vars/datadog_
prod
.yml"
roles
:
-
common
-
role
:
nginx
...
...
playbooks/edx-west/prod-jumpbox.yml
View file @
334e7cda
...
...
@@ -2,7 +2,7 @@
sudo
:
True
vars_files
:
-
"
{{
secure_dir
}}/vars/users_jumpbox.yml"
-
"
{{
secure_dir
}}/vars/datadog_
stage
.yml"
-
"
{{
secure_dir
}}/vars/datadog_
prod
.yml"
vars
:
secure_dir
:
'
../../../configuration-secure/ansible'
local_dir
:
'
../../../configuration-secure/ansible/local'
...
...
playbooks/edx-west/prod-ora.yml
View file @
334e7cda
...
...
@@ -15,7 +15,7 @@
-
"
{{
secure_dir
}}/vars/ora_prod_vars.yml"
-
"
{{
secure_dir
}}/vars/users.yml"
-
"
{{
secure_dir
}}/vars/edxapp_prod_users.yml"
-
"
{{
secure_dir
}}/vars/datadog_
stage
.yml"
-
"
{{
secure_dir
}}/vars/datadog_
prod
.yml"
roles
:
-
common
-
role
:
nginx
...
...
playbooks/edx-west/prod-worker.yml
View file @
334e7cda
...
...
@@ -14,7 +14,7 @@
-
"
{{
secure_dir
}}/vars/users.yml"
-
"
{{
secure_dir
}}/vars/edxapp_prod_users.yml"
-
"
{{
secure_dir
}}/vars/shib_prod_vars.yml"
-
"
{{
secure_dir
}}/vars/datadog_
stage
.yml"
-
"
{{
secure_dir
}}/vars/datadog_
prod
.yml"
roles
:
-
common
-
{
role
:
'
edxapp'
,
celery_worker
:
True
}
...
...
playbooks/edx-west/prod-xqueue.yml
View file @
334e7cda
# this gets all running prod webservers
#- hosts: tag_environment_prod:&tag_function_xqueue
# or we can get subsets of them by name
-
hosts
:
~tag_Name_xqueue1
1
_prod
-
hosts
:
~tag_Name_xqueue1
0
_prod
#- hosts: security_group_edx-prod-EdxappServerSecurityGroup-NSKCQTMZIPQB
sudo
:
True
vars
:
...
...
@@ -13,7 +13,7 @@
-
"
{{
secure_dir
}}/vars/xqueue_prod_vars.yml"
-
"
{{
secure_dir
}}/vars/users.yml"
-
"
{{
secure_dir
}}/vars/edxapp_prod_users.yml"
-
"
{{
secure_dir
}}/vars/datadog_
stage
.yml"
-
"
{{
secure_dir
}}/vars/datadog_
prod
.yml"
roles
:
-
common
-
role
:
nginx
...
...
playbooks/roles/datadog/files/usr/share/datadog/agent/nginx_log_http_response_counter.py
0 → 100755
View file @
334e7cda
"""Consumes nginx access logs and emits counts by response code
Run from datadog's 'dogstreams' functionality.
"""
import
re
import
statsd
#SHORT_HOSTNAME = '.'.join(socket.gethostname().split('.')[:2])
MONTHS_LOOKUP
=
{
'Jan'
:
1
,
'Feb'
:
2
,
'Mar'
:
3
,
'Apr'
:
4
,
'May'
:
5
,
'Jun'
:
6
,
'Jul'
:
7
,
'Aug'
:
8
,
'Sep'
:
9
,
'Oct'
:
10
,
'Nov'
:
11
,
'Dec'
:
12
}
RETURN_RE
=
re
.
compile
(
r'^(?:.+?) \[(?P<day>\d\d)\/(?P<month>\w\w\w)\/(?P<year>\d\d\d\d):(?P<hour>\d\d):(?P<minute>\d\d):(?P<second>\d\d) (?:[-+]\d+)\] "(?P<url>.+?)" (?P<value>\d\d\d) (?:.+)'
)
def
count_event_example
(
logger
,
line
):
"""Do not use this method as-is; it emits events too fast.
Events are supposed to be special things that happen periodically, rather
than common things we want to count a lot of. But here's a straightforward
example that shows how to emit events based on the contents of a log file."""
return
None
import
time
from
datetime
import
datetime
e
=
None
m
=
RETURN_RE
.
match
(
line
)
if
m
and
m
.
group
(
'url'
)
!=
'GET /heartbeat HTTP/1.1'
:
e
=
{
'msg_text'
:
line
,
'priority'
:
'low'
}
e
[
'timestamp'
]
=
int
(
time
.
mktime
(
datetime
(
month
=
MONTHS_LOOKUP
[
m
.
group
(
'month'
)],
year
=
int
(
m
.
group
(
'year'
)),
day
=
int
(
m
.
group
(
'day'
)),
hour
=
int
(
m
.
group
(
'hour'
)),
minute
=
int
(
m
.
group
(
'minute'
)),
second
=
int
(
m
.
group
(
'second'
)))
.
timetuple
()))
e
[
'msg_title'
]
=
'nginx HTTP '
+
m
.
group
(
'value'
)
e
[
'event_type'
]
=
'http.'
+
m
.
group
(
'value'
)
return
e
def
count
(
logger
,
line
):
"""Side-effectually count HTTP response codes by code and class.
Inspired by http://docs.datadoghq.com/guides/logs/ - this doesn't do
normal log event processing, but instead uses the dogstats API to do
simple counting. It's desirable to do it this way because we get folded
into the agent's daemonization, and don't have to write our own or rely
on cron."""
m
=
RETURN_RE
.
match
(
line
)
if
m
!=
None
:
statsd
.
statsd
.
increment
(
'nginx.response.'
+
m
.
group
(
'value'
)
+
'.count'
,
1
)
statsd
.
statsd
.
increment
(
'nginx.response.'
+
m
.
group
(
'value'
)[
0
]
+
'xx.count'
,
1
)
return
None
### Inspired by http://docs.datadoghq.com/guides/logs/
def
test
():
test_suite
=
[
(
'''10.0.0.65 - - [12/Nov/2013:06:30:53 +0000] "-" 400 0 "-" "-"'''
,
{
'msg_text'
:
'10.0.0.65 - - [12/Nov/2013:06:30:53 +0000] "-" 400 0 "-" "-"'
,
'priority'
:
'low'
,
'timestamp'
:
1384266653
,
'msg_title'
:
'400'
,
'event_type'
:
'400'
,
}),
(
'''10.0.0.65 - - [12/Nov/2013:06:30:54 +0000] "GET /heartbeat HTTP/1.1" 200 10097 "-" "ELB-HealthChecker/1.0"'''
,
{
'msg_text'
:
'10.0.0.65 - - [12/Nov/2013:06:30:54 +0000] "GET /heartbeat HTTP/1.1" 200 10097 "-" "ELB-HealthChecker/1.0"'
,
'priority'
:
'low'
,
'timestamp'
:
1384266654
,
'msg_title'
:
'200'
,
'event_type'
:
'200'
,
}),
(
'''10.0.0.65 - stanford [12/Nov/2013:19:35:30 +0000] "GET /courses/Education/EDUC115N/How_to_Learn_Math/courseware/627b094444a1487db5c1b3caaef096cf/d8d40562b7ec40789315e2ccefa5cc5b/ HTTP/1.1" 200 11219 "https://stage.class.stanford.edu/courses/Education/EDUC115N/How_to_Learn_Math/courseware/627b094444a1487db5c1b3caaef096cf/d8d40562b7ec40789315e2ccefa5cc5b/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"'''
,
{
'msg_text'
:
'10.0.0.65 - stanford [12/Nov/2013:19:35:30 +0000] "GET /courses/Education/EDUC115N/How_to_Learn_Math/courseware/627b094444a1487db5c1b3caaef096cf/d8d40562b7ec40789315e2ccefa5cc5b/ HTTP/1.1" 200 11219 "https://stage.class.stanford.edu/courses/Education/EDUC115N/How_to_Learn_Math/courseware/627b094444a1487db5c1b3caaef096cf/d8d40562b7ec40789315e2ccefa5cc5b/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"'
,
'priority'
:
'low'
,
'timestamp'
:
1384313730
,
'msg_title'
:
'200'
,
'event_type'
:
'200'
,
}),
(
'''10.0.0.65 - stanford [12/Nov/2013:19:36:54 +0000] "GET /jsi18n/ HTTP/1.1" 200 2170 "https://stage.class.stanford.edu/courses/Education/EDUC115N/How_to_Learn_Math/courseware/b5c2c03d98274010bdb655afa2eaed31/1e8b3bccf4c34f79b2e43ae64cd1f54c/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"'''
,
{
'msg_text'
:
'10.0.0.65 - stanford [12/Nov/2013:19:36:54 +0000] "GET /jsi18n/ HTTP/1.1" 200 2170 "https://stage.class.stanford.edu/courses/Education/EDUC115N/How_to_Learn_Math/courseware/b5c2c03d98274010bdb655afa2eaed31/1e8b3bccf4c34f79b2e43ae64cd1f54c/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"'
,
'priority'
:
'low'
,
'timestamp'
:
1384313814
,
'msg_title'
:
'200'
,
'event_type'
:
'200'
,
}),
]
# Set up the test logger
import
logging
logging
.
basicConfig
(
level
=
logging
.
DEBUG
)
for
pair
in
test_suite
:
actual
=
count_event_example
(
logging
,
pair
[
0
])
assert
pair
[
1
]
==
actual
,
"
%
s !=
%
s"
%
(
pair
[
1
],
actual
)
print
'test passes'
if
__name__
==
'__main__'
:
# For local testing, callable as "python /path/to/parsers.py"
test
()
playbooks/roles/datadog/handlers/main.yml
View file @
334e7cda
playbooks/roles/datadog/tasks/main.yml
View file @
334e7cda
...
...
@@ -45,11 +45,73 @@
-
ubuntu
when
:
ansible_distribution in common_debian_variants
-
name
:
datadog | datadog user to syslog group for log reading
user
:
append=yes groups=syslog name=dd-agent state=present
tags
:
-
datadog
-
ubuntu
-
name
:
datadog | install dogstatsd library
pip
:
name=dogstatsd-python state=present
tags
:
-
datadog
-
name
:
datadog | install log watcher scripts
copy
:
src=usr/share/datadog/agent/nginx_log_http_response_counter.py
dest=/usr/share/datadog/agent/nginx_log_http_response_counter.py
owner=root group=root mode=0644
tags
:
-
datadog
-
name
:
datadog | bootstrap config
shell
:
cp /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf creates=/etc/dd-agent/datadog.conf
tags
:
-
datadog
-
name
:
datadog | set config permissions
file
:
path=/etc/dd-agent/datadog.conf state=file owner=dd-agent group=root mode=0640
tags
:
-
datadog
-
name
:
datadog | enable nginx module
template
:
dest=/etc/dd-agent/conf.d/nginx.yaml src=nginx.yaml.j2 owner=root group=root mode=0644
notify
:
-
datadog | restart the datadog service
when
:
nginx_sites is defined
tags
:
-
datadog
# quoting intentional, missing space after line=api_key: also
# ansible wasn't handling the double quoted yaml properly
# otherwise.
-
name
:
datadog | set hostname if unset
lineinfile
:
dest="/etc/dd-agent/datadog.conf"
"line=hostname:{{ansible_hostname}}.{{environment_tag}}"
state=present
"regexp=^\#?\s*hostname:.+$"
notify
:
-
datadog | restart the datadog service
tags
:
-
datadog
# quoting intentional, missing space after line=api_key: also
# ansible wasn't handling the double quoted yaml properly
# otherwise.
-
name
:
datadog | update dogstreams
lineinfile
:
dest="/etc/dd-agent/datadog.conf"
"line=dogstreams:{{log_base_dir}}/nginx/access.log:nginx_log_http_response_counter:count"
state=present
insertafter=EOF
"regexp=^\#?\s*dogstreams:.+$"
when
:
nginx_sites is defined
notify
:
-
datadog | restart the datadog service
tags
:
-
datadog
# quoting intentional, missing space after line=api_key: also
# ansible wasn't handling the double quoted yaml properly
# otherwise.
...
...
@@ -62,3 +124,4 @@
-
datadog | restart the datadog service
tags
:
-
datadog
playbooks/roles/datadog/templates/nginx.yaml.j2
0 → 100644
View file @
334e7cda
init_config:
instances:
# For every instance, you have an `nginx_status_url` and (optionally)
# a list of tags.
- nginx_status_url: http://127.0.0.1/nginx_status/
tags:
- instance:{{ansible_hostname}}.{{environment_tag}}-nginx
#- nginx_status_url: http://example2.com:1234/nginx_status/
# tags:
# - instance:bar
playbooks/roles/nginx/tasks/main.yml
View file @
334e7cda
...
...
@@ -71,7 +71,7 @@
-
logging
-
update
-
name
:
nginx | Set up nginx
access
log rotation
-
name
:
nginx | Set up nginx
error
log rotation
template
:
dest=/etc/logrotate.d/nginx-error src=edx_logrotate_nginx_error.j2 owner=root group=root mode=644
tags
:
-
logging
...
...
playbooks/roles/nginx/templates/ora.j2
View file @
334e7cda
...
...
@@ -7,6 +7,9 @@ upstream app_server {
server {
listen {{ ORA_NGINX_PORT }} default_server;
access_log {{log_base_dir}}/nginx/access.log;
error_log {{log_base_dir}}/nginx/error.log error;
location / {
{% if ORA_BASIC_AUTH %}
...
...
@@ -43,4 +46,13 @@ server {
proxy_redirect off;
proxy_pass http://app_server;
}
# Monitoring support for datadog.
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1/32;
deny all;
}
}
playbooks/roles/nginx/templates/xqueue.j2
View file @
334e7cda
...
...
@@ -7,6 +7,9 @@ upstream xqueue_app_server {
server {
listen {{ XQUEUE_NGINX_PORT }} default_server;
access_log {{log_base_dir}}/nginx/access.log;
error_log {{log_base_dir}}/nginx/error.log error;
location / {
try_files $uri @proxy_to_app;
}
...
...
@@ -16,6 +19,14 @@ server {
try_files $uri @proxy_to_app;
}
# Monitoring support for datadog.
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1/32;
deny all;
}
location @proxy_to_app {
proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment