Merge pull request #3298 from edx/bilalahmad99/ntp_alert

adding logs if ntp is out of sync for [OPS-1395]

playbooks/roles/common/tasks/main.yml

@@ -158,3 +158,25 @@
     state: restarted
   become: True
   when: config_templates.changed
+
+- name: Add ntp alert script
+  template:
+    src: "log-ntp-alerts.sh.j2"
+    dest: "{{ COMMON_BIN_DIR }}/log-ntp-alerts.sh"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Set up a cron job to run the script
+  cron:
+    name: "log-ntp-alerts"
+    job: "{{ COMMON_BIN_DIR }}/log-ntp-alerts.sh >/dev/null 2>&1"
+
+- name: install logrotate configuration
+  template:
+    src: etc/logrotate.d/ntp.j2
+    dest: /etc/logrotate.d/ntp
+  tags:
+    - "install"
+    - "install:configuration"
+    - "logrotate"

playbooks/roles/common/templates/etc/logrotate.d/ntp.j2

+{{ COMMON_LOG_DIR }}/ntp.log {
+  compress
+  dateext
+  dateformat -%Y%m%d-%s
+  missingok
+  daily
+  rotate 3
+}

playbooks/roles/common/templates/log-ntp-alerts.sh.j2

+#!/bin/bash
+
+log_directory={{ COMMON_LOG_DIR }}
+reach=$(ntpq -c associations | awk '{print $5}' | grep yes)
+if [[ ${reach} == *"no"* ]]; then
+    echo $(date -u) $(hostname) "NTPD not synchronized - Please investigate" >> ${log_directory}/ntp.log
+fi
+
+limit=100   # limit in milliseconds
+offsets=$(ntpq -nc peers | tail -n +3 | cut -c 62-66 | tr -d '-')
+for offset in ${offsets}; do
+    if [ ${offset:-0} -ge ${limit:-100} ]; then
+        echo $(date -u) $(hostname) "An NTPD offset with value $offset is excessive - Please investigate" >> ${log_directory}/ntp.log
+        exit 1
+    fi
+done