adding logs if ntp is out of sync for [OPS-1395]

playbooks/roles/ecommerce/defaults/main.yml

@@ -204,6 +204,7 @@ ECOMMERCE_GUNICORN_WORKER_CLASS: "gevent"
 #
 ecommerce_service_name: "ecommerce"
 ecommerce_user: "{{ ecommerce_service_name }}"
+ecommerce_group: "{{ ecommerce_service_name }}"
 ecommerce_home: "{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}"
 ecommerce_code_dir: "{{ ecommerce_home }}/{{ ecommerce_service_name }}"
 ecommerce_venv_dir: "{{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}"

playbooks/roles/ecommerce/tasks/main.yml

@@ -226,3 +226,25 @@
   tags:
     - manage
     - manage:start
+
+- name: Add ntp alert script
+  template:
+    src: "edx/app/ecommerce/log-ntp-alerts.sh.j2"
+    dest: "{{ ecommerce_home }}/log-ntp-alerts.sh"
+    owner: "{{ ecommerce_user }}"
+    group: "{{ ecommerce_group }}"
+    mode: "0755"
+
+- name: Set up a cron job to run the script
+  cron:
+    name: "log-ntp-alerts"
+    job: "{{ ecommerce_home }}/log-ntp-alerts.sh >/dev/null 2>&1"
+
+- name: install logrotate configuration
+  template:
+    src: etc/logrotate.d/ecommerce.j2
+    dest: /etc/logrotate.d/ecommerce
+  tags:
+    - "install"
+    - "install:configuration"
+    - "logrotate"

playbooks/roles/ecommerce/templates/edx/app/ecommerce/log-ntp-alerts.sh.j2

+#!/bin/bash
+
+log_directory={{ ecommerce_log_dir }}
+reach=$(ntpq -c associations | awk '{print $5}' | grep yes)
+if [[ ${reach} == *"no"* ]]; then
+    echo "NTPD not synchronized - Please investigate" >> ${log_directory}/ntp.log
+fi
+
+limit=100   # limit in milliseconds
+offsets=$(ntpq -nc peers | tail -n +3 | cut -c 62-66 | tr -d '-')
+for offset in ${offsets}; do
+    if [ ${offset:-0} -ge ${limit:-100} ]; then
+        echo "An NTPD offset is excessive - Please investigate" >> ${log_directory}/ntp.log
+        exit 1
+    fi
+done

playbooks/roles/ecommerce/templates/etc/logrotate.d/ecommerce.j2

+{{ ecommerce_log_dir }}/*.log {
+  compress
+  dateext
+  dateformat -%Y%m%d-%s
+  missingok
+  daily
+  rotate 3
+}