Add support for hipchat notification from pre-supervisor script.

playbooks/roles/supervisor/defaults/main.yml

@@ -13,6 +13,11 @@
 ---
 SUPERVISOR_HTTP_BIND_IP: '127.0.0.1'
 
+# Used by the pre-supervisor script if you want to
+# notify a hipchat room with the output.
+SUPERVISOR_HIPCHAT_API_KEY: !!null
+SUPERVISOR_HIPCHAT_ROOM: default
+
 # do not override the bind_port since
 # all supervisors will then try to listen
 # on the same one
@@ -35,3 +40,7 @@ supervisor_cfg: "{{ supervisor_app_dir }}/supervisord.conf"
 # upstart service name and user
 supervisor_service: supervisor
 supervisor_service_user: "{{ common_web_user }}"
+
+supervisor_pip_pkgs:
+  - boto
+  - python-simple-hipchat

playbooks/roles/supervisor/files/pre_supervisor_checks.py

@@ -4,6 +4,8 @@ import boto
 import boto.utils
 import os
 import subprocess
+import hipchat
+import traceback
 
 
 if __name__ == '__main__':
@@ -14,24 +16,55 @@ if __name__ == '__main__':
     parser.add_argument("-e","--enabled",
         help="The location of the enabled services.")
 
+    hipchat_args = parser.add_argument_group("hipchat",
+            "Args for hipchat notification.")
+    hipchat_args.add_argument("-c","--hipchat-api-key",
+        help="Hipchat token if you want to receive notifications via hipchat.")
+    hipchat_args.add_argument("-r","--hipchat-room",
+        help="Room to send messages to.")
+
     args = parser.parse_args()
 
+    hc = None
+    report = []
+    hc_user = "PreSupervisor"
+
+    if args.hipchat_api_key:
+        hc = hipchat.HipChat(token=args.hipchat_api_key)
+
+    try:
         ec2 = boto.connect_ec2()
         instance_id = boto.utils.get_instance_metadata()['instance-id']
         reservations = ec2.get_all_instances(instance_ids=[instance_id])
-    report = []
         for reservation in reservations:
             for instance in reservation.instances:
                 if instance.id == instance_id:
+                    try:
                         services = instance.tags['services'].split(',')
+                    except KeyError as ke:
+                        msg = "Tag named 'services' not found on this instance({})".format(instance_id)
+                        raise Exception(msg)
+
                     for service in services:
                         # Link to available service.
-                    available_file = "{}/{}.conf".format(args.available, service)
-                    link_location = "{}/{}.conf".format(args.enabled, service)
+                        available_file = os.path.join(args.available, "{}.conf".format(service))
+                        link_location = os.path.join(args.enabled, "{}.conf".format(service))
                         if os.path.exists(available_file):
                             subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
                             report.append("Linking service: {}".format(service))
                         else:
-                        report.append("No conf available for service: {}".format(link_location))
-
+                            raise Exception("No conf available for service: {}".format(link_location))
+    except Exception as e:
+        msg = "{}: pre_supervisor failed with exception: {}".format(instance_id, traceback.format_exc())
+        print(msg)
+        if hc:
+            hc.message_room(room_id=args.hipchat_room,
+            message_from=hc_user,
+            message=msg)
+    finally:
         print("\n".join(report))
+        if hc:
+            hc.message_room(room_id=args.hipchat_room,
+                message_from=hc_user,
+                message="{}:\n{}".format(instance_id,"\n".join(report)))
+

playbooks/roles/supervisor/tasks/main.yml

@@ -94,10 +94,10 @@
 
 - name: install supervisor in its venv
   pip: >
-    name=boto virtualenv="{{supervisor_venv_dir}}" state=present
+    name={{ item }} virtualenv="{{supervisor_venv_dir}}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ supervisor_user }}"
-  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+  with_items: supervisor_pip_pkgs
 
 - name: create supervisor upstart job
   template: >

playbooks/roles/supervisor/templates/etc/init/pre_supervisor.conf.j2

@@ -4,4 +4,4 @@ start on runlevel [2345]
 task
 
 setuid {{ supervisor_user }}
-exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}}
+exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}} {% if SUPERVISOR_HIPCHAT_API_KEY %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %}