Merge pull request #1105 from edx/feanil/pre_sup_hipchat

Feanil/pre sup hipchat

playbooks/roles/supervisor/defaults/main.yml

@@ -13,6 +13,11 @@
 ---
 SUPERVISOR_HTTP_BIND_IP: '127.0.0.1'
 
+# Used by the pre-supervisor script if you want to
+# notify a hipchat room with the output.
+SUPERVISOR_HIPCHAT_API_KEY: !!null
+SUPERVISOR_HIPCHAT_ROOM: default
+
 # do not override the bind_port since
 # all supervisors will then try to listen
 # on the same one
@@ -35,3 +40,7 @@ supervisor_cfg: "{{ supervisor_app_dir }}/supervisord.conf"
 # upstart service name and user
 supervisor_service: supervisor
 supervisor_service_user: "{{ common_web_user }}"
+
+supervisor_pip_pkgs:
+  - boto
+  - python-simple-hipchat

playbooks/roles/supervisor/files/pre_supervisor_checks.py

-# Get the tags for this instance
 import argparse
 import boto
-import boto.utils
+from boto.utils import get_instance_metadata
+from boto.exception import AWSConnectionError
+import hipchat
 import os
 import subprocess
+import traceback
 
+# Services that should be checked for migrations.
+MIGRATION_COMMANDS = {
+        'lms': "{python} {code_dir}/manage.py lms migrate --noinput --settings=aws --db-dry-run --merge",
+        'cms': "{python} {code_dir}/manage.py cms migrate --noinput --settings=aws --db-dry-run --merge",
+        'xqueue': "{python} {code_dir}/manage.py xqueue migrate --noinput --settings=aws --db-dry-run --merge",
+    }
+HIPCHAT_USER = "PreSupervisor"
+
+def services_for_instance(instance_id):
+    """
+    Get the list of all services named by the services tag in this
+    instance's tags.
+    """
+    ec2 = boto.connect_ec2()
+    reservations = ec2.get_all_instances(instance_ids=[instance_id])
+    for reservation in reservations:
+        for instance in reservation.instances:
+            if instance.id == instance_id:
+                try:
+                    services = instance.tags['services'].split(',')
+                except KeyError as ke:
+                    msg = "Tag named 'services' not found on this instance({})".format(instance_id)
+                    raise Exception(msg)
+
+                for service in services:
+                    yield service
+
+def edp_for_instance(instance_id):
+    ec2 = boto.connect_ec2()
+    reservations = ec2.get_all_instances(instance_ids=[instance_id])
+    for reservation in reservations:
+        for instance in reservation.instances:
+            if instance.id == instance_id:
+                try:
+                    environment = instance.tags['environment']
+                    deployment = instance.tags['deployment']
+                    play = instance.tags['play']
+                except KeyError as ke:
+                    msg = "{} tag not found on this instance({})".format(ke.message, instance_id)
+                    raise Exception(msg)
+                return (environment, deployment, play)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -14,24 +57,95 @@ if __name__ == '__main__':
     parser.add_argument("-e","--enabled",
         help="The location of the enabled services.")
 
+    migration_args = parser.add_argument_group("edxapp_migrations",
+            "Args for running edxapp migration checks.")
+    migration_args.add_argument("--edxapp-code-dir",
+            help="Location of the edx-platform code.")
+    migration_args.add_argument("--edxapp-python",
+            help="Path to python to use for executing migration check.")
+
+    xq_migration_args = parser.add_argument_group("xqueue_migrations",
+            "Args for running xqueue migration checks.")
+    xq_migration_args.add_argument("--xqueue-code-dir",
+            help="Location of the edx-platform code.")
+    xq_migration_args.add_argument("--xqueue-python",
+            help="Path to python to use for executing migration check.")
+
+    hipchat_args = parser.add_argument_group("hipchat",
+            "Args for hipchat notification.")
+    hipchat_args.add_argument("-c","--hipchat-api-key",
+        help="Hipchat token if you want to receive notifications via hipchat.")
+    hipchat_args.add_argument("-r","--hipchat-room",
+        help="Room to send messages to.")
+
     args = parser.parse_args()
 
-    ec2 = boto.connect_ec2()
-    instance_id = boto.utils.get_instance_metadata()['instance-id']
-    reservations = ec2.get_all_instances(instance_ids=[instance_id])
     report = []
-    for reservation in reservations:
-        for instance in reservation.instances:
-            if instance.id == instance_id:
-                services = instance.tags['services'].split(',')
-                for service in services:
-                    # Link to available service.
-                    available_file = "{}/{}.conf".format(args.available, service)
-                    link_location = "{}/{}.conf".format(args.enabled, service)
-                    if os.path.exists(available_file):
-                        subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
-                        report.append("Linking service: {}".format(service))
-                    else:
-                        report.append("No conf available for service: {}".format(link_location))
-
-    print("\n".join(report))
+    prefix = None
+    notify = None
+
+    try:
+        if args.hipchat_api_key:
+            hc = hipchat.HipChat(token=args.hipchat_api_key)
+            notify = lambda message: hc.message_room(room_id=args.hipchat_room,
+                message_from=HIPCHAT_USER, message=message)
+    except Exception as e:
+        print("Failed to initialize hipchat, {}".format(e))
+        traceback.print_exc()
+
+    instance_id = get_instance_metadata()['instance-id']
+    prefix = instance_id
+
+    try:
+        environment, deployment, play = edp_for_instance(instance_id)
+        prefix = "{environment}-{deployment}-{play}-{instance_id}".format(
+            environment=environment,
+            deployment=deployment,
+            play=play,
+            instance_id=instance_id)
+        for service in services_for_instance(instance_id):
+            if service in MIGRATION_COMMANDS:
+                # Do extra migration related stuff.
+                if (service == 'lms' or service == 'cms') and args.edxapp_code_dir:
+                    cmd = MIGRATION_COMMANDS[service].format(python=args.edxapp_python,
+                        code_dir=args.edxapp_code_dir)
+                    if os.path.exists(args.edxapp_code_dir):
+                        os.chdir(args.edxapp_code_dir)
+                        # Run migration check command.
+                        output = subprocess.check_output(cmd, shell=True)
+                        if 'Migrating' in output:
+                            raise Exception("Migrations have not been run for {}".format(service))
+                elif service == 'xqueue' and args.xqueue_code_dir:
+                    cmd = MIGRATION_COMMANDS[service].format(python=args.xqueue_python,
+                        code_dir=xqueue_code_dir)
+                    if os.path.exists(args.xqueue_code_dir):
+                        os.chdir(args.xqueue_code_dir)
+                        # Run migration check command.
+                        output = subprocess.check_output(cmd, shell=True)
+                        if 'Migrating' in output:
+                            raise Exception("Migrations have not been run for {}".format(service))
+    
+            # Link to available service.
+            available_file = os.path.join(args.available, "{}.conf".format(service))
+            link_location = os.path.join(args.enabled, "{}.conf".format(service))
+            if os.path.exists(available_file):
+                subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
+                report.append("Linking service: {}".format(service))
+            else:
+                raise Exception("No conf available for service: {}".format(link_location))
+    except AWSConnectionError as ae:
+        msg = "{}: ERROR : {}".format(prefix, ae)
+        if notify:
+            notify(msg)
+            notify(traceback.format_exc())
+        raise ae
+    except Exception as e:
+        msg = "{}: ERROR : {}".format(prefix, e)
+        print(msg)
+        if notify:
+            notify(msg)
+    else:
+        msg = "{}: {}".format(prefix, " | ".join(report))
+        print(msg)
+        if notify:
+            notify(msg)

playbooks/roles/supervisor/tasks/main.yml

@@ -94,10 +94,10 @@
 
 - name: install supervisor in its venv
   pip: >
-    name=boto virtualenv="{{supervisor_venv_dir}}" state=present
+    name={{ item }} virtualenv="{{supervisor_venv_dir}}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ supervisor_user }}"
-  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+  with_items: supervisor_pip_pkgs
 
 - name: create supervisor upstart job
   template: >

playbooks/roles/supervisor/templates/etc/init/pre_supervisor.conf.j2

@@ -4,4 +4,5 @@ start on runlevel [2345]
 task
 
 setuid {{ supervisor_user }}
-exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}}
+
+exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}} {% if SUPERVISOR_HIPCHAT_API_KEY is defined %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %} {% if edxapp_code_dir is defined %}--edxapp-python {{ COMMON_BIN_DIR }}/python.edxapp --edxapp-code-dir {{ edxapp_code_dir }}{% endif %} {% if xqueue_code_dir is defined %}--xqueue-code-dir {{ xqueue_code_dir }} --xqueue-python {{ COMMON_BIN_DIR }}/python.xqueue {% endif %}