Add support for migration checks.

playbooks/roles/supervisor/files/pre_supervisor_checks.py

-# Get the tags for this instance
 import argparse
 import boto
-import boto.utils
+from boto.utils import get_instance_metadata
+from boto.exception import AWSConnectionError
+import hipchat
 import os
 import subprocess
-import hipchat
 import traceback
 
+# Services that should be checked for migrations.
+MIGRATION_SERVICES = [ 'lms', 'cms', 'xqueue' ]
+MIGRATION_COMMAND = "{python} {code_dir}/manage.py {env} migrate --noinput --settings=aws --db-dry-run --merge"
+HIPCHAT_USER = "PreSupervisor"
+
+def services_for_instance(instance_id):
+    """
+    Get the list of all services named by the services tag in this
+    instance's tags.
+    """
+    ec2 = boto.connect_ec2()
+    reservations = ec2.get_all_instances(instance_ids=[instance_id])
+    for reservation in reservations:
+        for instance in reservation.instances:
+            if instance.id == instance_id:
+                try:
+                    services = instance.tags['services'].split(',')
+                except KeyError as ke:
+                    msg = "Tag named 'services' not found on this instance({})".format(instance_id)
+                    raise Exception(msg)
+
+                for service in services:
+                    yield service
+
+def edp_for_instance(instance_id):
+    ec2 = boto.connect_ec2()
+    reservations = ec2.get_all_instances(instance_ids=[instance_id])
+    for reservation in reservations:
+        for instance in reservation.instances:
+            if instance.id == instance_id:
+                try:
+                    environment = instance.tags['environment']
+                    deployment = instance.tags['deployment']
+                    play = instance.tags['play']
+                except KeyError as ke:
+                    msg = "{} tag not found on this instance({})".format(ke.message, instance_id)
+                    raise Exception(msg)
+                return (environment, deployment, play)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -16,6 +54,20 @@ if __name__ == '__main__':
     parser.add_argument("-e","--enabled",
         help="The location of the enabled services.")
 
+    migration_args = parser.add_argument_group("edxapp_migrations",
+            "Args for running edxapp migration checks.")
+    migration_args.add_argument("--edxapp-code-dir",
+            help="Location of the edx-platform code.")
+    migration_args.add_argument("--edxapp-python",
+            help="Path to python to use for executing migration check.")
+
+    xq_migration_args = parser.add_argument_group("xqueue_migrations",
+            "Args for running xqueue migration checks.")
+    xq_migration_args.add_argument("--xqueue-code-dir",
+            help="Location of the edx-platform code.")
+    xq_migration_args.add_argument("--xqueue-python",
+            help="Path to python to use for executing migration check.")
+
     hipchat_args = parser.add_argument_group("hipchat",
             "Args for hipchat notification.")
     hipchat_args.add_argument("-c","--hipchat-api-key",
@@ -25,46 +77,68 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
-    hc = None
     report = []
-    hc_user = "PreSupervisor"
-
-    if args.hipchat_api_key:
-        hc = hipchat.HipChat(token=args.hipchat_api_key)
+    prefix = None
+    notify = None
 
     try:
-        ec2 = boto.connect_ec2()
-        instance_id = boto.utils.get_instance_metadata()['instance-id']
-        reservations = ec2.get_all_instances(instance_ids=[instance_id])
-        for reservation in reservations:
-            for instance in reservation.instances:
-                if instance.id == instance_id:
-                    try:
-                        services = instance.tags['services'].split(',')
-                    except KeyError as ke:
-                        msg = "Tag named 'services' not found on this instance({})".format(instance_id)
-                        raise Exception(msg)
+        if args.hipchat_api_key:
+            hc = hipchat.HipChat(token=args.hipchat_api_key)
+            notify = lambda message: hc.message_room(room_id=args.hipchat_room,
+                message_from=HIPCHAT_USER, message=message)
+    except Exception as e:
+        print("Failed to initialize hipchat, {}".format(e))
+        traceback.print_exc()
+
+    instance_id = get_instance_metadata()['instance-id']
+    prefix = instance_id
 
-                    for service in services:
-                        # Link to available service.
-                        available_file = os.path.join(args.available, "{}.conf".format(service))
-                        link_location = os.path.join(args.enabled, "{}.conf".format(service))
-                        if os.path.exists(available_file):
-                            subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
-                            report.append("Linking service: {}".format(service))
-                        else:
-                            raise Exception("No conf available for service: {}".format(link_location))
+    try:
+        edp = edp_for_instance(instance_id)
+        prefix = "{}-{}-{}-{}".format(edp[0], edp[1], edp[2], instance_id)
+        for service in services_for_instance(instance_id):
+            if service in MIGRATION_SERVICES:
+                # Do extra migration related stuff.
+                if (service == 'lms' or service == 'cms') and args.edxapp_code_dir:
+                    cmd = MIGRATION_COMMAND.format(python=args.edxapp_python,
+                        code_dir=args.edxapp_code_dir,
+                        env=service)
+                    if os.path.exists(args.edxapp_code_dir):
+                        # Run migration check command.
+                        output = subprocess.check_output(cmd, shell=True)
+                        if 'Migrating' in output:
+                            raise Exception("Migrations have not been run for {}".format(service))
+                elif service == 'xqueue' and args.xqueue_code_dir:
+                    cmd = MIGRATION_COMMAND.format(python=args.xqueue_python,
+                        code_dir=xqueue_code_dir,
+                        env=service)
+                    if os.path.exists(args.xqueue_code_dir):
+                        # Run migration check command.
+                        output = subprocess.check_output(cmd, shell=True)
+                        if 'Migrating' in output:
+                            raise Exception("Migrations have not been run for {}".format(service))
+    
+            # Link to available service.
+            available_file = os.path.join(args.available, "{}.conf".format(service))
+            link_location = os.path.join(args.enabled, "{}.conf".format(service))
+            if os.path.exists(available_file):
+                subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
+                report.append("Linking service: {}".format(service))
+            else:
+                raise Exception("No conf available for service: {}".format(link_location))
+    except AWSConnectionError as ae:
+        msg = "{}: ERROR : {}".format(prefix, ae)
+        if notify:
+            notify(msg)
+            notify(traceback.format_exc())
+        raise ae
     except Exception as e:
-        msg = "{}: pre_supervisor failed with exception: {}".format(instance_id, traceback.format_exc())
+        msg = "{}: ERROR : {}".format(prefix, e)
         print(msg)
-        if hc:
-            hc.message_room(room_id=args.hipchat_room,
-            message_from=hc_user,
-            message=msg)
-    finally:
-        print("\n".join(report))
-        if hc:
-            hc.message_room(room_id=args.hipchat_room,
-                message_from=hc_user,
-                message="{}:\n{}".format(instance_id,"\n".join(report)))
-
+        if notify:
+            notify(msg)
+    else:
+        msg = "{}: {}".format(prefix, " | ".join(report))
+        print(msg)
+        if notify:
+            notify(msg)

playbooks/roles/supervisor/templates/etc/init/pre_supervisor.conf.j2

@@ -4,4 +4,5 @@ start on runlevel [2345]
 task
 
 setuid {{ supervisor_user }}
-exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}} {% if SUPERVISOR_HIPCHAT_API_KEY %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %}
+
+exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}} {% if SUPERVISOR_HIPCHAT_API_KEY is defined %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %} {% if edxapp_code_dir is defined %}--edxapp-python {{ COMMON_BIN_DIR }}/python.edxapp --edxapp-code-dir {{ edxapp_code_dir }}{% endif %} {% if xqueue_code_dir is defined %}--xqueue-code-dir {{ xqueue_code_dir }} --xqueue-python {{ COMMON_BIN_DIR }}/python.xqueue {% endif %}