Wait up to 2 minutes for instance tags.

playbooks/roles/supervisor/files/pre_supervisor_checks.py

@@ -7,6 +7,7 @@ import os
 import subprocess
 import traceback
 import socket
+import time
 
 # Services that should be checked for migrations.
 MIGRATION_COMMANDS = {
@@ -16,6 +17,9 @@ MIGRATION_COMMANDS = {
     }
 HIPCHAT_USER = "PreSupervisor"
 
+# Max amount of time to wait for tags to be applied.
+MAX_BACKOFF = 120
+
 def services_for_instance(instance_id):
     """
     Get the list of all services named by the services tag in this
@@ -106,16 +110,31 @@ if __name__ == '__main__':
         # Needs to exit with 1 instead of 0 to prevent
         # services from starting.
         exit(1)
-
-    try:
-        environment, deployment, play = edp_for_instance(instance_id)
-        prefix = "{environment}-{deployment}-{play}-{instance_id}".format(
-            environment=environment,
-            deployment=deployment,
-            play=play,
-            instance_id=instance_id)
-    except:
-        print("Failed to get EDP for {}".format(instance_id))
+    time_left = MAX_BACKOFF
+    backoff = INITIAL_BACKOFF
+
+    environment = None
+    deployment = None
+    play = None
+    while time_left > 0:
+        try:
+            environment, deployment, play = edp_for_instance(instance_id)
+            prefix = "{environment}-{deployment}-{play}-{instance_id}".format(
+                environment=environment,
+                deployment=deployment,
+                play=play,
+                instance_id=instance_id)
+            break
+        except:
+            print("Failed to get EDP for {}".format(instance_id))
+            time.sleep(backoff)
+            backoff = backoff * 2
+
+    if environment is None or deployment is None or play is None:
+        msg = "Unable to retrieve environment, deployment, or play tag."
+        print(msg)
+        notify("{} : {}".format(prefix, msg))
+        exit(0)
 
     #get the hostname of the sandbox
     hostname = socket.gethostname()
@@ -130,7 +149,7 @@ if __name__ == '__main__':
                          "cluster": play,
                          "instance-id": instance_id,
                          "created": volume.create_time })
-        
+
     try:
         for service in services_for_instance(instance_id):
             if service in MIGRATION_COMMANDS:
@@ -153,7 +172,7 @@ if __name__ == '__main__':
                         output = subprocess.check_output(cmd, shell=True)
                         if 'Migrating' in output:
                             raise Exception("Migrations have not been run for {}".format(service))
-    
+
             # Link to available service.
             available_file = os.path.join(args.available, "{}.conf".format(service))
             link_location = os.path.join(args.enabled, "{}.conf".format(service))