Wait up to 2 minutes for instance tags.

playbooks/roles/supervisor/files/pre_supervisor_checks.py

@@ -7,6 +7,7 @@ import os
 import subprocess
 import traceback
 import socket
+import time
 
 # Services that should be checked for migrations.
 MIGRATION_COMMANDS = {
@@ -16,6 +17,9 @@ MIGRATION_COMMANDS = {
     }
 HIPCHAT_USER = "PreSupervisor"
 
+# Max amount of time to wait for tags to be applied.
+MAX_BACKOFF = 120
+
 def services_for_instance(instance_id):
     """
     Get the list of all services named by the services tag in this
@@ -106,7 +110,13 @@ if __name__ == '__main__':
         # Needs to exit with 1 instead of 0 to prevent
         # services from starting.
         exit(1)
+    time_left = MAX_BACKOFF
+    backoff = INITIAL_BACKOFF
 
+    environment = None
+    deployment = None
+    play = None
+    while time_left > 0:
         try:
             environment, deployment, play = edp_for_instance(instance_id)
             prefix = "{environment}-{deployment}-{play}-{instance_id}".format(
@@ -114,8 +124,17 @@ if __name__ == '__main__':
                 deployment=deployment,
                 play=play,
                 instance_id=instance_id)
+            break
         except:
             print("Failed to get EDP for {}".format(instance_id))
+            time.sleep(backoff)
+            backoff = backoff * 2
+
+    if environment is None or deployment is None or play is None:
+        msg = "Unable to retrieve environment, deployment, or play tag."
+        print(msg)
+        notify("{} : {}".format(prefix, msg))
+        exit(0)
 
     #get the hostname of the sandbox
     hostname = socket.gethostname()