update instance monitoring script

requirements.txt

@@ -17,6 +17,7 @@ requests==2.9.1
 datadog==0.8.0
 networkx==1.11
 pathlib2==2.1.0
+boto3==1.4.4
 
 # Needed for the mongo_* modules (playbooks/library/mongo_*)
 pymongo==3.1

util/cluster_instance_monitoring.py

@@ -2,9 +2,9 @@ import boto3
 import argparse
 import sys
 import yaml
-import pprint
+from pprint import pprint
 
-def find_active_instances(cluster_file):
+def find_active_instances(cluster_file, region):
     """
     Determines if a given cluster has at least one ASG and at least one active instance.
 
@@ -14,24 +14,19 @@ def find_active_instances(cluster_file):
         and cluster to find ASG's and active instances for. 
 
     """
-    pp = pprint.PrettyPrinter()
-
-    f = open(cluster_file)
-    cluster_map = yaml.safe_load(f)
-    f.close()
-
-    region = 'us-east-1'
+    with open(cluster_file, 'r') as f:
+        cluster_map = yaml.safe_load(f)
 
     asg = boto3.client('autoscaling', region)
     all_groups = asg.describe_auto_scaling_groups()
 
-    #all the asgs that match the specified environment, deployment, and cluster triples from the cluster map
-    all_matching_asgs = []
+    # dictionary that contains the environment/deployment/cluster triple as the key and the value is a list of the asgs that match the triple
+    all_matching_asgs = {}
 
-    #all the triples for which an autoscaling group does not exist 
+    # all the triples for which an autoscaling group does not exist 
     not_matching_triples = []
 
-    #check if there exists at least one ASG for each triple
+    # check if there exists at least one ASG for each triple
     for triple in cluster_map:
         #the asgs that match this particular triple
         cluster_asgs = []
@@ -53,29 +48,35 @@ def find_active_instances(cluster_file):
         if not cluster_asgs:
             not_matching_triples += [triple]
         else:
-            all_matching_asgs += cluster_asgs
+            triple_str = triple['env'] + '-' + triple['deployment'] + '-' + triple['cluster']
+            all_matching_asgs[triple_str] = cluster_asgs
 
-    #The ASG's for the triples that have no active instances
-    no_active_instances_asgs = []
+    #The triples that have no active instances
+    no_active_instances_triples = []
 
-    #check to make sure each ASG has at least 1 running instance
-    for asg in all_matching_asgs:
-        asg_has_active_instances = False
-        for instance in asg['Instances']:
-            if instance['LifecycleState'] == 'InService':
-                asg_has_active_instances = True
-                break
-        if not asg_has_active_instances:
-            no_active_instances_asgs += [asg]
+    #check that each triple has at least one active instance in at least one of its ASG's
+    for triple in all_matching_asgs:
+        asgs = all_matching_asgs[triple]
+        triple_has_active_instances = False
+        for asg in asgs:
+            for instance in asg['Instances']:
+                if instance['LifecycleState'] == 'InService':
+                    triple_has_active_instances = True
+        if not triple_has_active_instances:
+            no_active_instances_triples += [triple]
 
 
-    if no_active_instances_asgs or not_matching_triples:
+    if no_active_instances_triples or not_matching_triples:
         if not_matching_triples:
             print('Fail. There are no autoscaling groups found for the following cluster(s):')
-            pp.pprint(not_matching_triples)
-        if no_active_instances_asgs:
-            print("Fail. There are no active instances for the following ASG's:")
-            pp.pprint(no_active_instances_asgs)
+            pprint(not_matching_triples)
+        if no_active_instances_triples:
+            print("Fail. There are no active instances for the following cluster(s)")
+            for triple in no_active_instances_triples:
+                print('environment: ' + triple.split('-')[0])
+                print('deployment: ' + triple.split('-')[1])
+                print('cluster: ' + triple.split('-')[2])
+                print('----')
         sys.exit(1)
     
     print("Success. ASG's with active instances found for all of the cluster triples.")
@@ -85,7 +86,8 @@ def find_active_instances(cluster_file):
 if __name__=="__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('-f', '--file', help='Yaml file of env/deployment/cluster triples that we want to find active instances for', required=True)
+    parser.add_argument('-r', '--region', help="Region that we want to find ASG's and active instances in", default='us-east-1', required=True)
     args = parser.parse_args()
 
-    find_active_instances(args.file)
+    find_active_instances(args.file, args.region)
 

util/cluster_monitoring_requirements.txt

-boto3==1.4.4
\ No newline at end of file