Merge pull request #4282 from edx/jdmulloy/ops2545/redis_script_alarm_thresholds

Prevent reconfiguring alarms OPS-2545

util/jenkins/check-celery-queues.py

@@ -100,7 +100,8 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
 
     if len(all_queues) > max_metrics:
         # TODO: Use proper logging framework
-        print("Warning! Too many metrics, refusing to publish more than {}".format(max_metrics))
+        print("Warning! Too many metrics, refusing to publish more than {}"
+            .format(max_metrics))
 
     # Take first max_metrics number of queues from all_queues and remove
     # queues that aren't in redis
@@ -128,21 +129,26 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
         treat_missing_data = "missing"
         statistic = "Maximum"
         actions = [sns_arn]
-        alarm_name = "{} queue length over threshold".format(queue)
-
-        # Always create/configure alert to keep config up to date
-        cloudwatch.put_metric_alarm(AlarmName=alarm_name,
-                                    AlarmDescription=alarm_name,
-                                    Namespace=namespace,
-                                    MetricName=metric_name,
-                                    Dimensions=dimensions,
-                                    Period=period,
-                                    EvaluationPeriods=evaluation_periods,
-                                    TreatMissingData=treat_missing_data,
-                                    Threshold=threshold,
-                                    ComparisonOperator=comparison_operator,
-                                    Statistic=statistic,
-                                    AlarmActions=actions)
+        alarm_name = "{}-{} {} queue length over threshold".format(environment,
+                                                                   deploy,
+                                                                   queue)
+        if len(cloudwatch.describe_alarms_for_metric(Namespace=namespace,
+                                                     MetricName=metric_name,
+                                                     Dimensions=dimensions)
+                ['MetricAlarms']) < 1:
+            print('Creating new alarm "{}"'.format(alarm_name))
+            cloudwatch.put_metric_alarm(AlarmName=alarm_name,
+                                        AlarmDescription=alarm_name,
+                                        Namespace=namespace,
+                                        MetricName=metric_name,
+                                        Dimensions=dimensions,
+                                        Period=period,
+                                        EvaluationPeriods=evaluation_periods,
+                                        TreatMissingData=treat_missing_data,
+                                        Threshold=threshold,
+                                        ComparisonOperator=comparison_operator,
+                                        Statistic=statistic,
+                                        AlarmActions=actions)
 
 
 if __name__ == '__main__':