Merge pull request #4282 from edx/jdmulloy/ops2545/redis_script_alarm_thresholds

Prevent reconfiguring alarms OPS-2545

util/jenkins/check-celery-queues.py

@@ -100,7 +100,8 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
 
     if len(all_queues) > max_metrics:
         # TODO: Use proper logging framework
-        print("Warning! Too many metrics, refusing to publish more than {}".format(max_metrics))
+        print("Warning! Too many metrics, refusing to publish more than {}"
+            .format(max_metrics))
 
     # Take first max_metrics number of queues from all_queues and remove
     # queues that aren't in redis
@@ -128,9 +129,14 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
         treat_missing_data = "missing"
         statistic = "Maximum"
         actions = [sns_arn]
-        alarm_name = "{} queue length over threshold".format(queue)
-
-        # Always create/configure alert to keep config up to date
+        alarm_name = "{}-{} {} queue length over threshold".format(environment,
+                                                                   deploy,
+                                                                   queue)
+        if len(cloudwatch.describe_alarms_for_metric(Namespace=namespace,
+                                                     MetricName=metric_name,
+                                                     Dimensions=dimensions)
+                ['MetricAlarms']) < 1:
+            print('Creating new alarm "{}"'.format(alarm_name))
             cloudwatch.put_metric_alarm(AlarmName=alarm_name,
                                         AlarmDescription=alarm_name,
                                         Namespace=namespace,