Merge pull request #4624 from edx/jibsheet/loop-over-all-queues

Loop over all redis queues

util/jenkins/check-celery-queues.py

@@ -3,6 +3,7 @@ import click
 import boto3
 import botocore
 import backoff
+from itertools import zip_longest
 
 max_tries = 5
 
@@ -105,16 +106,8 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
         set(redis_queues).difference(existing_queues)
     )
 
-    if len(all_queues) > max_metrics:
-        # TODO: Use proper logging framework
-        print("Warning! Too many metrics, refusing to publish more than {}"
-              .format(max_metrics))
-
-    # Filter redis_queues out of all_queues and then take the max_metrics
-    # portion of that.
-    queues = [q for q in all_queues if q in redis_queues]
-    queues = queues[:max_metrics]
-
+    for queues in grouper(all_queues, max_metrics):
+        queues = [q for q in queues if q is not None]
         metric_data = []
         for queue in queues:
             metric_data.append({
@@ -161,6 +154,13 @@ def check_queues(host, port, environment, deploy, max_metrics, threshold,
                                         OKActions=actions,
                                         AlarmActions=actions)
 
+# Stolen right from the itertools recipes
+# https://docs.python.org/3/library/itertools.html#itertools-recipes
+def grouper(iterable, n, fillvalue=None):
+    "Collect data into fixed-length chunks or blocks"
+    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
+    args = [iter(iterable)] * n
+    return zip_longest(*args, fillvalue=fillvalue)
 
 if __name__ == '__main__':
     check_queues()