Add settings to cap infinite retries.

CHANGELOG.rst

@@ -9,6 +9,8 @@ LMS: Fix issue with CourseMode expiration dates
 
 LMS: Ported bulk emailing to the beta instructor dashboard.
 
+LMS: Add monitoring of bulk email subtasks to display progress on instructor dash.
+
 LMS: Add PaidCourseRegistration mode, where payment is required before course
 registration.
 

lms/djangoapps/bulk_email/tasks.py

@@ -469,8 +469,10 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
 
             else:
                 dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
-
-                log.debug('Email with id %s sent to %s', email_id, email)
+                if settings.BULK_EMAIL_LOG_SENT_EMAILS:
+                    log.info('Email with id %s sent to %s', email_id, email)
+                else:
+                    log.debug('Email with id %s sent to %s', email_id, email)
                 num_sent += 1
 
             # Pop the user that was emailed off the end of the list:
@@ -611,21 +613,23 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
              task_id, subtask_status['succeeded'], subtask_status['failed'], subtask_status['skipped'])
 
     # Calculate time until we retry this task (in seconds):
+    # The value for max_retries is increased by the number of times an "infinite-retry" exception
+    # has been retried.  We want the regular retries to trigger max-retry checking, but not these
+    # special retries.  So we count them separately.
     max_retries = _get_current_task().max_retries + subtask_status['retried_nomax']
     base_delay = _get_current_task().default_retry_delay
     if skip_retry_max:
-        retry_index = subtask_status['retried_nomax']
-        exp = min(retry_index, 5)
-        countdown = ((2 ** exp) * base_delay) * random.uniform(.5, 1.25)
+        # once we reach five retries, don't increase the countdown further.
+        retry_index = min(subtask_status['retried_nomax'], 5)
         exception_type = 'sending-rate'
     else:
         retry_index = subtask_status['retried_withmax']
-        countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.5)
         exception_type = 'transient'
 
-    # max_retries is increased by the number of times an "infinite-retry" exception
-    # has been retried.  We want the regular retries to trigger max-retry checking, but not these
-    # special retries.  So we count them separately.
+    # Skew the new countdown value by a random factor, so that not all
+    # retries are deferred by the same amount.
+    countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.25)
+
     log.warning('Task %s: email with id %d not delivered due to %s error %s, retrying send to %d recipients in %s seconds (with max_retry=%s)',
                 task_id, email_id, exception_type, current_exception, len(to_list), countdown, max_retries)
 
@@ -644,7 +648,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
             throw=True,
         )
     except RetryTaskError as retry_error:
-        # If retry call is successful, update with the current progress:
+        # If the retry call is successful, update with the current progress:
         log.exception('Task %s: email with id %d caused send_course_email task to retry.',
                       task_id, email_id)
         return subtask_status, retry_error

lms/envs/aws.py

@@ -93,10 +93,6 @@ CELERY_QUEUES = {
     DEFAULT_PRIORITY_QUEUE: {}
 }
 
-# We want Bulk Email running on the high-priority queue, so we define the
-# routing key that points to it.  At the moment, the name is the same.
-BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
-
 ########################## NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.
 
@@ -111,8 +107,6 @@ EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None)
 EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost')  # django default is localhost
 EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25)  # django default is 25
 EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False)  # django default is False
-EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100)
-EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000)
 SITE_NAME = ENV_TOKENS['SITE_NAME']
 SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE)
 SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
@@ -135,7 +129,6 @@ CACHES = ENV_TOKENS['CACHES']
 # Email overrides
 DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL)
 DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL)
-DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL)
 ADMINS = ENV_TOKENS.get('ADMINS', ADMINS)
 SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL)
 TECH_SUPPORT_EMAIL = ENV_TOKENS.get('TECH_SUPPORT_EMAIL', TECH_SUPPORT_EMAIL)
@@ -144,8 +137,18 @@ BUGS_EMAIL = ENV_TOKENS.get('BUGS_EMAIL', BUGS_EMAIL)
 PAYMENT_SUPPORT_EMAIL = ENV_TOKENS.get('PAYMENT_SUPPORT_EMAIL', PAYMENT_SUPPORT_EMAIL)
 PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CURRENCY',
                                                    PAID_COURSE_REGISTRATION_CURRENCY)
+
+# Bulk Email overrides
+DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL)
+EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100)
+EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000)
 BULK_EMAIL_DEFAULT_RETRY_DELAY = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_RETRY_DELAY', BULK_EMAIL_DEFAULT_RETRY_DELAY)
 BULK_EMAIL_MAX_RETRIES = ENV_TOKENS.get('BULK_EMAIL_MAX_RETRIES', BULK_EMAIL_MAX_RETRIES)
+BULK_EMAIL_INFINITE_RETRY_CAP = ENV_TOKENS.get('BULK_EMAIL_INFINITE_RETRY_CAP', BULK_EMAIL_INFINITE_RETRY_CAP)
+BULK_EMAIL_LOG_SENT_EMAILS = ENV_TOKENS.get('BULK_EMAIL_LOG_SENT_EMAILS', BULK_EMAIL_LOG_SENT_EMAILS)
+# We want Bulk Email running on the high-priority queue, so we define the
+# routing key that points to it.  At the moment, the name is the same.
+BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
 
 # Theme overrides
 THEME_NAME = ENV_TOKENS.get('THEME_NAME', None)

lms/envs/common.py

@@ -114,6 +114,7 @@ MITX_FEATURES = {
     # analytics experiments
     'ENABLE_INSTRUCTOR_ANALYTICS': False,
 
+    # bulk email available to instructors:
     'ENABLE_INSTRUCTOR_EMAIL': False,
 
     # enable analytics server.
@@ -333,7 +334,7 @@ TRACKING_BACKENDS = {
     }
 }
 
-# Backawrds compatibility with ENABLE_SQL_TRACKING_LOGS feature flag.
+# Backwards compatibility with ENABLE_SQL_TRACKING_LOGS feature flag.
 # In the future, adding the backend to TRACKING_BACKENDS enough.
 if MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
     TRACKING_BACKENDS.update({
@@ -414,12 +415,9 @@ HTTPS = 'on'
 ROOT_URLCONF = 'lms.urls'
 IGNORABLE_404_ENDS = ('favicon.ico')
 
-# Email
+# Platform Email
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 DEFAULT_FROM_EMAIL = 'registration@edx.org'
-DEFAULT_BULK_FROM_EMAIL = 'course-updates@edx.org'
-EMAILS_PER_TASK = 100
-EMAILS_PER_QUERY = 1000
 DEFAULT_FEEDBACK_EMAIL = 'feedback@edx.org'
 SERVER_EMAIL = 'devops@edx.org'
 TECH_SUPPORT_EMAIL = 'technical@edx.org'
@@ -800,11 +798,29 @@ CELERYD_HIJACK_ROOT_LOGGER = False
 
 ################################ Bulk Email ###################################
 
+DEFAULT_BULK_FROM_EMAIL = 'no-reply@courseupdates.edx.org'
+EMAILS_PER_TASK = 100
+EMAILS_PER_QUERY = 1000
+
+# Initial delay used for retrying tasks.  Additional retries use
+# longer delays.  Value is in seconds.
+BULK_EMAIL_DEFAULT_RETRY_DELAY = 30
+
+# Maximum number of retries per task for errors that are not related
+# to throttling.
+BULK_EMAIL_MAX_RETRIES = 5
+
+# Maximum number of retries per task for errors that are related to
+# throttling.  If this is not set, then there is no cap on such retries.
+BULK_EMAIL_INFINITE_RETRY_CAP = 1000
+
 # We want Bulk Email running on the high-priority queue, so we define the
 # routing key that points to it.  At the moment, the name is the same.
 BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
-BULK_EMAIL_DEFAULT_RETRY_DELAY = 15
-BULK_EMAIL_MAX_RETRIES = 5
+
+# Flag to indicate if individual email addresses should be logged as they are sent
+# a bulk email message.
+BULK_EMAIL_LOG_SENT_EMAILS = False
 
 ################################### APPS ######################################
 INSTALLED_APPS = (