Make BlockStructure tasks more resilient against failures.

https://openedx.atlassian.net/browse/TNL-5041

cms/envs/common.py

@@ -80,6 +80,7 @@ from lms.envs.common import (
 
     # django-debug-toolbar
     DEBUG_TOOLBAR_PATCH_SETTINGS,
+    BLOCK_STRUCTURES_SETTINGS,
 )
 from path import Path as path
 from warnings import simplefilter

lms/envs/common.py

@@ -1771,6 +1771,23 @@ CELERY_QUEUES = {
 # let logging work as configured:
 CELERYD_HIJACK_ROOT_LOGGER = False
 
+################################ Block Structures ###################################
+
+BLOCK_STRUCTURES_SETTINGS = dict(
+    # Delay, in seconds, after a new edit of a course is published
+    # before updating the block structures cache.  This is needed
+    # for a better chance at getting the latest changes when there
+    # are secondary reads in sharded mongoDB clusters. See TNL-5041
+    # for more info.
+    BLOCK_STRUCTURES_COURSE_PUBLISH_TASK_DELAY=30,
+
+    # Delay, in seconds, between retry attempts if a task fails.
+    BLOCK_STRUCTURES_TASK_DEFAULT_RETRY_DELAY=30,
+
+    # Maximum number of retries per task.
+    BLOCK_STRUCTURES_TASK_MAX_RETRIES=5,
+)
+
 ################################ Bulk Email ###################################
 
 # Suffix used to construct 'from' email address for bulk emails.

openedx/core/djangoapps/content/block_structure/signals.py

 """
 Signal handlers for invalidating cached data.
 """
+from django.conf import settings
 from django.dispatch.dispatcher import receiver
 
 from xmodule.modulestore.django import SignalHandler
@@ -19,7 +20,10 @@ def _listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable
 
     # The countdown=0 kwarg ensures the call occurs after the signal emitter
     # has finished all operations.
-    update_course_in_cache.apply_async([unicode(course_key)], countdown=0)
+    update_course_in_cache.apply_async(
+        [unicode(course_key)],
+        countdown=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_COURSE_PUBLISH_TASK_DELAY'],
+    )
 
 
 @receiver(SignalHandler.course_deleted)

openedx/core/djangoapps/content/block_structure/tasks.py

@@ -3,6 +3,7 @@ Asynchronous tasks related to the Course Blocks sub-application.
 """
 import logging
 from celery.task import task
+from django.conf import settings
 from opaque_keys.edx.keys import CourseKey
 
 from openedx.core.djangoapps.content.block_structure import api
@@ -10,7 +11,10 @@ from openedx.core.djangoapps.content.block_structure import api
 log = logging.getLogger('edx.celery.task')
 
 
-@task
+@task(
+    default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_TASK_DEFAULT_RETRY_DELAY'],
+    max_retries=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_TASK_MAX_RETRIES'],
+)
 def update_course_in_cache(course_key):
     """
     Updates the course blocks (in the database) for the specified course.