Merge pull request #16266 from edx/aed/EDUCATOR-1428

EDUCATOR-1428 | Add time_limit to course/subsection grading tasks.

lms/djangoapps/grades/signals/handlers.py

@@ -26,7 +26,7 @@ from ..constants import ScoreDatabaseTableEnum
 from ..course_grade_factory import CourseGradeFactory
 from .. import events
 from ..scores import weighted_score
-from ..tasks import RECALCULATE_GRADE_DELAY, recalculate_subsection_grade_v3
+from ..tasks import RECALCULATE_GRADE_DELAY_SECONDS, recalculate_subsection_grade_v3
 
 log = getLogger(__name__)
 
@@ -221,7 +221,7 @@ def enqueue_subsection_update(sender, **kwargs):  # pylint: disable=unused-argum
             event_transaction_type=unicode(get_event_transaction_type()),
             score_db_table=kwargs['score_db_table'],
         ),
-        countdown=RECALCULATE_GRADE_DELAY,
+        countdown=RECALCULATE_GRADE_DELAY_SECONDS,
     )
 
 

lms/djangoapps/grades/tasks.py

@@ -35,12 +35,15 @@ from .transformer import GradesTransformer
 
 log = getLogger(__name__)
 
+COURSE_GRADE_TIMEOUT_SECONDS = 1200
 KNOWN_RETRY_ERRORS = (  # Errors we expect occasionally, should be resolved on retry
     DatabaseError,
     ValidationError,
     DatabaseNotReadyError,
 )
-RECALCULATE_GRADE_DELAY = 2  # in seconds, to prevent excessive _has_db_updated failures. See TNL-6424.
+RECALCULATE_GRADE_DELAY_SECONDS = 2  # to prevent excessive _has_db_updated failures. See TNL-6424.
+RETRY_DELAY_SECONDS = 30
+SUBSECTION_GRADE_TIMEOUT_SECONDS = 300
 
 
 class _BaseTask(PersistOnFailureTask, LoggedTask):  # pylint: disable=abstract-method
@@ -72,7 +75,13 @@ def compute_all_grades_for_course(**kwargs):
             )
 
 
-@task(base=_BaseTask, bind=True, default_retry_delay=30, max_retries=1)
+@task(
+    bind=True,
+    base=_BaseTask,
+    default_retry_delay=RETRY_DELAY_SECONDS,
+    max_retries=1,
+    time_limit=COURSE_GRADE_TIMEOUT_SECONDS
+)
 def compute_grades_for_course_v2(self, **kwargs):
     """
     Compute grades for a set of students in the specified course.
@@ -113,7 +122,14 @@ def compute_grades_for_course(course_key, offset, batch_size, **kwargs):  # pyli
             raise result.error
 
 
-@task(bind=True, base=_BaseTask, default_retry_delay=30, routing_key=settings.RECALCULATE_GRADES_ROUTING_KEY)
+@task(
+    bind=True,
+    base=_BaseTask,
+    time_limit=SUBSECTION_GRADE_TIMEOUT_SECONDS,
+    max_retries=2,
+    default_retry_delay=RETRY_DELAY_SECONDS,
+    routing_key=settings.RECALCULATE_GRADES_ROUTING_KEY
+)
 def recalculate_subsection_grade_v3(self, **kwargs):
     """
     Latest version of the recalculate_subsection_grade task.  See docstring

lms/djangoapps/grades/tests/test_tasks.py

@@ -20,7 +20,7 @@ from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsec
 from lms.djangoapps.grades.services import GradesService
 from lms.djangoapps.grades.signals.signals import PROBLEM_WEIGHTED_SCORE_CHANGED
 from lms.djangoapps.grades.tasks import (
-    RECALCULATE_GRADE_DELAY,
+    RECALCULATE_GRADE_DELAY_SECONDS,
     _course_task_args,
     compute_grades_for_course_v2,
     recalculate_subsection_grade_v3
@@ -142,7 +142,7 @@ class RecalculateSubsectionGradeTest(HasCourseWithProblemsMixin, ModuleStoreTest
             return_value=None
         ) as mock_task_apply:
             PROBLEM_WEIGHTED_SCORE_CHANGED.send(sender=None, **send_args)
-            mock_task_apply.assert_called_once_with(countdown=RECALCULATE_GRADE_DELAY, kwargs=local_task_args)
+            mock_task_apply.assert_called_once_with(countdown=RECALCULATE_GRADE_DELAY_SECONDS, kwargs=local_task_args)
 
     @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send')
     def test_triggers_subsection_score_signal(self, mock_subsection_signal):