Recalculate Subsection Grade task retry logic

The task will now be retried if an IntegrityError is encountered, up to
the default of 3 times. Test included.

TNL-5739

lms/djangoapps/grades/tasks.py

@@ -5,6 +5,7 @@ This module contains tasks for asynchronous execution of grade updates.
 from celery import task
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.db.utils import IntegrityError
 
 from lms.djangoapps.course_blocks.api import get_course_blocks
 from lms.djangoapps.courseware.courses import get_course_by_id
@@ -17,7 +18,7 @@ from .transformer import GradesTransformer
 from .new.subsection_grade import SubsectionGradeFactory
 
 
-@task(routing_key=settings.RECALCULATE_GRADES_ROUTING_KEY)
+@task(default_retry_delay=30, routing_key=settings.RECALCULATE_GRADES_ROUTING_KEY)
 def recalculate_subsection_grade(user_id, course_id, usage_id):
     """
     Updates a saved subsection grade.
@@ -43,12 +44,15 @@ def recalculate_subsection_grade(user_id, course_id, usage_id):
         set()
     )
 
-    for subsection_usage_key in subsections_to_update:
-        transformed_subsection_structure = get_course_blocks(
-            student,
-            subsection_usage_key,
-            collected_block_structure=collected_block_structure,
-        )
-        subsection_grade_factory.update(
-            transformed_subsection_structure[subsection_usage_key], transformed_subsection_structure
-        )
+    try:
+        for subsection_usage_key in subsections_to_update:
+            transformed_subsection_structure = get_course_blocks(
+                student,
+                subsection_usage_key,
+                collected_block_structure=collected_block_structure,
+            )
+            subsection_grade_factory.update(
+                transformed_subsection_structure[subsection_usage_key], transformed_subsection_structure
+            )
+    except IntegrityError as exc:
+        raise recalculate_subsection_grade.retry(args=[user_id, course_id, usage_id], exc=exc)

lms/djangoapps/grades/tests/test_tasks.py

@@ -4,6 +4,7 @@ Tests for the functionality and infrastructure of grades tasks.
 
 import ddt
 from django.conf import settings
+from django.db.utils import IntegrityError
 from mock import patch
 from unittest import skip
 
@@ -154,3 +155,20 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
                         self.score_changed_kwargs['usage_id'],
                     )
                 )
+
+    @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade.retry')
+    @patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update')
+    def test_retry_on_integrity_error(self, mock_update, mock_retry):
+        """
+        Ensures that tasks will be retried if IntegrityErrors are encountered.
+        """
+        self.set_up_course()
+        mock_update.side_effect = IntegrityError("WHAMMY")
+        recalculate_subsection_grade.apply(
+            args=(
+                self.score_changed_kwargs['user_id'],
+                self.score_changed_kwargs['course_id'],
+                self.score_changed_kwargs['usage_id'],
+            )
+        )
+        self.assertTrue(mock_retry.called)