Merge pull request #61 from open-craft/assessment-targeted-review

Assessment targeted review

problem_builder/mentoring.py

@@ -34,6 +34,7 @@ from xblock.validation import ValidationMessage
 from .message import MentoringMessageBlock
 from .step import StepParentMixin, StepMixin
 
+from xblockutils.helpers import child_isinstance
 from xblockutils.resources import ResourceLoader
 from xblockutils.studio_editable import StudioEditableXBlockMixin, StudioContainerXBlockMixin
 
@@ -254,7 +255,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
     @property
     def score(self):
         """Compute the student score taking into account the weight of each step."""
-        steps = [self.runtime.get_block(step_id) for step_id in self.steps]
+        steps = self.get_steps()
         steps_map = {q.name: q for q in steps}
         total_child_weight = sum(float(step.weight) for step in steps)
         if total_child_weight == 0:
@@ -282,7 +283,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         self.migrate_fields()
 
         # Validate self.step:
-        num_steps = len(self.steps)
+        num_steps = len(self.get_steps())
         if self.step > num_steps:
             self.step = num_steps
 
@@ -323,8 +324,10 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         fragment.add_javascript_url(self.runtime.local_resource_url(self, js_file))
         fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/mentoring.js'))
         fragment.add_resource(loader.load_unicode('templates/html/mentoring_attempts.html'), "text/html")
-        fragment.add_resource(loader.load_unicode('templates/html/mentoring_grade.html'), "text/html")
-        fragment.add_resource(loader.load_unicode('templates/html/mentoring_review_questions.html'), "text/html")
+        if self.is_assessment:
+            fragment.add_resource(
+                loader.load_unicode('templates/html/mentoring_assessment_templates.html'), "text/html"
+            )
 
         self.include_theme_files(fragment)
         # Workbench doesn't have font awesome, so add it:
@@ -421,10 +424,30 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         Get the message to display to a student following a submission in assessment mode.
         """
         if not self.max_attempts_reached:
-            return self.get_message_content('on-assessment-review')
+            return self.get_message_content('on-assessment-review', or_default=True)
         else:
             return None
 
+    @property
+    def review_tips(self):
+        """ Get review tips, shown for wrong answers in assessment mode. """
+        if not self.is_assessment or self.step != len(self.steps):
+            return []  # Review tips are only used in assessment mode, and only on the last step.
+        review_tips = []
+        status_cache = dict(self.student_results)
+        for child in self.get_steps():
+            result = status_cache.get(child.name)
+            if result and result.get('status') != 'correct':
+                # The student got this wrong. Check if there is a review tip to show.
+                tip_html = child.get_review_tip()
+                if tip_html:
+                    review_tips.append(tip_html)
+        return review_tips
+
+    @property
+    def review_tips_json(self):
+        return json.dumps(self.review_tips)
+
     def show_extended_feedback(self):
         return self.extended_feedback and self.max_attempts_reached
 
@@ -488,8 +511,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         show_message = bool(self.student_results)
 
         # In standard mode, all children is visible simultaneously, so need collecting responses from all of them
-        for child_id in self.steps:
-            child = self.runtime.get_block(child_id)
+        for child in self.get_steps():
             child_result = child.get_last_result()
             results.append([child.name, child_result])
             completed = completed and (child_result.get('status', None) == 'correct')
@@ -512,8 +534,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         completed = True
         choices = dict(self.student_results)
         # Only one child should ever be of concern with this method.
-        for child_id in self.steps:
-            child = self.runtime.get_block(child_id)
+        for child in self.get_steps():
             if child.name and child.name in queries:
                 results = [child.name, child.get_results(choices[child.name])]
                 # Children may have their own definition of 'completed' which can vary from the general case
@@ -546,8 +567,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         submit_results = []
         previously_completed = self.completed
         completed = True
-        for child_id in self.steps:
-            child = self.runtime.get_block(child_id)
+        for child in self.get_steps():
             if child.name and child.name in submissions:
                 submission = submissions[child.name]
                 child_result = child.submit(submission)
@@ -604,6 +624,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         children = [child for child in children if not isinstance(child, MentoringMessageBlock)]
         steps = [child for child in children if isinstance(child, StepMixin)]  # Faster than the self.steps property
         assessment_message = None
+        review_tips = []
 
         for child in children:
             if child.name and child.name in submissions:
@@ -639,6 +660,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
             })
             event_data['final_grade'] = score.raw
             assessment_message = self.assessment_message
+            review_tips = self.review_tips
 
             self.num_attempts += 1
             self.completed = True
@@ -663,6 +685,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
             'partial': self.partial_json(stringify=False),
             'extended_feedback': self.show_extended_feedback() or '',
             'assessment_message': assessment_message,
+            'assessment_review_tips': review_tips,
         }
 
     @XBlock.json_handler
@@ -689,11 +712,16 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
     def max_attempts_reached(self):
         return self.max_attempts > 0 and self.num_attempts >= self.max_attempts
 
-    def get_message_content(self, message_type):
+    def get_message_content(self, message_type, or_default=False):
         for child_id in self.children:
-            child = self.runtime.get_block(child_id)
-            if isinstance(child, MentoringMessageBlock) and child.type == message_type:
-                return child.content
+            if child_isinstance(self, child_id, MentoringMessageBlock):
+                child = self.runtime.get_block(child_id)
+                if child.type == message_type:
+                    return child.content
+        if or_default:
+            # Return the default value since no custom message is set.
+            # Note the WYSIWYG editor usually wraps the .content HTML in a <p> tag so we do the same here.
+            return '<p>{}</p>'.format(MentoringMessageBlock.MESSAGE_TYPES[message_type]['default'])
 
     def validate(self):
         """

problem_builder/message.py

@@ -75,7 +75,8 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
             "display_name": _(u"Review with attempts left"),
             "long_display_name": _(u"Message shown during review when attempts remain"),
             "default": _(
-                u"You may try this assessment again, and only the latest score will be used."
+                u"Note: if you retake this assessment, only your final score counts. "
+                "If you would like to keep this score, please continue to the next unit."
             ),
             "description": _(
                 u"In assessment mode, this message will be shown when the student is reviewing "
@@ -84,6 +85,20 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
                 "used up all of their allowed attempts."
             ),
         },
+        "on-assessment-review-question": {
+            "display_name": _(u"Study tips if this question was wrong"),
+            "long_display_name": _(u"Study tips shown during assessment review if wrong"),
+            "default": _(
+                u"Review ____."
+            ),
+            "description": _(
+                u"In assessment mode, this message will be shown when the student is reviewing "
+                "their answers to the assessment, if the student got this specific question "
+                "wrong and is allowed to try again. "
+                "This message is ignored in standard mode and is not shown if the student has "
+                "used up all of their allowed attempts."
+            ),
+        },
     }
 
     content = String(
@@ -103,6 +118,10 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
             {"value": "incomplete", "display_name": MESSAGE_TYPES["incomplete"]["display_name"]},
             {"value": "max_attempts_reached", "display_name": MESSAGE_TYPES["max_attempts_reached"]["display_name"]},
             {"value": "on-assessment-review", "display_name": MESSAGE_TYPES["on-assessment-review"]["display_name"]},
+            {
+                "value": "on-assessment-review-question",
+                "display_name": MESSAGE_TYPES["on-assessment-review-question"]["display_name"]
+            },
         ),
     )
     editable_fields = ("content", )

problem_builder/public/css/problem-builder.css

@@ -2,13 +2,11 @@
     margin: 1em 0em;
 }
 
-.mentoring .messages,
-.mentoring .assessment-messages {
+.mentoring .messages {
     display: none;
 }
 
-.mentoring .messages .title1,
-.mentoring .assessment-messages .title1 {
+.mentoring .messages .title1 {
     color: #333333;
     text-transform: uppercase;
     font-weight: bold;
@@ -178,6 +176,26 @@
     display: none;
 }
 
+.mentoring .assessment-review-tips p.review-tips-intro {
+    margin-top: 1.2em;
+    margin-bottom: 0;
+    font-weight: bold;
+}
+
+.mentoring .assessment-review-tips .review-tips-list {
+    margin-top: 0;
+    padding-top: 0;
+}
+.mentoring .assessment-review-tips .review-tips-list li {
+    margin-left: 0.5em;
+    padding-left: 0;
+}
+.mentoring .assessment-review-tips .review-tips-list li p {
+    display: inline;
+    margin: 0;
+}
+
+
 .pb-clarification span.clarification i {
     font-style: normal;
 }

problem_builder/public/css/questionnaire-edit.css

@@ -4,3 +4,20 @@
   height: 30px;
   line-height: 30px;
 }
+
+.xblock .add-xblock-component .new-component .new-component-type .add-xblock-component-button.disabled,
+.xblock .add-xblock-component .new-component .new-component-type .add-xblock-component-button.disabled:hover {
+  background-color: #ccc;
+  border-color: #888;
+  cursor: default;
+}
+
+.xblock[data-block-type=pb-mcq] .submission-message-help p,
+.xblock[data-block-type=pb-mrq] .submission-message-help p,
+.xblock[data-block-type=pb-rating] .submission-message-help p {
+	border-top: 1px solid #ddd;
+	font-size: 0.85em;
+	font-style: italic;
+	margin-top: 1em;
+	padding-top: 0.3em;
+}

problem_builder/public/js/mentoring_assessment_view.js

 function MentoringAssessmentView(runtime, element, mentoring) {
     var gradeTemplate = _.template($('#xblock-grade-template').html());
-    var reviewQuestionsTemplate = _.template($('#xblock-review-questions-template').html());
-    var submitDOM, nextDOM, reviewDOM, tryAgainDOM, messagesDOM, reviewLinkDOM;
+    var reviewQuestionsTemplate = _.template($('#xblock-review-questions-template').html()); // Detailed list of which questions the user got wrong
+    var reviewTipsTemplate = _.template($('#xblock-review-tips-template').html()); // Tips about specific questions the user got wrong
+    var submitDOM, nextDOM, reviewDOM, tryAgainDOM, assessmentMessageDOM, reviewLinkDOM, reviewTipsDOM;
     var submitXHR;
     var checkmark;
     var active_child;
@@ -24,7 +25,8 @@ function MentoringAssessmentView(runtime, element, mentoring) {
 
         $('.grade').html('');
         $('.attempts').html('');
-        messagesDOM.empty().hide();
+        assessmentMessageDOM.html('');
+        reviewTipsDOM.empty().hide();
     }
 
     function no_more_attempts() {
@@ -65,9 +67,21 @@ function MentoringAssessmentView(runtime, element, mentoring) {
         }
 
         mentoring.renderAttempts();
-        if (data.assessment_message && (data.max_attempts === 0 || data.num_attempts < data.max_attempts)) {
-            mentoring.setContent(messagesDOM, data.assessment_message);
-            messagesDOM.show();
+        if (data.max_attempts === 0 || data.num_attempts < data.max_attempts) {
+            if (data.assessment_message) {
+                // Overall on-assessment-review message:
+                assessmentMessageDOM.html(data.assessment_message);
+            }
+            if (data.assessment_review_tips.length > 0) {
+                // on-assessment-review-question messages specific to questions the student got wrong:
+                reviewTipsDOM.html(reviewTipsTemplate({
+                    tips: data.assessment_review_tips
+                }));
+                reviewTipsDOM.show();
+            }
+        } else {
+            var msg = gettext("Note: you have used all attempts. Continue to the next unit.");
+            assessmentMessageDOM.html('').append($('<p></p>').html(msg));
         }
         $('a.question-link', element).click(reviewJump);
     }
@@ -102,7 +116,8 @@ function MentoringAssessmentView(runtime, element, mentoring) {
         tryAgainDOM = $(element).find('.submit .input-try-again');
         reviewLinkDOM = $(element).find('.review-link');
         checkmark = $('.assessment-checkmark', element);
-        messagesDOM = $('.assessment-messages', element);
+        assessmentMessageDOM = $('.assessment-message', element);
+        reviewTipsDOM = $('.assessment-review-tips', element);
 
         submitDOM.show();
         submitDOM.bind('click', submit);

problem_builder/public/js/questionnaire_edit.js

-function QuestionnaireEdit(runtime, element) {
-    'use strict';
-    ProblemBuilderUtil.transformClarifications(element);
-}

problem_builder/public/themes/apros.css

@@ -16,3 +16,8 @@
 .mentoring h3 {
     text-transform: uppercase;
 }
+
+.themed-xblock.mentoring .assessment-review-tips .review-tips-list li {
+    margin-left: 1.8em;
+    padding-left: 0;
+}

problem_builder/questionnaire.py

@@ -33,6 +33,7 @@ from xblockutils.studio_editable import StudioEditableXBlockMixin, StudioContain
 
 from .choice import ChoiceBlock
 from .mentoring import MentoringBlock
+from .message import MentoringMessageBlock
 from .step import StepMixin
 from .tip import TipBlock
 
@@ -180,8 +181,8 @@ class QuestionnaireAbstractBlock(StudioEditableXBlockMixin, StudioContainerXBloc
         fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/problem-builder.css'))
         fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/questionnaire-edit.css'))
         fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/util.js'))
-        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/questionnaire_edit.js'))
-        fragment.initialize_js('QuestionnaireEdit')
+        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/mentoring_edit.js'))
+        fragment.initialize_js('MentoringEditComponents')
         return fragment
 
     def validate_field_data(self, validation, data):
@@ -226,3 +227,11 @@ class QuestionnaireAbstractBlock(StudioEditableXBlockMixin, StudioContainerXBloc
                 break
             values_with_tips.update(values)
         return validation
+
+    def get_review_tip(self):
+        """ Get the text to show on the assessment review when the student gets this question wrong """
+        for child_id in self.children:
+            if child_isinstance(self, child_id, MentoringMessageBlock):
+                child = self.runtime.get_block(child_id)
+                if child.type == "on-assessment-review-question":
+                    return child.content

problem_builder/step.py

@@ -45,13 +45,19 @@ class StepParentMixin(object):
     An XBlock mixin for a parent block containing Step children
     """
 
-    @property
+    @lazy
     def steps(self):
         """
         Get the usage_ids of all of this XBlock's children that are "Steps"
         """
         return [_normalize_id(child_id) for child_id in self.children if child_isinstance(self, child_id, StepMixin)]
 
+    def get_steps(self):
+        """ Get the step children of this block, cached if possible. """
+        if getattr(self, "_steps_cache", None) is None:
+            self._steps_cache = [self.runtime.get_block(child_id) for child_id in self.steps]
+        return self._steps_cache
+
 
 class StepMixin(object):
     """

problem_builder/templates/html/mentoring.html

@@ -16,6 +16,8 @@
   {% endif %}
 
   <div class="{{self.mode}}-question-block">
+    <div class="assessment-message"></div>
+
     {{child_content|safe}}
 
     {% if self.display_submit %}
@@ -28,13 +30,12 @@
          data-num_attempts="{{ self.num_attempts }}"
          data-extended_feedback="{%if self.extended_feedback %}True{% endif %}"
          data-assessment_message="{{ self.assessment_message }}"
+         data-assessment_review_tips="{{ self.review_tips_json }}"
          data-correct="{{ self.correct_json }}"
          data-incorrect="{{ self.incorrect_json }}"
          data-partial="{{ self.partial_json }}">
     </div>
 
-    <div class="assessment-messages"></div>
-
     <div class="submit">
       {% if self.mode == 'assessment' %}
       <span class="assessment-checkmark fa icon-2x"></span>
@@ -52,6 +53,7 @@
     </div>
     {% endif %}
     <div class="messages"></div>
+    <div class="assessment-review-tips"></div>
   </div>
   <div class="review-link"><a href="#">Review final grade</a></div>
 </div>

problem_builder/templates/html/mentoring_grade.html → problem_builder/templates/html/mentoring_assessment_templates.html

 <script type="text/template" id="xblock-grade-template">
-  <% if (_.isNumber(max_attempts) && max_attempts > 0 && num_attempts >= max_attempts) {{ %>
-  <p><%= gettext("Note: you have used all attempts. Continue to the next unit.") %></p>
-  <% }} else {{ %>
-  <p><%= gettext("Note: if you retake this assessment, only your final score counts.") %></p>
-  <% }} %>
-
   <div class="grade-result">
     <h2>
       <%= _.template(gettext("You scored {percent}% on this assessment."), {percent: score}, {interpolate: /\{(.+?)\}/g}) %>
@@ -56,3 +50,24 @@
     <hr/>
   </div>
 </script>
+
+<!-- Template for extended feedback: Show extended feedback details when all attempts are used up. -->
+<script type="text/template" id="xblock-review-questions-template">
+<% var q, last_question; %>
+<ul class="review-list <%= label %>-list">
+    <% for (var question in questions) {{ q = questions[question]; last_question = question == questions.length - 1; %>
+    <li><a href="#" class="question-link" data-step="<%= q.number %>"><%= _.template(gettext("Question {number}"), {number: q.number}, {interpolate: /\{(.+?)\}/g}) %></a></li>
+    <% }} %>
+</ul>
+</script>
+
+<!-- Tips about specific questions the student got wrong. From pb-message[type=on-assessment-review-question] blocks -->
+<script type="text/template" id="xblock-review-tips-template">
+<p class="review-tips-intro"><%= gettext("You might consider reviewing the following items before your next assessment attempt:") %></p>
+<ul class="review-tips-list">
+    <% for (var tip_idx in tips) {{ %>
+        <li><%= tips[tip_idx] %></li>
+    <% }} %>
+</ul>
+</script>
+

problem_builder/templates/html/mentoring_review_questions.html

-<script type="text/template" id="xblock-review-questions-template">
-<% var q, last_question; %>
-<ul class="review-list <%= label %>-list">
-    <% for (var question in questions) {{ q = questions[question]; last_question = question == questions.length - 1; %>
-    <li><a href="#" class="question-link" data-step="<%= q.number %>"><%= _.template(gettext("Question {number}"), {number: q.number}, {interpolate: /\{(.+?)\}/g}) %></a></li>
-    <% }} %>
-</ul>
-</script>

problem_builder/templates/html/questionnaire_add_buttons.html

@@ -5,6 +5,7 @@
     <ul class="new-component-type">
       <li><a href="#" class="single-template add-xblock-component-button" data-category="pb-choice" data-boilerplate="studio_default">{% trans "Add Custom Choice" %}</a></li>
       <li><a href="#" class="single-template add-xblock-component-button" data-category="pb-tip">{% trans "Add Tip" %}</a></li>
+      <li><a href="#" class="single-template add-xblock-component-button" data-category="pb-message" data-boilerplate="on-assessment-review-question">{% trans "Message (Assessment Review)" %}</a></li>
     </ul>
   </div>
 </div>

problem_builder/tests/integration/test_assessment.py

@@ -255,7 +255,6 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         self.wait_until_text_in("You scored {percentage}% on this assessment.".format(**expected), mentoring)
         self.assert_persistent_elements_present(mentoring)
         if expected["max_attempts"] > 0 and expected["num_attempts"] < expected["max_attempts"]:
-            self.assertIn("Note: if you retake this assessment, only your final score counts.", mentoring.text)
             self.assertFalse(mentoring.find_elements_by_css_selector('.review-list'))
         elif extended_feedback:
             for q_type in ['correct', 'incorrect', 'partial']:
@@ -289,16 +288,14 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         self.assert_hidden(controls.review)
         self.assert_hidden(controls.review_link)
 
-    def assert_messages_text(self, mentoring, text):
-        messages = mentoring.find_element_by_css_selector('.assessment-messages')
-        self.assertEqual(messages.text, text)
-        self.assertTrue(messages.is_displayed())
+    def assert_message_text(self, mentoring, text):
+        message_wrapper = mentoring.find_element_by_css_selector('.assessment-message')
+        self.assertEqual(message_wrapper.text, text)
+        self.assertTrue(message_wrapper.is_displayed())
 
-    def assert_messages_empty(self, mentoring):
-        messages = mentoring.find_element_by_css_selector('.assessment-messages')
-        self.assertEqual(messages.text, '')
-        self.assertFalse(messages.find_elements_by_xpath('./*'))
-        self.assertFalse(messages.is_displayed())
+    def assert_no_message_text(self, mentoring):
+        message_wrapper = mentoring.find_element_by_css_selector('.assessment-message')
+        self.assertEqual(message_wrapper.text, '')
 
     def extended_feedback_checks(self, mentoring, controls, expected_results):
         # Multiple choice is third correctly answered question
@@ -358,15 +355,18 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         self.peek_at_review(mentoring, controls, expected_results, extended_feedback=extended_feedback)
 
         if max_attempts == 1:
-            self.assert_messages_empty(mentoring)
+            self.assert_message_text(mentoring, "Note: you have used all attempts. Continue to the next unit.")
             self.assert_disabled(controls.try_again)
             return
 
         # The on-assessment-review message is shown if attempts remain:
-        self.assert_messages_text(mentoring, "Assessment additional feedback message text")
+        self.assert_message_text(mentoring, "Assessment additional feedback message text")
         self.assert_clickable(controls.try_again)
         controls.try_again.click()
 
+        self.wait_until_hidden(controls.try_again)
+        self.assert_no_message_text(mentoring)
+
         self.freeform_answer(
             1, mentoring, controls, 'This is a different answer', CORRECT, saved_value='This is the answer'
         )
@@ -386,12 +386,62 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         else:
             self.assert_clickable(controls.try_again)
         if 1 <= max_attempts <= 2:
-            self.assert_messages_empty(mentoring)  # The on-assessment-review message is not shown if no attempts remain
+            self.assert_message_text(mentoring, "Note: you have used all attempts. Continue to the next unit.")
         else:
-            self.assert_messages_text(mentoring, "Assessment additional feedback message text")
+            self.assert_message_text(mentoring, "Assessment additional feedback message text")
         if extended_feedback:
             self.extended_feedback_checks(mentoring, controls, expected_results)
 
+    def test_review_tips(self):
+        params = {
+            "max_attempts": 3,
+            "extended_feedback": False,
+            "include_review_tips": True
+        }
+        mentoring, controls = self.load_assessment_scenario("assessment.xml", params)
+
+        # Get one question wrong and one partially wrong on attempt 1 of 3: ####################
+        self.freeform_answer(1, mentoring, controls, 'This is the answer', CORRECT)
+        self.single_choice_question(2, mentoring, controls, 'Maybe not', INCORRECT)
+        self.rating_question(3, mentoring, controls, "5 - Extremely good", CORRECT)
+        self.multiple_response_question(4, mentoring, controls, ("Its beauty",), PARTIAL, last=True)
+
+        # The review tips for MCQ 2 and the MRQ should be shown:
+        review_tips = mentoring.find_element_by_css_selector('.assessment-review-tips')
+        self.assertTrue(review_tips.is_displayed())
+        self.assertIn('You might consider reviewing the following items', review_tips.text)
+        self.assertIn('Take another look at', review_tips.text)
+        self.assertIn('Lesson 1', review_tips.text)
+        self.assertNotIn('Lesson 2', review_tips.text)  # This MCQ was correct
+        self.assertIn('Lesson 3', review_tips.text)
+        # The on-assessment-review message is also shown if attempts remain:
+        self.assert_message_text(mentoring, "Assessment additional feedback message text")
+
+        self.assert_clickable(controls.try_again)
+        controls.try_again.click()
+
+        # Get no questions wrong on attempt 2 of 3: ############################################
+        self.freeform_answer(1, mentoring, controls, 'This is the answer', CORRECT, saved_value='This is the answer')
+        self.single_choice_question(2, mentoring, controls, 'Yes', CORRECT)
+        self.rating_question(3, mentoring, controls, "5 - Extremely good", CORRECT)
+        user_selection = ("Its elegance", "Its beauty", "Its gracefulness")
+        self.multiple_response_question(4, mentoring, controls, user_selection, CORRECT, last=True)
+
+        self.assert_message_text(mentoring, "Assessment additional feedback message text")
+        self.assertFalse(review_tips.is_displayed())
+
+        self.assert_clickable(controls.try_again)
+        controls.try_again.click()
+
+        # Get some questions wrong again on attempt 3 of 3:
+        self.freeform_answer(1, mentoring, controls, 'This is the answer', CORRECT, saved_value='This is the answer')
+        self.single_choice_question(2, mentoring, controls, 'Maybe not', INCORRECT)
+        self.rating_question(3, mentoring, controls, "1 - Not good at all", INCORRECT)
+        self.multiple_response_question(4, mentoring, controls, ("Its beauty",), PARTIAL, last=True)
+
+        # The review tips will not be shown because no attempts remain:
+        self.assertFalse(review_tips.is_displayed())
+
     def test_single_question_assessment(self):
         """
         No 'Next Question' button on single question assessment.
@@ -405,7 +455,11 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         }
 
         self.peek_at_review(mentoring, controls, expected_results)
-        self.assert_messages_empty(mentoring)
+        self.assert_message_text(
+            mentoring,
+            "Note: if you retake this assessment, only your final score counts. "
+            "If you would like to keep this score, please continue to the next unit."
+        )
 
         self.wait_until_clickable(controls.try_again)
         controls.try_again.click()

problem_builder/tests/integration/xml_templates/assessment.xml

@@ -22,6 +22,11 @@
         <pb-tip values='["yes"]'>Great!</pb-tip>
         <pb-tip values='["maybenot"]'>Ah, damn.</pb-tip>
         <pb-tip values='["understand"]'><div id="test-custom-html">Really?</div></pb-tip>
+        {% if include_review_tips %}
+            <pb-message type="on-assessment-review-question">
+                <html>Take another look at <a href="#">Lesson 1</a></html>
+            </pb-message>
+        {% endif %}
     </pb-mcq>
 
     <pb-rating name="mcq_1_2" low="Not good at all" high="Extremely good" question="How much do you rate this MCQ?" correct_choices='["4","5"]'>
@@ -30,6 +35,11 @@
         <pb-tip values='["4","5"]'>I love good grades.</pb-tip>
         <pb-tip values='["1","2", "3"]'>Will do better next time...</pb-tip>
         <pb-tip values='["notwant"]'>Your loss!</pb-tip>
+        {% if include_review_tips %}
+            <pb-message type="on-assessment-review-question">
+                <html>Take another look at <a href="#">Lesson 2</a></html>
+            </pb-message>
+        {% endif %}
     </pb-rating>
 
     <pb-mrq name="mrq_1_1" question="What do you like in this MRQ?" required_choices='["gracefulness","elegance","beauty"]' message="Question Feedback Message">
@@ -41,6 +51,11 @@
         <pb-tip values='["gracefulness"]'>This MRQ is indeed very graceful</pb-tip>
         <pb-tip values='["elegance","beauty"]'>This is something everyone has to like about this MRQ</pb-tip>
         <pb-tip values='["bugs"]'>Nah, there aren't any!</pb-tip>
+        {% if include_review_tips %}
+            <pb-message type="on-assessment-review-question">
+                <html>Take another look at <a href="#">Lesson 3</a></html>
+            </pb-message>
+        {% endif %}
     </pb-mrq>
 
     <pb-message type="on-assessment-review">