Merge remote-tracking branch 'origin/master' into smarnach/review-link

.travis.yml

@@ -17,3 +17,5 @@ script:
     - python run_tests.py --with-coverage --cover-package=problem_builder
 notifications:
   email: false
+addons:
+  firefox: "36.0"

problem_builder/mentoring.py

@@ -26,7 +26,7 @@ import json
 from collections import namedtuple
 
 from xblock.core import XBlock
-from xblock.exceptions import NoSuchViewError
+from xblock.exceptions import NoSuchViewError, JsonHandlerError
 from xblock.fields import Boolean, Scope, String, Integer, Float, List
 from xblock.fragment import Fragment
 from xblock.validation import ValidationMessage
@@ -148,6 +148,7 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         # Has the student attempted this mentoring step?
         default=False,
         scope=Scope.user_state
+        # TODO: Does anything use this 'attempted' field? May want to delete it.
     )
     completed = Boolean(
         # Has the student completed this mentoring step?
@@ -376,15 +377,24 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         return {'result': 'ok'}
 
     def get_message(self, completed):
-        if self.max_attempts_reached:
-            return self.get_message_html('max_attempts_reached')
-        elif completed:
+        """
+        Get the message to display to a student following a submission in normal mode.
+        """
+        if completed:
+            # Student has achieved a perfect score
             return self.get_message_html('completed')
+        elif self.max_attempts_reached:
+            # Student has not achieved a perfect score and cannot try again
+            return self.get_message_html('max_attempts_reached')
         else:
+            # Student did not achieve a perfect score but can try again:
             return self.get_message_html('incomplete')
 
     @property
     def assessment_message(self):
+        """
+        Get the message to display to a student following a submission in assessment mode.
+        """
         if not self.max_attempts_reached:
             return self.get_message_html('on-assessment-review')
         else:
@@ -449,7 +459,6 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         return {
             'results': results,
             'completed': completed,
-            'attempted': self.attempted,
             'message': message,
             'step': step,
             'max_attempts': self.max_attempts,
@@ -459,12 +468,23 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
     @XBlock.json_handler
     def submit(self, submissions, suffix=''):
         log.info(u'Received submissions: {}'.format(submissions))
+        # server-side check that the user is allowed to submit:
+        if self.max_attempts_reached:
+            raise JsonHandlerError(403, "Maximum number of attempts already reached.")
+        elif self.has_missing_dependency:
+            raise JsonHandlerError(
+                403,
+                "You need to complete all previous steps before being able to complete the current one."
+            )
+
+        # This has now been attempted:
         self.attempted = True
 
         if self.is_assessment:
             return self.handle_assessment_submit(submissions, suffix)
 
         submit_results = []
+        previously_completed = self.completed
         completed = True
         for child_id in self.steps:
             child = self.runtime.get_block(child_id)
@@ -475,40 +495,32 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
                 child.save()
                 completed = completed and (child_result['status'] == 'correct')
 
-        message = self.get_message(completed)
-
-        # Once it has been completed once, keep completion even if user changes values
-        if self.completed:
-            completed = True
-
-        # server-side check to not set completion if the max_attempts is reached
-        if self.max_attempts_reached:
-            completed = False
-
-        if self.has_missing_dependency:
-            completed = False
-            message = 'You need to complete all previous steps before being able to complete the current one.'
-        elif completed and self.next_step == self.url_name:
+        if completed and self.next_step == self.url_name:
             self.next_step = self.followed_by
 
-        # Once it was completed, lock score
-        if not self.completed:
-            # save user score and results
+        # Update the score and attempts, unless the user had already achieved a perfect score ("completed"):
+        if not previously_completed:
+            # Update the results
             while self.student_results:
                 self.student_results.pop()
             for result in submit_results:
                 self.student_results.append(result)
 
+            # Save the user's latest score
             self.runtime.publish(self, 'grade', {
                 'value': self.score.raw,
                 'max_value': 1,
             })
 
-        if not self.completed and self.max_attempts > 0:
-            self.num_attempts += 1
+            # Mark this as having used an attempt:
+            if self.max_attempts > 0:
+                self.num_attempts += 1
 
-        self.completed = completed is True
+        # Save the completion status.
+        # Once it has been completed once, keep completion even if user changes values
+        self.completed = bool(completed) or previously_completed
 
+        message = self.get_message(completed)
         raw_score = self.score.raw
 
         self.runtime.publish(self, 'xblock.problem_builder.submitted', {
@@ -520,10 +532,9 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
         return {
             'results': submit_results,
             'completed': self.completed,
-            'attempted': self.attempted,
             'message': message,
             'max_attempts': self.max_attempts,
-            'num_attempts': self.num_attempts
+            'num_attempts': self.num_attempts,
         }
 
     def handle_assessment_submit(self, submissions, suffix):
@@ -561,14 +572,13 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
 
         if current_child == steps[-1]:
             log.info(u'Last assessment step submitted: {}'.format(submissions))
-            if not self.max_attempts_reached:
-                self.runtime.publish(self, 'grade', {
-                    'value': score.raw,
-                    'max_value': 1,
-                    'score_type': 'proficiency',
-                })
-                event_data['final_grade'] = score.raw
-                assessment_message = self.assessment_message
+            self.runtime.publish(self, 'grade', {
+                'value': score.raw,
+                'max_value': 1,
+                'score_type': 'proficiency',
+            })
+            event_data['final_grade'] = score.raw
+            assessment_message = self.assessment_message
 
             self.num_attempts += 1
             self.completed = True
@@ -581,7 +591,6 @@ class MentoringBlock(XBlock, StepParentMixin, StudioEditableXBlockMixin, StudioC
 
         return {
             'completed': completed,
-            'attempted': self.attempted,
             'max_attempts': self.max_attempts,
             'num_attempts': self.num_attempts,
             'step': self.step,

problem_builder/message.py

@@ -40,6 +40,52 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
     A message which can be conditionally displayed at the mentoring block level,
     for example upon completion of the block
     """
+    MESSAGE_TYPES = {
+        "completed": {
+            "display_name": _(u"Completed"),
+            "long_display_name": _(u"Message shown when complete"),
+            "default": _(u"Great job!"),
+            "description": _(
+                u"In standard mode, this message will be shown when the student achieves a "
+                "perfect score. "
+                "This message is ignored in assessment mode."
+            ),
+        },
+        "incomplete": {
+            "display_name": _(u"Incomplete"),
+            "long_display_name": _(u"Message shown when incomplete"),
+            "default": _(u"Not quite! You can try again, though."),
+            "description": _(
+                u"In standard mode, this message will be shown when the student gets at least "
+                "one question wrong, but is allowed to try again. "
+                "This message is ignored in assessment mode."
+            ),
+        },
+        "max_attempts_reached": {
+            "display_name": _(u"Reached max. # of attempts"),
+            "long_display_name": _(u"Message shown when student reaches max. # of attempts"),
+            "default": _(u"Sorry, you have used up all of your allowed submissions."),
+            "description": _(
+                u"In standard mode, this message will be shown when the student has used up "
+                "all of their allowed attempts without achieving a perfect score. "
+                "This message is ignored in assessment mode."
+            ),
+        },
+        "on-assessment-review": {
+            "display_name": _(u"Review with attempts left"),
+            "long_display_name": _(u"Message shown during review when attempts remain"),
+            "default": _(
+                u"You may try this assessment again, and only the latest score will be used."
+            ),
+            "description": _(
+                u"In assessment mode, this message will be shown when the student is reviewing "
+                "their answers to the assessment, if the student is allowed to try again. "
+                "This message is ignored in standard mode and is not shown if the student has "
+                "used up all of their allowed attempts."
+            ),
+        },
+    }
+
     content = String(
         display_name=_("Message"),
         help=_("Message to display upon completion"),
@@ -53,10 +99,10 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
         scope=Scope.content,
         default="completed",
         values=(
-            {"display_name": "Completed", "value": "completed"},
-            {"display_name": "Incompleted", "value": "incomplete"},
-            {"display_name": "Reached max. # of attemps", "value": "max_attempts_reached"},
-            {"display_name": "Review with attempts left", "value": "on-assessment-review"}
+            {"value": "completed", "display_name": MESSAGE_TYPES["completed"]["display_name"]},
+            {"value": "incomplete", "display_name": MESSAGE_TYPES["incomplete"]["display_name"]},
+            {"value": "max_attempts_reached", "display_name": MESSAGE_TYPES["max_attempts_reached"]["display_name"]},
+            {"value": "on-assessment-review", "display_name": MESSAGE_TYPES["on-assessment-review"]["display_name"]},
         ),
     )
     editable_fields = ("content", )
@@ -67,34 +113,44 @@ class MentoringMessageBlock(XBlock, StudioEditableXBlockMixin):
 
     def mentoring_view(self, context=None):
         """ Render this message for use by a mentoring block. """
-        html = u'<div class="message {msg_type}">{content}</div>'.format(msg_type=self.type, content=self.content)
+        html = u'<div class="submission-message {msg_type}">{content}</div>'.format(
+            msg_type=self.type,
+            content=self.content
+        )
         return Fragment(html)
 
     def student_view(self, context=None):
         """ Normal view of this XBlock, identical to mentoring_view """
         return self.mentoring_view(context)
 
+    def author_view(self, context=None):
+        fragment = self.mentoring_view(context)
+        fragment.content += u'<div class="submission-message-help"><p>{}</p></div>'.format(self.help_text)
+        return fragment
+
     @property
     def display_name_with_default(self):
-        if self.type == 'max_attempts_reached':
-            max_attempts = self.get_parent().max_attempts
-            return self._(u"Message when student reaches max. # of attempts ({limit})").format(
-                limit=self._(u"unlimited") if max_attempts == 0 else max_attempts
-            )
-        if self.type == 'completed':
-            return self._(u"Message shown when complete")
-        if self.type == 'incomplete':
-            return self._(u"Message shown when incomplete")
-        if self.type == 'on-assessment-review':
-            return self._(u"Message shown during review when attempts remain")
-        return u"INVALID MESSAGE"
+        try:
+            return self._(self.MESSAGE_TYPES[self.type]["long_display_name"])
+        except KeyError:
+            return u"INVALID MESSAGE"
+
+    @property
+    def help_text(self):
+        try:
+            return self._(self.MESSAGE_TYPES[self.type]["description"])
+        except KeyError:
+            return u"This message is not a valid message type!"
 
     @classmethod
     def get_template(cls, template_id):
         """
         Used to interact with Studio's create_xblock method to instantiate pre-defined templates.
         """
-        return {'data': {'type': template_id, 'content': "Message goes here."}}
+        return {'data': {
+            'type': template_id,
+            'content': cls.MESSAGE_TYPES[template_id]["default"],
+        }}
 
     @classmethod
     def parse_xml(cls, node, runtime, keys, id_generator):

problem_builder/public/css/mentoring_edit.css

@@ -26,3 +26,11 @@
   border-color: #888;
   cursor: default;
 }
+
+.xblock[data-block-type=problem-builder] .submission-message-help p {
+	border-top: 1px solid #ddd;
+	font-size: 0.85em;
+	font-style: italic;
+	margin-top: 1em;
+	padding-top: 0.3em;
+}

problem_builder/public/js/answer.js

@@ -24,7 +24,6 @@ function AnswerBlock(runtime, element) {
         handleSubmit: function(result) {
 
             var checkmark = $('.answer-checkmark', element);
-            $(element).find('.message').text((result || {}).error || '');
 
             this.clearResult();
 

problem_builder/public/js/mentoring_edit.js

@@ -5,7 +5,7 @@ function MentoringEditComponents(runtime, element) {
     var updateButtons = function() {
         $buttons.each(function() {
             var msg_type = $(this).data('boilerplate');
-            $(this).toggleClass('disabled', $('.xblock .message.'+msg_type).length > 0);
+            $(this).toggleClass('disabled', $('.xblock .submission-message.'+msg_type).length > 0);
         });
     };
     updateButtons();

problem_builder/public/js/mentoring_standard_view.js

@@ -32,6 +32,24 @@ function MentoringStandardView(runtime, element, mentoring) {
         submitDOM.attr('disabled', 'disabled');
     }
 
+    function handleSubmitError(jqXHR, textStatus, errorThrown) {
+        if (textStatus == "error") {
+            var errMsg = errorThrown;
+            // Check if there's a more specific JSON error message:
+            if (jqXHR.responseText) {
+                // Is there a more specific error message we can show?
+                try {
+                    errMsg = JSON.parse(jqXHR.responseText).error;
+                } catch (error) { errMsg = jqXHR.responseText.substr(0, 300); }
+            }
+
+            mentoring.setContent(messagesDOM, errMsg);
+            messagesDOM.show();
+
+            submitDOM.attr('disabled', 'disabled');
+        }
+    }
+
     function calculate_results(handler_name) {
         var data = {};
         var children = mentoring.children;
@@ -45,11 +63,7 @@ function MentoringStandardView(runtime, element, mentoring) {
         if (submitXHR) {
             submitXHR.abort();
         }
-        submitXHR = $.post(handlerUrl, JSON.stringify(data)).success(handleSubmitResults);
-    }
-
-    function get_results() {
-        calculate_results('get_results');
+        submitXHR = $.post(handlerUrl, JSON.stringify(data)).success(handleSubmitResults).error(handleSubmitError);
     }
 
     function submit() {

problem_builder/tests/integration/base_test.py

@@ -65,13 +65,33 @@ class MentoringBaseTest(SeleniumBaseTest, PopupCheckMixin):
     default_css_selector = 'div.mentoring'
 
 
-class MentoringAssessmentBaseTest(SeleniumXBlockTest, PopupCheckMixin):
+class MentoringBaseTemplateTest(SeleniumXBlockTest, PopupCheckMixin):
     """
-    Base class for tests of assessment mode
+    Base class for mentoring tests that use templated XML.
+    All new tests should inherit from this rather than MentoringBaseTest
     """
     module_name = __name__
     default_css_selector = 'div.mentoring'
 
+    def load_scenario(self, xml_file, params=None):
+        params = params or {}
+        scenario = loader.render_template("xml_templates/{}".format(xml_file), params)
+        self.set_scenario_xml(scenario)
+        return self.go_to_view("student_view")
+
+    def click_submit(self, mentoring):
+        """ Click the submit button and wait for the response """
+        submit = mentoring.find_element_by_css_selector('.submit input.input-main')
+        self.assertTrue(submit.is_displayed())
+        self.assertTrue(submit.is_enabled())
+        submit.click()
+        self.wait_until_disabled(submit)
+
+
+class MentoringAssessmentBaseTest(MentoringBaseTemplateTest):
+    """
+    Base class for tests of assessment mode
+    """
     @staticmethod
     def question_text(number):
         if number:

problem_builder/tests/integration/test_assessment.py

@@ -394,7 +394,7 @@ class MentoringAssessmentTest(MentoringAssessmentBaseTest):
         self.peek_at_review(mentoring, controls, expected_results)
         self.assert_messages_empty(mentoring)
 
+        self.wait_until_clickable(controls.try_again)
         controls.try_again.click()
-        # this is a wait and assertion all together - it waits until expected text is in mentoring block
-        # and it fails with PrmoiseFailed exception if it's not
-        self.wait_until_text_in(self.question_text(0), mentoring)
+        self.wait_until_hidden(controls.try_again)
+        self.assertIn(self.question_text(0), mentoring.text)

problem_builder/tests/integration/test_dashboard.py

@@ -18,7 +18,7 @@
 # "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
 #
 from mock import Mock, patch
-from xblockutils.base_test import SeleniumXBlockTest
+from .base_test import MentoringBaseTemplateTest
 
 
 class MockSubmissionsAPI(object):
@@ -50,77 +50,20 @@ class MockSubmissionsAPI(object):
         return []
 
 
-class TestDashboardBlock(SeleniumXBlockTest):
+class TestDashboardBlock(MentoringBaseTemplateTest):
     """
     Test the Student View of a dashboard XBlock linked to some problem builder blocks
     """
     def setUp(self):
         super(TestDashboardBlock, self).setUp()
         # Set up our scenario:
-        self.set_scenario_xml("""
-            <vertical_demo>
-                <problem-builder display_name="Step 1">
-                    <pb-mcq display_name="1.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                    <pb-mcq display_name="1.2 Second MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                    <pb-mcq display_name="1.3 Third MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                    <html_demo> This message here should be ignored. </html_demo>
-                </problem-builder>
-                <problem-builder display_name="Step 2">
-                    <pb-mcq display_name="2.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="4">Option 4</pb-choice>
-                        <pb-choice value="5">Option 5</pb-choice>
-                        <pb-choice value="6">Option 6</pb-choice>
-                    </pb-mcq>
-                    <pb-mcq display_name="2.2 Second MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                    <pb-mcq display_name="2.3 Third MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                </problem-builder>
-                <problem-builder display_name="Step 3">
-                    <pb-mcq display_name="3.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="1">Option 1</pb-choice>
-                        <pb-choice value="2">Option 2</pb-choice>
-                        <pb-choice value="3">Option 3</pb-choice>
-                        <pb-choice value="4">Option 4</pb-choice>
-                    </pb-mcq>
-                    <pb-mcq display_name="3.2 MCQ with non-numeric values"
-                      question="Which option?" correct_choices='["1","2","3","4"]'>
-                        <pb-choice value="A">Option A</pb-choice>
-                        <pb-choice value="B">Option B</pb-choice>
-                        <pb-choice value="C">Option C</pb-choice>
-                    </pb-mcq>
-                </problem-builder>
-                <pb-dashboard mentoring_ids='["dummy-value"]'>
-                </pb-dashboard>
-            </vertical_demo>
-        """)
+        self.load_scenario('dashboard.xml')
 
         # Apply a whole bunch of patches that are needed in lieu of the LMS/CMS runtime and edx-submissions:
+
         def get_mentoring_blocks(dashboard_block, mentoring_ids, ignore_errors=True):
             return [dashboard_block.runtime.get_block(key) for key in dashboard_block.get_parent().children[:-1]]
+
         mock_submisisons_api = MockSubmissionsAPI()
         patches = (
             (
@@ -174,10 +117,7 @@ class TestDashboardBlock(SeleniumXBlockTest):
             for idx, mcq in enumerate(mcqs):
                 choices = mcq.find_elements_by_css_selector('.choices .choice label')
                 choices[idx].click()
-            submit = pb.find_element_by_css_selector('.submit input.input-main')
-            self.assertTrue(submit.is_enabled())
-            submit.click()
-            self.wait_until_disabled(submit)
+            self.click_submit(pb)
 
         # Reload the page:
         self.go_to_view("student_view")

problem_builder/tests/integration/test_messages.py

+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
+#
+# This software's license gives you freedom; you can copy, convey,
+# propagate, redistribute and/or modify this program under the terms of
+# the GNU Affero General Public License (AGPL) as published by the Free
+# Software Foundation (FSF), either version 3 of the License, or (at your
+# option) any later version of the AGPL published by the FSF.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program in a file in the toplevel directory called
+# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
+#
+from .base_test import MentoringBaseTemplateTest
+import ddt
+
+
+COMPLETED, INCOMPLETE, MAX_REACHED = "completed", "incomplete", "max_attempts_reached"
+MESSAGES = {
+    COMPLETED: u"Great job! (completed message)",
+    INCOMPLETE: u"Not quite! You can try again, though. (incomplete message)",
+    MAX_REACHED: (
+        u"Sorry, you have used up all of your allowed submissions. (max_attempts_reached message)"
+    ),
+}
+
+
+@ddt.ddt
+class MessagesTest(MentoringBaseTemplateTest):
+    """
+    Test the various types of message that can be added to a problem.
+    """
+    def expect_message(self, msg_type, mentoring):
+        """
+        Assert that the message of the specified type is shown to the user.
+        """
+        messages_element = mentoring.find_element_by_css_selector('.messages')
+        if msg_type is None:
+            self.assertFalse(messages_element.is_displayed())
+        else:
+            self.assertTrue(messages_element.is_displayed())
+            message_text = messages_element.text.strip()
+            self.assertTrue(message_text.startswith("FEEDBACK"))
+            message_text = message_text[8:].lstrip()
+            self.assertEqual(MESSAGES[msg_type], message_text)
+
+    def click_choice(self, container, choice_text):
+        """ Click on the choice label with the specified text """
+        for label in container.find_elements_by_css_selector('.choices .choice label'):
+            if choice_text in label.text:
+                label.click()
+                break
+
+    @ddt.data(
+        ("One", COMPLETED),
+        ("Two", COMPLETED),
+        ("I don't understand", MAX_REACHED),
+    )
+    @ddt.unpack
+    def test_one_shot(self, choice_text, expected_message_type):
+        """
+        Test a question that has max_attempts set to 1
+        """
+        mentoring = self.load_scenario("messages.xml", {"max_attempts": 1})
+        self.expect_message(None, mentoring)
+        self.click_choice(mentoring, choice_text)
+        self.click_submit(mentoring)
+        self.expect_message(expected_message_type, mentoring)
+
+    @ddt.data(
+        (2, "One", COMPLETED),
+        (2, "I don't understand", MAX_REACHED),
+        (0, "I don't understand", INCOMPLETE),
+        (10, "I don't understand", INCOMPLETE),
+    )
+    @ddt.unpack
+    def test_retry(self, max_attempts, choice_text, expected_message_type):
+        """
+        Test submitting a wrong answer, seeing a message, then submitting another answer.
+        In each case, max_attempts is not 1.
+        """
+        mentoring = self.load_scenario("messages.xml", {"max_attempts": max_attempts})
+        # First, there should be no message.
+        self.expect_message(None, mentoring)
+        # Let's get the question wrong:
+        self.click_choice(mentoring, "I don't understand")
+        self.click_submit(mentoring)
+        # We should now see the INCOMPLETE message:
+        self.expect_message(INCOMPLETE, mentoring)
+
+        # Now, answer as directed and expect the given message:
+        self.click_choice(mentoring, "One")  # Make sure we change our choice so we will be able to submit
+        self.click_choice(mentoring, choice_text)
+        self.click_submit(mentoring)
+        self.expect_message(expected_message_type, mentoring)

problem_builder/tests/integration/test_progression.py

@@ -68,6 +68,7 @@ class MentoringProgressionTest(MentoringBaseTest):
         answer = mentoring.find_element_by_css_selector('textarea')
         answer.send_keys('This is the answer')
         submit = mentoring.find_element_by_css_selector('.submit input.input-main')
+        self.assertTrue(submit.is_displayed() and submit.is_enabled())
         submit.click()
         self.wait_until_disabled(submit)
 
@@ -90,6 +91,7 @@ class MentoringProgressionTest(MentoringBaseTest):
         answer = mentoring.find_element_by_css_selector('textarea')
         answer.send_keys('This is the answer')
         submit = mentoring.find_element_by_css_selector('.submit input.input-main')
+        self.assertTrue(submit.is_displayed() and submit.is_enabled())
         submit.click()
         self.wait_until_disabled(submit)
         self.assert_warning_is_hidden(mentoring)
@@ -106,7 +108,11 @@ class MentoringProgressionTest(MentoringBaseTest):
 
         # Complete step 2 - no more warnings anywhere
         submit = mentoring.find_element_by_css_selector('.submit input.input-main')
-        submit.click()  # Already filled the textarea in previous step
+        answer = mentoring.find_element_by_css_selector('textarea')
+        self.assertEqual(answer.text, "")  # Earlier attempt to submit did not save
+        answer.send_keys('This is the answer')
+        self.assertTrue(submit.is_displayed() and submit.is_enabled())
+        submit.click()
         self.wait_until_disabled(submit)
 
         messages = mentoring.find_element_by_css_selector('.messages')

problem_builder/tests/integration/xml_templates/dashboard.xml

+<vertical_demo>
+    <problem-builder display_name="Step 1">
+        <pb-mcq display_name="1.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+        <pb-mcq display_name="1.2 Second MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+        <pb-mcq display_name="1.3 Third MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+        <html_demo> This message here should be ignored. </html_demo>
+    </problem-builder>
+    <problem-builder display_name="Step 2">
+        <pb-mcq display_name="2.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="4">Option 4</pb-choice>
+            <pb-choice value="5">Option 5</pb-choice>
+            <pb-choice value="6">Option 6</pb-choice>
+        </pb-mcq>
+        <pb-mcq display_name="2.2 Second MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+        <pb-mcq display_name="2.3 Third MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+    </problem-builder>
+    <problem-builder display_name="Step 3">
+        <pb-mcq display_name="3.1 First MCQ" question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="1">Option 1</pb-choice>
+            <pb-choice value="2">Option 2</pb-choice>
+            <pb-choice value="3">Option 3</pb-choice>
+            <pb-choice value="4">Option 4</pb-choice>
+        </pb-mcq>
+        <pb-mcq display_name="3.2 MCQ with non-numeric values"
+          question="Which option?" correct_choices='["1","2","3","4"]'>
+            <pb-choice value="A">Option A</pb-choice>
+            <pb-choice value="B">Option B</pb-choice>
+            <pb-choice value="C">Option C</pb-choice>
+        </pb-mcq>
+    </problem-builder>
+    <pb-dashboard mentoring_ids='["dummy-value"]'>
+    </pb-dashboard>
+</vertical_demo>

problem_builder/tests/integration/xml_templates/messages.xml

+<problem-builder url_name="messages-test" display_name="A Simple Problem Set" max_attempts="{{max_attempts}}">
+    <html_demo>
+        <p>This is a test of messages</p>
+    </html_demo>
+
+    <pb-mcq name="mcq_1_1" question="What is 1+0x1?" correct_choices='["one", "two"]'>
+        <pb-choice value="one">One</pb-choice>
+        <pb-choice value="two">Two</pb-choice>
+        <pb-choice value="huh">I don't understand</pb-choice>
+
+        <pb-tip values='["one"]'>Yep, if you interpret 'x' as multiplication, this equals one.</pb-tip>
+        <pb-tip values='["two"]'>You must be a programmer. If you interpret '0x' as a hexadecimal prefix, this equals two.</pb-tip>
+    </pb-mcq>
+
+    <pb-message type="completed">Great job! (completed message)</pb-message>
+    <pb-message type="incomplete">Not quite! You can try again, though. (incomplete message)</pb-message>
+    <pb-message type="max_attempts_reached">Sorry, you have used up all of your allowed submissions. (max_attempts_reached message)</pb-message>
+    <pb-message type="on-assessment-review">You may try this assessment again, and only the latest score will be used. (on-assessment-review message)</pb-message>
+</problem-builder>