Merge pull request #87 from open-craft/ekolpakov/assessment-submit

Click handlers and backend processing for assessment mode

.travis.yml

@@ -12,7 +12,8 @@ install:
     - "pip install dist/xblock-drag-and-drop-v2-2.0.7.tar.gz"
 script:
     - pep8 drag_and_drop_v2 tests --max-line-length=120
-    - pylint drag_and_drop_v2 tests
+    - pylint drag_and_drop_v2
+    - pylint tests --rcfile=tests/pylintrc
     - python run_tests.py
 notifications:
     email: false

README.md

@@ -96,11 +96,19 @@ and Drop component to a lesson, then click the `EDIT` button.
 
 ![Edit view](/doc/img/edit-view.png)
 
-In the first step, you can set some basic properties of the component,
-such as the title, the mode, the maximum number of attempts, the maximum score,
+In the first step, you can set some basic properties of the component, such as
+the title, the problem mode, the maximum number of attempts, the maximum score,
 the problem text to render above the background image, the introductory feedback
 (shown initially), and the final feedback (shown after the learner successfully
-completes the drag and drop problem).
+completes the drag and drop problem, or when the learner runs out of attempts).
+
+There are two problem modes available:
+
+* **Standard**: In this mode, the learner gets immediate feedback on each
+  attempt to place an item, and the number of attempts is not limited.
+* **Assessment**: In this mode, the learner places all items on the board and
+  then clicks a "Submit" button to get feedback.  The number of attempts can be
+  limited.
 
 ![Drop zone edit](/doc/img/edit-view-zones.png)
 
@@ -126,13 +134,14 @@ potentially, overlap the zones below.
 
 ![Drag item edit](/doc/img/edit-view-items.png)
 
-In the final step, you define the background and text color for drag
-items, as well as the drag items themselves. A drag item can contain
-either text or an image. You can define custom success and error
-feedback for each item. The feedback text is displayed in a popup
-after the learner drops the item on a zone - the success feedback is
-shown if the item is dropped on a correct zone, while the error
-feedback is shown when dropping the item on an incorrect drop zone.
+In the final step, you define the background and text color for drag items, as
+well as the drag items themselves. A drag item can contain either text or an
+image. You can define custom success and error feedback for each item. In
+standard mode, the feedback text is displayed in a popup after the learner drops
+the item on a zone - the success feedback is shown if the item is dropped on a
+correct zone, while the error feedback is shown when dropping the item on an
+incorrect drop zone.  In assessment mode, the success and error feedback texts
+are not used.
 
 You can select any number of zones for an item to belong to using
 the checkboxes; all zones defined in the previous step are available.

drag_and_drop_v2/__init__.py

+""" Drag and Drop v2 XBlock """
 from .drag_and_drop_v2 import DragAndDropBlock

drag_and_drop_v2/default_data.py

+""" Default data for Drag and Drop v2 XBlock """
 from .utils import _
 
 TARGET_IMG_DESCRIPTION = _(
@@ -64,9 +65,7 @@ DEFAULT_DATA = {
                 "incorrect": ITEM_INCORRECT_FEEDBACK,
                 "correct": ITEM_CORRECT_FEEDBACK.format(zone=TOP_ZONE_TITLE)
             },
-      "zones": [
-        TOP_ZONE_ID
-      ],
+            "zones": [TOP_ZONE_ID],
             "imageURL": "",
             "id": 0,
         },
@@ -76,9 +75,7 @@ DEFAULT_DATA = {
                 "incorrect": ITEM_INCORRECT_FEEDBACK,
                 "correct": ITEM_CORRECT_FEEDBACK.format(zone=MIDDLE_ZONE_TITLE)
             },
-      "zones": [
-        MIDDLE_ZONE_ID
-      ],
+            "zones": [MIDDLE_ZONE_ID],
             "imageURL": "",
             "id": 1,
         },
@@ -88,9 +85,7 @@ DEFAULT_DATA = {
                 "incorrect": ITEM_INCORRECT_FEEDBACK,
                 "correct": ITEM_CORRECT_FEEDBACK.format(zone=BOTTOM_ZONE_TITLE)
             },
-      "zones": [
-        BOTTOM_ZONE_ID
-      ],
+            "zones": [BOTTOM_ZONE_ID],
             "imageURL": "",
             "id": 2,
         },
@@ -100,11 +95,7 @@ DEFAULT_DATA = {
                 "incorrect": "",
                 "correct": ITEM_ANY_ZONE_FEEDBACK
             },
-      "zones": [
-        TOP_ZONE_ID,
-        BOTTOM_ZONE_ID,
-        MIDDLE_ZONE_ID
-      ],
+            "zones": [TOP_ZONE_ID, BOTTOM_ZONE_ID, MIDDLE_ZONE_ID],
             "imageURL": "",
             "id": 3
         },

drag_and_drop_v2/drag_and_drop_v2.py

 # -*- coding: utf-8 -*-
 #
-
+""" Drag and Drop v2 XBlock """
 # Imports ###########################################################
 
-import json
-import webob
 import copy
+import json
 import urllib
+import webob
 
 from xblock.core import XBlock
 from xblock.exceptions import JsonHandlerError
@@ -15,7 +15,7 @@ from xblock.fragment import Fragment
 from xblockutils.resources import ResourceLoader
 from xblockutils.settings import XBlockWithSettingsMixin, ThemableXBlockMixin
 
-from .utils import _  # pylint: disable=unused-import
+from .utils import _, DummyTranslationService, FeedbackMessage, FeedbackMessages
 from .default_data import DEFAULT_DATA
 
 
@@ -35,6 +35,22 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
     STANDARD_MODE = "standard"
     ASSESSMENT_MODE = "assessment"
 
+    SOLUTION_CORRECT = "correct"
+    SOLUTION_PARTIAL = "partial"
+    SOLUTION_INCORRECT = "incorrect"
+
+    GRADE_FEEDBACK_CLASSES = {
+        SOLUTION_CORRECT: FeedbackMessages.MessageClasses.CORRECT_SOLUTION,
+        SOLUTION_PARTIAL: FeedbackMessages.MessageClasses.PARTIAL_SOLUTION,
+        SOLUTION_INCORRECT: FeedbackMessages.MessageClasses.INCORRECT_SOLUTION,
+    }
+
+    PROBLEM_FEEDBACK_CLASSES = {
+        SOLUTION_CORRECT: FeedbackMessages.MessageClasses.CORRECT_SOLUTION,
+        SOLUTION_PARTIAL: None,
+        SOLUTION_INCORRECT: None
+    }
+
     display_name = String(
         display_name=_("Title"),
         help=_("The title of the drag and drop problem. The title is displayed to learners."),
@@ -127,7 +143,7 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         default={},
     )
 
-    num_attempts = Integer(
+    attempts = Integer(
         help=_("Number of attempts learner used"),
         scope=Scope.user_state,
         default=0
@@ -139,6 +155,12 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         default=False,
     )
 
+    grade = Float(
+        help=_("Keeps maximum achieved score by student"),
+        scope=Scope.user_state,
+        default=0
+    )
+
     block_settings_key = 'drag-and-drop-v2'
     has_score = True
 
@@ -179,6 +201,9 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         """
 
         def items_without_answers():
+            """
+            Removes feedback and answer from items
+            """
             items = copy.deepcopy(self.data.get('items', ''))
             for item in items:
                 del item['feedback']
@@ -278,6 +303,9 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
 
     @XBlock.json_handler
     def studio_submit(self, submissions, suffix=''):
+        """
+        Handles studio save.
+        """
         self.display_name = submissions['display_name']
         self.mode = submissions['mode']
         self.max_attempts = submissions['max_attempts']
@@ -294,43 +322,271 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         }
 
     @XBlock.json_handler
-    def do_attempt(self, attempt, suffix=''):
-        item = self._get_item_definition(attempt['val'])
+    def drop_item(self, item_attempt, suffix=''):
+        """
+        Handles dropping item into a zone.
+        """
+        self._validate_drop_item(item_attempt)
 
-        state = None
-        zone = None
-        feedback = item['feedback']['incorrect']
-        overall_feedback = None
-        is_correct = False
+        if self.mode == self.ASSESSMENT_MODE:
+            return self._drop_item_assessment(item_attempt)
+        elif self.mode == self.STANDARD_MODE:
+            return self._drop_item_standard(item_attempt)
+        else:
+            raise JsonHandlerError(
+                500,
+                self.i18n_service.gettext("Unknown DnDv2 mode {mode} - course is misconfigured").format(self.mode)
+            )
 
-        if self._is_attempt_correct(attempt):  # Student placed item in a correct zone
-            is_correct = True
-            feedback = item['feedback']['correct']
-            state = {
-                'zone': attempt['zone'],
-                'correct': True,
-                'x_percent': attempt['x_percent'],
-                'y_percent': attempt['y_percent'],
+    @XBlock.json_handler
+    def do_attempt(self, data, suffix=''):
+        """
+        Checks submitted solution and returns feedback.
+
+        Raises:
+             * JsonHandlerError with 400 error code in standard mode.
+             * JsonHandlerError with 409 error code if no more attempts left
+        """
+        self._validate_do_attempt()
+
+        self.attempts += 1
+        self._mark_complete_and_publish_grade()  # must happen before _get_feedback
+
+        overall_feedback_msgs, misplaced_ids = self._get_feedback()
+
+        for item_id in misplaced_ids:
+            del self.item_state[item_id]
+
+        return {
+            'attempts': self.attempts,
+            'misplaced_items': list(misplaced_ids),
+            'overall_feedback': self._present_overall_feedback(overall_feedback_msgs)
         }
 
-        if state:
-            self.item_state[str(item['id'])] = state
-            zone = self._get_zone_by_uid(state['zone'])
+    @XBlock.json_handler
+    def publish_event(self, data, suffix=''):
+        """
+        Handler to publish XBlock event from frontend
+        """
+        try:
+            event_type = data.pop('event_type')
+        except KeyError:
+            return {'result': 'error', 'message': 'Missing event_type in JSON data'}
+
+        self.runtime.publish(self, event_type, data)
+        return {'result': 'success'}
+
+    @XBlock.json_handler
+    def reset(self, data, suffix=''):
+        """
+        Resets problem to initial state
+        """
+        self.item_state = {}
+        return self._get_user_state()
+
+    @XBlock.json_handler
+    def expand_static_url(self, url, suffix=''):
+        """ AJAX-accessible handler for expanding URLs to static [image] files """
+        return {'url': self._expand_static_url(url)}
+
+    @property
+    def i18n_service(self):
+        """ Obtains translation service """
+        i18n_service = self.runtime.service(self, "i18n")
+        if i18n_service:
+            return i18n_service
         else:
-            zone = self._get_zone_by_uid(attempt['zone'])
+            return DummyTranslationService()
+
+    @property
+    def target_img_expanded_url(self):
+        """ Get the expanded URL to the target image (the image items are dragged onto). """
+        if self.data.get("targetImg"):
+            return self._expand_static_url(self.data["targetImg"])
+        else:
+            return self.default_background_image_url
+
+    @property
+    def target_img_description(self):
+        """ Get the description for the target image (the image items are dragged onto). """
+        return self.data.get("targetImgDescription", "")
+
+    @property
+    def default_background_image_url(self):
+        """ The URL to the default background image, shown when no custom background is used """
+        return self.runtime.local_resource_url(self, "public/img/triangle.png")
+
+    @property
+    def attempts_remain(self):
+        """
+        Checks if current student still have more attempts.
+        """
+        return self.max_attempts is None or self.max_attempts == 0 or self.attempts < self.max_attempts
+
+    @XBlock.handler
+    def get_user_state(self, request, suffix=''):
+        """ GET all user-specific data, and any applicable feedback """
+        data = self._get_user_state()
+
+        return webob.Response(body=json.dumps(data), content_type='application/json')
+
+    def _validate_do_attempt(self):
+        """
+        Validates if `do_attempt` handler should be executed
+        """
+        if self.mode != self.ASSESSMENT_MODE:
+            raise JsonHandlerError(
+                400,
+                self.i18n_service.gettext("do_attempt handler should only be called for assessment mode")
+            )
+        if not self.attempts_remain:
+            raise JsonHandlerError(
+                409,
+                self.i18n_service.gettext("Max number of attempts reached")
+            )
+
+    def _get_feedback(self):
+        """
+        Builds overall feedback for both standard and assessment modes
+        """
+        answer_correctness = self._answer_correctness()
+        is_correct = answer_correctness == self.SOLUTION_CORRECT
+
+        if self.mode == self.STANDARD_MODE or not self.attempts:
+            feedback_key = 'finish' if is_correct else 'start'
+            return [FeedbackMessage(self.data['feedback'][feedback_key], None)], set()
+
+        required_ids, placed_ids, correct_ids = self._get_item_raw_stats()
+        missing_ids = required_ids - placed_ids
+        misplaced_ids = placed_ids - correct_ids
+
+        feedback_msgs = []
+
+        def _add_msg_if_exists(ids_list, message_template, message_class):
+            """ Adds message to feedback messages if corresponding items list is not empty """
+            if ids_list:
+                message = message_template(len(ids_list), self.i18n_service.ngettext)
+                feedback_msgs.append(FeedbackMessage(message, message_class))
+
+        _add_msg_if_exists(
+            correct_ids, FeedbackMessages.correctly_placed, FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+        )
+        _add_msg_if_exists(misplaced_ids, FeedbackMessages.misplaced, FeedbackMessages.MessageClasses.MISPLACED)
+        _add_msg_if_exists(missing_ids, FeedbackMessages.not_placed, FeedbackMessages.MessageClasses.NOT_PLACED)
+
+        if misplaced_ids and self.attempts_remain:
+            feedback_msgs.append(
+                FeedbackMessage(FeedbackMessages.MISPLACED_ITEMS_RETURNED, None)
+            )
+
+        if self.attempts_remain and (misplaced_ids or missing_ids):
+            problem_feedback_message = self.data['feedback']['start']
+        else:
+            problem_feedback_message = self.data['feedback']['finish']
+
+        problem_feedback_class = self.PROBLEM_FEEDBACK_CLASSES.get(answer_correctness, None)
+        grade_feedback_class = self.GRADE_FEEDBACK_CLASSES.get(answer_correctness, None)
+
+        feedback_msgs.append(FeedbackMessage(problem_feedback_message, problem_feedback_class))
+
+        if not self.attempts_remain:
+            feedback_msgs.append(
+                FeedbackMessage(FeedbackMessages.FINAL_ATTEMPT_TPL.format(score=self.grade), grade_feedback_class)
+            )
+
+        return feedback_msgs, misplaced_ids
+
+    @staticmethod
+    def _present_overall_feedback(feedback_messages):
+        """
+        Transforms feedback messages into format expected by frontend code
+        """
+        return [
+            {"message": msg.message, "message_class": msg.message_class}
+            for msg in feedback_messages
+            if msg.message
+        ]
+
+    def _drop_item_standard(self, item_attempt):
+        """
+        Handles dropping item to a zone in standard mode.
+        """
+        item = self._get_item_definition(item_attempt['val'])
+
+        is_correct = self._is_attempt_correct(item_attempt)  # Student placed item in a correct zone
+        if is_correct:  # In standard mode state is only updated when attempt is correct
+            self.item_state[str(item['id'])] = self._make_state_from_attempt(item_attempt, is_correct)
+
+        self._mark_complete_and_publish_grade()  # must happen before _get_feedback
+        self._publish_item_dropped_event(item_attempt, is_correct)
+
+        item_feedback_key = 'correct' if is_correct else 'incorrect'
+        item_feedback = item['feedback'][item_feedback_key]
+        overall_feedback, __ = self._get_feedback()
+
+        return {
+            'correct': is_correct,
+            'finished': self._is_answer_correct(),
+            'overall_feedback': self._present_overall_feedback(overall_feedback),
+            'feedback': item_feedback
+        }
+
+    def _drop_item_assessment(self, item_attempt):
+        """
+        Handles dropping item into a zone in assessment mode
+        """
+        if not self.attempts_remain:
+            raise JsonHandlerError(409, self.i18n_service.gettext("Max number of attempts reached"))
+
+        item = self._get_item_definition(item_attempt['val'])
+
+        is_correct = self._is_attempt_correct(item_attempt)
+        # State is always updated in assessment mode to store intermediate item positions
+        self.item_state[str(item['id'])] = self._make_state_from_attempt(item_attempt, is_correct)
+
+        self._publish_item_dropped_event(item_attempt, is_correct)
+
+        return {}
+
+    def _validate_drop_item(self, item):
+        """
+        Validates `drop_item` parameters
+        """
+        zone = self._get_zone_by_uid(item['zone'])
         if not zone:
             raise JsonHandlerError(400, "Item zone data invalid.")
 
-        if self._is_finished():
-            overall_feedback = self.data['feedback']['finish']
+    @staticmethod
+    def _make_state_from_attempt(attempt, correct):
+        """
+        Converts "attempt" data coming from browser into "state" entry stored in item_state
+        """
+        return {
+            'zone': attempt['zone'],
+            'correct': correct,
+            'x_percent': attempt['x_percent'],
+            'y_percent': attempt['y_percent'],
+        }
 
-        # don't publish the grade if the student has already completed the problem
-        if not self.completed:
-            if self._is_finished():
-                self.completed = True
+    def _mark_complete_and_publish_grade(self):
+        """
+        Helper method to update `self.completed` and submit grade event if appropriate conditions met.
+        """
+        # There's no going back from "completed" status to "incomplete"
+        self.completed = self.completed or self._is_answer_correct() or not self.attempts_remain
+        grade = self._get_grade()
+        # ... and from higher grade to lower
+        if grade > self.grade:
+            self.grade = grade
+            self._publish_grade()
+
+    def _publish_grade(self):
+        """
+        Publishes grade
+        """
         try:
             self.runtime.publish(self, 'grade', {
-                    'value': self._get_grade(),
+                'value': self.grade,
                 'max_value': self.weight,
             })
         except NotImplementedError:
@@ -338,6 +594,14 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
             # so we have to figure that we're running in Studio for now
             pass
 
+    def _publish_item_dropped_event(self, attempt, is_correct):
+        """
+        Publishes item dropped event.
+        """
+        item = self._get_item_definition(attempt['val'])
+        # attempt should already be validated here - not doing the check for existing zone again
+        zone = self._get_zone_by_uid(attempt['zone'])
+
         self.runtime.publish(self, 'edx.drag_and_drop_v2.item.dropped', {
             'item_id': item['id'],
             'location': zone.get("title"),
@@ -345,24 +609,6 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
             'is_correct': is_correct,
         })
 
-        if self.mode == self.ASSESSMENT_MODE:
-            # In assessment mode we don't send any feedback on drop.
-            result = {}
-        else:
-            result = {
-                'correct': is_correct,
-                'finished': self._is_finished(),
-                'overall_feedback': overall_feedback,
-                'feedback': feedback
-            }
-
-        return result
-
-    @XBlock.json_handler
-    def reset(self, data, suffix=''):
-        self.item_state = {}
-        return self._get_user_state()
-
     def _is_attempt_correct(self, attempt):
         """
         Check if the item was placed correctly.
@@ -389,35 +635,6 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
                 pass
         return url
 
-    @XBlock.json_handler
-    def expand_static_url(self, url, suffix=''):
-        """ AJAX-accessible handler for expanding URLs to static [image] files """
-        return {'url': self._expand_static_url(url)}
-
-    @property
-    def target_img_expanded_url(self):
-        """ Get the expanded URL to the target image (the image items are dragged onto). """
-        if self.data.get("targetImg"):
-            return self._expand_static_url(self.data["targetImg"])
-        else:
-            return self.default_background_image_url
-
-    @property
-    def target_img_description(self):
-        """ Get the description for the target image (the image items are dragged onto). """
-        return self.data.get("targetImgDescription", "")
-
-    @property
-    def default_background_image_url(self):
-        """ The URL to the default background image, shown when no custom background is used """
-        return self.runtime.local_resource_url(self, "public/img/triangle.png")
-
-    @XBlock.handler
-    def get_user_state(self, request, suffix=''):
-        """ GET all user-specific data, and any applicable feedback """
-        data = self._get_user_state()
-        return webob.Response(body=json.dumps(data), content_type='application/json')
-
     def _get_user_state(self):
         """ Get all user-specific data, and any applicable feedback """
         item_state = self._get_item_state()
@@ -436,24 +653,38 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
                 else:
                     item['zone'] = 'unknown'
 
-        is_finished = self._is_finished()
+            # In assessment mode, if item is placed correctly and than the page is refreshed, "correct"
+            # will spill to the frontend, making item "disabled", thus allowing students to obtain answer by trial
+            # and error + refreshing the page. In order to avoid that, we remove "correct" from an item here
+            if self.mode == self.ASSESSMENT_MODE:
+                del item["correct"]
+
+        overall_feedback_msgs, __ = self._get_feedback()
+        if self.mode == self.STANDARD_MODE:
+            is_finished = self._is_answer_correct()
+        else:
+            is_finished = not self.attempts_remain
+
         return {
             'items': item_state,
             'finished': is_finished,
-            'num_attempts': self.num_attempts,
-            'overall_feedback': self.data['feedback']['finish' if is_finished else 'start'],
+            'attempts': self.attempts,
+            'overall_feedback': self._present_overall_feedback(overall_feedback_msgs)
         }
 
     def _get_item_state(self):
         """
-        Returns the user item state.
+        Returns a copy of the user item state.
         Converts to a dict if data is stored in legacy tuple form.
         """
+
+        # IMPORTANT: this method should always return a COPY of self.item_state - it is called from get_user_state
+        # handler and manipulated there to hide correctness of items placed
         state = {}
 
         for item_id, item in self.item_state.iteritems():
             if isinstance(item, dict):
-                state[item_id] = item
+                state[item_id] = item.copy()  # items are manipulated in _get_user_state, so we protect actual data
             else:
                 state[item_id] = {'top': item[0], 'left': item[1]}
 
@@ -512,17 +743,28 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         Returns a tuple representing the number of correctly-placed items,
         and the total number of items that must be placed on the board (non-decoy items).
         """
-        all_items = self.data['items']
-        item_state = self._get_item_state()
+        required_items, __, correct_items = self._get_item_raw_stats()
 
-        required_items = [str(item['id']) for item in all_items if self._get_item_zones(item['id']) != []]
-        placed_items = [item for item in required_items if item in item_state]
-        correct_items = [item for item in placed_items if item_state[item]['correct']]
+        return len(correct_items), len(required_items)
 
-        required_count = len(required_items)
-        correct_count = len(correct_items)
+    def _get_item_raw_stats(self):
+        """
+        Returns a 3-tuple containing required, placed and correct items.
 
-        return correct_count, required_count
+        Returns:
+            tuple: (required_items, placed_items, correct_items)
+                * required_items - IDs of items that must be placed on the board
+                * placed_items - IDs of items actually placed on the board
+                * correct_items - IDs of items that were placed correctly
+        """
+        all_items = [str(item['id']) for item in self.data['items']]
+        item_state = self._get_item_state()
+
+        required_items = set(item_id for item_id in all_items if self._get_item_zones(int(item_id)) != [])
+        placed_items = set(item_id for item_id in all_items if item_id in item_state)
+        correct_items = set(item_id for item_id in placed_items if item_state[item_id]['correct'])
+
+        return required_items, placed_items, correct_items
 
     def _get_grade(self):
         """
@@ -531,31 +773,32 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
         correct_count, required_count = self._get_item_stats()
         return correct_count / float(required_count) * self.weight
 
-    def _is_finished(self):
+    def _answer_correctness(self):
         """
-        All items are at their correct place and a value has been
-        submitted for each item that expects a value.
+        Checks answer correctness:
+
+        Returns:
+            string: Correct/Incorrect/Partial
+                * Correct: All items are at their correct place.
+                * Partial: Some items are at their correct place.
+                * Incorrect: None items are at their correct place.
         """
         correct_count, required_count = self._get_item_stats()
-        return correct_count == required_count
-
-    @XBlock.json_handler
-    def publish_event(self, data, suffix=''):
-        try:
-            event_type = data.pop('event_type')
-        except KeyError:
-            return {'result': 'error', 'message': 'Missing event_type in JSON data'}
+        if correct_count == required_count:
+            return self.SOLUTION_CORRECT
+        elif correct_count == 0:
+            return self.SOLUTION_INCORRECT
+        else:
+            return self.SOLUTION_PARTIAL
 
-        self.runtime.publish(self, event_type, data)
-        return {'result': 'success'}
+    def _is_answer_correct(self):
+        """
+        Helper - checks if answer is correct
 
-    def _get_unique_id(self):
-        usage_id = self.scope_ids.usage_id
-        try:
-            return usage_id.name
-        except AttributeError:
-            # workaround for xblock workbench
-            return usage_id
+        Returns:
+            bool: True if current answer is correct
+        """
+        return self._answer_correctness() == self.SOLUTION_CORRECT
 
     @staticmethod
     def workbench_scenarios():

drag_and_drop_v2/public/css/drag_and_drop.css

@@ -284,6 +284,57 @@
     border-top: solid 1px #bdbdbd;
 }
 
+.xblock--drag-and-drop .feedback p {
+    margin: 2px 0;
+    padding: 0.5em;
+    border: 2px solid #999;
+}
+
+.xblock--drag-and-drop .feedback p.correct {
+    color: #166e36;
+    border: 2px solid #166e36;
+}
+
+.xblock--drag-and-drop .feedback p.partial {
+    color: #166e36;
+    border: 2px solid #166e36;
+}
+
+.xblock--drag-and-drop .feedback p.incorrect {
+    color: #b20610;
+    border: 2px solid #b20610;
+}
+
+
+/* Font Awesome icons have different width - margin-right tries to compensate it */
+.xblock--drag-and-drop .feedback p:before {
+    content: "\f129";
+    font-family: FontAwesome;
+    margin-right: 0.7em;
+    margin-left: 0.3em;
+}
+
+.xblock--drag-and-drop .feedback p.correct:before {
+    content: "\f00c";
+    font-family: FontAwesome;
+    margin-right: 0.3em;
+    margin-left: 0;
+}
+
+.xblock--drag-and-drop .feedback p.partial:before {
+    content: "\f069";
+    font-family: FontAwesome;
+    margin-right: 0.3em;
+    margin-left: 0;
+}
+
+.xblock--drag-and-drop .feedback p.incorrect:before {
+    content: "\f00d";
+    font-family: FontAwesome;
+    margin-right: 0.45em;
+    margin-left: 0.1em;
+}
+
 .xblock--drag-and-drop .popup {
     position: absolute;
     display: none;

drag_and_drop_v2/public/js/drag_and_drop.js

@@ -227,12 +227,21 @@ function DragAndDropTemplates(configuration) {
     };
 
     var feedbackTemplate = function(ctx) {
-        var feedback_display = ctx.feedback_html ? 'block' : 'none';
         var properties = { attributes: { 'aria-live': 'polite' } };
+        var messages = ctx.overall_feedback_messages || [];
+        var feedback_display = messages.length > 0 ? 'block' : 'none';
+        var feedback_messages = messages.map(function(message) {
+            var selector = "p.message";
+            if (message.message_class) {
+                selector += "."+message.message_class;
+            }
+            return h(selector, {innerHTML: message.message}, []);
+        });
+
         return (
             h('section.feedback', properties, [
                 h('h3.title1', { style: { display: feedback_display } }, gettext('Feedback')),
-                h('p.message', { style: { display: feedback_display }, innerHTML: ctx.feedback_html })
+                h('div.messages', { style: { display: feedback_display } }, feedback_messages)
             ])
         );
     };
@@ -266,20 +275,23 @@ function DragAndDropTemplates(configuration) {
     var submitAnswerTemplate = function(ctx) {
         var attemptsUsedId = "attempts-used-"+configuration.url_name;
         var attemptsUsedDisplay = (ctx.max_attempts && ctx.max_attempts > 0) ? 'inline': 'none';
-        var button_enabled = ctx.items.some(function(item) {return item.is_placed;}) &&
-            (ctx.max_attempts === null || ctx.max_attempts > ctx.num_attempts);
 
         return (
           h("section.action-toolbar-item.submit-answer", {}, [
               h(
                   "button.btn-brand.submit-answer-button",
-                  {disabled: !button_enabled, attributes: {"aria-describedby": attemptsUsedId}},
+                  {
+                      disabled: ctx.disable_submit_button || ctx.submit_spinner, 
+                      attributes: {"aria-describedby": attemptsUsedId}},
+                  [
+                      (ctx.submit_spinner ? h("span.fa.fa-spin.fa-spinner") : null),
                       gettext("Submit")
+                  ]
               ),
               h(
                   "span.attempts-used#"+attemptsUsedId, {style: {display: attemptsUsedDisplay}},
                   gettext("You have used {used} of {total} attempts.")
-                      .replace("{used}", ctx.num_attempts).replace("{total}", ctx.max_attempts)
+                      .replace("{used}", ctx.attempts).replace("{total}", ctx.max_attempts)
               )
           ])
         );
@@ -444,6 +456,7 @@ function DragAndDropBlock(runtime, element, configuration) {
             // Set up event handlers:
 
             $(document).on('keydown mousedown touchstart', closePopup);
+            $element.on('click', '.submit-answer-button', doAttempt);
             $element.on('click', '.keyboard-help-button', showKeyboardHelp);
             $element.on('keydown', '.keyboard-help-button', function(evt) {
                 runOnKey(evt, RET, showKeyboardHelp);
@@ -897,7 +910,7 @@ function DragAndDropBlock(runtime, element, configuration) {
         if (!zone) {
             return;
         }
-        var url = runtime.handlerUrl(element, 'do_attempt');
+        var url = runtime.handlerUrl(element, 'drop_item');
         var data = {
             val: item_id,
             zone: zone,
@@ -969,6 +982,45 @@ function DragAndDropBlock(runtime, element, configuration) {
         });
     };
 
+    var doAttempt = function(evt) {
+        evt.preventDefault();
+        state.submit_spinner = true;
+        applyState();
+
+        $.ajax({
+            type: 'POST',
+            url: runtime.handlerUrl(element, "do_attempt"),
+            data: '{}'
+        }).done(function(data){
+            state.attempts = data.attempts;
+            state.overall_feedback = data.overall_feedback;
+            if (attemptsRemain()) {
+                data.misplaced_items.forEach(function(misplaced_item_id) {
+                    delete state.items[misplaced_item_id]
+                });
+            } else {
+                state.finished = true;
+            }
+            focusFirstDraggable();
+        }).always(function() {
+            state.submit_spinner = false;
+            applyState();
+        });
+    };
+
+    var canSubmitAttempt = function() {
+        return Object.keys(state.items).length > 0 && attemptsRemain();
+    };
+
+    var canReset = function() {
+        return Object.keys(state.items).length > 0 &&
+            (configuration.mode !== DragAndDropBlock.ASSESSMENT_MODE || attemptsRemain())
+    };
+
+    var attemptsRemain = function() {
+        return !configuration.max_attempts || configuration.max_attempts > state.attempts;
+    };
+
     var render = function() {
         var items = configuration.items.map(function(item) {
             var item_user_state = state.items[item.id];
@@ -1028,7 +1080,7 @@ function DragAndDropBlock(runtime, element, configuration) {
             show_title: configuration.show_title,
             mode: configuration.mode,
             max_attempts: configuration.max_attempts,
-            num_attempts: state.num_attempts,
+            attempts: state.attempts,
             problem_html: configuration.problem_text,
             show_problem_header: configuration.show_problem_header,
             show_submit_answer: configuration.mode == DragAndDropBlock.ASSESSMENT_MODE,
@@ -1042,8 +1094,10 @@ function DragAndDropBlock(runtime, element, configuration) {
             last_action_correct: state.last_action_correct,
             item_bank_focusable: item_bank_focusable,
             popup_html: state.feedback || '',
-            feedback_html: $.trim(state.overall_feedback),
-            disable_reset_button: Object.keys(state.items).length == 0,
+            overall_feedback_messages: state.overall_feedback,
+            disable_reset_button: !canReset(),
+            disable_submit_button: !canSubmitAttempt(),
+            submit_spinner: state.submit_spinner
         };
 
         return renderView(context);

drag_and_drop_v2/translations/en/LC_MESSAGES/text.po

@@ -286,10 +286,6 @@ msgid ""
 msgstr ""
 
 #: templates/html/js_templates.html
-msgid "Zones"
-msgstr ""
-
-#: templates/html/js_templates.html
 msgid "Use text that is clear and descriptive of the item to be placed"
 msgstr ""
 
@@ -323,6 +319,7 @@ msgstr ""
 msgid "Final Feedback"
 msgstr ""
 
+#: templates/html/js_templates.html
 #: templates/html/drag_and_drop_edit.html
 msgid "Zones"
 msgstr ""
@@ -421,7 +418,10 @@ msgid "Correctly placed in: {zone_title}"
 msgstr ""
 
 #: public/js/drag_and_drop.js
-msgid "Reset problem"
+msgid "Reset"
+msgstr ""
+
+msgid "Submit"
 msgstr ""
 
 #: public/js/drag_and_drop.js
@@ -485,3 +485,29 @@ msgstr ""
 #: public/js/drag_and_drop_edit.js
 msgid "Error: "
 msgstr ""
+
+#: utils.py:18
+msgid "Final attempt was used, highest score is {score}"
+msgstr ""
+
+#: utils.py:19
+msgid "Misplaced items were returned to item bank."
+msgstr ""
+
+#: utils.py:24
+msgid "Correctly placed {correct_count} item."
+msgid_plural "Correctly placed {correct_count} items."
+msgstr[0] ""
+msgstr[1] ""
+
+#: utils.py:32
+msgid "Misplaced {misplaced_count} item."
+msgid_plural "Misplaced {misplaced_count} items."
+msgstr[0] ""
+msgstr[1] ""
+
+#: utils.py:40
+msgid "Did not place {missing_count} required item."
+msgid_plural "Did not place {missing_count} required items."
+msgstr[0] ""
+msgstr[1] ""

drag_and_drop_v2/translations/eo/LC_MESSAGES/text.po

@@ -164,6 +164,7 @@ msgstr ""
 "Ìf thé välüé ïs nöt sét, ïnfïnïté ättémpts äré ällöwéd. Ⱡ'σяєм #"
 
 #: drag_and_drop_v2.py
+#: templates/html/drag_and_drop_edit.html
 msgid "Show title"
 msgstr "Shöw tïtlé Ⱡ'σяєм ιρѕυм ∂σłσ#"
 
@@ -173,6 +174,7 @@ msgstr ""
 "Dïspläý thé tïtlé tö thé léärnér? Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢тє#"
 
 #: drag_and_drop_v2.py
+#: templates/html/drag_and_drop_edit.html
 msgid "Problem text"
 msgstr "Prößlém téxt Ⱡ'σяєм ιρѕυм ∂σłσя ѕ#"
 
@@ -349,6 +351,7 @@ msgstr ""
 "äütömätïç wïdth): Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σ#"
 
 #: templates/html/js_templates.html
+#: templates/html/drag_and_drop_edit.html
 msgid "Zones"
 msgstr "Zönés Ⱡ'σяєм ιρѕ#"
 
@@ -379,22 +382,10 @@ msgid "Problem mode"
 msgstr "Prößlém mödé Ⱡ'σяєм ιρѕυм ∂σłσя ѕ#"
 
 #: templates/html/drag_and_drop_edit.html
-msgid "Show title"
-msgstr "Shöw tïtlé Ⱡ'σяєм ιρѕυм ∂σłσ#"
-
-#: templates/html/drag_and_drop_edit.html
 msgid "Maximum score"
 msgstr "Mäxïmüm sçöré Ⱡ'σяєм ιρѕυм ∂σłσя ѕι#"
 
 #: templates/html/drag_and_drop_edit.html
-msgid "Problem text"
-msgstr "Prößlém téxt Ⱡ'σяєм ιρѕυм ∂σłσя ѕ#"
-
-#: templates/html/drag_and_drop_edit.html
-msgid "Show \"Problem\" heading"
-msgstr "Shöw \"Prößlém\" héädïng Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢#"
-
-#: templates/html/drag_and_drop_edit.html
 msgid "Introductory Feedback"
 msgstr "Ìntrödüçtörý Féédßäçk Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, #"
 
@@ -403,10 +394,6 @@ msgid "Final Feedback"
 msgstr "Fïnäl Féédßäçk Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт#"
 
 #: templates/html/drag_and_drop_edit.html
-msgid "Zones"
-msgstr "Zönés Ⱡ'σяєм ιρѕ#"
-
-#: templates/html/drag_and_drop_edit.html
 msgid "Background URL"
 msgstr "Bäçkgröünd ÛRL Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт#"
 
@@ -506,8 +493,11 @@ msgid "Correctly placed in: {zone_title}"
 msgstr "Çörréçtlý pläçéd ïn: {zone_title} Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, #"
 
 #: public/js/drag_and_drop.js
-msgid "Reset problem"
-msgstr "Rését prößlém Ⱡ'σяєм ιρѕυм ∂σłσя ѕι#"
+msgid "Reset"
+msgstr "Rését Ⱡ'σяєм ιρѕυм ∂σłσя ѕι#"
+
+msgid "Submit"
+msgstr "Süßmït Ⱡ'σяєм ιρѕυм ∂σłσя ѕι#"
 
 #: public/js/drag_and_drop.js
 msgid "Feedback"
@@ -584,3 +574,30 @@ msgstr "Nöné Ⱡ'σяєм ι#"
 #: public/js/drag_and_drop_edit.js
 msgid "Error: "
 msgstr "Érrör:  Ⱡ'σяєм ιρѕυм #"
+
+
+#: utils.py:18
+msgid "Fïnäl ättémpt wäs üséd, hïghést sçöré ïs {score} Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢тєтυя #"
+msgstr ""
+
+#: utils.py:19
+msgid "Mïspläçéd ïtéms wéré rétürnéd tö ïtém ßänk. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢тєтυя #"
+msgstr ""
+
+#: utils.py:24
+msgid "Çörréçtlý pläçéd {correct_count} ïtém. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕ#"
+msgid_plural "Çörréçtlý pläçéd {correct_count} ïtéms. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє#"
+msgstr[0] ""
+msgstr[1] ""
+
+#: utils.py:32
+msgid "Mïspläçéd {misplaced_count} ïtém. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт,#"
+msgid_plural "Mïspläçéd {misplaced_count} ïtéms. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, #"
+msgstr[0] ""
+msgstr[1] ""
+
+#: utils.py:40
+msgid "Dïd nöt pläçé {missing_count} réqüïréd ïtém. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢#"
+msgid_plural "Dïd nöt pläçé {missing_count} réqüïréd ïtéms. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, ¢σηѕє¢т#"
+msgstr[0] ""
+msgstr[1] ""
\ No newline at end of file

drag_and_drop_v2/utils.py

 # -*- coding: utf-8 -*-
-#
+""" Drag and Drop v2 XBlock - Utils """
+from collections import namedtuple
 
 
-# Make '_' a no-op so we can scrape strings
 def _(text):
+    """ Dummy `gettext` replacement to make string extraction tools scrape strings marked for translation """
     return text
+
+
+def ngettext_fallback(text_singular, text_plural, number):
+    """ Dummy `ngettext` replacement to make string extraction tools scrape strings marked for translation """
+    if number == 1:
+        return text_singular
+    else:
+        return text_plural
+
+
+class DummyTranslationService(object):
+    """
+    Dummy drop-in replacement for i18n XBlock service
+    """
+    gettext = _
+    ngettext = ngettext_fallback
+
+
+class FeedbackMessages(object):
+    """
+    Feedback messages collection
+    """
+    class MessageClasses(object):
+        """
+        Namespace for message classes
+        """
+        CORRECT_SOLUTION = "correct"
+        PARTIAL_SOLUTION = "partial"
+        INCORRECT_SOLUTION = "incorrect"
+
+        CORRECTLY_PLACED = CORRECT_SOLUTION
+        MISPLACED = INCORRECT_SOLUTION
+        NOT_PLACED = INCORRECT_SOLUTION
+
+    FINAL_ATTEMPT_TPL = _('Final attempt was used, highest score is {score}')
+    MISPLACED_ITEMS_RETURNED = _('Misplaced item(s) were returned to item bank.')
+
+    @staticmethod
+    def correctly_placed(number, ngettext=ngettext_fallback):
+        """
+        Formats "correctly placed items" message
+        """
+        return ngettext(
+            'Correctly placed {correct_count} item.',
+            'Correctly placed {correct_count} items.',
+            number
+        ).format(correct_count=number)
+
+    @staticmethod
+    def misplaced(number, ngettext=ngettext_fallback):
+        """
+        Formats "misplaced items" message
+        """
+        return ngettext(
+            'Misplaced {misplaced_count} item. Misplaced item was returned to item bank.',
+            'Misplaced {misplaced_count} items. Misplaced items were returned to item bank.',
+            number
+        ).format(misplaced_count=number)
+
+    @staticmethod
+    def not_placed(number, ngettext=ngettext_fallback):
+        """
+        Formats "did not place required items" message
+        """
+        return ngettext(
+            'Did not place {missing_count} required item.',
+            'Did not place {missing_count} required items.',
+            number
+        ).format(missing_count=number)
+
+
+FeedbackMessage = namedtuple("FeedbackMessage", ["message", "message_class"])  # pylint: disable=invalid-name

pylintrc

@@ -6,18 +6,19 @@ max-line-length=120
 
 [MESSAGES CONTROL]
 disable=
-    attribute-defined-outside-init,
     locally-disabled,
-    missing-docstring,
     too-many-ancestors,
-    too-many-arguments,
-    too-many-branches,
     too-many-instance-attributes,
     too-few-public-methods,
     too-many-public-methods,
-    unused-argument,
-    invalid-name,
-    no-member
+    unused-argument
 
 [SIMILARITIES]
 min-similarity-lines=4
+
+[OPTIONS]
+good-names=_,__,log,loader
+method-rgx=_?[a-z_][a-z0-9_]{2,40}$
+function-rgx=_?[a-z_][a-z0-9_]{2,40}$
+method-name-hint=_?[a-z_][a-z0-9_]{2,40}$
+function-name-hint=_?[a-z_][a-z0-9_]{2,40}$

tests/integration/test_interaction.py

@@ -17,6 +17,7 @@ from drag_and_drop_v2.default_data import (
     ITEM_CORRECT_FEEDBACK, ITEM_INCORRECT_FEEDBACK, ITEM_NO_ZONE_FEEDBACK,
     ITEM_ANY_ZONE_FEEDBACK, START_FEEDBACK, FINISH_FEEDBACK
 )
+from drag_and_drop_v2.utils import FeedbackMessages
 from .test_base import BaseIntegrationTest
 
 
@@ -481,6 +482,23 @@ class DefaultAssessmentDataTestMixin(DefaultDataTestMixin):
         """.format(max_attempts=self.MAX_ATTEMPTS)
 
 
+class AssessmentTestMixin(object):
+    """
+    Provides helper methods for assessment tests
+    """
+    @staticmethod
+    def _wait_until_enabled(element):
+        wait = WebDriverWait(element, 2)
+        wait.until(lambda e: e.is_displayed() and e.get_attribute('disabled') is None)
+
+    def click_submit(self):
+        submit_button = self._get_submit_button()
+
+        self._wait_until_enabled(submit_button)
+
+        submit_button.click()
+
+
 @ddt
 class StandardInteractionTest(DefaultDataTestMixin, InteractionTestBase, BaseIntegrationTest):
     """
@@ -512,7 +530,9 @@ class StandardInteractionTest(DefaultDataTestMixin, InteractionTestBase, BaseInt
 
 
 @ddt
-class AssessmentInteractionTest(DefaultAssessmentDataTestMixin, InteractionTestBase, BaseIntegrationTest):
+class AssessmentInteractionTest(
+    DefaultAssessmentDataTestMixin, AssessmentTestMixin, InteractionTestBase, BaseIntegrationTest
+):
     """
     Testing interactions with Drag and Drop XBlock against default data in assessment mode.
     All interactions are tested using mouse (action_key=None) and four different keyboard action keys.
@@ -562,6 +582,97 @@ class AssessmentInteractionTest(DefaultAssessmentDataTestMixin, InteractionTestB
 
         self.assertEqual(submit_button.get_attribute('disabled'), None)
 
+    def test_misplaced_items_returned_to_bank(self):
+        """
+        Test items placed to incorrect zones are returned to item bank after submitting solution
+        """
+        correct_items = {0: TOP_ZONE_ID}
+        misplaced_items = {1: BOTTOM_ZONE_ID, 2: MIDDLE_ZONE_ID}
+
+        for item_id, zone_id in correct_items.iteritems():
+            self.place_item(item_id, zone_id)
+
+        for item_id, zone_id in misplaced_items.iteritems():
+            self.place_item(item_id, zone_id)
+
+        self.click_submit()
+        for item_id in correct_items:
+            self.assert_placed_item(item_id, TOP_ZONE_TITLE, assessment_mode=True)
+
+        for item_id in misplaced_items:
+            self.assert_reverted_item(item_id)
+
+    def test_max_attempts_reached_submit_and_reset_disabled(self):
+        """
+        Test "Submit" and "Reset" buttons are disabled when no more attempts remaining
+        """
+        self.place_item(0, TOP_ZONE_ID)
+
+        submit_button, reset_button = self._get_submit_button(), self._get_reset_button()
+
+        attempts_info = self._get_attempts_info()
+
+        for index in xrange(self.MAX_ATTEMPTS):
+            expected_text = "You have used {num} of {max} attempts.".format(num=index, max=self.MAX_ATTEMPTS)
+            self.assertEqual(attempts_info.text, expected_text)  # precondition check
+            self.assertEqual(submit_button.get_attribute('disabled'), None)
+            self.assertEqual(reset_button.get_attribute('disabled'), None)
+            self.click_submit()
+
+        self.assertEqual(submit_button.get_attribute('disabled'), 'true')
+        self.assertEqual(reset_button.get_attribute('disabled'), 'true')
+
+    def test_do_attempt_feedback_is_updated(self):
+        """
+        Test updating overall feedback after submitting solution in assessment mode
+        """
+        # used keyboard mode to avoid bug/feature with selenium "selecting" everything instead of dragging an element
+        self.place_item(0, TOP_ZONE_ID, Keys.RETURN)
+
+        self.click_submit()
+
+        feedback_lines = [
+            "FEEDBACK",
+            FeedbackMessages.correctly_placed(1),
+            FeedbackMessages.not_placed(3),
+            START_FEEDBACK
+
+        ]
+        expected_feedback = "\n".join(feedback_lines)
+        self.assertEqual(self._get_feedback().text, expected_feedback)
+
+        self.place_item(1, BOTTOM_ZONE_ID, Keys.RETURN)
+        self.click_submit()
+
+        feedback_lines = [
+            "FEEDBACK",
+            FeedbackMessages.correctly_placed(1),
+            FeedbackMessages.misplaced(1),
+            FeedbackMessages.not_placed(2),
+            FeedbackMessages.MISPLACED_ITEMS_RETURNED,
+            START_FEEDBACK
+        ]
+        expected_feedback = "\n".join(feedback_lines)
+        self.assertEqual(self._get_feedback().text, expected_feedback)
+
+        # reach final attempt
+        for _ in xrange(self.MAX_ATTEMPTS-3):
+            self.click_submit()
+
+        self.place_item(1, MIDDLE_ZONE_ID, Keys.RETURN)
+        self.place_item(2, BOTTOM_ZONE_ID, Keys.RETURN)
+        self.place_item(3, TOP_ZONE_ID, Keys.RETURN)
+
+        self.click_submit()
+        feedback_lines = [
+            "FEEDBACK",
+            FeedbackMessages.correctly_placed(4),
+            FINISH_FEEDBACK,
+            FeedbackMessages.FINAL_ATTEMPT_TPL.format(score=1.0)
+        ]
+        expected_feedback = "\n".join(feedback_lines)
+        self.assertEqual(self._get_feedback().text, expected_feedback)
+
 
 class MultipleValidOptionsInteractionTest(DefaultDataTestMixin, InteractionTestBase, BaseIntegrationTest):
 

tests/pylintrc

+[REPORTS]
+reports=no
+
+[FORMAT]
+max-line-length=120
+
+[MESSAGES CONTROL]
+disable=
+    attribute-defined-outside-init,
+    locally-disabled,
+    missing-docstring,
+    abstract-class-little-used,
+    too-many-ancestors,
+    too-few-public-methods,
+    too-many-public-methods,
+    invalid-name,
+    no-member
+
+[SIMILARITIES]
+min-similarity-lines=4
+
+[OPTIONS]
+max-args=6

tests/unit/data/assessment/config_out.json

@@ -47,16 +47,22 @@
           "id": 1
         },
         {
+          "displayName": "3",
+          "imageURL": "",
+          "expandedImageURL": "",
+          "id": 2
+        },
+        {
           "displayName": "X",
           "imageURL": "/static/test_url_expansion",
           "expandedImageURL": "/course/test-course/assets/test_url_expansion",
-          "id": 2
+          "id": 3
         },
         {
           "displayName": "",
           "imageURL": "http://placehold.it/200x100",
           "expandedImageURL": "http://placehold.it/200x100",
-          "id": 3
+          "id": 4
         }
     ]
 }

tests/unit/data/assessment/data.json

@@ -41,6 +41,16 @@
       "id": 1
     },
     {
+      "displayName": "3",
+      "feedback": {
+        "incorrect": "No 3",
+        "correct": "Yes 3"
+      },
+      "zone": "zone-2",
+      "imageURL": "",
+      "id": 2
+    },
+    {
       "displayName": "X",
       "feedback": {
         "incorrect": "",
@@ -48,7 +58,7 @@
       },
       "zone": "none",
       "imageURL": "/static/test_url_expansion",
-      "id": 2
+      "id": 3
     },
     {
       "displayName": "",
@@ -58,7 +68,7 @@
       },
       "zone": "none",
       "imageURL": "http://placehold.it/200x100",
-      "id": 3
+      "id": 4
     }
   ],
 

tests/unit/test_advanced.py

 # Imports ###########################################################
 
+import ddt
 import json
+import mock
+import random
 import unittest
 
 from xblockutils.resources import ResourceLoader
 
-from ..utils import make_block, TestCaseMixin
+from drag_and_drop_v2.utils import FeedbackMessages
+
+from ..utils import make_block, TestCaseMixin, generate_max_and_attempts
 
 
 # Globals ###########################################################
@@ -19,6 +24,8 @@ class BaseDragAndDropAjaxFixture(TestCaseMixin):
     ZONE_1 = None
     ZONE_2 = None
 
+    OVERALL_FEEDBACK_KEY = 'overall_feedback'
+
     FEEDBACK = {
         0: {"correct": None, "incorrect": None},
         1: {"correct": None, "incorrect": None},
@@ -37,6 +44,10 @@ class BaseDragAndDropAjaxFixture(TestCaseMixin):
             setattr(self.block, field, initial_settings[field])
         self.block.data = self.initial_data()
 
+    @staticmethod
+    def _make_feedback_message(message=None, message_class=None):
+        return {"message": message, "message_class": message_class}
+
     @classmethod
     def initial_data(cls):
         return json.loads(loader.load_unicode('data/{}/data.json'.format(cls.FOLDER)))
@@ -62,34 +73,34 @@ class StandardModeFixture(BaseDragAndDropAjaxFixture):
     """
     Common tests for drag and drop in standard mode
     """
-    def test_do_attempt_wrong_with_feedback(self):
+    def test_drop_item_wrong_with_feedback(self):
         item_id, zone_id = 0, self.ZONE_2
         data = {"val": item_id, "zone": zone_id, "x_percent": "33%", "y_percent": "11%"}
-        res = self.call_handler('do_attempt', data)
+        res = self.call_handler(self.DROP_ITEM_HANDLER, data)
         self.assertEqual(res, {
-            "overall_feedback": None,
+            "overall_feedback": [self._make_feedback_message(message=self.INITIAL_FEEDBACK)],
             "finished": False,
             "correct": False,
             "feedback": self.FEEDBACK[item_id]["incorrect"]
         })
 
-    def test_do_attempt_wrong_without_feedback(self):
+    def test_drop_item_wrong_without_feedback(self):
         item_id, zone_id = 2, self.ZONE_1
         data = {"val": item_id, "zone": zone_id, "x_percent": "33%", "y_percent": "11%"}
-        res = self.call_handler('do_attempt', data)
+        res = self.call_handler(self.DROP_ITEM_HANDLER, data)
         self.assertEqual(res, {
-            "overall_feedback": None,
+            "overall_feedback": [self._make_feedback_message(message=self.INITIAL_FEEDBACK)],
             "finished": False,
             "correct": False,
             "feedback": self.FEEDBACK[item_id]["incorrect"]
         })
 
-    def test_do_attempt_correct(self):
+    def test_drop_item_correct(self):
         item_id, zone_id = 0, self.ZONE_1
         data = {"val": item_id, "zone": zone_id, "x_percent": "33%", "y_percent": "11%"}
-        res = self.call_handler('do_attempt', data)
+        res = self.call_handler(self.DROP_ITEM_HANDLER, data)
         self.assertEqual(res, {
-            "overall_feedback": None,
+            "overall_feedback": [self._make_feedback_message(message=self.INITIAL_FEEDBACK)],
             "finished": False,
             "correct": True,
             "feedback": self.FEEDBACK[item_id]["correct"]
@@ -98,43 +109,43 @@ class StandardModeFixture(BaseDragAndDropAjaxFixture):
     def test_grading(self):
         published_grades = []
 
-        def mock_publish(self, event, params):
+        def mock_publish(_, event, params):
             if event == 'grade':
                 published_grades.append(params)
         self.block.runtime.publish = mock_publish
 
-        self.call_handler('do_attempt', {
+        self.call_handler(self.DROP_ITEM_HANDLER, {
             "val": 0, "zone": self.ZONE_1, "y_percent": "11%", "x_percent": "33%"
         })
 
         self.assertEqual(1, len(published_grades))
         self.assertEqual({'value': 0.5, 'max_value': 1}, published_grades[-1])
 
-        self.call_handler('do_attempt', {
+        self.call_handler(self.DROP_ITEM_HANDLER, {
             "val": 1, "zone": self.ZONE_2, "y_percent": "90%", "x_percent": "42%"
         })
 
         self.assertEqual(2, len(published_grades))
         self.assertEqual({'value': 1, 'max_value': 1}, published_grades[-1])
 
-    def test_do_attempt_final(self):
+    def test_drop_item_final(self):
         data = {"val": 0, "zone": self.ZONE_1, "x_percent": "33%", "y_percent": "11%"}
-        self.call_handler('do_attempt', data)
+        self.call_handler(self.DROP_ITEM_HANDLER, data)
 
         expected_state = {
             "items": {
                 "0": {"x_percent": "33%", "y_percent": "11%", "correct": True, "zone": self.ZONE_1}
             },
             "finished": False,
-            "num_attempts": 0,
-            'overall_feedback': self.initial_feedback(),
+            "attempts": 0,
+            'overall_feedback': [self._make_feedback_message(message=self.INITIAL_FEEDBACK)],
         }
         self.assertEqual(expected_state, self.call_handler('get_user_state', method="GET"))
 
         data = {"val": 1, "zone": self.ZONE_2, "x_percent": "22%", "y_percent": "22%"}
-        res = self.call_handler('do_attempt', data)
+        res = self.call_handler(self.DROP_ITEM_HANDLER, data)
         self.assertEqual(res, {
-            "overall_feedback": self.FINAL_FEEDBACK,
+            "overall_feedback": [self._make_feedback_message(message=self.FINAL_FEEDBACK)],
             "finished": True,
             "correct": True,
             "feedback": self.FEEDBACK[1]["correct"]
@@ -150,23 +161,225 @@ class StandardModeFixture(BaseDragAndDropAjaxFixture):
                 }
             },
             "finished": True,
-            "num_attempts": 0,
-            'overall_feedback': self.FINAL_FEEDBACK,
+            "attempts": 0,
+            'overall_feedback': [self._make_feedback_message(self.FINAL_FEEDBACK)],
         }
         self.assertEqual(expected_state, self.call_handler('get_user_state', method="GET"))
 
+    def test_do_attempt_not_available(self):
+        """
+        Tests that do_attempt handler returns 400 error for standard mode DnDv2
+        """
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, expect_json=False)
+
+        self.assertEqual(res.status_code, 400)
 
+
+@ddt.ddt
 class AssessmentModeFixture(BaseDragAndDropAjaxFixture):
     """
     Common tests for drag and drop in assessment mode
     """
-    def test_do_attempt_in_assessment_mode(self):
-        item_id, zone_id = 0, self.ZONE_1
-        data = {"val": item_id, "zone": zone_id, "x_percent": "33%", "y_percent": "11%"}
-        res = self.call_handler('do_attempt', data)
-        # In assessment mode, the do_attempt doesn't return any data.
+    @staticmethod
+    def _make_submission(item_id, zone_id):
+        x_percent, y_percent = str(random.randint(0, 100)) + '%', str(random.randint(0, 100)) + '%'
+        data = {"val": item_id, "zone": zone_id, "x_percent": x_percent, "y_percent": y_percent}
+        return data
+
+    def _submit_solution(self, solution):
+        for item_id, zone_id in solution.iteritems():
+            data = self._make_submission(item_id, zone_id)
+            self.call_handler(self.DROP_ITEM_HANDLER, data)
+
+    def _submit_complete_solution(self):  # pylint: disable=no-self-use
+        raise NotImplementedError()
+
+    def _submit_partial_solution(self):  # pylint: disable=no-self-use
+        raise NotImplementedError()
+
+    def _reset_problem(self):
+        self.call_handler(self.RESET_HANDLER, data={})
+        self.assertEqual(self.block.item_state, {})
+
+    def _set_final_attempt(self):
+        self.block.max_attempts = 5
+        self.block.attempts = 4
+
+    def test_multiple_drop_item(self):
+        item_zone_map = {0: self.ZONE_1, 1: self.ZONE_2}
+        for item_id, zone_id in item_zone_map.iteritems():
+            data = self._make_submission(item_id, zone_id)
+            x_percent, y_percent = data['x_percent'], data['y_percent']
+            res = self.call_handler(self.DROP_ITEM_HANDLER, data)
+
             self.assertEqual(res, {})
 
+            expected_item_state = {'zone': zone_id, 'correct': True, 'x_percent': x_percent, 'y_percent': y_percent}
+
+            self.assertIn(str(item_id), self.block.item_state)
+            self.assertEqual(self.block.item_state[str(item_id)], expected_item_state)
+
+        # make sure item_state is appended to, not reset
+        for item_id in item_zone_map:
+            self.assertIn(str(item_id), self.block.item_state)
+
+    def test_get_user_state_no_attempts(self):
+        self.block.attempts = 0
+
+        res = self.call_handler(self.USER_STATE_HANDLER, data={})
+        expected_feedback = [
+            self._make_feedback_message(self.INITIAL_FEEDBACK)
+        ]
+        self.assertEqual(res[self.OVERALL_FEEDBACK_KEY], expected_feedback)
+
+    # pylint: disable=star-args
+    @ddt.data(
+        (None, 10, False),
+        (0, 12, False),
+        *(generate_max_and_attempts())
+    )
+    @ddt.unpack
+    def test_do_attempt_validation(self, max_attempts, attempts, expect_validation_error):
+        self.block.max_attempts = max_attempts
+        self.block.attempts = attempts
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={}, expect_json=False)
+
+        if expect_validation_error:
+            self.assertEqual(res.status_code, 409)
+        else:
+            self.assertEqual(res.status_code, 200)
+
+    @ddt.data(*[random.randint(0, 100) for _ in xrange(10)])  # pylint: disable=star-args
+    def test_do_attempt_raises_number_of_attempts(self, attempts):
+        self.block.attempts = attempts
+        self.block.max_attempts = attempts + 1
+
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertEqual(self.block.attempts, attempts + 1)
+        self.assertEqual(res['attempts'], self.block.attempts)
+
+    def test_do_attempt_correct_mark_complete_and_publish_grade(self):
+        self._submit_complete_solution()
+
+        with mock.patch('workbench.runtime.WorkbenchRuntime.publish', mock.Mock()) as patched_publish:
+            self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+            self.assertTrue(self.block.completed)
+            patched_publish.assert_called_once_with(self.block, 'grade', {
+                'value': self.block.weight,
+                'max_value': self.block.weight,
+            })
+
+    def test_do_attempt_incorrect_publish_grade(self):
+        correctness = self._submit_partial_solution()
+
+        with mock.patch('workbench.runtime.WorkbenchRuntime.publish', mock.Mock()) as patched_publish:
+            self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+            self.assertFalse(self.block.completed)
+            patched_publish.assert_called_once_with(self.block, 'grade', {
+                'value': self.block.weight * correctness,
+                'max_value': self.block.weight,
+            })
+
+    def test_do_attempt_post_correct_no_publish_grade(self):
+        self._submit_complete_solution()
+
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})  # sets self.complete
+
+        self._reset_problem()
+
+        with mock.patch('workbench.runtime.WorkbenchRuntime.publish', mock.Mock()) as patched_publish:
+            self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+            self.assertTrue(self.block.completed)
+            self.assertFalse(patched_publish.called)
+
+    def test_get_user_state_finished_after_final_attempt(self):
+        self._set_final_attempt()
+        self._submit_partial_solution()
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+        self.assertFalse(self.block.attempts_remain)  # precondition check
+
+        res = self.call_handler(self.USER_STATE_HANDLER, data={})
+        self.assertTrue(res['finished'])
+
+    def test_do_attempt_incorrect_final_attempt_publish_grade(self):
+        self._set_final_attempt()
+
+        correctness = self._submit_partial_solution()
+        expected_grade = self.block.weight * correctness
+
+        with mock.patch('workbench.runtime.WorkbenchRuntime.publish', mock.Mock()) as patched_publish:
+            res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+            self.assertTrue(self.block.completed)
+            patched_publish.assert_called_once_with(self.block, 'grade', {
+                'value': expected_grade,
+                'max_value': self.block.weight,
+            })
+
+            expected_grade_feedback = self._make_feedback_message(
+                FeedbackMessages.FINAL_ATTEMPT_TPL.format(score=expected_grade),
+                FeedbackMessages.MessageClasses.PARTIAL_SOLUTION
+            )
+            self.assertIn(expected_grade_feedback, res[self.OVERALL_FEEDBACK_KEY])
+
+    def test_do_attempt_incorrect_final_attempt_after_correct(self):
+        self._submit_complete_solution()
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+        self.assertTrue(self.block.completed)  # precondition check
+        self.assertEqual(self.block.grade, 1.0)  # precondition check
+
+        self._reset_problem()
+
+        self._set_final_attempt()
+
+        self._submit_partial_solution()
+
+        with mock.patch('workbench.runtime.WorkbenchRuntime.publish', mock.Mock()) as patched_publish:
+            res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+
+            expected_grade_feedback = self._make_feedback_message(
+                FeedbackMessages.FINAL_ATTEMPT_TPL.format(score=1.0),
+                FeedbackMessages.MessageClasses.PARTIAL_SOLUTION
+            )
+            self.assertFalse(patched_publish.called)
+            self.assertIn(expected_grade_feedback, res[self.OVERALL_FEEDBACK_KEY])
+            self.assertEqual(self.block.grade, 1.0)
+
+    def test_do_attempt_misplaced_ids(self):
+        misplaced_ids = self._submit_incorrect_solution()
+
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertTrue(res['misplaced_items'], misplaced_ids)
+        self.assertIn(
+            self._make_feedback_message(FeedbackMessages.MISPLACED_ITEMS_RETURNED),
+            res[self.OVERALL_FEEDBACK_KEY]
+        )
+
+    def test_do_attempt_shows_final_feedback_at_last_attempt(self):
+        self._set_final_attempt()
+
+        self._submit_partial_solution()
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        expected_message = self._make_feedback_message(self.FINAL_FEEDBACK)
+        self.assertIn(expected_message, res[self.OVERALL_FEEDBACK_KEY])
+
+    def test_get_user_state_does_not_include_correctness(self):
+        self._submit_complete_solution()
+        original_item_state = self.block.item_state
+
+        res = self.call_handler(self.USER_STATE_HANDLER)
+
+        item_data = res['items']
+        for item in item_data:
+            self.assertNotIn('correct', item)
+
+        self.assertEqual(self.block.item_state, original_item_state)
+
 
 class TestDragAndDropHtmlData(StandardModeFixture, unittest.TestCase):
     FOLDER = "html"
@@ -180,6 +393,7 @@ class TestDragAndDropHtmlData(StandardModeFixture, unittest.TestCase):
         2: {"correct": "", "incorrect": ""}
     }
 
+    INITIAL_FEEDBACK = "HTML <strong>Intro</strong> Feed"
     FINAL_FEEDBACK = "Final <strong>feedback</strong>!"
 
 
@@ -195,6 +409,7 @@ class TestDragAndDropPlainData(StandardModeFixture, unittest.TestCase):
         2: {"correct": "", "incorrect": ""}
     }
 
+    INITIAL_FEEDBACK = "This is the initial feedback."
     FINAL_FEEDBACK = "This is the final feedback."
 
 
@@ -203,6 +418,8 @@ class TestOldDataFormat(TestDragAndDropPlainData):
     Make sure we can work with the slightly-older format for 'data' field values.
     """
     FOLDER = "old"
+
+    INITIAL_FEEDBACK = "Intro Feed"
     FINAL_FEEDBACK = "Final Feed"
 
     ZONE_1 = "Zone 1"
@@ -221,4 +438,102 @@ class TestDragAndDropAssessmentData(AssessmentModeFixture, unittest.TestCase):
         2: {"correct": "", "incorrect": ""}
     }
 
+    INITIAL_FEEDBACK = "This is the initial feedback."
     FINAL_FEEDBACK = "This is the final feedback."
+
+    def _submit_complete_solution(self):
+        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2, 2: self.ZONE_2})
+
+    def _submit_partial_solution(self):
+        self._submit_solution({0: self.ZONE_1})
+        return 1.0 / 3.0
+
+    def _submit_incorrect_solution(self):
+        self._submit_solution({0: self.ZONE_2, 1: self.ZONE_1})
+        return 0, 1
+
+    def test_do_attempt_feedback_incorrect_not_placed(self):
+        self._submit_solution({0: self.ZONE_2, 1: self.ZONE_2})
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        overall_feedback = res[self.OVERALL_FEEDBACK_KEY]
+        expected_overall_feedback = [
+            self._make_feedback_message(
+                FeedbackMessages.correctly_placed(1), FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+            ),
+            self._make_feedback_message(FeedbackMessages.misplaced(1), FeedbackMessages.MessageClasses.MISPLACED),
+            self._make_feedback_message(FeedbackMessages.not_placed(1), FeedbackMessages.MessageClasses.NOT_PLACED),
+            self._make_feedback_message(FeedbackMessages.MISPLACED_ITEMS_RETURNED, None),
+            self._make_feedback_message(self.INITIAL_FEEDBACK, None),
+        ]
+        self.assertEqual(overall_feedback, expected_overall_feedback)
+
+    def test_do_attempt_feedback_not_placed(self):
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        overall_feedback = res[self.OVERALL_FEEDBACK_KEY]
+        expected_overall_feedback = [
+            self._make_feedback_message(FeedbackMessages.not_placed(3), FeedbackMessages.MessageClasses.NOT_PLACED),
+            self._make_feedback_message(self.INITIAL_FEEDBACK, None),
+        ]
+        self.assertEqual(overall_feedback, expected_overall_feedback)
+
+    def test_do_attempt_feedback_correct_and_decoy(self):
+        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2, 3: self.ZONE_2})  # incorrect solution - decoy placed
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        overall_feedback = res[self.OVERALL_FEEDBACK_KEY]
+        expected_overall_feedback = [
+            self._make_feedback_message(
+                FeedbackMessages.correctly_placed(2), FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+            ),
+            self._make_feedback_message(FeedbackMessages.misplaced(1), FeedbackMessages.MessageClasses.MISPLACED),
+            self._make_feedback_message(FeedbackMessages.not_placed(1), FeedbackMessages.MessageClasses.NOT_PLACED),
+            self._make_feedback_message(FeedbackMessages.MISPLACED_ITEMS_RETURNED, None),
+            self._make_feedback_message(self.INITIAL_FEEDBACK, None),
+        ]
+        self.assertEqual(overall_feedback, expected_overall_feedback)
+
+    def test_do_attempt_feedback_correct(self):
+        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2, 2: self.ZONE_2})  # correct solution
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        overall_feedback = res[self.OVERALL_FEEDBACK_KEY]
+        expected_overall_feedback = [
+            self._make_feedback_message(
+                FeedbackMessages.correctly_placed(3), FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+            ),
+            self._make_feedback_message(self.FINAL_FEEDBACK, FeedbackMessages.MessageClasses.CORRECT_SOLUTION),
+        ]
+        self.assertEqual(overall_feedback, expected_overall_feedback)
+
+    def test_do_attempt_feedback_partial(self):
+        self._submit_solution({0: self.ZONE_1})  # partial solution
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        overall_feedback = res[self.OVERALL_FEEDBACK_KEY]
+        expected_overall_feedback = [
+            self._make_feedback_message(
+                FeedbackMessages.correctly_placed(1), FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+            ),
+            self._make_feedback_message(FeedbackMessages.not_placed(2), FeedbackMessages.MessageClasses.NOT_PLACED),
+            self._make_feedback_message(self.INITIAL_FEEDBACK, None),
+        ]
+        self.assertEqual(overall_feedback, expected_overall_feedback)
+
+    def test_do_attempt_keeps_highest_score(self):
+        self.assertFalse(self.block.completed)  # precondition check
+        expected_score = 2.0 / 3.0
+
+        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2})  # partial solution, 0.66 score
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertEqual(self.block.grade, expected_score)
+
+        self._reset_problem()
+        # make it a last attempt so we can check feedback
+        self._set_final_attempt()
+
+        self._submit_solution({0: self.ZONE_1})  # partial solution, 0.33 score
+        res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertEqual(self.block.grade, expected_score)
+
+        expected_feedback = self._make_feedback_message(
+            FeedbackMessages.FINAL_ATTEMPT_TPL.format(score=expected_score),
+            FeedbackMessages.MessageClasses.PARTIAL_SOLUTION
+        )
+        self.assertIn(expected_feedback, res[self.OVERALL_FEEDBACK_KEY])

tests/unit/test_basics.py

@@ -69,20 +69,20 @@ class BasicTests(TestCaseMixin, unittest.TestCase):
             self.assertEqual(self.call_handler("get_user_state"), {
                 'items': {},
                 'finished': False,
-                "num_attempts": 0,
-                'overall_feedback': START_FEEDBACK,
+                "attempts": 0,
+                'overall_feedback': [{"message": START_FEEDBACK, "message_class": None}]
             })
         assert_user_state_empty()
 
         # Drag three items into the correct spot:
         data = {"val": 0, "zone": TOP_ZONE_ID, "x_percent": "33%", "y_percent": "11%"}
-        self.call_handler('do_attempt', data)
+        self.call_handler(self.DROP_ITEM_HANDLER, data)
         data = {"val": 1, "zone": MIDDLE_ZONE_ID, "x_percent": "67%", "y_percent": "80%"}
-        self.call_handler('do_attempt', data)
+        self.call_handler(self.DROP_ITEM_HANDLER, data)
         data = {"val": 2, "zone": BOTTOM_ZONE_ID, "x_percent": "99%", "y_percent": "95%"}
-        self.call_handler('do_attempt', data)
+        self.call_handler(self.DROP_ITEM_HANDLER, data)
         data = {"val": 3, "zone": MIDDLE_ZONE_ID, "x_percent": "67%", "y_percent": "80%"}
-        self.call_handler('do_attempt', data)
+        self.call_handler(self.DROP_ITEM_HANDLER, data)
 
         # Check the result:
         self.assertTrue(self.block.completed)
@@ -100,8 +100,8 @@ class BasicTests(TestCaseMixin, unittest.TestCase):
                 '3': {'x_percent': '67%', 'y_percent': '80%', 'correct': True, "zone": MIDDLE_ZONE_ID},
             },
             'finished': True,
-            "num_attempts": 0,
-            'overall_feedback': FINISH_FEEDBACK,
+            "attempts": 0,
+            'overall_feedback': [{"message": FINISH_FEEDBACK, "message_class": None}],
         })
 
         # Reset to initial conditions

tests/utils.py

 import json
+import random
 import re
 
 from mock import patch
@@ -31,10 +32,23 @@ def make_block():
     return drag_and_drop_v2.DragAndDropBlock(runtime, field_data, scope_ids=scope_ids)
 
 
+def generate_max_and_attempts(count=100):
+    for _ in xrange(count):
+        max_attempts = random.randint(1, 100)
+        attempts = random.randint(0, 100)
+        expect_validation_error = max_attempts <= attempts
+        yield max_attempts, attempts, expect_validation_error
+
+
 class TestCaseMixin(object):
     """ Helpful mixins for unittest TestCase subclasses """
     maxDiff = None
 
+    DROP_ITEM_HANDLER = 'drop_item'
+    DO_ATTEMPT_HANDLER = 'do_attempt'
+    RESET_HANDLER = 'reset'
+    USER_STATE_HANDLER = 'get_user_state'
+
     def patch_workbench(self):
         self.apply_patch(
             'workbench.runtime.WorkbenchRuntime.local_resource_url',