[SOL-1988] Account for decoy items in score

In order to take into account the placement of decoy items, calculate a
student's score as:

score = ((correctly placed items) + (correctly unplaced decoys)) / (total items)

README.md

@@ -151,6 +151,32 @@ You can leave all of the checkboxes unchecked in order to create a
 You can define an arbitrary number of drag items, each of which may
 be attached to any number of zones.
 
+Scoring
+-------
+
+Student assessment scores for the Drag and Drop XBlock are calculated according
+to the following formula:
+
+    score = (C + D) / T
+
+Where *C* is the number of correctly placed regular items, *D* is the number of
+decoy items that were correctly left unplaced, and *T* is the total number of
+items available.
+
+Example: consider a Drag and Drop instance configured with a total of four
+items, of which three are regular items and one is a decoy.  If a learner
+places two of the normal items correctly and one incorrectly (`C = 2`), and
+wrongly places the decoy item onto a drop zone (`D = 0`), that learner's score
+will be `50%`, as given by:
+
+    score = (2 + 0) / 4
+
+If the learner were to then move the decoy item back to the bank (`D = 1`) and
+move the wrongly placed regular item to the correct dropzone (`C = 3`), their
+score would be `100%`:
+
+    score = (3 + 1) / 4
+
 Demo Course
 -----------
 

drag_and_drop_v2/drag_and_drop_v2.py

@@ -16,7 +16,7 @@ from xblock.fragment import Fragment
 from xblockutils.resources import ResourceLoader
 from xblockutils.settings import XBlockWithSettingsMixin, ThemableXBlockMixin
 
-from .utils import _, DummyTranslationService, FeedbackMessage, FeedbackMessages
+from .utils import _, DummyTranslationService, FeedbackMessage, FeedbackMessages, ItemStats
 from .default_data import DEFAULT_DATA
 
 
@@ -456,9 +456,9 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
             feedback_key = 'finish' if is_correct else 'start'
             return [FeedbackMessage(self.data['feedback'][feedback_key], None)], set()
 
-        required_ids, placed_ids, correct_ids = self._get_item_raw_stats()
-        missing_ids = required_ids - placed_ids
-        misplaced_ids = placed_ids - correct_ids
+        items = self._get_item_raw_stats()
+        missing_ids = items.required - items.placed
+        misplaced_ids = items.placed - items.correctly_placed
 
         feedback_msgs = []
 
@@ -469,7 +469,7 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
                 feedback_msgs.append(FeedbackMessage(message, message_class))
 
         _add_msg_if_exists(
-            correct_ids, FeedbackMessages.correctly_placed, FeedbackMessages.MessageClasses.CORRECTLY_PLACED
+            items.correctly_placed, FeedbackMessages.correctly_placed, FeedbackMessages.MessageClasses.CORRECTLY_PLACED
         )
         _add_msg_if_exists(misplaced_ids, FeedbackMessages.misplaced, FeedbackMessages.MessageClasses.MISPLACED)
         _add_msg_if_exists(missing_ids, FeedbackMessages.not_placed, FeedbackMessages.MessageClasses.NOT_PLACED)
@@ -740,31 +740,39 @@ class DragAndDropBlock(XBlock, XBlockWithSettingsMixin, ThemableXBlockMixin):
 
     def _get_item_stats(self):
         """
-        Returns a tuple representing the number of correctly-placed items,
-        and the total number of items that must be placed on the board (non-decoy items).
+        Returns a tuple representing the number of correctly placed items,
+        and the total number of items required (including decoy items).
         """
-        required_items, __, correct_items = self._get_item_raw_stats()
+        items = self._get_item_raw_stats()
 
-        return len(correct_items), len(required_items)
+        correct_count = len(items.correctly_placed) + len(items.decoy_in_bank)
+        total_count = len(items.required) + len(items.decoy)
+
+        return correct_count, total_count
 
     def _get_item_raw_stats(self):
         """
-        Returns a 3-tuple containing required, placed and correct items.
+        Returns a named tuple containing required, decoy, placed, correctly
+        placed, and correctly unplaced decoy items.
 
         Returns:
-            tuple: (required_items, placed_items, correct_items)
-                * required_items - IDs of items that must be placed on the board
-                * placed_items - IDs of items actually placed on the board
-                * correct_items - IDs of items that were placed correctly
+            namedtuple: (required, placed, correctly_placed, decoy, decoy_in_bank)
+                * required - IDs of items that must be placed on the board
+                * placed - IDs of items actually placed on the board
+                * correctly_placed - IDs of items that were placed correctly
+                * decoy - IDs of decoy items
+                * decoy_in_bank - IDs of decoy items that were unplaced
         """
-        all_items = [str(item['id']) for item in self.data['items']]
         item_state = self._get_item_state()
 
-        required_items = set(item_id for item_id in all_items if self._get_item_zones(int(item_id)) != [])
-        placed_items = set(item_id for item_id in all_items if item_id in item_state)
-        correct_items = set(item_id for item_id in placed_items if item_state[item_id]['correct'])
+        all_items = set(str(item['id']) for item in self.data['items'])
+        required = set(item_id for item_id in all_items if self._get_item_zones(int(item_id)) != [])
+        placed = set(item_id for item_id in all_items if item_id in item_state)
+        correctly_placed = set(item_id for item_id in placed if item_state[item_id]['correct'])
+        decoy = all_items - required
+        decoy_in_bank = set(item_id for item_id in decoy if item_id not in item_state)
 
-        return required_items, placed_items, correct_items
+        return ItemStats(required, placed, correctly_placed, decoy, decoy_in_bank)
 
     def _get_grade(self):
         """

drag_and_drop_v2/utils.py

@@ -78,3 +78,7 @@ class FeedbackMessages(object):
 
 
 FeedbackMessage = namedtuple("FeedbackMessage", ["message", "message_class"])  # pylint: disable=invalid-name
+ItemStats = namedtuple(  # pylint: disable=invalid-name
+    'ItemStats',
+    ["required", "placed", "correctly_placed", "decoy", "decoy_in_bank"]
+)

tests/integration/test_interaction.py

@@ -673,6 +673,26 @@ class AssessmentInteractionTest(
         expected_feedback = "\n".join(feedback_lines)
         self.assertEqual(self._get_feedback().text, expected_feedback)
 
+    def test_grade(self):
+        """
+        Test grading after submitting solution in assessment mode
+        """
+        mock = Mock()
+        context = patch.object(WorkbenchRuntime, 'publish', mock)
+        context.start()
+        self.addCleanup(context.stop)
+        self.publish = mock
+
+        self.place_item(0, TOP_ZONE_ID, Keys.RETURN)  # Correctly placed item
+        self.place_item(1, BOTTOM_ZONE_ID, Keys.RETURN)  # Incorrectly placed item
+        self.place_item(4, MIDDLE_ZONE_ID, Keys.RETURN)  # Incorrectly placed decoy
+        self.click_submit()
+
+        events = self.publish.call_args_list
+        published_grade = next((event[0][2] for event in events if event[0][1] == 'grade'))
+        expected_grade = {'max_value': 1, 'value': (1.0 / 5.0)}
+        self.assertEqual(published_grade, expected_grade)
+
 
 class MultipleValidOptionsInteractionTest(DefaultDataTestMixin, InteractionTestBase, BaseIntegrationTest):
 
@@ -716,7 +736,7 @@ class EventsFiredTest(DefaultDataTestMixin, InteractionTestBase, BaseIntegration
         },
         {
             'name': 'grade',
-            'data': {'max_value': 1, 'value': (1.0 / 4)},
+            'data': {'max_value': 1, 'value': (2.0 / 5)},
         },
         {
             'name': 'edx.drag_and_drop_v2.item.dropped',

tests/unit/test_advanced.py

@@ -119,7 +119,7 @@ class StandardModeFixture(BaseDragAndDropAjaxFixture):
         })
 
         self.assertEqual(1, len(published_grades))
-        self.assertEqual({'value': 0.5, 'max_value': 1}, published_grades[-1])
+        self.assertEqual({'value': 0.75, 'max_value': 1}, published_grades[-1])
 
         self.call_handler(self.DROP_ITEM_HANDLER, {
             "val": 1, "zone": self.ZONE_2, "y_percent": "90%", "x_percent": "42%"
@@ -446,7 +446,7 @@ class TestDragAndDropAssessmentData(AssessmentModeFixture, unittest.TestCase):
 
     def _submit_partial_solution(self):
         self._submit_solution({0: self.ZONE_1})
-        return 1.0 / 3.0
+        return 3.0 / 5.0
 
     def _submit_incorrect_solution(self):
         self._submit_solution({0: self.ZONE_2, 1: self.ZONE_1})
@@ -518,9 +518,9 @@ class TestDragAndDropAssessmentData(AssessmentModeFixture, unittest.TestCase):
 
     def test_do_attempt_keeps_highest_score(self):
         self.assertFalse(self.block.completed)  # precondition check
-        expected_score = 2.0 / 3.0
+        expected_score = 4.0 / 5.0
 
-        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2})  # partial solution, 0.66 score
+        self._submit_solution({0: self.ZONE_1, 1: self.ZONE_2})  # partial solution, 0.8 score
         self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
         self.assertEqual(self.block.grade, expected_score)
 
@@ -528,7 +528,7 @@ class TestDragAndDropAssessmentData(AssessmentModeFixture, unittest.TestCase):
         # make it a last attempt so we can check feedback
         self._set_final_attempt()
 
-        self._submit_solution({0: self.ZONE_1})  # partial solution, 0.33 score
+        self._submit_solution({0: self.ZONE_1})  # partial solution, 0.6 score
         res = self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
         self.assertEqual(self.block.grade, expected_score)
 
@@ -537,3 +537,25 @@ class TestDragAndDropAssessmentData(AssessmentModeFixture, unittest.TestCase):
             FeedbackMessages.MessageClasses.PARTIAL_SOLUTION
         )
         self.assertIn(expected_feedback, res[self.OVERALL_FEEDBACK_KEY])
+
+    def test_do_attempt_check_score_with_decoy(self):
+        self.assertFalse(self.block.completed)  # precondition check
+        expected_score = 4.0 / 5.0
+        self._submit_solution({
+            0: self.ZONE_1,
+            1: self.ZONE_2,
+            2: self.ZONE_2,
+            3: self.ZONE_1,
+        })  # incorrect solution, 0.8 score
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertEqual(self.block.grade, expected_score)
+
+    def test_do_attempt_zero_score_with_all_decoys(self):
+        self.assertFalse(self.block.completed)  # precondition check
+        expected_score = 0
+        self._submit_solution({
+            3: self.ZONE_1,
+            4: self.ZONE_2,
+        })  # incorrect solution, 0 score
+        self.call_handler(self.DO_ATTEMPT_HANDLER, data={})
+        self.assertEqual(self.block.grade, expected_score)