fix tolerance rounding error (TNL-904)

common/lib/capa/capa/tests/test_util.py

@@ -81,6 +81,29 @@ class UtilTest(unittest.TestCase):
         self.assertFalse(result)
         result = compare_with_tolerance(infinity, infinity, '1.0', False)
         self.assertTrue(result)
+        # Test absolute tolerance for smaller values
+        result = compare_with_tolerance(100.01, 100.0, 0.01, False)
+        self.assertTrue(result)
+        result = compare_with_tolerance(100.001, 100.0, 0.001, False)
+        self.assertTrue(result)
+        result = compare_with_tolerance(100.01, 100.0, '0.01%', False)
+        self.assertTrue(result)
+        result = compare_with_tolerance(100.002, 100.0, 0.001, False)
+        self.assertFalse(result)
+        result = compare_with_tolerance(0.4, 0.44, 0.01, False)
+        self.assertFalse(result)
+        result = compare_with_tolerance(100.01, 100.0, 0.010, False)
+        self.assertTrue(result)
+
+        # Test complex_number instructor_complex
+        result = compare_with_tolerance(0.4, complex(0.44, 0), 0.01, False)
+        self.assertFalse(result)
+        result = compare_with_tolerance(100.01, complex(100.0, 0), 0.010, False)
+        self.assertTrue(result)
+        result = compare_with_tolerance(110.1, complex(100.0, 0), '10.0', False)
+        self.assertFalse(result)
+        result = compare_with_tolerance(111.0, complex(100.0, 0), '10%', True)
+        self.assertTrue(result)
 
     def test_sanitize_html(self):
         """

common/lib/capa/capa/util.py

@@ -2,9 +2,10 @@
 Utility functions for capa.
 """
 import bleach
+from decimal import Decimal
 
 from calc import evaluator
-from cmath import isinf
+from cmath import isinf, isnan
 #-----------------------------------------------------------------------------
 #
 # Utility functions used in CAPA responsetypes
@@ -64,6 +65,23 @@ def compare_with_tolerance(student_complex, instructor_complex, tolerance=defaul
         # `tolerance` both equal to infinity. Then, below we would have
         # `inf <= inf` which is a fail. Instead, compare directly.
         return student_complex == instructor_complex
+
+    # because student_complex and instructor_complex are not necessarily
+    # complex here, we enforce it here:
+    student_complex = complex(student_complex)
+    instructor_complex = complex(instructor_complex)
+
+    # if both the instructor and student input are real,
+    # compare them as Decimals to avoid rounding errors
+    if not (instructor_complex.imag or student_complex.imag):
+        # if either of these are not a number, short circuit and return False
+        if isnan(instructor_complex.real) or isnan(student_complex.real):
+            return False
+        student_decimal = Decimal(str(student_complex.real))
+        instructor_decimal = Decimal(str(instructor_complex.real))
+        tolerance_decimal = Decimal(str(tolerance))
+        return abs(student_decimal - instructor_decimal) <= tolerance_decimal
+
     else:
         # v1 and v2 are, in general, complex numbers:
         # there are some notes about backward compatibility issue: see responsetypes.get_staff_ans()).