Merge pull request #1589 from MITx/peter/symbolic

Added support for superscripts in variables

common/static/js/capa/symbolic_mathjax_preprocessor.js

+/* This file defines a processor in between the student's math input
+   (AsciiMath) and what is read by MathJax. It allows for our own
+   customizations, such as use of the syntax "a_b__x" in superscripts, or
+   possibly coloring certain variables, etc&.
+
+   It is used in the <textline> definition like the following:
+
+     <symbolicresponse expect="a_b^c + b_x__d" size="30">
+       <textline math="1"
+         preprocessorClassName="SymbolicMathjaxPreprocessor"
+	 preprocessorSrc="/static/js/capa/symbolic_mathjax_preprocessor.js"/>
+     </symbolicresponse>
+*/
+window.SymbolicMathjaxPreprocessor = function () {
+  this.fn = function (eqn) {
+    // flags and config
+    var superscriptsOn = true;
+
+    if (superscriptsOn) {
+      // find instances of "__" and make them superscripts ("^") and tag them
+      // as such. Specifcally replace instances of "__X" or "__{XYZ}" with
+      // "^{CHAR$1}", marking superscripts as different from powers
+
+      // a zero width space--this is an invisible character that no one would
+      // use, that gets passed through MathJax and to the server
+      var c = "\u200b";
+      eqn = eqn.replace(/__(?:([^\{])|\{([^\}]+)\})/g, '^{' + c + '$1$2}');
+
+      // NOTE: MathJax supports '\class{name}{mathcode}' but not for asciimath
+      // input, which is too bad. This would be preferable to this char tag
+    }
+
+    return eqn;
+  };
+};

doc/public/course_data_formats/symbolic_response.rst

+#################
+Symbolic Response
+#################
+
+This document plans to document features that the current symbolic response
+supports. In general it allows the input and validation of math expressions,
+up to commutativity and some identities.
+
+
+********
+Features
+********
+
+This is a partial list of features, to be revised as we go along:
+    * sub and superscripts: an expression following the ``^`` character
+      indicates exponentiation. To use superscripts in variables, the syntax
+      is ``b_x__d`` for the variable ``b`` with subscript ``x`` and super
+      ``d``. 
+
+      An example of a problem::
+
+      <symbolicresponse expect="a_b^c + b_x__d" size="30">
+        <textline math="1"
+           preprocessorClassName="SymbolicMathjaxPreprocessor"
+           preprocessorSrc="/static/js/capa/symbolic_mathjax_preprocessor.js"/>
+      </symbolicresponse>
+
+      It's a bit of a pain to enter that.
+
+    * The script-style math variant. What would be outputted in latex if you
+      entered ``\mathcal{N}``. This is used in some variables.
+
+      An example::
+
+          <symbolicresponse expect="scriptN_B + x" size="30">
+            <textline math="1"/>
+          </symbolicresponse>
+
+      There is no fancy preprocessing needed, but if you had superscripts or
+      something, you would need to include that part.

lms/lib/symmath/formula.py

@@ -74,6 +74,15 @@ def to_latex(x):
     # LatexPrinter._print_dot = _print_dot
     xs = latex(x)
     xs = xs.replace(r'\XI', 'XI')	 # workaround for strange greek
+
+    # substitute back into latex form for scripts
+    # literally something of the form
+    # 'scriptN' becomes '\\mathcal{N}'
+    # note: can't use something akin to the _print_hat method above because we sometimes get 'script(N)__B' or more complicated terms
+    xs = re.sub(r'script([a-zA-Z0-9]+)',
+                '\\mathcal{\\1}',
+                xs)
+
     #return '<math>%s{}{}</math>' % (xs[1:-1])
     if xs[0] == '$':
         return '[mathjax]%s[/mathjax]<br>' % (xs[1:-1])	 # for sympy v6
@@ -106,6 +115,7 @@ def my_sympify(expr, normphase=False, matrix=False, abcsym=False, do_qubit=False
                   'i': sympy.I,			# lowercase i is also sqrt(-1)
                   'Q': sympy.Symbol('Q'),	 # otherwise it is a sympy "ask key"
                   'I': sympy.Symbol('I'),	 # otherwise it is sqrt(-1)
+                  'N': sympy.Symbol('N'),	 # or it is some kind of sympy function
                   #'X':sympy.sympify('Matrix([[0,1],[1,0]])'),
                   #'Y':sympy.sympify('Matrix([[0,-I],[I,0]])'),
                   #'Z':sympy.sympify('Matrix([[1,0],[0,-1]])'),
@@ -247,6 +257,127 @@ class formula(object):
                 fix_hat(k)
         fix_hat(xml)
 
+        def flatten_pmathml(xml):
+            ''' Give the text version of certain PMathML elements
+
+            Sometimes MathML will be given with each letter separated (it
+            doesn't know if its implicit multiplication or what). From an xml
+            node, find the (text only) variable name it represents. So it takes
+            <mrow>
+              <mi>m</mi>
+              <mi>a</mi>
+              <mi>x</mi>
+            </mrow>
+            and returns 'max', for easier use later on.
+            '''
+            tag = gettag(xml)
+            if tag == 'mn': return xml.text
+            elif tag == 'mi': return xml.text
+            elif tag == 'mrow': return ''.join([flatten_pmathml(y) for y in xml])
+            raise Exception, '[flatten_pmathml] unknown tag %s' % tag
+
+        def fix_mathvariant(parent):
+            '''Fix certain kinds of math variants
+
+            Literally replace <mstyle mathvariant="script"><mi>N</mi></mstyle>
+            with 'scriptN'. There have been problems using script_N or script(N)
+            '''
+            for child in parent:
+                if (gettag(child) == 'mstyle' and child.get('mathvariant') == 'script'):
+                    newchild = etree.Element('mi')
+                    newchild.text = 'script%s' % flatten_pmathml(child[0])
+                    parent.replace(child, newchild)
+                fix_mathvariant(child)
+        fix_mathvariant(xml)
+
+
+        # find "tagged" superscripts
+        # they have the character \u200b in the superscript
+        # replace them with a__b so snuggle doesn't get confused
+        def fix_superscripts(xml):
+            ''' Look for and replace sup elements with 'X__Y' or 'X_Y__Z'
+
+            In the javascript, variables with '__X' in them had an invisible
+            character inserted into the sup (to distinguish from powers)
+            E.g. normal:
+            <msubsup>
+              <mi>a</mi>
+              <mi>b</mi>
+              <mi>c</mi>
+            </msubsup>
+            to be interpreted '(a_b)^c' (nothing done by this method)
+
+            And modified:
+            <msubsup>
+              <mi>b</mi>
+              <mi>x</mi>
+              <mrow>
+                <mo>&#x200B;</mo>
+                <mi>d</mi>
+              </mrow>
+            </msubsup>
+            to be interpreted 'a_b__c'
+
+            also:
+            <msup>
+              <mi>x</mi>
+              <mrow>
+                <mo>&#x200B;</mo>
+                <mi>B</mi>
+              </mrow>
+            </msup>
+            to be 'x__B'
+            '''
+            for k in xml:
+                tag = gettag(k)
+
+                # match things like the last example--
+                # the second item in msub is an mrow with the first
+                # character equal to \u200b
+                if (tag == 'msup' and
+                    len(k) == 2 and gettag(k[1]) == 'mrow' and
+                    gettag(k[1][0]) == 'mo' and k[1][0].text == u'\u200b'): # whew
+
+                    # replace the msup with 'X__Y'
+                    k[1].remove(k[1][0])
+                    newk = etree.Element('mi')
+                    newk.text = '%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]))
+                    xml.replace(k, newk)
+
+                # match things like the middle example-
+                # the third item in msubsup is an mrow with the first
+                # character equal to \u200b
+                if (tag == 'msubsup' and
+                    len(k) == 3 and gettag(k[2]) == 'mrow' and
+                    gettag(k[2][0]) == 'mo' and k[2][0].text == u'\u200b'): # whew
+
+                    # replace the msubsup with 'X_Y__Z'
+                    k[2].remove(k[2][0])
+                    newk = etree.Element('mi')
+                    newk.text = '%s_%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]), flatten_pmathml(k[2]))
+                    xml.replace(k, newk)
+
+                fix_superscripts(k)
+        fix_superscripts(xml)
+
+        # Snuggle returns an error when it sees an <msubsup>
+        # replace such elements with an <msup>, except the first element is of
+        # the form a_b. I.e. map a_b^c => (a_b)^c
+        def fix_msubsup(parent):
+            for child in parent:
+                # fix msubsup
+                if (gettag(child) == 'msubsup' and len(child) == 3):
+                    newchild = etree.Element('msup')
+                    newbase = etree.Element('mi')
+                    newbase.text = '%s_%s' % (flatten_pmathml(child[0]), flatten_pmathml(child[1]))
+                    newexp = child[2]
+                    newchild.append(newbase)
+                    newchild.append(newexp)
+                    parent.replace(child, newchild)
+
+                fix_msubsup(child)
+        fix_msubsup(xml)
+
         self.xml = xml
         return self.xml
 
@@ -257,6 +388,7 @@ class formula(object):
         try:
             xml = self.preprocess_pmathml(self.expr)
         except Exception, err:
+            log.warning('Err %s while preprocessing; expr=%s' % (err, self.expr))
             return "<html>Error! Cannot process pmathml</html>"
         pmathml = etree.tostring(xml, pretty_print=True)
         self.the_pmathml = pmathml

lms/lib/symmath/test_formula.py

+"""
+Tests of symbolic math
+"""
+
+
+import unittest
+import formula
+import re
+from lxml import etree
+
+def stripXML(xml):
+    xml = xml.replace('\n', '')
+    xml = re.sub(r'\> +\<', '><', xml)
+    return xml
+
+class FormulaTest(unittest.TestCase):
+    # for readability later
+    mathml_start = '<math xmlns="http://www.w3.org/1998/Math/MathML"><mstyle displaystyle="true">'
+    mathml_end = '</mstyle></math>'
+
+    def setUp(self):
+        self.formulaInstance = formula.formula('')
+
+    def test_replace_mathvariants(self):
+        expr = '''
+<mstyle mathvariant="script">
+  <mi>N</mi>
+</mstyle>'''
+
+        expected = '<mi>scriptN</mi>'
+
+        # wrap
+        expr = stripXML(self.mathml_start + expr + self.mathml_end)
+        expected = stripXML(self.mathml_start + expected + self.mathml_end)
+
+        # process the expression
+        xml = etree.fromstring(expr)
+        xml = self.formulaInstance.preprocess_pmathml(xml)
+        test = etree.tostring(xml)
+
+        # success?
+        self.assertEqual(test, expected)
+
+
+    def test_fix_simple_superscripts(self):
+        expr = '''
+<msup>
+  <mi>a</mi>
+  <mrow>
+    <mo>&#x200B;</mo>
+    <mi>b</mi>
+  </mrow>
+</msup>'''
+
+        expected = '<mi>a__b</mi>'
+
+        # wrap
+        expr = stripXML(self.mathml_start + expr + self.mathml_end)
+        expected = stripXML(self.mathml_start + expected + self.mathml_end)
+
+        # process the expression
+        xml = etree.fromstring(expr)
+        xml = self.formulaInstance.preprocess_pmathml(xml)
+        test = etree.tostring(xml)
+
+        # success?
+        self.assertEqual(test, expected)
+
+    def test_fix_complex_superscripts(self):
+        expr = '''
+<msubsup>
+  <mi>a</mi>
+  <mi>b</mi>
+  <mrow>
+    <mo>&#x200B;</mo>
+    <mi>c</mi>
+  </mrow>
+</msubsup>'''
+
+        expected = '<mi>a_b__c</mi>'
+
+        # wrap
+        expr = stripXML(self.mathml_start + expr + self.mathml_end)
+        expected = stripXML(self.mathml_start + expected + self.mathml_end)
+
+        # process the expression
+        xml = etree.fromstring(expr)
+        xml = self.formulaInstance.preprocess_pmathml(xml)
+        test = etree.tostring(xml)
+
+        # success?
+        self.assertEqual(test, expected)
+
+
+    def test_fix_msubsup(self):
+        expr = '''
+<msubsup>
+  <mi>a</mi>
+  <mi>b</mi>
+  <mi>c</mi>
+</msubsup>'''
+
+        expected = '<msup><mi>a_b</mi><mi>c</mi></msup>' # which is (a_b)^c
+
+        # wrap
+        expr = stripXML(self.mathml_start + expr + self.mathml_end)
+        expected = stripXML(self.mathml_start + expected + self.mathml_end)
+
+        # process the expression
+        xml = etree.fromstring(expr)
+        xml = self.formulaInstance.preprocess_pmathml(xml)
+        test = etree.tostring(xml)
+
+        # success?
+        self.assertEqual(test, expected)