Merge pull request #1100 from MITx/fix/brian/nonascii-html-xmodule

Fix/brian/nonascii html xmodule

common/djangoapps/mitxmako/shortcuts.py

@@ -12,10 +12,6 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-import logging
-
-log = logging.getLogger("mitx." + __name__)
-
 from django.template import Context
 from django.http import HttpResponse
 
@@ -42,7 +38,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
         context_dictionary.update(context)
     # fetch and render template
     template = middleware.lookup[namespace].get_template(template_name)
-    return template.render(**context_dictionary)
+    return template.render_unicode(**context_dictionary)
 
 
 def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs):

common/djangoapps/mitxmako/template.py

@@ -54,5 +54,4 @@ class Template(MakoTemplate):
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
                 
-        return super(Template, self).render(**context_dictionary)
-        
+        return super(Template, self).render_unicode(**context_dictionary)

common/lib/xmodule/xmodule/error_module.py

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
         '''
         try:
             xml = etree.fromstring(self.definition['data']['contents'])
-            return etree.tostring(xml)
+            return etree.tostring(xml, encoding='unicode')
         except etree.XMLSyntaxError:
             # still not valid.
             root = etree.Element('error')
             root.text = self.definition['data']['contents']
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
-            return etree.tostring(root)
+            return etree.tostring(root, encoding='unicode')
 
 
 class NonStaffErrorDescriptor(ErrorDescriptor):

common/lib/xmodule/xmodule/html_module.py

@@ -7,15 +7,14 @@ from lxml import etree
 from lxml.html import rewrite_links
 from path import path
 
-from .x_module import XModule
 from pkg_resources import resource_string
-from .xml_module import XmlDescriptor, name_to_pathname
-from .editing_module import EditingDescriptor
-from .stringify import stringify_children
-from .html_checker import check_html
-from xmodule.modulestore import Location
-
 from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
+from xmodule.editing_module import EditingDescriptor
+from xmodule.html_checker import check_html
+from xmodule.modulestore import Location
+from xmodule.stringify import stringify_children
+from xmodule.x_module import XModule
+from xmodule.xml_module import XmlDescriptor, name_to_pathname
 
 log = logging.getLogger("mitx.courseware")
 
@@ -123,7 +122,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
             try:
                 with system.resources_fs.open(filepath) as file:
-                    html = file.read()
+                    html = file.read().decode('utf-8')
                     # Log a warning if we can't parse the file, but don't error
                     if not check_html(html):
                         msg = "Couldn't parse html in {0}.".format(filepath)
@@ -164,7 +163,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.definition['data'])
+            file.write(self.definition['data'].encode('utf-8'))
 
         # write out the relative name
         relname = path(pathname).basename()

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 make_name_unique(xml_data)
 
                 descriptor = XModuleDescriptor.load_from_xml(
-                    etree.tostring(xml_data), self, self.org,
+                    etree.tostring(xml_data, encoding='unicode'), self, self.org,
                     self.course, xmlstore.default_class)
             except Exception as err:
                 print err, self.load_error_modules
@@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase):
                 self.load_error_modules,
             )
 
-            course_descriptor = system.process_xml(etree.tostring(course_data))
+            course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead

common/lib/xmodule/xmodule/raw_module.py

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     """
     @classmethod
     def definition_from_xml(cls, xml_object, system):
-        return {'data': etree.tostring(xml_object, pretty_print=True)}
+        return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
 
     def definition_to_xml(self, resource_fs):
         try:

common/lib/xmodule/xmodule/seq_module.py

@@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         children = []
         for child in xml_object:
             try:
-                children.append(system.process_xml(etree.tostring(child)).location.url())
+                children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
             except:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 continue

common/lib/xmodule/xmodule/stringify.py

@@ -22,7 +22,7 @@ def stringify_children(node):
     #                 next element.
     parts = [node.text]
     for c in node.getchildren():
-        parts.append(etree.tostring(c, with_tail=True))
+        parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
 
     # filter removes possible Nones in texts and tails
-    return ''.join(filter(None, parts))
+    return u''.join(filter(None, parts))

common/lib/xmodule/xmodule/template_module.py

@@ -58,7 +58,7 @@ class CustomTagDescriptor(RawDescriptor):
         params = dict(xmltree.items())
         with system.resources_fs.open('custom_tags/{name}'
                                    .format(name=template_name)) as template:
-            return Template(template.read()).render(**params)
+            return Template(template.read().decode('utf-8')).render(**params)
 
 
     def __init__(self, system, definition, **kwargs):

common/lib/xmodule/xmodule/xml_module.py

-from xmodule.x_module import (XModuleDescriptor, policy_key)
-from xmodule.modulestore import Location
-from lxml import etree
 import json
 import copy
 import logging
-import traceback
-from collections import namedtuple
-from fs.errors import ResourceNotFoundError
 import os
 import sys
+from collections import namedtuple
+from lxml import etree
+
+from xmodule.x_module import (XModuleDescriptor, policy_key)
+from xmodule.modulestore import Location
 
 log = logging.getLogger(__name__)
 
+# assume all XML files are persisted as utf-8.
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=True, remove_blank_text=True,
+                                 encoding='utf-8')
 
 def name_to_pathname(name):
     """
@@ -366,7 +367,7 @@ class XmlDescriptor(XModuleDescriptor):
             filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
-                file.write(etree.tostring(xml_object, pretty_print=True))
+                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
 
             # And return just a pointer with the category and filename.
             record_object = etree.Element(self.category)
@@ -381,7 +382,7 @@ class XmlDescriptor(XModuleDescriptor):
             record_object.set('org', self.location.org)
             record_object.set('course', self.location.course)
 
-        return etree.tostring(record_object, pretty_print=True)
+        return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
 
     def definition_to_xml(self, resource_fs):
         """

common/test/data/full/about/faq.html

@@ -6,7 +6,7 @@
     <p>No - anyone and everyone is welcome to take this course.</p>
   </li>
   <li>What textbook should I buy?
-    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) -- Volume II, which was written specifically for this course.</p>
+    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) &mdash; Volume II, which was written specifically for this course.</p>
   </li>
   <li>Does Harvard award credentials or reports regarding my work in this course?
     <p>Princeton does not award credentials or issue reports for student work in this course. However, Coursera could maintain a record of your score on the assessments and, with your permission, verify that score for authorized parties.</p>

common/test/data/full/chapter/Overview.xml

@@ -2,7 +2,7 @@
   <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
-    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab… </html>
     <html slug="html_5555" filename="html_5555"/>
     <problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
   </vertical>

common/test/data/full/custom_tags/book

-More information given in <a href="/book/${page}">the text</a>. 
+More information given in… <a href="/book/${page}">the text</a>. 

common/test/data/full/custom_tags/discuss

-<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag} </a>
\ No newline at end of file
+<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag}… </a>
\ No newline at end of file

common/test/data/full/custom_tags/slides

-Lecture Slides Handout [<a href="">Clean </a>][<a href="">Annotated</a>]
+Lecture Slides Handout [<a href="">Clean… </a>][<a href="">Annotated…</a>]

common/test/data/full/html/FirstOrderSystemHint.xml

-Hint
+Hint…
 <br/><br/>
 Remember that the time evolution of any variable \(x(t)\) governed by
 a first-order system with a time-constant \(\tau\) for a time \(t) between an initial

common/test/data/full/html/Week_13_Tutorials.html

@@ -4,14 +4,14 @@
     <section class="tutorials">
       <h2> Basic Tutorials </h2>
       <ul>
-        <li><a href="/section/wk13_solder">Soldering</a> -- Steve
+        <li><a href="/section/wk13_solder">Soldering</a> &mdash; Steve
        Finberg, one of the pioneers in from Draper Lab, talks about
        soldering. </li>
       </ul>
       <h2> Bonus Tutorials </h2>
       <ul>
         <li><a href="/section/wk13_FreqResp">Frequency Response
-       Curves</a> -- We explain several techniques for understanding
+       Curves</a> &mdash; We explain several techniques for understanding
        and approximating Bode plots. </li>
       </ul>
     </section>

common/test/data/full/html/Week_1_Tutorials.html

@@ -41,7 +41,7 @@
         <li><a href="/section/problem_1_3">OCW Problem 1-3 </a> - Reverse engineer a black-box resistor network</li>
       </ul>
       <hr/>
-      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. </p>
+      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. Gratuitous &ge; entity.</p>
     </section>
   </body>
 </html>

common/test/data/full/html/html_5555.html

-    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab. </html>

common/test/data/full/html/linearity_clarify.html

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/linearity_clarify.xml

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/schematic_tutorial.html

@@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram.  Release
 the mouse when the component is in the correct position.
 </td>
 </tr>
-
+<!-- note that entities like &mdash; may be used. -->
 <tr>
 <td>Move a component</td>
 <td>Click to select a component in the diagram (it will turn green)
 and then drag it to its new location.  You can use shift-click to add
 a component to the current selection.  Or you can click somewhere in
 the diagram that is not on top of a component and drag out a selection
-rectangle -- components intersecting the rectangle will be added to
+rectangle &mdash; components intersecting the rectangle will be added to
 the current selection.
 </td>
 </tr>
@@ -63,7 +63,7 @@ engineeering notation:
 <td>Add a wire</td>
 <td>Wires start at connection points, the open circles that
 appear at the terminals of components or the ends of wires.
-Click on a connection point to start a wire -- a green wire
+Click on a connection point to start a wire &mdash; a green wire
 will appear with one end anchored at the starting point.
 Drag the mouse and release the mouse button when the other
 end of the wire is positioned as you wish.  Once a wire has

common/test/data/full/html/units_hint.html

-Hint
+Hint…
 <br/><br/>
 Be careful of units here.  Make sure you notice multipliers such
-as u, k, m, M.
+as u (or &mu;), k, m, M.

common/test/data/full/info/updates.html

@@ -9,8 +9,9 @@
 <li> <h2>May 2 </h2>
 <section class="update-description">
 <ul>
-<li> We have opened the show-answer button on the midterm. </li>
-<li> There was a four hour outage in posting ability on the discussion board Monday night. It has been fixed. We apologise for the inconvenience.</li>
+<!-- utf-8 characters are acceptable… as are HTML entities -->
+<li> We have opened the show-answer button on the midterm… </li>
+<li> There was a four hour outage in posting ability on the discussion board Monday night&hellip; It has been fixed. We apologise for the inconvenience.</li>
 </ul>
 </li>
 <li> <h2>April 30 </h2>

common/test/data/full/problem/Circuit_Sandbox.xml

-<problem><startouttext/><p/>Here's a sandbox where you can experiment with all the components
+<problem><!-- include ellipses to test non-ascii characters --><startouttext/><p/>Here's a sandbox where you can experiment with all the components
 we'll discuss in 6.002x.  If you click on CHECK below, your diagram
-will be saved on the server and you can return at some later time.
+will be saved on the server and you can return at some later time…
 <endouttext/><schematicresponse><p/><center><schematic name="work" value="" width="800" height="600"/></center><answer type="loncapa/python">
 correct = ['correct']
 </answer></schematicresponse></problem>

common/test/data/full/problem/H1P3_Poor_Workmanship.xml

@@ -78,7 +78,8 @@ So the total heating power in Joe's shop was:
 <numericalresponse answer="$Pbad"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>
 <startouttext/>
 <br/>
-No wonder Joe was cold.
+<!-- add non-ascii utf-8 character here -->
+No wonder Joe was cold…
 <endouttext/>
 
 </problem>

common/test/data/full/problem/Lab_0_Using_the_Tools.xml

@@ -94,7 +94,7 @@ scope probes to nodes A, B and C and edit their properties so that the
 plots will be different colors. Now run a transient analysis for 5ms.
 Move the mouse over the plot until the marker (a vertical dashed line
 that follows the mouse when it's over the plot) is at approximately
-1.25ms.  Please report the measured voltages for nodes A, B and C.
+1.25ms.  Please report the measured voltages for nodes A, B and C…
 
 <br/>
 <div style="margin-left: 4em;">

common/test/data/full/problem/Sample_Algebraic_Problem.xml

@@ -6,7 +6,7 @@ z = "A*x^2 + sqrt(y)"
 Enter the algebraic expression \(A x^2 + \sqrt{y}\) in the box below.  The
 entry is case sensitive.  The product must be indicated with an
 asterisk, and the exponentation with a caret, so you must write 
-"A*x^2 + sqrt(y)".
+"A*x^2 + sqrt(y)"…
 <endouttext/>
 <formularesponse type="cs" samples="A,x,y@1,1,1:3,3,3#10" answer="$z"><responseparam description="Numerical Tolerance" type="tolerance" default="0.00001" name="tol"/><textline size="40"/></formularesponse>
 

common/test/data/full/problem/Sample_Numeric_Problem.xml

 <problem><startouttext/>
 Enter the numerical value of the expression \(x + y\) where 
-\(x = 3\) and \(y = 5\).
+\(x = 3\) and \(y = 5\)…
 <endouttext/>
 
 <numericalresponse answer="8"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>

common/test/data/full/problem/choiceresponse_demo.xml

 <problem display_name="S3E2: Lorentz Force">
 
 <startouttext/>
-<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.</p>
+<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…</p>
 <center><img width="400" src="/static/images/LSQimages/LSQ_W01_8.png"/></center>
 
 <p>a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)</p>
 <endouttext/>
 <choiceresponse>
 <checkboxgroup>
-<choice correct="true"><text>Magnetic field strength</text></choice>
-<choice correct="false"><text>Electric field strength</text></choice>
-<choice correct="true"><text>Electric charge of the electron</text></choice>
-<choice correct="false"><text>Radius of the electron</text></choice>
-<choice correct="false"><text>Mass of the electron</text></choice>
-<choice correct="true"><text>Velocity of the electron</text></choice>
+<!-- include ellipses to test non-ascii characters -->
+<choice correct="true"><text>Magnetic field strength…</text></choice>
+<choice correct="false"><text>Electric field strength…</text></choice>
+<choice correct="true"><text>Electric charge of the electron…</text></choice>
+<choice correct="false"><text>Radius of the electron…</text></choice>
+<choice correct="false"><text>Mass of the electron…</text></choice>
+<choice correct="true"><text>Velocity of the electron…</text></choice>
 
 </checkboxgroup>
 </choiceresponse>

common/test/data/full/problem/codeinput_demo.xml

@@ -2,7 +2,8 @@
 <problem display_name="L4 Problem 1">
   <text>
     <p>
-      <b class="bfseries">Part 1: Function Types</b>
+      <!-- include ellipses to test non-ascii characters -->
+      <b class="bfseries">Part 1: Function Types…</b>
     </p>
     <p>
 For each of the following functions, specify the type of its <b class="bfseries">output</b>. You can assume each function is called with an appropriate argument, as specified by its docstring. </p>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -3,12 +3,13 @@
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
     <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
-    <html slug="html_68"> S1E4 has been removed. </html>
+    <!-- utf-8 characters acceptable, but not HTML entities -->
+    <html slug="html_68"> S1E4 has been removed…</html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
-    <html slug="html_95">Minor correction: Six elements (five resistors)</html>
+    <html slug="html_95">Minor correction: Six elements (five resistors)…</html>
     <customtag tag="S1" slug="discuss_96" impl="discuss"/>
   </vertical>
 </sequential>

common/test/data/full/vertical/vertical_89.xml

 <sequential>
   <html slug="html_90">
-    <h1> </h1>
+    <!-- UTF-8 characters are acceptable… HTML entities are not -->
+    <h1>Inline content…</h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
   <customtag tag="S1" slug="discuss_91" impl="discuss"/>

common/test/data/full/video/welcome.xml

-<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>
+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome…"/>

common/test/data/toy/html/secret/toylab.html

 <b>Lab 2A: Superposition Experiment</b>
 
+<<<<<<< Updated upstream
 <p>Isn't the toy course great?</p>
+
+<p>Let's add some markup that uses non-ascii characters.
+For example, we should be able to write words like encyclop&aelig;dia, or foreign words like fran&ccedil;ais.
+Looking beyond latin-1, we should handle math symbols:  &pi;r&sup2 &le; &#8734.
+And it shouldn't matter if we use entities or numeric codes &mdash; &Omega; &ne; &pi; &equiv; &#937; &#8800; &#960;.
+</p>
+=======
+<p>Isn't the toy course great? — &le;</p>
+>>>>>>> Stashed changes

lms/djangoapps/courseware/tests/tests.py

-import copy
+import logging
+log = logging.getLogger("mitx." + __name__)
+
 import json
-import os
-import sys
 import time
 
-from nose import SkipTest
-from path import path
-from pprint import pprint
 from urlparse import urlsplit, urlunsplit
 
 from django.contrib.auth.models import User, Group
-from django.core.handlers.wsgi import WSGIRequest
 from django.test import TestCase
-from django.test.client import Client, RequestFactory
+from django.test.client import RequestFactory
 from django.conf import settings
 from django.core.urlresolvers import reverse
-from mock import patch, Mock
 from override_settings import override_settings
 
 import xmodule.modulestore.django
@@ -26,9 +21,11 @@ from courseware.access import _course_staff_group_name
 from courseware.models import StudentModuleCache
 
 from student.models import Registration
+from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.timeparse import stringify_time
 
 def parse_json(response):
@@ -45,26 +42,6 @@ def registration(email):
     '''look up registration object by email'''
     return Registration.objects.get(user__email=email)
 
-
-# A bit of a hack--want mongo modulestore for these tests, until
-# jump_to works with the xmlmodulestore or we have an even better solution
-# NOTE: this means this test requires mongo to be running.
-
-def mongo_store_config(data_dir):
-    return {
-    'default': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
-        'OPTIONS': {
-            'default_class': 'xmodule.raw_module.RawDescriptor',
-            'host': 'localhost',
-            'db': 'xmodule',
-            'collection': 'modulestore',
-            'fs_root': data_dir,
-            'render_template': 'mitxmako.shortcuts.render_to_string',
-        }
-    }
-}
-
 def xml_store_config(data_dir):
     return {
     'default': {
@@ -76,14 +53,9 @@ def xml_store_config(data_dir):
     }
 }
 
-
 TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
-TEST_DATA_MONGO_MODULESTORE = mongo_store_config(TEST_DATA_DIR)
 TEST_DATA_XML_MODULESTORE = xml_store_config(TEST_DATA_DIR)
 
-REAL_DATA_DIR = settings.GITHUB_REPO_ROOT
-REAL_DATA_MODULESTORE = mongo_store_config(REAL_DATA_DIR)
-
 class ActivateLoginTestCase(TestCase):
     '''Check that we can activate and log in'''
 
@@ -233,10 +205,17 @@ class PageLoader(ActivateLoginTestCase):
     def check_pages_load(self, course_name, data_dir, modstore):
         """Make all locations in course load"""
         print "Checking course {0} in {1}".format(course_name, data_dir)
-        import_from_xml(modstore, data_dir, [course_name])
-
-        # enroll in the course before trying to access pages
-        courses = modstore.get_courses()
+        default_class='xmodule.hidden_module.HiddenDescriptor'
+        load_error_modules=True
+        module_store = XMLModuleStore(
+                                      data_dir,
+                                      default_class=default_class,
+                                      course_dirs=[course_name],
+                                      load_error_modules=load_error_modules,
+                                      )
+
+       # enroll in the course before trying to access pages
+        courses = module_store.get_courses()
         self.assertEqual(len(courses), 1)
         course = courses[0]
         self.enroll(course)
@@ -245,36 +224,55 @@ class PageLoader(ActivateLoginTestCase):
         n = 0
         num_bad = 0
         all_ok = True
-        for descriptor in modstore.get_items(
-                Location(None, None, None, None, None)):
+        for descriptor in module_store.modules[course_id].itervalues(): 
             n += 1
             print "Checking ", descriptor.location.url()
             #print descriptor.__class__, descriptor.location
             resp = self.client.get(reverse('jump_to',
                                    kwargs={'course_id': course_id,
-                                           'location': descriptor.location.url()}))
+                                           'location': descriptor.location.url()}), follow=True)
+            # check status codes first
             msg = str(resp.status_code)
+            if resp.status_code != 200:
+                msg = "ERROR " + msg  + ": " + descriptor.location.url()
+                all_ok = False
+                num_bad += 1
+            elif resp.redirect_chain[0][1] != 302:
+                msg = "ERROR on redirect from " + descriptor.location.url()
+                all_ok = False
+                num_bad += 1
 
-            if resp.status_code != 302:
-                msg = "ERROR " + msg
+            # check content to make sure there were no rendering failures
+            content = resp.content
+            if content.find("this module is temporarily unavailable")>=0:
+                msg = "ERROR unavailable module " 
+                all_ok = False
+                num_bad += 1
+            elif isinstance(descriptor, ErrorDescriptor):
+                msg = "ERROR error descriptor loaded: " 
+                msg = msg + descriptor.definition['data']['error_msg']
                 all_ok = False
                 num_bad += 1
             print msg
             self.assertTrue(all_ok)  # fail fast
 
         print "{0}/{1} good".format(n - num_bad, n)
+        log.info( "{0}/{1} good".format(n - num_bad, n))
         self.assertTrue(all_ok)
 
 
-@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+
+@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
 class TestCoursesLoadTestCase(PageLoader):
     '''Check that all pages in test courses load properly'''
 
     def setUp(self):
         ActivateLoginTestCase.setUp(self)
         xmodule.modulestore.django._MODULESTORES = {}
-        xmodule.modulestore.django.modulestore().collection.drop()
-
+#        xmodule.modulestore.django.modulestore().collection.drop()
+#        store = xmodule.modulestore.django.modulestore()
+        # is there a way to empty the store?
+        
     def test_toy_course_loads(self):
         self.check_pages_load('toy', TEST_DATA_DIR, modulestore())
 
@@ -615,35 +613,6 @@ class TestViewAuth(PageLoader):
         self.unenroll(self.toy)
         self.assertTrue(self.try_enroll(self.toy))
 
-
-@override_settings(MODULESTORE=REAL_DATA_MODULESTORE)
-class RealCoursesLoadTestCase(PageLoader):
-    '''Check that all pages in real courses load properly'''
-
-    def setUp(self):
-        ActivateLoginTestCase.setUp(self)
-        xmodule.modulestore.django._MODULESTORES = {}
-        xmodule.modulestore.django.modulestore().collection.drop()
-
-    def test_real_courses_loads(self):
-        '''See if any real courses are available at the REAL_DATA_DIR.
-        If they are, check them.'''
-
-        # TODO: Disabled test for now..  Fix once things are cleaned up.
-        raise SkipTest
-        # TODO: adjust staticfiles_dirs
-        if not os.path.isdir(REAL_DATA_DIR):
-            # No data present.  Just pass.
-            return
-
-        courses = [course_dir for course_dir in os.listdir(REAL_DATA_DIR)
-                   if os.path.isdir(REAL_DATA_DIR / course_dir)]
-        for course in courses:
-            self.check_pages_load(course, REAL_DATA_DIR, modulestore())
-
-
-    # ========= TODO: check ajax interaction here too?
-
 @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)
 class TestCourseGrader(PageLoader):
     """Check that a course gets graded properly"""

lms/djangoapps/courseware/views.py

@@ -293,7 +293,6 @@ def index(request, course_id, chapter=None, section=None,
 
     return result
 
-
 @ensure_csrf_cookie
 def jump_to(request, course_id, location):
     '''
@@ -318,12 +317,18 @@ def jump_to(request, course_id, location):
     except NoPathToItem:
         raise Http404("This location is not in any class: {0}".format(location))
 
+    # choose the appropriate view (and provide the necessary args) based on the 
+    # args provided by the redirect.
     # Rely on index to do all error handling and access control.
-    return redirect('courseware_position',
-                    course_id=course_id,
-                    chapter=chapter,
-                    section=section,
-                    position=position)
+    if chapter is None:
+        return redirect('courseware', course_id=course_id)
+    elif section is None:
+        return redirect('courseware_chapter', course_id=course_id, chapter=chapter)
+    elif position is None:
+        return redirect('courseware_section', course_id=course_id, chapter=chapter, section=section)
+    else:
+        return redirect('courseware_position', course_id=course_id, chapter=chapter, section=section, position=position)
+        
 @ensure_csrf_cookie
 def course_info(request, course_id):
     """