Add more non-ascii characters to full dataset, and make fixes in xmodule code.

common/djangoapps/mitxmako/shortcuts.py

@@ -42,7 +42,12 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
         context_dictionary.update(context)
     # fetch and render template
     template = middleware.lookup[namespace].get_template(template_name)
-    return template.render_unicode(**context_dictionary)
+#    return template.render_unicode(**context_dictionary)
+
+    output = template.render_unicode(**context_dictionary)
+#    log.info('  render_to_string of "{0}" as "{1}r"'.format(type(output), output))
+    return output
+#    return template.render(**context_dictionary)
 
 
 def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs):

common/djangoapps/mitxmako/template.py

@@ -12,6 +12,9 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import logging
+log = logging.getLogger("mitx." + __name__)
+
 from django.conf import settings
 from mako.template import Template as MakoTemplate
 
@@ -54,5 +57,9 @@ class Template(MakoTemplate):
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
                 
-        return super(Template, self).render_unicode(**context_dictionary)
+#        return super(Template, self).render_unicode(**context_dictionary)
+#        return super(Template, self).render(**context_dictionary)
  
+        output = super(Template, self).render(**context_dictionary)
+        log.info('  render_to_string of "{0}" as "{1}"'.format(type(output), output))       
+        return output

common/lib/xmodule/xmodule/html_module.py

@@ -30,8 +30,10 @@ class HtmlModule(XModule):
     
     def get_html(self):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
+        input = self.html
         output = rewrite_links(self.html, self.rewrite_content_links)
-        log.info('  HTMLModule converting markup "{0}" to "{1}"'.format(self.html, output))
+#        log.info('  HTMLModule converting markup from "{0}" as  "{1}r"'.format(type(input), input))
+#        log.info('  HTMLModule converting markup to "{0}" as "{1}r"'.format(type(output), output))
         return output
 
     def __init__(self, system, location, definition, descriptor,
@@ -166,7 +168,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.definition['data'].encode('utf-8'))
+            file.write(self.definition['data']) # .encode('utf-8'))
 
         # write out the relative name
         relname = path(pathname).basename()

common/test/data/full/custom_tags/book

-More information given in <a href="/book/${page}">the text</a>. 
+More information given in… <a href="/book/${page}">the text</a>. 

common/test/data/full/custom_tags/discuss

-<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag} </a>
\ No newline at end of file
+<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag}… </a>
\ No newline at end of file

common/test/data/full/custom_tags/slides

-Lecture Slides Handout [<a href="">Clean </a>][<a href="">Annotated</a>]
+Lecture Slides Handout [<a href="">Clean… </a>][<a href="">Annotated…</a>]

common/test/data/full/problem/Circuit_Sandbox.xml

-<problem><startouttext/><p/>Here's a sandbox where you can experiment with all the components
+<problem><!-- include ellipses to test non-ascii characters --><startouttext/><p/>Here's a sandbox where you can experiment with all the components
 we'll discuss in 6.002x.  If you click on CHECK below, your diagram
-will be saved on the server and you can return at some later time.
+will be saved on the server and you can return at some later time…
 <endouttext/><schematicresponse><p/><center><schematic name="work" value="" width="800" height="600"/></center><answer type="loncapa/python">
 correct = ['correct']
 </answer></schematicresponse></problem>

common/test/data/full/problem/choiceresponse_demo.xml

 <problem display_name="S3E2: Lorentz Force">
 
 <startouttext/>
-<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.</p>
+<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…</p>
 <center><img width="400" src="/static/images/LSQimages/LSQ_W01_8.png"/></center>
 
 <p>a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)</p>
 <endouttext/>
 <choiceresponse>
 <checkboxgroup>
-<choice correct="true"><text>Magnetic field strength</text></choice>
-<choice correct="false"><text>Electric field strength</text></choice>
-<choice correct="true"><text>Electric charge of the electron</text></choice>
-<choice correct="false"><text>Radius of the electron</text></choice>
-<choice correct="false"><text>Mass of the electron</text></choice>
-<choice correct="true"><text>Velocity of the electron</text></choice>
+<!-- include ellipses to test non-ascii characters -->
+<choice correct="true"><text>Magnetic field strength…</text></choice>
+<choice correct="false"><text>Electric field strength…</text></choice>
+<choice correct="true"><text>Electric charge of the electron…</text></choice>
+<choice correct="false"><text>Radius of the electron…</text></choice>
+<choice correct="false"><text>Mass of the electron…</text></choice>
+<choice correct="true"><text>Velocity of the electron…</text></choice>
 
 </checkboxgroup>
 </choiceresponse>

common/test/data/full/problem/codeinput_demo.xml

@@ -2,7 +2,8 @@
 <problem display_name="L4 Problem 1">
   <text>
     <p>
-      <b class="bfseries">Part 1: Function Types</b>
+      <!-- include ellipses to test non-ascii characters -->
+      <b class="bfseries">Part 1: Function Types…</b>
     </p>
     <p>
 For each of the following functions, specify the type of its <b class="bfseries">output</b>. You can assume each function is called with an appropriate argument, as specified by its docstring. </p>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -3,12 +3,12 @@
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
     <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
-    <html slug="html_68"> S1E4 has been removed. </html>
+    <html slug="html_68"> S1E4 has been removed… </html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
-    <html slug="html_95">Minor correction: Six elements (five resistors)</html>
+    <html slug="html_95">Minor correction: Six elements (five resistors)…</html>
     <customtag tag="S1" slug="discuss_96" impl="discuss"/>
   </vertical>
 </sequential>

common/test/data/full/vertical/vertical_89.xml

 <sequential>
   <html slug="html_90">
-    <h1> </h1>
+    <h1>Inline content…</h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
   <customtag tag="S1" slug="discuss_91" impl="discuss"/>

common/test/data/full/video/welcome.xml

-<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>
+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome…"/>

lms/djangoapps/courseware/tests/tests.py

+import logging
+log = logging.getLogger("mitx." + __name__)
+
 import json
 import os
 import time
@@ -20,6 +23,7 @@ from courseware.access import _course_staff_group_name
 from courseware.models import StudentModuleCache
 
 from student.models import Registration
+from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import Location
 from xmodule.modulestore.xml_importer import import_from_xml
@@ -279,6 +283,7 @@ class PageLoader(ActivateLoginTestCase):
         print "Checking course {0} in {1}".format(course_name, data_dir)
         default_class='xmodule.hidden_module.HiddenDescriptor'  # 'xmodule.raw_module.RawDescriptor',
         load_error_modules=True
+#        load_error_modules=False
         module_store = XMLModuleStore(
                                       data_dir,
                                       default_class=default_class,
@@ -308,24 +313,40 @@ class PageLoader(ActivateLoginTestCase):
         for descriptor in module_store.modules[course_id].itervalues(): 
             n += 1
             print "Checking ", descriptor.location.url()
+            log.info('Checking the content returned for page %s', descriptor.location.url())
             #print descriptor.__class__, descriptor.location
             resp = self.client.get(reverse('jump_to',
                                    kwargs={'course_id': course_id,
                                            'location': descriptor.location.url()}), follow=True)
             msg = str(resp.status_code)
             if resp.status_code != 200:
-                msg = "ERROR " + msg # + ": " + str(resp.request['PATH_INFO'])
+                msg = "ERROR " + msg  + ": " + descriptor.location.url()
                 all_ok = False
                 num_bad += 1
             elif resp.redirect_chain[0][1] != 302:
-                msg = "ERROR " + msg
+                msg = "ERROR on redirect from " + descriptor.location.url()
+                all_ok = False
+                num_bad += 1
+            content = resp.content
+#            contentlines = content.splitlines()
+            if content.find("this module is temporarily unavailable")>=0:
+                msg = "ERROR unavailable module " 
+                all_ok = False
+                num_bad += 1
+            elif isinstance(descriptor, ErrorDescriptor):
+                msg = "ERROR error descriptor loaded: " 
+                msg = msg + descriptor.definition['data']['error_msg']
                 all_ok = False
                 num_bad += 1
+            log.info('Output the content returned for page %s', descriptor.location.url())
+            log.info('Content returned: %s', content)
             print msg
 #            self.assertTrue(all_ok)  # fail fast
 
         print "{0}/{1} good".format(n - num_bad, n)
-        self.assertTrue(all_ok)
+        log.info( "{0}/{1} good".format(n - num_bad, n))
+#        self.assertTrue(all_ok)
+        self.assertTrue(false)
 
 
 #@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
@@ -343,8 +364,8 @@ class TestCoursesLoadTestCase(PageLoader):
     def test_toy_course_loads(self):
         self.check_xml_pages_load('toy', TEST_DATA_DIR, modulestore())
 
-#    def test_full_course_loads(self):
-#        self.check_pages_load('full', TEST_DATA_DIR, modulestore())
+    def test_full_course_loads(self):
+        self.check_xml_pages_load('full', TEST_DATA_DIR, modulestore())
 
 
 @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE)