Merge pull request #7 from mitocw/mitx-hotfix-rc-20140919

Mitx hotfix rc 20140919

cms/startup.py

 """
 Module with code executed during Studio startup
 """
+
 from django.conf import settings
 
 # Force settings to run so that the python path is modified
@@ -13,6 +14,11 @@ def run():
     """
     Executed during django startup
     """
+
+    # Patch the xml libs.
+    from safe_lxml import defuse_xml_libs
+    defuse_xml_libs()
+
     autostartup()
 
     add_mimetypes()

common/djangoapps/edxmako/shortcuts.py

@@ -92,7 +92,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
     context_instance['marketing_link'] = marketing_link
 
     # In various testing contexts, there might not be a current request context.
-    if edxmako.middleware.REQUEST_CONTEXT.context is not None:
+    if getattr(edxmako.middleware.REQUEST_CONTEXT, "context", None):
         for d in edxmako.middleware.REQUEST_CONTEXT.context:
             context_dictionary.update(d)
     for d in context_instance:

common/djangoapps/edxmako/template.py

@@ -48,7 +48,7 @@ class Template(MakoTemplate):
         context_dictionary = {}
 
         # In various testing contexts, there might not be a current request context.
-        if edxmako.middleware.REQUEST_CONTEXT.context is not None:
+        if getattr(edxmako.middleware.REQUEST_CONTEXT, "context", None):
             for d in edxmako.middleware.REQUEST_CONTEXT.context:
                 context_dictionary.update(d)
         for d in context_instance:

common/djangoapps/edxmako/tests.py

 
 from mock import patch, Mock
+import unittest
+
+from django.conf import settings
 from django.http import HttpResponse
 from django.test import TestCase
 from django.test.utils import override_settings
@@ -7,7 +10,7 @@ from django.test.client import RequestFactory
 from django.core.urlresolvers import reverse
 import edxmako.middleware
 from edxmako import add_lookup, LOOKUP
-from edxmako.shortcuts import marketing_link
+from edxmako.shortcuts import marketing_link, render_to_string
 from student.tests.factories import UserFactory
 from util.testing import UrlResetMixin
 
@@ -71,6 +74,26 @@ class MakoMiddlewareTest(TestCase):
         # requestcontext should be None.
         self.assertIsNone(edxmako.middleware.REQUEST_CONTEXT.context)
 
+    @unittest.skipUnless(settings.ROOT_URLCONF == 'lms.urls', 'Test only valid in lms')
+    @patch("edxmako.middleware.REQUEST_CONTEXT")
+    def test_render_to_string_when_no_global_context_lms(self, context_mock):
+        """
+        Test render_to_string() when makomiddleware has not initialized
+        the threadlocal REQUEST_CONTEXT.context. This is meant to run in LMS.
+        """
+        del context_mock.context
+        self.assertIn("this module is temporarily unavailable", render_to_string("courseware/error-message.html", None))
+
+    @unittest.skipUnless(settings.ROOT_URLCONF == 'cms.urls', 'Test only valid in cms')
+    @patch("edxmako.middleware.REQUEST_CONTEXT")
+    def test_render_to_string_when_no_global_context_cms(self, context_mock):
+        """
+        Test render_to_string() when makomiddleware has not initialized
+        the threadlocal REQUEST_CONTEXT.context. This is meant to run in CMS.
+        """
+        del context_mock.context
+        self.assertIn("We're having trouble rendering your component", render_to_string("html_error.html", None))
+
 
 def mako_middleware_process_request(request):
     """

common/lib/safe_lxml/safe_lxml/__init__.py

+"""
+Defuse vulnerabilities in XML packages.
+"""
+
+
+def defuse_xml_libs():
+    """
+    Monkey patch and defuse all stdlib xml packages and lxml.
+    """
+    from defusedxml import defuse_stdlib
+    defuse_stdlib()
+
+    import lxml
+    import lxml.etree
+    from . import etree as safe_etree
+
+    lxml.etree = safe_etree

common/lib/safe_lxml/safe_lxml/etree.py

+"""
+Safer version of lxml.etree.
+
+It overrides some unsafe functions from lxml.etree with safer versions from defusedxml.
+It also includes a safer XMLParser.
+
+For processing xml always prefer this over using lxml.etree directly.
+"""
+
+from lxml.etree import *  # pylint: disable=wildcard-import, unused-wildcard-import
+from lxml.etree import XMLParser as _XMLParser
+
+# This should be imported after lxml.etree so that it overrides the following attributes.
+from defusedxml.lxml import parse, fromstring, XML
+
+
+class XMLParser(_XMLParser):  # pylint: disable=function-redefined
+    """
+    A safer version of XMLParser which by default disables entity resolution.
+    """
+
+    def __init__(self, *args, **kwargs):
+        if "resolve_entities" not in kwargs:
+            kwargs["resolve_entities"] = False
+        super(XMLParser, self).__init__(*args, **kwargs)

common/lib/safe_lxml/setup.py

+"""
+Setup.py for safe_lxml.
+"""
+
+from setuptools import setup
+
+setup(
+    name="safe_lxml",
+    version="1.0",
+    packages=["safe_lxml"],
+    install_requires=[
+        "lxml",
+        "defusedxml"
+    ],
+)

common/static/coffee/src/discussion/views/discussion_thread_view_inline.coffee

@@ -45,7 +45,7 @@ if Backbone?
 
     expandPost: (event) =>
       @$el.addClass('expanded')
-      @$el.find('.post-body').html(@model.get('body'))
+      @$el.find('.post-body').text(@model.get('body'))
       @showView.convertMath()
       @$el.find('.expand-post').css('display', 'none')
       @$el.find('.collapse-post').css('display', 'block')
@@ -60,7 +60,7 @@ if Backbone?
       if postTop < curScroll
         $('html, body').animate({scrollTop: postTop})
       @$el.removeClass('expanded')
-      @$el.find('.post-body').html(@model.get('abbreviatedBody'))
+      @$el.find('.post-body').text(@model.get('abbreviatedBody'))
       @showView.convertMath()
       @$el.find('.expand-post').css('display', 'block')
       @$el.find('.collapse-post').css('display', 'none')

lms/djangoapps/dashboard/tests/test_sysadmin.py

@@ -552,6 +552,33 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
                 'course_id': 'Not/Real/Testing'}))
         self.assertEqual(404, response.status_code)
 
+    def test_gitlog_no_logs(self):
+        """
+        Make sure the template behaves well when rendered despite there not being any logs.
+        (This is for courses imported using methods other than the git_add_course command)
+        """
+
+        self._setstaff_login()
+        self._mkdir(getattr(settings, 'GIT_REPO_DIR'))
+
+        self._add_edx4edx()
+
+        # Simulate a lack of git import logs
+        import_logs = CourseImportLog.objects.all()
+        import_logs.delete()
+
+        response = self.client.get(
+            reverse('gitlogs_detail', kwargs={
+                'course_id': 'MITx/edx4edx/edx4edx'
+            })
+        )
+        self.assertIn(
+            'No git import logs have been recorded for this course.',
+            response.content
+        )
+
+        self._rm_edx4edx()
+
     def test_gitlog_courseteam_access(self):
         """
         Ensure course team users are allowed to access only their own course.

lms/djangoapps/psychometrics/management/commands/init_psychometrics.py

@@ -52,9 +52,9 @@ class Command(BaseCommand):
 
                 # get attempt times from tracking log
                 uname = sm.student.username
-                tset = TrackingLog.objects.using(db).filter(username=uname, event_type__contains='save_problem_check')
+                tset = TrackingLog.objects.using(db).filter(username=uname, event_type__contains='problem_check')
                 tset = tset.filter(event_source='server')
-                tset = tset.filter(event__contains="'%s'" % url)
+                tset = tset.filter(event__contains="'%s'" % usage_key)
                 checktimes = [x.dtcreated for x in tset]
                 pmd.checktimes = checktimes
                 if not len(checktimes) == pmd.attempts:

lms/djangoapps/psychometrics/psychoanalyze.py

@@ -10,6 +10,7 @@ import logging
 import json
 import math
 import numpy as np
+from opaque_keys.edx.locator import BlockUsageLocator
 from scipy.optimize import curve_fit
 
 from django.conf import settings
@@ -129,7 +130,14 @@ def problems_with_psychometric_data(course_id):
     '''
     pmdset = PsychometricData.objects.using(db).filter(studentmodule__course_id=course_id)
     plist = [p['studentmodule__module_state_key'] for p in pmdset.values('studentmodule__module_state_key').distinct()]
-    problems = dict((p, pmdset.filter(studentmodule__module_state_key=p).count()) for p in plist)
+    problems = dict(
+        (
+            p,
+            pmdset.filter(
+                studentmodule__module_state_key=BlockUsageLocator.from_string(p)
+            ).count()
+        ) for p in plist
+    )
 
     return problems
 
@@ -138,7 +146,9 @@ def problems_with_psychometric_data(course_id):
 
 def generate_plots_for_problem(problem):
 
-    pmdset = PsychometricData.objects.using(db).filter(studentmodule__module_state_key=problem)
+    pmdset = PsychometricData.objects.using(db).filter(
+        studentmodule__module_state_key=BlockUsageLocator.from_string(problem)
+    )
     nstudents = pmdset.count()
     msg = ""
     plots = []

lms/startup.py

@@ -19,6 +19,11 @@ def run():
     """
     Executed during django startup
     """
+
+    # Patch the xml libs.
+    from safe_lxml import defuse_xml_libs
+    defuse_xml_libs()
+
     autostartup()
 
     add_mimetypes()

lms/templates/sysadmin_dashboard_gitlogs.html

@@ -97,6 +97,8 @@ textarea {
               logs = cilset[:10]
           else:
               logs = cilset[:2]
+
+          cil = None
           %>
           % for cil in logs:
               <%
@@ -116,7 +118,14 @@ textarea {
           %if course_id is not None:
               <tr>
                 <td colspan="3">
-                  <pre>${cil.import_log | h}</pre>
+                  <pre>
+                    %if cil is not None:
+                        ${cil.import_log | h}
+                    %else:
+                        ## Translators: git is a version-control system; see http://git-scm.com/about
+                        ${_('No git import logs have been recorded for this course.')}
+                    %endif
+                  </pre>
                 </td>
               </tr>
           %endif

requirements/edx/base.txt

@@ -11,6 +11,7 @@ html5lib==0.999
 boto==2.13.3
 celery==3.0.19
 dealer==0.2.3
+defusedxml==0.4.1
 distribute>=0.6.28, <0.7
 django-babel-underscore==0.1.0
 django-celery==3.0.17

requirements/edx/local.txt

@@ -3,6 +3,7 @@
 -e common/lib/calc
 -e common/lib/capa
 -e common/lib/chem
+-e common/lib/safe_lxml
 -e common/lib/sandbox-packages
 -e common/lib/symmath
 -e common/lib/xmodule