Merge pull request #880 from edx/db/update-requests-lib

Update `requests` library to 1.2.3

common/djangoapps/terrain/browser.py

@@ -12,7 +12,7 @@ from django.core.management import call_command
 from django.conf import settings
 from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-from requests import put
+import requests
 from base64 import encodestring
 from json import dumps
 
@@ -54,12 +54,12 @@ def set_job_status(jobid, passed=True):
     """
     Sets the job status on sauce labs
     """
-    body_content = dumps({"passed": passed})
     config = get_username_and_key()
+    url = 'http://saucelabs.com/rest/v1/{}/jobs/{}'.format(config['username'], world.jobid)
+    body_content = dumps({"passed": passed})
     base64string = encodestring('{}:{}'.format(config['username'], config['access-key']))[:-1]
-    result = put('http://saucelabs.com/rest/v1/{}/jobs/{}'.format(config['username'], world.jobid),
-        data=body_content,
-        headers={"Authorization": "Basic {}".format(base64string)})
+    headers = {"Authorization": "Basic {}".format(base64string)}
+    result = requests.put(url, data=body_content, headers=headers)
     return result.status_code == 200
 
 

common/lib/capa/capa/tests/__init__.py

@@ -36,6 +36,7 @@ def test_system():
         user=Mock(),
         filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")),
         debug=True,
+        hostname="edx.org",
         xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10},
         node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
         anonymous_student_id='student',

common/lib/capa/capa/xqueue_interface.py

@@ -64,7 +64,8 @@ class XQueueInterface(object):
     def __init__(self, url, django_auth, requests_auth=None):
         self.url = url
         self.auth = django_auth
-        self.session = requests.session(auth=requests_auth)
+        self.session = requests.Session()
+        self.session.auth = requests_auth
 
     def send_to_queue(self, header, body, files_to_upload=None):
         """

common/lib/sample-post.py

@@ -39,7 +39,7 @@ username = prompt('username on server', 'victor@edx.org')
 password = prompt('password', 'abc123', safe=True)
 
 print "get csrf cookie"
-session = requests.session()
+session = requests.Session()
 r = session.get(server + '/')
 r.raise_for_status()
 

common/lib/xmodule/xmodule/js/src/lti/lti.js

@@ -16,9 +16,9 @@ window.LTI = (function () {
 
         // If the Form's action attribute is set (i.e. we can perform a normal
         // submit), then we submit the form and make the frame shown.
-        if (form.attr('action')) {
+        if (form.attr('action') && form.attr('action') !== 'http://www.example.com') {
             form.submit();
-            element.find('.lti').addClass('rendered')
+            element.find('.lti').addClass('rendered');
         }
     }
 

common/lib/xmodule/xmodule/lti_module.py

 """
 Module that allows to insert LTI tools to page.
 
-Module uses current edx-platform 0.14.2 version of requests (oauth part).
-Please update code when upgrading requests.
-
 Protocol is oauth1, LTI version is 1.1.1:
 http://www.imsglobal.org/LTI/v1p1p1/ltiIMGv1p1p1.html
 """
 
 import logging
-import requests
+import oauthlib.oauth1
 import urllib
 
 from xmodule.editing_module import MetadataOnlyEditingDescriptor
@@ -41,9 +38,12 @@ class LTIFields(object):
         vbid=put_book_id_here
         book_location=page/put_page_number_here
 
+    Default non-empty url for `launch_url` is needed due to oauthlib demand (url scheme should be presented)::
+
+    https://github.com/idan/oauthlib/blob/master/oauthlib/oauth1/rfc5849/signature.py#L136
     """
     lti_id = String(help="Id of the tool", default='', scope=Scope.settings)
-    launch_url = String(help="URL of the tool", default='', scope=Scope.settings)
+    launch_url = String(help="URL of the tool", default='http://www.example.com', scope=Scope.settings)
     custom_parameters = List(help="Custom parameters (vbid, book_location, etc..)", scope=Scope.settings)
 
 
@@ -192,7 +192,7 @@ class LTIModule(LTIFields, XModule):
         Also *anonymous student id* is passed to template and therefore to LTI provider.
         """
 
-        client = requests.auth.Client(
+        client = oauthlib.oauth1.Client(
             client_key=unicode(client_key),
             client_secret=unicode(client_secret)
         )
@@ -215,14 +215,30 @@ class LTIModule(LTIFields, XModule):
         # appending custom parameter for signing
         body.update(custom_parameters)
 
-        # This is needed for body encoding:
-        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+        headers = {
+            # This is needed for body encoding:
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+
+        try:
+            __, headers, __ = client.sign(
+                unicode(self.launch_url),
+                http_method=u'POST',
+                body=body,
+                headers=headers)
+        except ValueError:  # scheme not in url
+            # Stubbing headers for now:
+            headers = {
+                u'Content-Type': u'application/x-www-form-urlencoded',
+                u'Authorization': u'oAuth '  # cont..
+                    u'oauth_nonce="80966668944732164491378916897", '  # cont..
+                    u'oauth_timestamp="1378916897", '        # cont..
+                    u'oauth_version="1.0", '                 # cont..
+                    u'oauth_signature_method="HMAC-SHA1", '  # cont..
+                    u'oauth_consumer_key="", '               # cont..
+                    u'oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"',
+            }
 
-        __, headers, __ = client.sign(
-            unicode(self.launch_url),
-            http_method=u'POST',
-            body=body,
-            headers=headers)
         params = headers['Authorization']
         # parse headers to pass to template as part of context:
         params = dict([param.strip().replace('"', '').split('=') for param in params.split(',')])
@@ -230,8 +246,8 @@ class LTIModule(LTIFields, XModule):
         params[u'oauth_nonce'] = params[u'OAuth oauth_nonce']
         del params[u'OAuth oauth_nonce']
 
-        # 0.14.2 (current) version of requests oauth library encodes signature,
-        # with 'Content-Type': 'application/x-www-form-urlencoded'
+        # oauthlib encodes signature with
+        # 'Content-Type': 'application/x-www-form-urlencoded'
         # so '='' becomes '%3D'.
         # We send form via browser, so browser will encode it again,
         # So we need to decode signature back:

common/lib/xmodule/xmodule/open_ended_grading_classes/grading_service_module.py

@@ -25,7 +25,7 @@ class GradingService(object):
     def __init__(self, config):
         self.username = config['username']
         self.password = config['password']
-        self.session = requests.session()
+        self.session = requests.Session()
         self.system = config['system']
 
     def _login(self):
@@ -42,7 +42,7 @@ class GradingService(object):
 
         response.raise_for_status()
 
-        return response.json
+        return response.json()
 
     def post(self, url, data, allow_redirects=False):
         """
@@ -88,9 +88,10 @@ class GradingService(object):
         Returns the result of operation().  Does not catch exceptions.
         """
         response = operation()
-        if (response.json
-            and response.json.get('success') is False
-            and response.json.get('error') == 'login_required'):
+        resp_json = response.json()
+        if (resp_json
+                and resp_json.get('success') is False
+                and resp_json.get('error') == 'login_required'):
             # apparrently we aren't logged in.  Try to fix that.
             r = self._login()
             if r and not r.get('success'):

common/lib/xmodule/xmodule/tests/__init__.py

@@ -62,6 +62,7 @@ def get_test_system(course_id=''):
         user=Mock(is_staff=False),
         filestore=Mock(),
         debug=True,
+        hostname="edx.org",
         xqueue={'interface': None, 'callback_url': '/', 'default_queuename': 'testqueue', 'waittime': 10, 'construct_callback' : Mock(side_effect="/")},
         node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
         xblock_field_data=lambda descriptor: descriptor._field_data,

common/lib/xmodule/xmodule/x_module.py

@@ -833,7 +833,7 @@ class ModuleSystem(Runtime):
     def __init__(
             self, ajax_url, track_function, get_module, render_template,
             replace_urls, xblock_field_data, user=None, filestore=None,
-            debug=False, xqueue=None, publish=None, node_path="",
+            debug=False, hostname="", xqueue=None, publish=None, node_path="",
             anonymous_student_id='', course_id=None,
             open_ended_grading_interface=None, s3_interface=None,
             cache=None, can_execute_unsafe_code=None, replace_course_urls=None,
@@ -897,6 +897,7 @@ class ModuleSystem(Runtime):
         self.get_module = get_module
         self.render_template = render_template
         self.DEBUG = self.debug = debug
+        self.HOSTNAME = self.hostname = hostname
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path

docs/shared/requirements.txt

@@ -36,6 +36,7 @@ mako==0.7.3
 Markdown==2.2.1
 networkx==1.7
 nltk==2.0.4
+oauthlib==0.5.1
 paramiko==1.9.0
 path.py==3.0.1
 Pillow==1.7.8
@@ -48,7 +49,7 @@ python-memcached==1.48
 python-openid==2.2.5
 pytz==2012h
 PyYAML==3.10
-requests==0.14.2
+requests==1.2.3
 Shapely==1.2.16
 sorl-thumbnail==11.12
 South==0.7.6

lms/djangoapps/courseware/mock_lti_server/mock_lti_server.py

 from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
 import urlparse
-from requests.packages.oauthlib.oauth1.rfc5849 import signature
+from oauthlib.oauth1.rfc5849 import signature
 import mock
 from logging import getLogger
 logger = getLogger(__name__)

lms/djangoapps/courseware/mock_youtube_server/mock_youtube_server.py

 from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
 import urlparse
-from requests.packages.oauthlib.oauth1.rfc5849 import signature
 import mock
 import threading
 import json

lms/djangoapps/courseware/module_render.py

@@ -347,6 +347,8 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
         filestore=descriptor.system.resources_fs,
         get_module=inner_get_module,
         user=user,
+        debug=settings.DEBUG,
+        hostname=settings.SITE_NAME,
         # TODO (cpennington): This should be removed when all html from
         # a module is coming through get_html and is therefore covered
         # by the replace_static_urls code below
@@ -380,7 +382,6 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
 
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
-    system.set('DEBUG', settings.DEBUG)
     if settings.MITX_FEATURES.get('ENABLE_PSYCHOMETRICS'):
         system.set(
             'psychometrics_handler',  # set callback for updating PsychometricsData

lms/djangoapps/courseware/tests/test_lti.py

 """LTI integration tests"""
 
-import requests
+import oauthlib
 from . import BaseTestXmodule
 from collections import OrderedDict
 import mock
@@ -11,7 +11,8 @@ class TestLTI(BaseTestXmodule):
     Integration test for lti xmodule.
 
     It checks overall code, by assuring that context that goes to template is correct.
-    As part of that, checks oauth signature generation by mocking signing function of `requests` library.
+    As part of that, checks oauth signature generation by mocking signing function
+    of `oauthlib` library.
     """
     CATEGORY = "lti"
 
@@ -43,7 +44,7 @@ class TestLTI(BaseTestXmodule):
             u'oauth_signature': mocked_decoded_signature
         }
 
-        saved_sign = requests.auth.Client.sign
+        saved_sign = oauthlib.oauth1.Client.sign
 
         def mocked_sign(self, *args, **kwargs):
             """
@@ -60,7 +61,7 @@ class TestLTI(BaseTestXmodule):
             headers[u'Authorization'] = ', '.join([k+'="'+v+'"' for k, v in old_parsed.items()])
             return None, headers, None
 
-        patcher = mock.patch.object(requests.auth.Client, "sign", mocked_sign)
+        patcher = mock.patch.object(oauthlib.oauth1.Client, "sign", mocked_sign)
         patcher.start()
         self.addCleanup(patcher.stop)
 
@@ -74,6 +75,6 @@ class TestLTI(BaseTestXmodule):
             'input_fields': self.correct_headers,
             'element_class': self.item_module.location.category,
             'element_id': self.item_module.location.html_id(),
-            'launch_url': '',  # default value
+            'launch_url': 'http://www.example.com',  # default value
         }
         self.assertDictEqual(generated_context, expected_context)

lms/djangoapps/instructor/tests/test_api.py

@@ -4,6 +4,7 @@ Unit tests for instructor.api methods.
 # pylint: disable=E1111
 import unittest
 import json
+import requests
 from urllib import quote
 from django.conf import settings
 from django.test import TestCase
@@ -756,7 +757,7 @@ class TestInstructorAPIAnalyticsProxy(ModuleStoreTestCase, LoginEnrollmentTestCa
     class FakeProxyResponse(object):
         """ Fake successful requests response object. """
         def __init__(self):
-            self.status_code = instructor.views.api.codes.OK
+            self.status_code = requests.status_codes.codes.OK
             self.content = '{"test_content": "robot test content"}'
 
     class FakeBadProxyResponse(object):

lms/djangoapps/instructor/views/api.py

@@ -9,7 +9,6 @@ Many of these GETs may become PUTs in the future.
 import re
 import logging
 import requests
-from requests.status_codes import codes
 from collections import OrderedDict
 from django.conf import settings
 from django_future.csrf import ensure_csrf_cookie

requirements/edx/base.txt

@@ -39,6 +39,7 @@ mako==0.7.3
 Markdown==2.2.1
 networkx==1.7
 nltk==2.0.4
+oauthlib==0.5.1
 paramiko==1.9.0
 path.py==3.0.1
 Pillow==1.7.8
@@ -53,7 +54,7 @@ python-memcached==1.48
 python-openid==2.2.5
 pytz==2012h
 PyYAML==3.10
-requests==0.14.2
+requests==1.2.3
 scipy==0.11.0
 Shapely==1.2.16
 sorl-thumbnail==11.12