Upgrade django-rest-framework version to edX fork, which is DRF v3.6.3

with a custom patch needed by edx-platform.
Upgrade django-filter as well to v1.0.4
Import DjangoFilterBackend from the correct module - django_filter.
Add django-filter to INSTALLED_APPS.

cms/envs/common.py

@@ -1045,6 +1045,9 @@ INSTALLED_APPS = (
 
     # Waffle related utilities
     'openedx.core.djangoapps.waffle_utils',
+
+    # DRF filters
+    'django_filters',
 )
 
 

lms/djangoapps/discussion_api/tests/test_views.py

 """
 Tests for Discussion API views
 """
+from __future__ import unicode_literals
+
 import json
 from datetime import datetime
 from urlparse import urlparse
@@ -376,7 +378,7 @@ class ThreadViewSetListTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase, Pro
             results=expected_threads,
             count=1,
             num_pages=2,
-            next_link="http://testserver/api/discussion/v1/threads/?course_id=x%2Fy%2Fz&page=2",
+            next_link="http://testserver/api/discussion/v1/threads/?course_id=x%2Fy%2Fz&following=&page=2",
             previous_link=None
         )
         expected_response.update({"text_search_rewrite": None})

lms/djangoapps/discussion_api/tests/utils.py

 """
 Discussion API test utilities
 """
+from __future__ import unicode_literals
+
 import hashlib
 import json
 import re

lms/djangoapps/experiments/views.py

 from django.contrib.auth import get_user_model
 from django.db import transaction
+from django_filters.rest_framework import DjangoFilterBackend
 from edx_rest_framework_extensions.authentication import JwtAuthentication
 from rest_framework import permissions, viewsets
 from rest_framework.decorators import list_route
-from rest_framework.filters import DjangoFilterBackend
 from rest_framework.response import Response
 
 from experiments import filters, serializers

lms/djangoapps/teams/views.py

@@ -499,7 +499,7 @@ class TeamsListView(ExpandableFieldViewMixin, GenericAPIView):
             return Response(status=status.HTTP_403_FORBIDDEN)
 
         data = request.data.copy()
-        data['course_id'] = course_key
+        data['course_id'] = unicode(course_key)
 
         serializer = CourseTeamCreationSerializer(data=data)
         add_serializer_errors(serializer, data, field_errors)

lms/envs/common.py

@@ -2248,6 +2248,9 @@ INSTALLED_APPS = (
     'openedx.features.enterprise_support',
 
     'experiments',
+
+    # DRF filters
+    'django_filters',
 )
 
 ######################### CSRF #########################################

openedx/core/djangoapps/user_api/accounts/api.py

@@ -187,7 +187,7 @@ def update_account_settings(requesting_user, update, username=None):
         # We have not found a way using signals to get the language proficiency changes (grouped by user).
         # As a workaround, store old and new values here and emit them after save is complete.
         if "language_proficiencies" in update:
-            old_language_proficiencies = legacy_profile_serializer.data["language_proficiencies"]
+            old_language_proficiencies = list(existing_user_profile.language_proficiencies.values('code'))
 
         for serializer in user_serializer, legacy_profile_serializer:
             serializer.save()

openedx/core/djangoapps/user_api/accounts/tests/test_views.py

@@ -454,7 +454,7 @@ class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase):
         ("country", "GB", "XY", u'"XY" is not a valid choice.'),
         ("year_of_birth", 2009, "not_an_int", u"A valid integer is required."),
         ("name", "bob", "z" * 256, u"Ensure this value has at most 255 characters (it has 256)."),
-        ("name", u"ȻħȺɍłɇs", "z   ", "The name field must be at least 2 characters long."),
+        ("name", u"ȻħȺɍłɇs", "z   ", u"The name field must be at least 2 characters long."),
         ("goals", "Smell the roses"),
         ("mailing_address", "Sesame Street"),
         # Note that we store the raw data, so it is up to client to escape the HTML.
@@ -677,16 +677,25 @@ class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase):
             self.assertItemsEqual(response.data["language_proficiencies"], proficiencies)
 
     @ddt.data(
-        (u"not_a_list", {u'non_field_errors': [u'Expected a list of items but got type "unicode".']}),
-        ([u"not_a_JSON_object"], [{u'non_field_errors': [u'Invalid data. Expected a dictionary, but got unicode.']}]),
-        ([{}], [OrderedDict([('code', [u'This field is required.'])])]),
+        (
+            u"not_a_list",
+            {u'non_field_errors': [u'Expected a list of items but got type "unicode".']}
+        ),
+        (
+            [u"not_a_JSON_object"],
+            [{u'non_field_errors': [u'Invalid data. Expected a dictionary, but got unicode.']}]
+        ),
+        (
+            [{}],
+            [{'code': [u'This field is required.']}]
+        ),
         (
             [{u"code": u"invalid_language_code"}],
-            [OrderedDict([('code', [u'"invalid_language_code" is not a valid choice.'])])]
+            [{'code': [u'"invalid_language_code" is not a valid choice.']}]
         ),
         (
             [{u"code": u"kw"}, {u"code": u"el"}, {u"code": u"kw"}],
-            ['The language_proficiencies field must consist of unique languages']
+            [u'The language_proficiencies field must consist of unique languages']
         ),
     )
     @ddt.unpack

openedx/core/djangoapps/user_api/helpers.py

@@ -10,7 +10,7 @@ from functools import wraps
 
 from django import forms
 from django.core.serializers.json import DjangoJSONEncoder
-from django.http import HttpResponseBadRequest
+from django.http import HttpResponseBadRequest, HttpRequest
 from django.utils.encoding import force_text
 from django.utils.functional import Promise
 
@@ -407,26 +407,32 @@ def shim_student_view(view_func, check_logged_in=False):
     """
     @wraps(view_func)
     def _inner(request):  # pylint: disable=missing-docstring
-        # Ensure that the POST querydict is mutable
-        request.POST = request.POST.copy()
+        # Make a copy of the current POST request to modify.
+        modified_request = request.POST.copy()
+        if isinstance(request, HttpRequest):
+            # Works for an HttpRequest but not a rest_framework.request.Request.
+            request.POST = modified_request
+        else:
+            # The request must be a rest_framework.request.Request.
+            request._data = modified_request
 
         # The login and registration handlers in student view try to change
         # the user's enrollment status if these parameters are present.
         # Since we want the JavaScript client to communicate directly with
         # the enrollment API, we want to prevent the student views from
         # updating enrollments.
-        if "enrollment_action" in request.POST:
-            del request.POST["enrollment_action"]
-        if "course_id" in request.POST:
-            del request.POST["course_id"]
+        if "enrollment_action" in modified_request:
+            del modified_request["enrollment_action"]
+        if "course_id" in modified_request:
+            del modified_request["course_id"]
 
         # Include the course ID if it's specified in the analytics info
         # so it can be included in analytics events.
-        if "analytics" in request.POST:
+        if "analytics" in modified_request:
             try:
-                analytics = json.loads(request.POST["analytics"])
+                analytics = json.loads(modified_request["analytics"])
                 if "enroll_course_id" in analytics:
-                    request.POST["course_id"] = analytics.get("enroll_course_id")
+                    modified_request["course_id"] = analytics.get("enroll_course_id")
             except (ValueError, TypeError):
                 LOGGER.error(
                     u"Could not parse analytics object sent to user API: {analytics}".format(

openedx/core/djangoapps/user_api/preferences/api.py

@@ -113,6 +113,7 @@ def update_user_preferences(requesting_user, update, user=None):
     for preference_key in update.keys():
         preference_value = update[preference_key]
         if preference_value is not None:
+            preference_value = unicode(preference_value)
             try:
                 serializer = create_user_preference_serializer(user, preference_key, preference_value)
                 validate_user_preference_serializer(serializer, preference_key, preference_value)
@@ -129,6 +130,7 @@ def update_user_preferences(requesting_user, update, user=None):
     for preference_key in update.keys():
         preference_value = update[preference_key]
         if preference_value is not None:
+            preference_value = unicode(preference_value)
             try:
                 serializer = serializers[preference_key]
 
@@ -152,7 +154,7 @@ def set_user_preference(requesting_user, preference_key, preference_value, usern
         requesting_user (User): The user requesting to modify account information. Only the user with username
             'username' has permissions to modify account information.
         preference_key (str): The key for the user preference.
-        preference_value (str): The value to be stored. Non-string values will be converted to strings.
+        preference_value (str): The value to be stored. Non-string values are converted to strings.
         username (str): Optional username specifying which account should be updated. If not specified,
             `requesting_user.username` is assumed.
 
@@ -166,6 +168,8 @@ def set_user_preference(requesting_user, preference_key, preference_value, usern
         UserAPIInternalError: the operation failed due to an unexpected error.
     """
     existing_user = _get_authorized_user(requesting_user, username)
+    if preference_value is not None:
+        preference_value = unicode(preference_value)
     serializer = create_user_preference_serializer(existing_user, preference_key, preference_value)
     validate_user_preference_serializer(serializer, preference_key, preference_value)
 

openedx/core/djangoapps/user_api/serializers.py

@@ -39,13 +39,14 @@ class UserSerializer(serializers.HyperlinkedModelSerializer):
 
 class UserPreferenceSerializer(serializers.HyperlinkedModelSerializer):
     """
-    Serializer that generates a represenation of a UserPreference entity
+    Serializer that generates a representation of a UserPreference entity.
     """
     user = UserSerializer()
 
     class Meta(object):
         model = UserPreference
         depth = 1
+        fields = ('user', 'key', 'value', 'url')
 
 
 class RawUserPreferenceSerializer(serializers.ModelSerializer):
@@ -57,6 +58,7 @@ class RawUserPreferenceSerializer(serializers.ModelSerializer):
     class Meta(object):
         model = UserPreference
         depth = 1
+        fields = ('user', 'key', 'value', 'url')
 
 
 class ReadOnlyFieldsSerializerMixin(object):

openedx/core/djangoapps/user_api/tests/test_views.py

@@ -359,7 +359,7 @@ class UserPreferenceViewSetTest(CacheIsolationTestCase, UserApiTestCase):
         self.assertHttpMethodNotAllowed(self.request_with_auth("put", self.LIST_URI))
 
     def test_patch_list_not_allowed(self):
-        raise SkipTest("Django 1.4's test client does not support patch")
+        self.assertHttpMethodNotAllowed(self.request_with_auth("patch", self.LIST_URI))
 
     def test_delete_list_not_allowed(self):
         self.assertHttpMethodNotAllowed(self.request_with_auth("delete", self.LIST_URI))
@@ -450,7 +450,7 @@ class UserPreferenceViewSetTest(CacheIsolationTestCase, UserApiTestCase):
         self.assertHttpMethodNotAllowed(self.request_with_auth("put", self.detail_uri))
 
     def test_patch_detail_not_allowed(self):
-        raise SkipTest("Django 1.4's test client does not support patch")
+        self.assertHttpMethodNotAllowed(self.request_with_auth("patch", self.detail_uri))
 
     def test_delete_detail_not_allowed(self):
         self.assertHttpMethodNotAllowed(self.request_with_auth("delete", self.detail_uri))

openedx/core/djangoapps/user_api/views.py

@@ -10,10 +10,11 @@ from django.utils.decorators import method_decorator
 from django.utils.translation import ugettext as _
 from django.views.decorators.csrf import csrf_exempt, csrf_protect, ensure_csrf_cookie
 from django_countries import countries
+from django_filters.rest_framework import DjangoFilterBackend
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx import locator
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
-from rest_framework import authentication, filters, generics, status, viewsets
+from rest_framework import authentication, generics, status, viewsets
 from rest_framework.exceptions import ParseError
 from rest_framework.views import APIView
 
@@ -1054,7 +1055,7 @@ class UserPreferenceViewSet(viewsets.ReadOnlyModelViewSet):
     authentication_classes = (authentication.SessionAuthentication,)
     permission_classes = (ApiKeyHeaderPermission,)
     queryset = UserPreference.objects.all()
-    filter_backends = (filters.DjangoFilterBackend,)
+    filter_backends = (DjangoFilterBackend,)
     filter_fields = ("key", "user")
     serializer_class = UserPreferenceSerializer
     paginate_by = 10

openedx/core/lib/api/authentication.py

@@ -5,11 +5,10 @@ import logging
 import django.utils.timezone
 from oauth2_provider import models as dot_models
 from provider.oauth2 import models as dop_models
-from rest_framework import exceptions as drf_exceptions
+from rest_framework.exceptions import AuthenticationFailed
 from rest_framework.authentication import SessionAuthentication
 from rest_framework_oauth.authentication import OAuth2Authentication
 
-from openedx.core.lib.api.exceptions import AuthenticationFailed
 
 OAUTH2_TOKEN_ERROR = u'token_error'
 OAUTH2_TOKEN_ERROR_EXPIRED = u'token_expired'
@@ -89,27 +88,25 @@ class OAuth2AuthenticationAllowInactiveUser(OAuth2Authentication):
         succeeds, raises an AuthenticationFailed (HTTP 401) if authentication
         fails or None if the user did not try to authenticate using an access
         token.
-
-        Overrides base class implementation to return edX-style error
-        responses.
         """
 
         try:
             return super(OAuth2AuthenticationAllowInactiveUser, self).authenticate(*args, **kwargs)
-        except AuthenticationFailed:
-            # AuthenticationFailed is a subclass of drf_exceptions.AuthenticationFailed,
-            # but we don't want to post-process the exception detail for our own class.
-            raise
-        except drf_exceptions.AuthenticationFailed as exc:
-            if 'No credentials provided' in exc.detail:
-                error_code = OAUTH2_TOKEN_ERROR_NOT_PROVIDED
-            elif 'Token string should not contain spaces' in exc.detail:
-                error_code = OAUTH2_TOKEN_ERROR_MALFORMED
+        except AuthenticationFailed as exc:
+            if isinstance(exc.detail, dict):
+                developer_message = exc.detail['developer_message']
+                error_code = exc.detail['error_code']
             else:
-                error_code = OAUTH2_TOKEN_ERROR
+                developer_message = exc.detail
+                if 'No credentials provided' in developer_message:
+                    error_code = OAUTH2_TOKEN_ERROR_NOT_PROVIDED
+                elif 'Token string should not contain spaces' in developer_message:
+                    error_code = OAUTH2_TOKEN_ERROR_MALFORMED
+                else:
+                    error_code = OAUTH2_TOKEN_ERROR
             raise AuthenticationFailed({
                 u'error_code': error_code,
-                u'developer_message': exc.detail
+                u'developer_message': developer_message
             })
 
     def authenticate_credentials(self, request, access_token):

openedx/core/lib/api/exceptions.py

-"""
-Custom exceptions, that allow details to be passed as dict values (which can be
-converted to JSON, like other API responses.
-"""
-
-from rest_framework import exceptions
-
-
-# TODO: Override Throttled, UnsupportedMediaType, ValidationError.  These types require
-# more careful handling of arguments.
-
-
-class _DictAPIException(exceptions.APIException):
-    """
-    Intermediate class to allow exceptions to pass dict detail values.  Use by
-    subclassing this along with another subclass of `exceptions.APIException`.
-    """
-    def __init__(self, detail):
-        if isinstance(detail, dict):
-            self.detail = detail
-        else:
-            super(_DictAPIException, self).__init__(detail)
-
-
-class AuthenticationFailed(exceptions.AuthenticationFailed, _DictAPIException):
-    """
-    Override of DRF's AuthenticationFailed exception to allow dictionary responses.
-    """
-    pass
-
-
-class MethodNotAllowed(exceptions.MethodNotAllowed, _DictAPIException):
-    """
-    Override of DRF's MethodNotAllowed exception to allow dictionary responses.
-    """
-    def __init__(self, method, detail=None):
-        if isinstance(detail, dict):
-            self.detail = detail
-        else:
-            super(MethodNotAllowed, self).__init__(method, detail)
-
-
-class NotAcceptable(exceptions.NotAcceptable, _DictAPIException):
-    """
-    Override of DRF's NotAcceptable exception to allow dictionary responses.
-    """
-
-    def __init__(self, detail=None, available_renderers=None):
-        self.available_renderers = available_renderers
-        if isinstance(detail, dict):
-            self.detail = detail
-        else:
-            super(NotAcceptable, self).__init__(detail, available_renderers)
-
-
-class NotAuthenticated(exceptions.NotAuthenticated, _DictAPIException):
-    """
-    Override of DRF's NotAuthenticated exception to allow dictionary responses.
-    """
-    pass
-
-
-class NotFound(exceptions.NotFound, _DictAPIException):
-    """
-    Override of DRF's NotFound exception to allow dictionary responses.
-    """
-    pass
-
-
-class ParseError(exceptions.ParseError, _DictAPIException):
-    """
-    Override of DRF's ParseError exception to allow dictionary responses.
-    """
-    pass
-
-
-class PermissionDenied(exceptions.PermissionDenied, _DictAPIException):
-    """
-    Override of DRF's PermissionDenied exception to allow dictionary responses.
-    """
-    pass

openedx/core/lib/api/tests/test_exceptions.py

@@ -6,67 +6,35 @@ from django.test import TestCase
 from nose.plugins.attrib import attr
 from rest_framework import exceptions as drf_exceptions
 
-from .. import exceptions
-
 
 @attr(shard=2)
 @ddt.ddt
 class TestDictExceptionsAllowDictDetails(TestCase):
     """
-    Standard DRF exceptions coerce detail inputs to strings.  We want to use
-    dicts to allow better customization of error messages.  Demonstrate that
-    we can provide dictionaries as exception details, and that custom
-    classes subclass the relevant DRF exceptions, to provide consistent
-    exception catching behavior.
+    Test that standard DRF exceptions can return dictionaries in error details.
     """
 
-    def test_drf_errors_coerce_strings(self):
-        # Demonstrate the base issue we are trying to solve.
+    def test_drf_errors_are_not_coerced_to_strings(self):
+        # Demonstrate that dictionaries in exceptions are not coerced to strings.
         exc = drf_exceptions.AuthenticationFailed({u'error_code': -1})
-        self.assertEqual(exc.detail, u"{u'error_code': -1}")
+        self.assertNotIsInstance(exc.detail, basestring)
 
     @ddt.data(
-        exceptions.AuthenticationFailed,
-        exceptions.NotAuthenticated,
-        exceptions.NotFound,
-        exceptions.ParseError,
-        exceptions.PermissionDenied,
+        drf_exceptions.AuthenticationFailed,
+        drf_exceptions.NotAuthenticated,
+        drf_exceptions.NotFound,
+        drf_exceptions.ParseError,
+        drf_exceptions.PermissionDenied,
     )
     def test_exceptions_allows_dict_detail(self, exception_class):
         exc = exception_class({u'error_code': -1})
-        self.assertEqual(exc.detail, {u'error_code': -1})
+        self.assertEqual(exc.detail, {u'error_code': u'-1'})
 
     def test_method_not_allowed_allows_dict_detail(self):
-        exc = exceptions.MethodNotAllowed(u'POST', {u'error_code': -1})
-        self.assertEqual(exc.detail, {u'error_code': -1})
+        exc = drf_exceptions.MethodNotAllowed(u'POST', {u'error_code': -1})
+        self.assertEqual(exc.detail, {u'error_code': u'-1'})
 
     def test_not_acceptable_allows_dict_detail(self):
-        exc = exceptions.NotAcceptable({u'error_code': -1}, available_renderers=['application/json'])
-        self.assertEqual(exc.detail, {u'error_code': -1})
+        exc = drf_exceptions.NotAcceptable({u'error_code': -1}, available_renderers=['application/json'])
+        self.assertEqual(exc.detail, {u'error_code': u'-1'})
         self.assertEqual(exc.available_renderers, ['application/json'])
-
-
-@attr(shard=2)
-@ddt.ddt
-class TestDictExceptionSubclassing(TestCase):
-    """
-    Custom exceptions should subclass standard DRF exceptions, so code that
-    catches the DRF exceptions also catches ours.
-    """
-
-    @ddt.data(
-        (exceptions.AuthenticationFailed, drf_exceptions.AuthenticationFailed),
-        (exceptions.NotAcceptable, drf_exceptions.NotAcceptable),
-        (exceptions.NotAuthenticated, drf_exceptions.NotAuthenticated),
-        (exceptions.NotFound, drf_exceptions.NotFound),
-        (exceptions.ParseError, drf_exceptions.ParseError),
-        (exceptions.PermissionDenied, drf_exceptions.PermissionDenied),
-    )
-    @ddt.unpack
-    def test_exceptions_subclass_drf_exceptions(self, exception_class, drf_exception_class):
-        exc = exception_class({u'error_code': -1})
-        self.assertIsInstance(exc, drf_exception_class)
-
-    def test_method_not_allowed_subclasses_drf_exception(self):
-        exc = exceptions.MethodNotAllowed(u'POST', {u'error_code': -1})
-        self.assertIsInstance(exc, drf_exceptions.MethodNotAllowed)

requirements/edx/base.txt

@@ -20,7 +20,7 @@ django-celery==3.2.1
 django-config-models==0.1.5
 django-countries==4.0
 django-extensions==1.5.9
-django-filter==0.11.0
+django-filter==1.0.4
 django-ipware==1.1.0
 django-model-utils==2.3.1
 django-mptt==0.7.4
@@ -35,9 +35,7 @@ django-statici18n==1.4.0
 django-storages==1.4.1
 django-method-override==0.1.0
 django-user-tasks==0.1.4
-# We need a fix to DRF 3.2.x, for now use it from our own cherry-picked repo
-#djangorestframework>=3.1,<3.2
-git+https://github.com/edx/django-rest-framework.git@3c72cb5ee5baebc4328947371195eae2077197b0#egg=djangorestframework==3.2.3
+git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
 django==1.8.18
 django-waffle==0.12.0
 djangorestframework-jwt==1.8.0

requirements/edx/github.txt

@@ -75,9 +75,9 @@ git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
 -e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2
 -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock
 git+https://github.com/edx/edx-ora2.git@1.4.6#egg=ora2==1.4.6
--e git+https://github.com/edx/edx-submissions.git@2.0.0#egg=edx-submissions==2.0.0
+git+https://github.com/edx/edx-submissions.git@2.0.1#egg=edx-submissions==2.0.1
 git+https://github.com/edx/ease.git@release-2015-07-14#egg=ease==0.1.3
-git+https://github.com/edx/edx-val.git@0.0.17#egg=edxval==0.0.17
+git+https://github.com/edx/edx-val.git@0.0.18#egg=edxval==0.0.18
 git+https://github.com/edx/RecommenderXBlock.git@0e744b393cf1f8b886fe77bc697e7d9d78d65cd6#egg=recommender-xblock==1.2
 git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1
 -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock