Merge pull request #11277 from edx/PERF-251

Add the basis of configuring a cache TTL for course assets.

cms/envs/common.py

@@ -306,6 +306,7 @@ simplefilter('ignore')
 
 MIDDLEWARE_CLASSES = (
     'request_cache.middleware.RequestCache',
+    'clean_headers.middleware.CleanHeadersMiddleware',
     'django.middleware.cache.UpdateCacheMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
@@ -749,6 +750,7 @@ INSTALLED_APPS = (
 
     # For CMS
     'contentstore',
+    'contentserver',
     'course_creators',
     'external_auth',
     'student',  # misleading name due to sharing with lms

common/djangoapps/clean_headers/__init__.py

+"""
+This middleware is used for cleaning headers from a response before it is sent to the end user.
+
+Due to the nature of how middleware runs, a piece of middleware high in the chain cannot ensure
+that response headers won't be present on the final response body, as middleware further down
+the chain could be adding them.
+
+This middleware is intended to sit as close as possible to the top of the list, so that it has
+a chance on the reponse going out to strip the intended headers.
+"""
+
+
+def remove_headers_from_response(response, *headers):
+    """Removes the given headers from the response using the clean_headers middleware."""
+    response.clean_headers = headers

common/djangoapps/clean_headers/decorators.py

+"""
+Middleware decorator for removing headers.
+"""
+
+from functools import wraps
+
+
+def clean_headers(*headers):
+    """
+    Decorator that removes any headers specified from the response.
+    Usage:
+        @clean_headers("Vary")
+        def myview(request):
+            ...
+
+    The CleanHeadersMiddleware must be used and placed as closely as possible to the top
+    of the middleware chain, ideally after any caching middleware but before everything else.
+
+    This decorator is not safe for multiple uses: each call will overwrite any previously set values.
+    """
+    def _decorator(func):
+        """
+        Decorates the given function.
+        """
+        @wraps(func)
+        def _inner(*args, **kwargs):
+            """
+            Alters the response.
+            """
+            response = func(*args, **kwargs)
+            response.clean_headers = headers
+            return response
+
+        return _inner
+
+    return _decorator

common/djangoapps/clean_headers/middleware.py

+"""
+Middleware used for cleaning headers from a response before it is sent to the end user.
+"""
+
+
+class CleanHeadersMiddleware(object):
+    """
+    Middleware that can drop headers present in a response.
+
+    This can be used, for example, to remove headers i.e. drop any Vary headers to improve cache performance.
+    """
+
+    def process_response(self, _request, response):
+        """
+        Processes the given response, potentially stripping out any unwanted headers.
+        """
+
+        if len(getattr(response, 'clean_headers', [])) > 0:
+            for header in response.clean_headers:
+                try:
+                    del response[header]
+                except KeyError:
+                    pass
+
+        return response

common/djangoapps/clean_headers/tests/test_decorators.py

+"""Tests for clean_headers decorator. """
+from django.http import HttpResponse, HttpRequest
+from django.test import TestCase
+from clean_headers.decorators import clean_headers
+
+
+def fake_view(_request):
+    """Fake view that returns an empty response."""
+    return HttpResponse()
+
+
+class TestCleanHeaders(TestCase):
+    """Test the `clean_headers` decorator."""
+
+    def test_clean_headers(self):
+        request = HttpRequest()
+        wrapper = clean_headers('Vary', 'Accept-Encoding')
+        wrapped_view = wrapper(fake_view)
+        response = wrapped_view(request)
+        self.assertEqual(len(response.clean_headers), 2)

common/djangoapps/clean_headers/tests/test_middleware.py

+"""Tests for clean_headers middleware."""
+from django.http import HttpResponse, HttpRequest
+from django.test import TestCase
+from clean_headers.middleware import CleanHeadersMiddleware
+
+
+class TestCleanHeadersMiddlewareProcessResponse(TestCase):
+    """Test the `clean_headers` middleware. """
+    def setUp(self):
+        super(TestCleanHeadersMiddlewareProcessResponse, self).setUp()
+        self.middleware = CleanHeadersMiddleware()
+
+    def test_cleans_intended_headers(self):
+        fake_request = HttpRequest()
+
+        fake_response = HttpResponse()
+        fake_response['Vary'] = 'Cookie'
+        fake_response['Accept-Encoding'] = 'gzip'
+        fake_response.clean_headers = ['Vary']
+
+        result = self.middleware.process_response(fake_request, fake_response)
+        self.assertNotIn('Vary', result)
+        self.assertEquals('gzip', result['Accept-Encoding'])
+
+    def test_does_not_mangle_undecorated_response(self):
+        fake_request = HttpRequest()
+
+        fake_response = HttpResponse()
+        fake_response['Vary'] = 'Cookie'
+        fake_response['Accept-Encoding'] = 'gzip'
+
+        result = self.middleware.process_response(fake_request, fake_response)
+        self.assertEquals('Cookie', result['Vary'])
+        self.assertEquals('gzip', result['Accept-Encoding'])

common/djangoapps/contentserver/admin.py

+"""
+Django admin page for CourseAssetCacheTtlConfig, which allows you to configure the TTL
+that gets used when sending cachability headers back with request course assets.
+"""
+from django.contrib import admin
+from config_models.admin import ConfigurationModelAdmin
+from .models import CourseAssetCacheTtlConfig
+
+
+class CourseAssetCacheTtlConfigAdmin(ConfigurationModelAdmin):
+    """
+    Basic configuration for cache TTL.
+    """
+    list_display = [
+        'cache_ttl'
+    ]
+
+    def get_list_display(self, request):
+        """
+        Restore default list_display behavior.
+
+        ConfigurationModelAdmin overrides this, but in a way that doesn't
+        respect the ordering. This lets us customize it the usual Django admin
+        way.
+        """
+        return self.list_display
+
+
+admin.site.register(CourseAssetCacheTtlConfig, CourseAssetCacheTtlConfigAdmin)

common/djangoapps/contentserver/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+#pylint: skip-file
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+from django.conf import settings
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CourseAssetCacheTtlConfig',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
+                ('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
+                ('cache_ttl', models.PositiveIntegerField(default=0, help_text=b'The time, in seconds, to report that a course asset is allowed to be cached for.')),
+                ('changed_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Changed by')),
+            ],
+        ),
+    ]

common/djangoapps/contentserver/models.py

+"""
+Models for contentserver
+"""
+
+from django.db.models.fields import PositiveIntegerField
+from config_models.models import ConfigurationModel
+
+
+class CourseAssetCacheTtlConfig(ConfigurationModel):
+    """Configuration for the TTL of course assets."""
+
+    class Meta(object):
+        app_label = 'contentserver'
+
+    cache_ttl = PositiveIntegerField(
+        default=0,
+        help_text="The time, in seconds, to report that a course asset is allowed to be cached for."
+    )
+
+    @classmethod
+    def get_cache_ttl(cls):
+        """Gets the cache TTL for course assets, if present"""
+        return cls.current().cache_ttl
+
+    def __repr__(self):
+        return '<CourseAssetCacheTtlConfig(cache_ttl={})>'.format(self.get_cache_ttl())
+
+    def __unicode__(self):
+        return unicode(repr(self))

common/djangoapps/contentserver/tests/test.py → common/djangoapps/contentserver/test/test_contentserver.py

@@ -2,6 +2,8 @@
 Tests for StaticContentServer
 """
 import copy
+
+import datetime
 import ddt
 import logging
 import unittest
@@ -10,6 +12,7 @@ from uuid import uuid4
 from django.conf import settings
 from django.test.client import Client
 from django.test.utils import override_settings
+from mock import patch
 
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.django import modulestore
@@ -17,7 +20,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.xml_importer import import_course_from_xml
 
-from contentserver.middleware import parse_range_header
+from contentserver.middleware import parse_range_header, HTTP_DATE_FORMAT, StaticContentServer
 from student.models import CourseEnrollment
 
 log = logging.getLogger(__name__)
@@ -136,8 +139,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         first_byte = self.length_unlocked / 4
         last_byte = self.length_unlocked / 2
         resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
-            first=first_byte, last=last_byte)
-        )
+            first=first_byte, last=last_byte))
 
         self.assertEqual(resp.status_code, 206)  # HTTP_206_PARTIAL_CONTENT
         self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format(
@@ -151,8 +153,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         first_byte = self.length_unlocked / 4
         last_byte = self.length_unlocked / 2
         resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}, -100'.format(
-            first=first_byte, last=last_byte)
-        )
+            first=first_byte, last=last_byte))
 
         self.assertEqual(resp.status_code, 200)
         self.assertNotIn('Content-Range', resp)
@@ -178,8 +179,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         416 Requested Range Not Satisfiable.
         """
         resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
-            first=(self.length_unlocked / 2), last=(self.length_unlocked / 4))
-        )
+            first=(self.length_unlocked / 2), last=(self.length_unlocked / 4)))
         self.assertEqual(resp.status_code, 416)
 
     def test_range_request_malformed_out_of_bounds(self):
@@ -188,10 +188,88 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         outputs 416 Requested Range Not Satisfiable.
         """
         resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
-            first=(self.length_unlocked), last=(self.length_unlocked))
-        )
+            first=(self.length_unlocked), last=(self.length_unlocked)))
         self.assertEqual(resp.status_code, 416)
 
+    @patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
+    def test_cache_headers_with_ttl_unlocked(self, mock_get_cache_ttl):
+        """
+        Tests that when a cache TTL is set, an unlocked asset will be sent back with
+        the correct cache control/expires headers.
+        """
+        mock_get_cache_ttl.return_value = 10
+
+        resp = self.client.get(self.url_unlocked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertIn('Expires', resp)
+        self.assertEquals('public, max-age=10, s-maxage=10', resp['Cache-Control'])
+
+    @patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
+    def test_cache_headers_with_ttl_locked(self, mock_get_cache_ttl):
+        """
+        Tests that when a cache TTL is set, a locked asset will be sent back without
+        any cache control/expires headers.
+        """
+        mock_get_cache_ttl.return_value = 10
+
+        CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
+        self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
+
+        self.client.login(username=self.non_staff_usr, password=self.non_staff_pwd)
+        resp = self.client.get(self.url_locked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertNotIn('Expires', resp)
+        self.assertEquals('private, no-cache, no-store', resp['Cache-Control'])
+
+    @patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
+    def test_cache_headers_without_ttl_unlocked(self, mock_get_cache_ttl):
+        """
+        Tests that when a cache TTL is not set, an unlocked asset will be sent back without
+        any cache control/expires headers.
+        """
+        mock_get_cache_ttl.return_value = 0
+
+        resp = self.client.get(self.url_unlocked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertNotIn('Expires', resp)
+        self.assertNotIn('Cache-Control', resp)
+
+    @patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
+    def test_cache_headers_without_ttl_locked(self, mock_get_cache_ttl):
+        """
+        Tests that when a cache TTL is not set, a locked asset will be sent back with a
+        cache-control header that indicates this asset should not be cached.
+        """
+        mock_get_cache_ttl.return_value = 0
+
+        CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
+        self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
+
+        self.client.login(username=self.non_staff_usr, password=self.non_staff_pwd)
+        resp = self.client.get(self.url_locked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertNotIn('Expires', resp)
+        self.assertEquals('private, no-cache, no-store', resp['Cache-Control'])
+
+    def test_get_expiration_value(self):
+        start_dt = datetime.datetime.strptime("Thu, 01 Dec 1983 20:00:00 GMT", HTTP_DATE_FORMAT)
+        near_expire_dt = StaticContentServer.get_expiration_value(start_dt, 55)
+        self.assertEqual("Thu, 01 Dec 1983 20:00:55 GMT", near_expire_dt)
+
+    def test_response_no_vary_header_unlocked(self):
+        resp = self.client.get(self.url_unlocked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertNotIn('Vary', resp)
+
+    def test_response_no_vary_header_locked(self):
+        CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
+        self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
+
+        self.client.login(username=self.non_staff_usr, password=self.non_staff_pwd)
+        resp = self.client.get(self.url_locked)
+        self.assertEqual(resp.status_code, 200)
+        self.assertNotIn('Vary', resp)
+
 
 @ddt.ddt
 class ParseRangeHeaderTestCase(unittest.TestCase):

lms/envs/common.py

@@ -1072,6 +1072,7 @@ simplefilter('ignore')
 
 MIDDLEWARE_CLASSES = (
     'request_cache.middleware.RequestCache',
+    'clean_headers.middleware.CleanHeadersMiddleware',
     'microsite_configuration.middleware.MicrositeMiddleware',
     'django_comment_client.middleware.AjaxExceptionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -1756,6 +1757,9 @@ INSTALLED_APPS = (
     'pipeline',
     'static_replace',
 
+    # For content serving
+    'contentserver',
+
     # Theming
     'openedx.core.djangoapps.theming',