Merge pull request #11025 from edx/PERF-224

[PERF-224] Serve course assets from a CDN

common/djangoapps/static_replace/__init__.py

@@ -5,6 +5,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.contrib.staticfiles import finders
 from django.conf import settings
 
+from static_replace.models import AssetBaseUrlConfig
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.contentstore.content import StaticContent
@@ -180,7 +181,8 @@ def replace_static_urls(text, data_directory=None, course_id=None, static_asset_
             else:
                 # if not, then assume it's courseware specific content and then look in the
                 # Mongo-backed database
-                url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
+                base_url = AssetBaseUrlConfig.get_base_url()
+                url = StaticContent.get_canonicalized_asset_path(course_id, rest, base_url)
 
                 if AssetLocator.CANONICAL_NAMESPACE in url:
                     url = url.replace('block@', 'block/', 1)

common/djangoapps/static_replace/admin.py

+"""
+Django admin page for AssetBaseUrlConfig, which allows you to set the base URL
+that gets prepended to asset URLs in order to serve them from, say, a CDN.
+"""
+from django.contrib import admin
+
+from config_models.admin import ConfigurationModelAdmin
+from .models import AssetBaseUrlConfig
+
+
+class AssetBaseUrlConfigAdmin(ConfigurationModelAdmin):
+    """
+    Basic configuration for asset base URL.
+    """
+    list_display = [
+        'base_url'
+    ]
+
+    def get_list_display(self, request):
+        """
+        Restore default list_display behavior.
+
+        ConfigurationModelAdmin overrides this, but in a way that doesn't
+        respect the ordering. This lets us customize it the usual Django admin
+        way.
+        """
+        return self.list_display
+
+
+admin.site.register(AssetBaseUrlConfig, AssetBaseUrlConfigAdmin)

common/djangoapps/static_replace/models.py

+"""
+Models for static_replace
+"""
+
+from django.db.models.fields import TextField
+from config_models.models import ConfigurationModel
+
+
+class AssetBaseUrlConfig(ConfigurationModel):
+    """Configuration for the base URL used for static assets."""
+
+    class Meta(object):
+        app_label = 'static_replace'
+
+    base_url = TextField(
+        blank=True,
+        help_text="The alternative hostname to serve static assets from.  Should be in the form of hostname[:port]."
+    )
+
+    @classmethod
+    def get_base_url(cls):
+        """Gets the base URL to use for serving static assets, if present"""
+        return cls.current().base_url
+
+    def __repr__(self):
+        return '<AssetBaseUrlConfig(base_url={})>'.format(self.get_base_url())
+
+    def __unicode__(self):
+        return unicode(repr(self))

common/lib/xmodule/xmodule/contentstore/content.py

 import re
 import uuid
+
+from xmodule.assetstore.assetmgr import AssetManager
+
 XASSET_LOCATION_TAG = 'c4x'
 XASSET_SRCREF_PREFIX = 'xasset:'
 
@@ -16,6 +19,8 @@ from urllib import urlencode
 from opaque_keys.edx.locator import AssetLocator
 from opaque_keys.edx.keys import CourseKey, AssetKey
 from opaque_keys import InvalidKeyError
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.exceptions import NotFoundError
 from PIL import Image
 
 
@@ -137,32 +142,79 @@ class StaticContent(object):
                 return AssetKey.from_string(path[1:])
 
     @staticmethod
-    def convert_legacy_static_url_with_course_id(path, course_id):
+    def get_asset_key_from_path(course_key, path):
+        """
+        Parses a path, extracting an asset key or creating one.
+
+        Args:
+            course_key: key to the course which owns this asset
+            path: the path to said content
+
+        Returns:
+            AssetKey: the asset key that represents the path
+        """
+
+        # Clean up the path, removing any static prefix and any leading slash.
+        if path.startswith('/static/'):
+            path = path[len('/static/'):]
+
+        path = path.lstrip('/')
+
+        try:
+            return AssetKey.from_string(path)
+        except InvalidKeyError:
+            # If we couldn't parse the path, just let compute_location figure it out.
+            # It's most likely a path like /image.png or something.
+            return StaticContent.compute_location(course_key, path)
+
+    @staticmethod
+    def get_canonicalized_asset_path(course_key, path, base_url):
         """
-        Returns a path to a piece of static content when we are provided with a filepath and
-        a course_id
+        Returns a fully-qualified path to a piece of static content.
+
+        If a static asset CDN is configured, this path will include it.
+        Otherwise, the path will simply be relative.
+
+        Args:
+            course_key: key to the course which owns this asset
+            path: the path to said content
+
+        Returns:
+            string: fully-qualified path to asset
         """
-        # Generate url of urlparse.path component
-        scheme, netloc, orig_path, params, query, fragment = urlparse(path)
-        loc = StaticContent.compute_location(course_id, orig_path)
-        loc_url = StaticContent.serialize_asset_key_with_slash(loc)
-
-        # parse the query params for "^/static/" and replace with the location url
-        orig_query = parse_qsl(query)
-        new_query_list = []
-        for query_name, query_value in orig_query:
+
+        # Break down the input path.
+        _, _, relative_path, params, query_string, fragment = urlparse(path)
+
+        # Convert our path to an asset key if it isn't one already.
+        asset_key = StaticContent.get_asset_key_from_path(course_key, relative_path)
+
+        # Check the status of the asset to see if this can be served via CDN aka publicly.
+        serve_from_cdn = False
+        try:
+            content = AssetManager.find(asset_key, as_stream=True)
+            is_locked = getattr(content, "locked", True)
+            serve_from_cdn = not is_locked
+        except (ItemNotFoundError, NotFoundError):
+            # If we can't find the item, just treat it as if it's locked.
+            serve_from_cdn = False
+
+        # Update any query parameter values that have asset paths in them. This is for assets that
+        # require their own after-the-fact values, like a Flash file that needs the path of a config
+        # file passed to it e.g. /static/visualization.swf?configFile=/static/visualization.xml
+        query_params = parse_qsl(query_string)
+        updated_query_params = []
+        for query_name, query_value in query_params:
             if query_value.startswith("/static/"):
-                new_query = StaticContent.compute_location(
-                    course_id,
-                    query_value[len('/static/'):],
-                )
-                new_query_url = StaticContent.serialize_asset_key_with_slash(new_query)
-                new_query_list.append((query_name, new_query_url))
+                new_query_value = StaticContent.get_canonicalized_asset_path(course_key, query_value, base_url)
+                updated_query_params.append((query_name, new_query_value))
             else:
-                new_query_list.append((query_name, query_value))
+                updated_query_params.append((query_name, query_value))
+
+        serialized_asset_key = StaticContent.serialize_asset_key_with_slash(asset_key)
+        base_url = base_url if serve_from_cdn else ''
 
-        # Reconstruct with new path
-        return urlunparse((scheme, netloc, loc_url, params, urlencode(new_query_list), fragment))
+        return urlunparse((None, base_url, serialized_asset_key, params, urlencode(updated_query_params), fragment))
 
     def stream_data(self):
         yield self._data

common/lib/xmodule/xmodule/tests/test_content.py

@@ -4,6 +4,7 @@ import os
 import unittest
 import ddt
 from path import Path as path
+
 from xmodule.contentstore.content import StaticContent, StaticContentStream
 from xmodule.contentstore.content import ContentStore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
@@ -94,11 +95,6 @@ class ContentTest(unittest.TestCase):
         content = StaticContent('loc', 'name', 'content_type', 'data')
         self.assertIsNone(content.thumbnail_location)
 
-    def test_static_url_generation_from_courseid(self):
-        course_key = SlashSeparatedCourseKey('foo', 'bar', 'bz')
-        url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', course_key)
-        self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
-
     @ddt.data(
         (u"monsters__.jpg", u"monsters__.jpg"),
         (u"monsters__.png", u"monsters__-png.jpg"),
@@ -122,9 +118,9 @@ class ContentTest(unittest.TestCase):
         self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
 
     def test_get_location_from_path(self):
-        asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
+        asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg')
         self.assertEqual(
-            AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
+            AssetLocation(u'a', u'b', None, u'asset', u'images_course_image.jpg', None),
             asset_location
         )
 

lms/djangoapps/courseware/tests/test_module_render.py

@@ -1104,7 +1104,7 @@ class TestHtmlModifiers(ModuleStoreTestCase):
         result_fragment = module.render(STUDENT_VIEW)
 
         self.assertIn(
-            '/c4x/{org}/{course}/asset/_file.jpg'.format(
+            '/c4x/{org}/{course}/asset/file.jpg'.format(
                 org=self.course.location.org,
                 course=self.course.location.course,
             ),