[PERF-224] Support to serve static assets from a CDN.

A base URL can now be configured which is, potentially, prepended to an
asset URL.  This allows a CDN, or caching server, to front static asset
requests, taking load off of the contentstore and speeding up page load
times.

Asset URL generation respects locked vs unlocked assets, and will not
generate links to locked assets that would traverse a CDN (even though
the authorization component of the contentserver middleware wouldn't
allow those links to work anyways).

common/djangoapps/static_replace/__init__.py

@@ -5,6 +5,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage
 from django.contrib.staticfiles import finders
 from django.conf import settings
 
+from static_replace.models import AssetBaseUrlConfig
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.contentstore.content import StaticContent
@@ -180,7 +181,8 @@ def replace_static_urls(text, data_directory=None, course_id=None, static_asset_
             else:
                 # if not, then assume it's courseware specific content and then look in the
                 # Mongo-backed database
-                url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
+                base_url = AssetBaseUrlConfig.get_base_url()
+                url = StaticContent.get_canonicalized_asset_path(course_id, rest, base_url)
 
                 if AssetLocator.CANONICAL_NAMESPACE in url:
                     url = url.replace('block@', 'block/', 1)

common/djangoapps/static_replace/admin.py

+"""
+Django admin page for AssetBaseUrlConfig, which allows you to set the base URL
+that gets prepended to asset URLs in order to serve them from, say, a CDN.
+"""
+from django.contrib import admin
+
+from config_models.admin import ConfigurationModelAdmin
+from .models import AssetBaseUrlConfig
+
+
+class AssetBaseUrlConfigAdmin(ConfigurationModelAdmin):
+    """
+    Basic configuration for asset base URL.
+    """
+    list_display = [
+        'base_url'
+    ]
+
+    def get_list_display(self, request):
+        """
+        Restore default list_display behavior.
+
+        ConfigurationModelAdmin overrides this, but in a way that doesn't
+        respect the ordering. This lets us customize it the usual Django admin
+        way.
+        """
+        return self.list_display
+
+
+admin.site.register(AssetBaseUrlConfig, AssetBaseUrlConfigAdmin)

common/djangoapps/static_replace/models.py

+"""
+Models for static_replace
+"""
+
+from django.db.models.fields import TextField
+from config_models.models import ConfigurationModel
+
+
+class AssetBaseUrlConfig(ConfigurationModel):
+    """Configuration for the base URL used for static assets."""
+
+    class Meta(object):
+        app_label = 'static_replace'
+
+    base_url = TextField(
+        blank=True,
+        help_text="The alternative hostname to serve static assets from.  Should be in the form of hostname[:port]."
+    )
+
+    @classmethod
+    def get_base_url(cls):
+        """Gets the base URL to use for serving static assets, if present"""
+        return cls.current().base_url
+
+    def __repr__(self):
+        return '<AssetBaseUrlConfig(base_url={})>'.format(self.get_base_url())
+
+    def __unicode__(self):
+        return unicode(repr(self))

common/lib/xmodule/xmodule/contentstore/content.py

 import re
 import uuid
+
+from xmodule.assetstore.assetmgr import AssetManager
+
 XASSET_LOCATION_TAG = 'c4x'
 XASSET_SRCREF_PREFIX = 'xasset:'
 
@@ -16,6 +19,8 @@ from urllib import urlencode
 from opaque_keys.edx.locator import AssetLocator
 from opaque_keys.edx.keys import CourseKey, AssetKey
 from opaque_keys import InvalidKeyError
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.exceptions import NotFoundError
 from PIL import Image
 
 
@@ -137,32 +142,79 @@ class StaticContent(object):
                 return AssetKey.from_string(path[1:])
 
     @staticmethod
-    def convert_legacy_static_url_with_course_id(path, course_id):
+    def get_asset_key_from_path(course_key, path):
+        """
+        Parses a path, extracting an asset key or creating one.
+
+        Args:
+            course_key: key to the course which owns this asset
+            path: the path to said content
+
+        Returns:
+            AssetKey: the asset key that represents the path
+        """
+
+        # Clean up the path, removing any static prefix and any leading slash.
+        if path.startswith('/static/'):
+            path = path[len('/static/'):]
+
+        path = path.lstrip('/')
+
+        try:
+            return AssetKey.from_string(path)
+        except InvalidKeyError:
+            # If we couldn't parse the path, just let compute_location figure it out.
+            # It's most likely a path like /image.png or something.
+            return StaticContent.compute_location(course_key, path)
+
+    @staticmethod
+    def get_canonicalized_asset_path(course_key, path, base_url):
         """
-        Returns a path to a piece of static content when we are provided with a filepath and
-        a course_id
+        Returns a fully-qualified path to a piece of static content.
+
+        If a static asset CDN is configured, this path will include it.
+        Otherwise, the path will simply be relative.
+
+        Args:
+            course_key: key to the course which owns this asset
+            path: the path to said content
+
+        Returns:
+            string: fully-qualified path to asset
         """
-        # Generate url of urlparse.path component
-        scheme, netloc, orig_path, params, query, fragment = urlparse(path)
-        loc = StaticContent.compute_location(course_id, orig_path)
-        loc_url = StaticContent.serialize_asset_key_with_slash(loc)
-
-        # parse the query params for "^/static/" and replace with the location url
-        orig_query = parse_qsl(query)
-        new_query_list = []
-        for query_name, query_value in orig_query:
+
+        # Break down the input path.
+        _, _, relative_path, params, query_string, fragment = urlparse(path)
+
+        # Convert our path to an asset key if it isn't one already.
+        asset_key = StaticContent.get_asset_key_from_path(course_key, relative_path)
+
+        # Check the status of the asset to see if this can be served via CDN aka publicly.
+        serve_from_cdn = False
+        try:
+            content = AssetManager.find(asset_key, as_stream=True)
+            is_locked = getattr(content, "locked", True)
+            serve_from_cdn = not is_locked
+        except (ItemNotFoundError, NotFoundError):
+            # If we can't find the item, just treat it as if it's locked.
+            serve_from_cdn = False
+
+        # Update any query parameter values that have asset paths in them. This is for assets that
+        # require their own after-the-fact values, like a Flash file that needs the path of a config
+        # file passed to it e.g. /static/visualization.swf?configFile=/static/visualization.xml
+        query_params = parse_qsl(query_string)
+        updated_query_params = []
+        for query_name, query_value in query_params:
             if query_value.startswith("/static/"):
-                new_query = StaticContent.compute_location(
-                    course_id,
-                    query_value[len('/static/'):],
-                )
-                new_query_url = StaticContent.serialize_asset_key_with_slash(new_query)
-                new_query_list.append((query_name, new_query_url))
+                new_query_value = StaticContent.get_canonicalized_asset_path(course_key, query_value, base_url)
+                updated_query_params.append((query_name, new_query_value))
             else:
-                new_query_list.append((query_name, query_value))
+                updated_query_params.append((query_name, query_value))
+
+        serialized_asset_key = StaticContent.serialize_asset_key_with_slash(asset_key)
+        base_url = base_url if serve_from_cdn else ''
 
-        # Reconstruct with new path
-        return urlunparse((scheme, netloc, loc_url, params, urlencode(new_query_list), fragment))
+        return urlunparse((None, base_url, serialized_asset_key, params, urlencode(updated_query_params), fragment))
 
     def stream_data(self):
         yield self._data

common/lib/xmodule/xmodule/tests/test_content.py

@@ -4,6 +4,7 @@ import os
 import unittest
 import ddt
 from path import Path as path
+
 from xmodule.contentstore.content import StaticContent, StaticContentStream
 from xmodule.contentstore.content import ContentStore
 from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
@@ -94,11 +95,6 @@ class ContentTest(unittest.TestCase):
         content = StaticContent('loc', 'name', 'content_type', 'data')
         self.assertIsNone(content.thumbnail_location)
 
-    def test_static_url_generation_from_courseid(self):
-        course_key = SlashSeparatedCourseKey('foo', 'bar', 'bz')
-        url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', course_key)
-        self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
-
     @ddt.data(
         (u"monsters__.jpg", u"monsters__.jpg"),
         (u"monsters__.png", u"monsters__-png.jpg"),
@@ -122,9 +118,9 @@ class ContentTest(unittest.TestCase):
         self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
 
     def test_get_location_from_path(self):
-        asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
+        asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg')
         self.assertEqual(
-            AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
+            AssetLocation(u'a', u'b', None, u'asset', u'images_course_image.jpg', None),
             asset_location
         )
 

lms/djangoapps/courseware/tests/test_module_render.py

@@ -1104,7 +1104,7 @@ class TestHtmlModifiers(ModuleStoreTestCase):
         result_fragment = module.render(STUDENT_VIEW)
 
         self.assertIn(
-            '/c4x/{org}/{course}/asset/_file.jpg'.format(
+            '/c4x/{org}/{course}/asset/file.jpg'.format(
                 org=self.course.location.org,
                 course=self.course.location.course,
             ),