Merge pull request #4196 from edx/oleg/video-caching

Redirect Chinese students to a Chinese CDN for video.

CHANGELOG.rst

@@ -5,6 +5,8 @@ These are notable changes in edx-platform.  This is a rolling list of changes,
 in roughly chronological order, most recent first.  Add your entries at or near
 the top.  Include a label indicating the component affected.
 
+Blades: Redirect Chinese students to a Chinese CDN for video. BLD-1052.
+
 Studio: Move Peer Assessment into advanced problems menu.
 
 Studio: Support creation and editing of split_test instances (Content Experiments)

cms/djangoapps/models/settings/course_metadata.py

@@ -10,6 +10,7 @@ class CourseMetadata(object):
     The objects have no predefined attrs but instead are obj encodings of the
     editable metadata.
     '''
+    # The list of fields that wouldn't be shown in Advanced Settings.
     FILTERED_LIST = ['xml_attributes',
                      'start',
                      'end',
@@ -21,6 +22,7 @@ class CourseMetadata(object):
                      'show_timezone',
                      'format',
                      'graded',
+                     'video_speed_optimizations',
     ]
 
     @classmethod

common/djangoapps/geoinfo/middleware.py

+"""
+Middleware to identify the country of origin of page requests.
+
+Middleware adds `country_code` in session.
+
+Usage:
+
+# To enable the Geoinfo feature on a per-view basis, use:
+decorator `django.utils.decorators.decorator_from_middleware(middleware_class)`
+
+"""
+
+import logging
+import pygeoip
+
+from ipware.ip import get_real_ip
+from django.conf import settings
+
+log = logging.getLogger(__name__)
+
+
+class CountryMiddleware(object):
+    """
+    Identify the country by IP address.
+    """
+    def process_request(self, request):
+        """
+        Identify the country by IP address.
+
+        Store country code in session.
+        """
+        new_ip_address = get_real_ip(request)
+        old_ip_address = request.session.get('ip_address', None)
+
+        if new_ip_address != old_ip_address:
+            country_code = pygeoip.GeoIP(settings.GEOIP_PATH).country_code_by_addr(new_ip_address)
+            request.session['country_code'] = country_code
+            request.session['ip_address'] = new_ip_address
+            log.debug('Country code for IP: %s is set to %s', new_ip_address, country_code)

common/djangoapps/geoinfo/tests/test_middleware.py

+"""
+Tests for CountryMiddleware.
+"""
+
+from mock import Mock, patch
+import pygeoip
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from django.test.client import RequestFactory
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from student.models import CourseEnrollment
+from student.tests.factories import UserFactory, AnonymousUserFactory
+
+from django.contrib.sessions.middleware import SessionMiddleware
+from geoinfo.middleware import CountryMiddleware
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class CountryMiddlewareTests(TestCase):
+    """
+    Tests of CountryMiddleware.
+    """
+    def setUp(self):
+        self.country_middleware = CountryMiddleware()
+        self.session_middleware = SessionMiddleware()
+        self.authenticated_user = UserFactory.create()
+        self.anonymous_user = AnonymousUserFactory.create()
+        self.request_factory = RequestFactory()
+        self.patcher = patch.object(pygeoip.GeoIP, 'country_code_by_addr', self.mock_country_code_by_addr)
+        self.patcher.start()
+
+    def tearDown(self):
+        self.patcher.stop()
+
+    def mock_country_code_by_addr(self, ip_addr):
+        """
+        Gives us a fake set of IPs
+        """
+        ip_dict = {
+            '117.79.83.1': 'CN',
+            '117.79.83.100': 'CN',
+            '4.0.0.0': 'SD',
+        }
+        return ip_dict.get(ip_addr, 'US')
+
+    def test_country_code_added(self):
+        request = self.request_factory.get('/somewhere',
+                                            HTTP_X_FORWARDED_FOR='117.79.83.1')
+        request.user = self.authenticated_user
+        self.session_middleware.process_request(request)
+        # No country code exists before request.
+        self.assertNotIn('country_code', request.session)
+        self.assertNotIn('ip_address', request.session)
+        self.country_middleware.process_request(request)
+        # Country code added to session.
+        self.assertEqual('CN', request.session.get('country_code'))
+        self.assertEqual('117.79.83.1', request.session.get('ip_address'))
+
+    def test_ip_address_changed(self):
+        request = self.request_factory.get('/somewhere',
+                                            HTTP_X_FORWARDED_FOR='4.0.0.0')
+        request.user = self.anonymous_user
+        self.session_middleware.process_request(request)
+        request.session['country_code'] = 'CN'
+        request.session['ip_address'] = '117.79.83.1'
+        self.country_middleware.process_request(request)
+        # Country code is changed.
+        self.assertEqual('SD', request.session.get('country_code'))
+        self.assertEqual('4.0.0.0', request.session.get('ip_address'))
+
+    def test_ip_address_is_not_changed(self):
+        request = self.request_factory.get('/somewhere',
+                                            HTTP_X_FORWARDED_FOR='117.79.83.1')
+        request.user = self.anonymous_user
+        self.session_middleware.process_request(request)
+        request.session['country_code'] = 'CN'
+        request.session['ip_address'] = '117.79.83.1'
+        self.country_middleware.process_request(request)
+        # Country code is not changed.
+        self.assertEqual('CN', request.session.get('country_code'))
+        self.assertEqual('117.79.83.1', request.session.get('ip_address'))
+
+    def test_same_country_different_ip(self):
+        request = self.request_factory.get('/somewhere',
+                                            HTTP_X_FORWARDED_FOR='117.79.83.100')
+        request.user = self.anonymous_user
+        self.session_middleware.process_request(request)
+        request.session['country_code'] = 'CN'
+        request.session['ip_address'] = '117.79.83.1'
+        self.country_middleware.process_request(request)
+        # Country code is not changed.
+        self.assertEqual('CN', request.session.get('country_code'))
+        self.assertEqual('117.79.83.100', request.session.get('ip_address'))

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -113,6 +113,11 @@ class InheritanceMixin(XBlockMixin):
         default=[],
         scope=Scope.settings
     )
+    video_speed_optimizations = Boolean(
+        help="Enable Video CDN.",
+        default=True,
+        scope=Scope.settings
+    )
 
 
 def compute_inherited_metadata(descriptor):

common/lib/xmodule/xmodule/tests/__init__.py

@@ -91,6 +91,7 @@ def get_test_system(course_id=SlashSeparatedCourseKey('org', 'course', 'run')):
         error_descriptor_class=ErrorDescriptor,
         get_user_role=Mock(is_staff=False),
         descriptor_runtime=get_test_descriptor_system(),
+        user_location=Mock(),
     )
 
 

common/lib/xmodule/xmodule/tests/test_video.py

@@ -15,12 +15,12 @@ the course, section, subsection, unit, etc.
 
 import unittest
 import datetime
-from mock import Mock
+from mock import Mock, patch
 
 from . import LogicTest
 from lxml import etree
 from opaque_keys.edx.locations import Location
-from xmodule.video_module import VideoDescriptor, create_youtube_string
+from xmodule.video_module import VideoDescriptor, create_youtube_string, get_video_from_cdn
 from .test_import import DummySystem
 from xblock.field_data import DictFieldData
 from xblock.fields import ScopeIds
@@ -563,3 +563,33 @@ class VideoExportTestCase(unittest.TestCase):
         expected = '<video url_name="SampleProblem1"/>\n'
 
         self.assertEquals(expected, etree.tostring(xml, pretty_print=True))
+
+
+class VideoCdnTest(unittest.TestCase):
+    """
+    Tests for Video CDN.
+    """
+    @patch('requests.get')
+    def test_get_video_success(self, cdn_response):
+        """
+        Test successful CDN request.
+        """
+        original_video_url = "http://www.original_video.com/original_video.mp4"
+        cdn_response_video_url = "http://www.cdn_video.com/cdn_video.mp4"
+        cdn_response_content = '{{"sources":["{cdn_url}"]}}'.format(cdn_url=cdn_response_video_url)
+        cdn_response.return_value=Mock(status_code=200, content=cdn_response_content)
+        fake_cdn_url = 'http://fake_cdn.com/'
+        self.assertEqual(
+            get_video_from_cdn(fake_cdn_url, original_video_url),
+            cdn_response_video_url
+        )
+
+    @patch('requests.get')
+    def test_get_no_video_exists(self, cdn_response):
+        """
+        Test if no alternative video in CDN exists.
+        """
+        original_video_url = "http://www.original_video.com/original_video.mp4"
+        cdn_response.return_value=Mock(status_code=404)
+        fake_cdn_url = 'http://fake_cdn.com/'
+        self.assertIsNone(get_video_from_cdn(fake_cdn_url, original_video_url))

common/lib/xmodule/xmodule/video_module/video_module.py

@@ -36,7 +36,7 @@ from xmodule.editing_module import TabsEditingDescriptor
 from xmodule.raw_module import EmptyDataRawDescriptor
 from xmodule.xml_module import is_pointer_tag, name_to_pathname, deserialize_field
 
-from .video_utils import create_youtube_string
+from .video_utils import create_youtube_string, get_video_from_cdn
 from .video_xfields import VideoFields
 from .video_handlers import VideoStudentViewHandlers, VideoStudioViewHandlers
 
@@ -93,12 +93,25 @@ class VideoModule(VideoFields, VideoStudentViewHandlers, XModule):
     ]}
     js_module_name = "Video"
 
+
     def get_html(self):
         track_url = None
         download_video_link = None
         transcript_download_format = self.transcript_download_format
         sources = filter(None, self.html5_sources)
 
+        # If the user comes from China use China CDN for html5 videos.
+        # 'CN' is China ISO 3166-1 country code.
+        # Video caching is disabled for Studio. User_location is always None in Studio.
+        # CountryMiddleware disabled for Studio.
+        cdn_url = getattr(settings, 'VIDEO_CDN_URL', {}).get(self.system.user_location)
+
+        if getattr(self, 'video_speed_optimizations', True) and cdn_url:
+            for index, source_url in enumerate(sources):
+                new_url = get_video_from_cdn(cdn_url, source_url)
+                if new_url:
+                    sources[index] = new_url
+
         if self.download_video:
             if self.source:
                 download_video_link = self.source

common/lib/xmodule/xmodule/video_module/video_utils.py

 """
 Module containts utils specific for video_module but not for transcripts.
 """
+import json
+import logging
+import urllib
+import requests
+
+from requests.exceptions import RequestException
+
+log = logging.getLogger(__name__)
 
 
 def create_youtube_string(module):
@@ -23,3 +31,41 @@ def create_youtube_string(module):
         in zip(youtube_speeds, youtube_ids)
         if pair[1]
     ])
+
+
+def get_video_from_cdn(cdn_base_url, original_video_url):
+    """
+    Get video URL from CDN.
+
+    `original_video_url` is the existing video url.
+    Currently `cdn_base_url` equals 'http://api.xuetangx.com/edx/video?s3_url='
+    Example of CDN outcome:
+        {
+            "sources":
+                [
+                    "http://cm12.c110.play.bokecc.com/flvs/ca/QxcVl/u39EQbA0Ra-20.mp4",
+                    "http://bm1.42.play.bokecc.com/flvs/ca/QxcVl/u39EQbA0Ra-20.mp4"
+                ],
+            "s3_url": "http://s3.amazonaws.com/BESTech/CS169/download/CS169_v13_w5l2s3.mp4"
+        }
+    where `s3_url` is requested original video url and `sources` is the list of
+    alternative links.
+    """
+
+    if not cdn_base_url:
+        return None
+
+    request_url = cdn_base_url + urllib.quote(original_video_url)
+
+    try:
+        cdn_response = requests.get(request_url, timeout=0.5)
+    except RequestException as err:
+        log.warning("Error requesting from CDN server at %s", request_url)
+        log.exception(err)
+        return None
+
+    if cdn_response.status_code == 200:
+        cdn_content = json.loads(cdn_response.content)
+        return cdn_content['sources'][0]
+    else:
+        return None

common/lib/xmodule/xmodule/x_module.py

@@ -1244,7 +1244,7 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):  # pylin
             cache=None, can_execute_unsafe_code=None, replace_course_urls=None,
             replace_jump_to_id_urls=None, error_descriptor_class=None, get_real_user=None,
             field_data=None, get_user_role=None, rebind_noauth_module_to_user=None,
-            **kwargs):
+            user_location=None, **kwargs):
         """
         Create a closure around the system environment.
 
@@ -1340,6 +1340,7 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):  # pylin
         self.xmodule_instance = None
 
         self.get_real_user = get_real_user
+        self.user_location = user_location
 
         self.get_user_role = get_user_role
         self.descriptor_runtime = descriptor_runtime

lms/djangoapps/courseware/module_render.py

@@ -218,17 +218,19 @@ def get_module_for_descriptor(user, request, descriptor, field_data_cache, cours
     track_function = make_track_function(request)
     xqueue_callback_url_prefix = get_xqueue_callback_url_prefix(request)
 
+    user_location = getattr(request, 'session', {}).get('country_code')
+
     return get_module_for_descriptor_internal(user, descriptor, field_data_cache, course_id,
                                               track_function, xqueue_callback_url_prefix,
                                               position, wrap_xmodule_display, grade_bucket_type,
-                                              static_asset_path)
+                                              static_asset_path, user_location)
 
 
 def get_module_system_for_user(user, field_data_cache,
                                # Arguments preceding this comment have user binding, those following don't
                                descriptor, course_id, track_function, xqueue_callback_url_prefix,
                                position=None, wrap_xmodule_display=True, grade_bucket_type=None,
-                               static_asset_path=''):
+                               static_asset_path='', user_location=None):
     """
     Helper function that returns a module system and student_data bound to a user and a descriptor.
 
@@ -310,7 +312,7 @@ def get_module_system_for_user(user, field_data_cache,
         return get_module_for_descriptor_internal(user, descriptor, field_data_cache, course_id,
                                                   track_function, make_xqueue_callback,
                                                   position, wrap_xmodule_display, grade_bucket_type,
-                                                  static_asset_path)
+                                                  static_asset_path, user_location)
 
     def handle_grade_event(block, event_type, event):
         user_id = event.get('user_id', user.id)
@@ -379,7 +381,7 @@ def get_module_system_for_user(user, field_data_cache,
         (inner_system, inner_student_data) = get_module_system_for_user(
             real_user, field_data_cache_real_user,  # These have implicit user bindings, rest of args considered not to
             module.descriptor, course_id, track_function, xqueue_callback_url_prefix, position, wrap_xmodule_display,
-            grade_bucket_type, static_asset_path
+            grade_bucket_type, static_asset_path, user_location
         )
         # rebinds module to a different student.  We'll change system, student_data, and scope_ids
         module.descriptor.bind_for_student(
@@ -500,6 +502,7 @@ def get_module_system_for_user(user, field_data_cache,
         get_user_role=lambda: get_user_role(user, course_id),
         descriptor_runtime=descriptor.runtime,
         rebind_noauth_module_to_user=rebind_noauth_module_to_user,
+        user_location=user_location,
     )
 
     # pass position specified in URL to module through ModuleSystem
@@ -525,7 +528,7 @@ def get_module_system_for_user(user, field_data_cache,
 def get_module_for_descriptor_internal(user, descriptor, field_data_cache, course_id,  # pylint: disable=invalid-name
                                        track_function, xqueue_callback_url_prefix,
                                        position=None, wrap_xmodule_display=True, grade_bucket_type=None,
-                                       static_asset_path=''):
+                                       static_asset_path='', user_location=None):
     """
     Actually implement get_module, without requiring a request.
 
@@ -541,7 +544,7 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
     (system, student_data) = get_module_system_for_user(
         user, field_data_cache,  # These have implicit user bindings, the rest of args are considered not to
         descriptor, course_id, track_function, xqueue_callback_url_prefix, position, wrap_xmodule_display,
-        grade_bucket_type, static_asset_path
+        grade_bucket_type, static_asset_path, user_location
     )
 
     descriptor.bind_for_student(system, LmsFieldData(descriptor._field_data, student_data))  # pylint: disable=protected-access

lms/djangoapps/courseware/tests/test_video_mongo.py

@@ -365,6 +365,105 @@ class TestGetHtmlMethod(BaseTestXmodule):
                 self.item_descriptor.xmodule_runtime.render_template('video.html', expected_context)
             )
 
+    @patch('xmodule.video_module.video_module.get_video_from_cdn')
+    def test_get_html_cdn_source(self, mocked_get_video):
+        """
+        Test if sources got from CDN.
+        """
+        def side_effect(*args, **kwargs):
+            cdn = {
+            'http://example.com/example.mp4': 'http://cdn_example.com/example.mp4',
+            'http://example.com/example.webm': 'http://cdn_example.com/example.webm',
+            }
+            return cdn.get(args[1])
+
+        mocked_get_video.side_effect = side_effect
+
+        SOURCE_XML = """
+            <video show_captions="true"
+            display_name="A Name"
+            sub="a_sub_file.srt.sjson" source="{source}"
+            download_video="{download_video}"
+            start_time="01:00:03" end_time="01:00:10"
+            >
+                {sources}
+            </video>
+        """
+        cases = [
+            #
+            {
+                'download_video': 'true',
+                'source': 'example_source.mp4',
+                'sources': """
+                    <source src="http://example.com/example.mp4"/>
+                    <source src="http://example.com/example.webm"/>
+                """,
+                'result': {
+                    'download_video_link': u'example_source.mp4',
+                    'sources': json.dumps(
+                        [
+                            u'http://cdn_example.com/example.mp4',
+                            u'http://cdn_example.com/example.webm'
+                        ]
+                    ),
+                },
+            },
+        ]
+
+        initial_context = {
+            'data_dir': getattr(self, 'data_dir', None),
+            'show_captions': 'true',
+            'handout': None,
+            'display_name': u'A Name',
+            'download_video_link': None,
+            'end': 3610.0,
+            'id': None,
+            'sources': '[]',
+            'speed': 'null',
+            'general_speed': 1.0,
+            'start': 3603.0,
+            'saved_video_position': 0.0,
+            'sub': u'a_sub_file.srt.sjson',
+            'track': None,
+            'youtube_streams': '1.00:OEoXaMPEzfM',
+            'autoplay': settings.FEATURES.get('AUTOPLAY_VIDEOS', True),
+            'yt_test_timeout': 1500,
+            'yt_api_url': 'www.youtube.com/iframe_api',
+            'yt_test_url': 'gdata.youtube.com/feeds/api/videos/',
+            'transcript_download_format': 'srt',
+            'transcript_download_formats_list': [{'display_name': 'SubRip (.srt) file', 'value': 'srt'}, {'display_name': 'Text (.txt) file', 'value': 'txt'}],
+            'transcript_language': u'en',
+            'transcript_languages': '{"en": "English"}',
+        }
+
+        for data in cases:
+            DATA = SOURCE_XML.format(
+                download_video=data['download_video'],
+                source=data['source'],
+                sources=data['sources']
+            )
+            self.initialize_module(data=DATA)
+            self.item_descriptor.xmodule_runtime.user_location = 'CN'
+
+            context = self.item_descriptor.render('student_view').content
+
+            expected_context = dict(initial_context)
+            expected_context.update({
+                'transcript_translation_url': self.item_descriptor.xmodule_runtime.handler_url(
+                    self.item_descriptor, 'transcript', 'translation'
+                ).rstrip('/?'),
+                'transcript_available_translations_url': self.item_descriptor.xmodule_runtime.handler_url(
+                    self.item_descriptor, 'transcript', 'available_translations'
+                ).rstrip('/?'),
+                'ajax_url': self.item_descriptor.xmodule_runtime.ajax_url + '/save_user_state',
+                'id': self.item_descriptor.location.html_id(),
+            })
+            expected_context.update(data['result'])
+
+            self.assertEqual(
+                context,
+                self.item_descriptor.xmodule_runtime.render_template('video.html', expected_context)
+            )
 
 class TestVideoDescriptorInitialization(BaseTestXmodule):
     """

lms/envs/aws.py

@@ -282,6 +282,10 @@ if FEATURES.get('AUTH_USE_CAS'):
 
 HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = ENV_TOKENS.get('HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS',{})
 
+# Video Caching. Pairing country codes with CDN URLs.
+# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='}
+VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {})
+
 ############################## SECURE AUTH ITEMS ###############
 # Secret things: passwords, access keys, etc.
 

lms/envs/common.py

@@ -784,6 +784,7 @@ MIDDLEWARE_CLASSES = (
 
     # Allows us to dark-launch particular languages
     'dark_lang.middleware.DarkLangMiddleware',
+    'geoinfo.middleware.CountryMiddleware',
     'embargo.middleware.EmbargoMiddleware',
 
     # Allows us to set user preferences

lms/envs/test.py

@@ -326,3 +326,7 @@ VERIFY_STUDENT["SOFTWARE_SECURE"] = {
         "API_ACCESS_KEY": "BBBBBBBBBBBBBBBBBBBB",
         "API_SECRET_KEY": "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC",
 }
+
+VIDEO_CDN_URL = {
+    'CN': 'http://api.xuetangx.com/edx/video?s3_url='
+}