Implement shim for mobile video events.

This PR addresses the following issues:

1) All requests return a 200 OK unless there is an authorization failure. This is deliberate in case the secret key is compromised.
2) Push all of the nasty logic necessary to generate compatible video events into the LMS instead of trying to do that mapping on the mobile devices.
3) Stop using the deprecated "action" field in the segment.io event. According to their support team this field should not be used anymore and is just around for backwards compatibility reasons.

Fixes: AN-3818

common/djangoapps/track/shim.py

 """Map new event context values to old top-level field values. Ensures events can be parsed by legacy parsers."""
 
+import json
+import logging
+
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import UsageKey
+
+
+log = logging.getLogger(__name__)
+
 CONTEXT_FIELDS_TO_INCLUDE = [
     'username',
     'session',
@@ -13,33 +22,39 @@ class LegacyFieldMappingProcessor(object):
     """Ensures all required fields are included in emitted events"""
 
     def __call__(self, event):
+        context = event.get('context', {})
         if 'context' in event:
-            context = event['context']
             for field in CONTEXT_FIELDS_TO_INCLUDE:
-                if field in context:
-                    event[field] = context[field]
-                else:
-                    event[field] = ''
+                self.move_from_context(field, event)
             remove_shim_context(event)
 
-        if 'event_type' in event.get('context', {}):
-            event['event_type'] = event['context']['event_type']
-            del event['context']['event_type']
-        else:
-            event['event_type'] = event.get('name', '')
-
         if 'data' in event:
             event['event'] = event['data']
             del event['data']
         else:
             event['event'] = {}
 
-        if 'timestamp' in event:
+        if 'timestamp' in context:
+            event['time'] = context['timestamp']
+            del context['timestamp']
+        elif 'timestamp' in event:
             event['time'] = event['timestamp']
+
+        if 'timestamp' in event:
             del event['timestamp']
 
-        event['event_source'] = 'server'
-        event['page'] = None
+        self.move_from_context('event_type', event, event.get('name', ''))
+        self.move_from_context('event_source', event, 'server')
+        self.move_from_context('page', event, None)
+
+    def move_from_context(self, field, event, default_value=''):
+        """Move a field from the context to the top level of the event."""
+        context = event.get('context', {})
+        if field in context:
+            event[field] = context[field]
+            del context[field]
+        else:
+            event[field] = default_value
 
 
 def remove_shim_context(event):
@@ -52,3 +67,66 @@ def remove_shim_context(event):
         for field in context_fields_to_remove:
             if field in context:
                 del context[field]
+
+
+NAME_TO_EVENT_TYPE_MAP = {
+    'edx.video.played': 'play_video',
+    'edx.video.paused': 'pause_video',
+    'edx.video.stopped': 'stop_video',
+    'edx.video.loaded': 'load_video',
+    'edx.video.transcript.shown': 'show_transcript',
+    'edx.video.transcript.hidden': 'hide_transcript',
+}
+
+
+class VideoEventProcessor(object):
+    """
+    Converts new format video events into the legacy video event format.
+
+    Mobile devices cannot actually emit events that exactly match their counterparts emitted by the LMS javascript
+    video player. Instead of attempting to get them to do that, we instead insert a shim here that converts the events
+    they *can* easily emit and converts them into the legacy format.
+
+    TODO: Remove this shim and perform the conversion as part of some batch canonicalization process.
+
+    """
+
+    def __call__(self, event):
+        name = event.get('name')
+        if not name:
+            return
+
+        if name not in NAME_TO_EVENT_TYPE_MAP:
+            return
+
+        event['event_type'] = NAME_TO_EVENT_TYPE_MAP[name]
+
+        if 'event' not in event:
+            return
+
+        payload = event['event']
+
+        if 'module_id' in payload:
+            module_id = payload['module_id']
+            try:
+                usage_key = UsageKey.from_string(module_id)
+            except InvalidKeyError:
+                log.warning('Unable to parse module_id "%s"', module_id, exc_info=True)
+            else:
+                payload['id'] = usage_key.html_id()
+
+            del payload['module_id']
+
+        if 'current_time' in payload:
+            payload['currentTime'] = payload.pop('current_time')
+
+        event['event'] = json.dumps(payload)
+
+        if 'context' not in event:
+            return
+
+        context = event['context']
+
+        if 'browser_page' in context:
+            page, _sep, _tail = context.pop('browser_page').rpartition('/')
+            event['page'] = page

common/djangoapps/track/tests/__init__.py

+"""Helpers for tests related to emitting events to the tracking logs."""
+
+from datetime import datetime
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from freezegun import freeze_time
+from pytz import UTC
+
+from eventtracking import tracker
+from eventtracking.django import DjangoTracker
+
+
+FROZEN_TIME = datetime(2013, 10, 3, 8, 24, 55, tzinfo=UTC)
+IN_MEMORY_BACKEND_CONFIG = {
+    'mem': {
+        'ENGINE': 'track.tests.InMemoryBackend'
+    }
+}
+
+
+class InMemoryBackend(object):
+    """A backend that simply stores all events in memory"""
+
+    def __init__(self):
+        super(InMemoryBackend, self).__init__()
+        self.events = []
+
+    def send(self, event):
+        """Store the event in a list"""
+        self.events.append(event)
+
+
+@freeze_time(FROZEN_TIME)
+@override_settings(
+    EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND_CONFIG
+)
+class EventTrackingTestCase(TestCase):
+    """
+    Supports capturing of emitted events in memory and inspecting them.
+
+    Each test gets a "clean slate" and can retrieve any events emitted during their execution.
+
+    """
+
+    # Make this more robust to the addition of new events that the test doesn't care about.
+
+    def setUp(self):
+        super(EventTrackingTestCase, self).setUp()
+
+        self.tracker = DjangoTracker()
+        tracker.register_tracker(self.tracker)
+
+    @property
+    def backend(self):
+        """A reference to the in-memory backend that stores the events."""
+        return self.tracker.backends['mem']
+
+    def get_event(self, idx=0):
+        """Retrieve an event emitted up to this point in the test."""
+        return self.backend.events[idx]
+
+    def assert_no_events_emitted(self):
+        """Ensure no events were emitted at this point in the test."""
+        self.assertEquals(len(self.backend.events), 0)
+
+    def assert_events_emitted(self):
+        """Ensure at least one event has been emitted at this point in the test."""
+        self.assertGreaterEqual(len(self.backend.events), 1)

common/djangoapps/track/tests/test_shim.py

 """Ensure emitted events contain the fields legacy processors expect to find."""
 
-from datetime import datetime
-
-from freezegun import freeze_time
 from mock import sentinel
-from django.test import TestCase
 from django.test.utils import override_settings
-from pytz import UTC
-
-from eventtracking.django import DjangoTracker
 
+from track.tests import EventTrackingTestCase, FROZEN_TIME
 
-IN_MEMORY_BACKEND = {
-    'mem': {
-        'ENGINE': 'track.tests.test_shim.InMemoryBackend'
-    }
-}
 
 LEGACY_SHIM_PROCESSOR = [
     {
@@ -23,20 +12,14 @@ LEGACY_SHIM_PROCESSOR = [
     }
 ]
 
-FROZEN_TIME = datetime(2013, 10, 3, 8, 24, 55, tzinfo=UTC)
 
-
-@freeze_time(FROZEN_TIME)
-class LegacyFieldMappingProcessorTestCase(TestCase):
+class LegacyFieldMappingProcessorTestCase(EventTrackingTestCase):
     """Ensure emitted events contain the fields legacy processors expect to find."""
 
     @override_settings(
-        EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND,
         EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR,
     )
     def test_event_field_mapping(self):
-        django_tracker = DjangoTracker()
-
         data = {sentinel.key: sentinel.value}
 
         context = {
@@ -49,16 +32,15 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
             'user_id': sentinel.user_id,
             'course_id': sentinel.course_id,
             'org_id': sentinel.org_id,
-            'event_type': sentinel.event_type,
             'client_id': sentinel.client_id,
         }
-        with django_tracker.context('test', context):
-            django_tracker.emit(sentinel.name, data)
+        with self.tracker.context('test', context):
+            self.tracker.emit(sentinel.name, data)
 
-        emitted_event = django_tracker.backends['mem'].get_event()
+        emitted_event = self.get_event()
 
         expected_event = {
-            'event_type': sentinel.event_type,
+            'event_type': sentinel.name,
             'name': sentinel.name,
             'context': {
                 'user_id': sentinel.user_id,
@@ -79,15 +61,12 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
         self.assertEqual(expected_event, emitted_event)
 
     @override_settings(
-        EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND,
         EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR,
     )
     def test_missing_fields(self):
-        django_tracker = DjangoTracker()
+        self.tracker.emit(sentinel.name)
 
-        django_tracker.emit(sentinel.name)
-
-        emitted_event = django_tracker.backends['mem'].get_event()
+        emitted_event = self.get_event()
 
         expected_event = {
             'event_type': sentinel.name,
@@ -104,19 +83,3 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
             'session': '',
         }
         self.assertEqual(expected_event, emitted_event)
-
-
-class InMemoryBackend(object):
-    """A backend that simply stores all events in memory"""
-
-    def __init__(self):
-        super(InMemoryBackend, self).__init__()
-        self.events = []
-
-    def send(self, event):
-        """Store the event in a list"""
-        self.events.append(event)
-
-    def get_event(self):
-        """Return the first event that was emitted."""
-        return self.events[0]

lms/envs/aws.py

@@ -396,9 +396,8 @@ STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUD
 TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))
 EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {}))
 TRACKING_SEGMENTIO_WEBHOOK_SECRET = AUTH_TOKENS.get("TRACKING_SEGMENTIO_WEBHOOK_SECRET", TRACKING_SEGMENTIO_WEBHOOK_SECRET)
-TRACKING_SEGMENTIO_ALLOWED_ACTIONS = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_ACTIONS", TRACKING_SEGMENTIO_ALLOWED_ACTIONS)
-TRACKING_SEGMENTIO_ALLOWED_CHANNELS = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_CHANNELS", TRACKING_SEGMENTIO_ALLOWED_CHANNELS)
-
+TRACKING_SEGMENTIO_ALLOWED_TYPES = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_TYPES", TRACKING_SEGMENTIO_ALLOWED_TYPES)
+TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP)
 
 # Student identity verification settings
 VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT)

lms/envs/common.py

@@ -506,6 +506,9 @@ EVENT_TRACKING_BACKENDS = {
 EVENT_TRACKING_PROCESSORS = [
     {
         'ENGINE': 'track.shim.LegacyFieldMappingProcessor'
+    },
+    {
+        'ENGINE': 'track.shim.VideoEventProcessor'
     }
 ]
 
@@ -524,8 +527,11 @@ if FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
     })
 
 TRACKING_SEGMENTIO_WEBHOOK_SECRET = None
-TRACKING_SEGMENTIO_ALLOWED_ACTIONS = ['Track', 'Screen']
-TRACKING_SEGMENTIO_ALLOWED_CHANNELS = ['mobile']
+TRACKING_SEGMENTIO_ALLOWED_TYPES = ['track']
+TRACKING_SEGMENTIO_SOURCE_MAP = {
+    'analytics-android': 'mobile',
+    'analytics-ios': 'mobile',
+}
 
 ######################## GOOGLE ANALYTICS ###########################
 GOOGLE_ANALYTICS_ACCOUNT = None

lms/urls.py

@@ -28,7 +28,7 @@ urlpatterns = ('',  # nopep8
     url(r'^reject_name_change$', 'student.views.reject_name_change'),
     url(r'^pending_name_changes$', 'student.views.pending_name_changes'),
     url(r'^event$', 'track.views.user_track'),
-    url(r'^segmentio/event$', 'track.views.segmentio.track_segmentio_event'),
+    url(r'^segmentio/event$', 'track.views.segmentio.segmentio_event'),
     url(r'^t/(?P<template>[^/]*)$', 'static_template_view.views.index'),   # TODO: Is this used anymore? What is STATIC_GRAB?
 
     url(r'^accounts/login$', 'student.views.accounts_login', name="accounts_login"),