Merge pull request #5781 from mulby/mobile-eventing-v2

Implement shim for mobile video events.

common/djangoapps/track/shim.py

 """Map new event context values to old top-level field values. Ensures events can be parsed by legacy parsers."""
 
+import json
+import logging
+
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import UsageKey
+
+
+log = logging.getLogger(__name__)
+
 CONTEXT_FIELDS_TO_INCLUDE = [
     'username',
     'session',
@@ -13,33 +22,39 @@ class LegacyFieldMappingProcessor(object):
     """Ensures all required fields are included in emitted events"""
 
     def __call__(self, event):
+        context = event.get('context', {})
         if 'context' in event:
-            context = event['context']
             for field in CONTEXT_FIELDS_TO_INCLUDE:
-                if field in context:
-                    event[field] = context[field]
-                else:
-                    event[field] = ''
+                self.move_from_context(field, event)
             remove_shim_context(event)
 
-        if 'event_type' in event.get('context', {}):
-            event['event_type'] = event['context']['event_type']
-            del event['context']['event_type']
-        else:
-            event['event_type'] = event.get('name', '')
-
         if 'data' in event:
             event['event'] = event['data']
             del event['data']
         else:
             event['event'] = {}
 
-        if 'timestamp' in event:
+        if 'timestamp' in context:
+            event['time'] = context['timestamp']
+            del context['timestamp']
+        elif 'timestamp' in event:
             event['time'] = event['timestamp']
+
+        if 'timestamp' in event:
             del event['timestamp']
 
-        event['event_source'] = 'server'
-        event['page'] = None
+        self.move_from_context('event_type', event, event.get('name', ''))
+        self.move_from_context('event_source', event, 'server')
+        self.move_from_context('page', event, None)
+
+    def move_from_context(self, field, event, default_value=''):
+        """Move a field from the context to the top level of the event."""
+        context = event.get('context', {})
+        if field in context:
+            event[field] = context[field]
+            del context[field]
+        else:
+            event[field] = default_value
 
 
 def remove_shim_context(event):
@@ -52,3 +67,66 @@ def remove_shim_context(event):
         for field in context_fields_to_remove:
             if field in context:
                 del context[field]
+
+
+NAME_TO_EVENT_TYPE_MAP = {
+    'edx.video.played': 'play_video',
+    'edx.video.paused': 'pause_video',
+    'edx.video.stopped': 'stop_video',
+    'edx.video.loaded': 'load_video',
+    'edx.video.transcript.shown': 'show_transcript',
+    'edx.video.transcript.hidden': 'hide_transcript',
+}
+
+
+class VideoEventProcessor(object):
+    """
+    Converts new format video events into the legacy video event format.
+
+    Mobile devices cannot actually emit events that exactly match their counterparts emitted by the LMS javascript
+    video player. Instead of attempting to get them to do that, we instead insert a shim here that converts the events
+    they *can* easily emit and converts them into the legacy format.
+
+    TODO: Remove this shim and perform the conversion as part of some batch canonicalization process.
+
+    """
+
+    def __call__(self, event):
+        name = event.get('name')
+        if not name:
+            return
+
+        if name not in NAME_TO_EVENT_TYPE_MAP:
+            return
+
+        event['event_type'] = NAME_TO_EVENT_TYPE_MAP[name]
+
+        if 'event' not in event:
+            return
+
+        payload = event['event']
+
+        if 'module_id' in payload:
+            module_id = payload['module_id']
+            try:
+                usage_key = UsageKey.from_string(module_id)
+            except InvalidKeyError:
+                log.warning('Unable to parse module_id "%s"', module_id, exc_info=True)
+            else:
+                payload['id'] = usage_key.html_id()
+
+            del payload['module_id']
+
+        if 'current_time' in payload:
+            payload['currentTime'] = payload.pop('current_time')
+
+        event['event'] = json.dumps(payload)
+
+        if 'context' not in event:
+            return
+
+        context = event['context']
+
+        if 'browser_page' in context:
+            page, _sep, _tail = context.pop('browser_page').rpartition('/')
+            event['page'] = page

common/djangoapps/track/tests/__init__.py

+"""Helpers for tests related to emitting events to the tracking logs."""
+
+from datetime import datetime
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from freezegun import freeze_time
+from pytz import UTC
+
+from eventtracking import tracker
+from eventtracking.django import DjangoTracker
+
+
+FROZEN_TIME = datetime(2013, 10, 3, 8, 24, 55, tzinfo=UTC)
+IN_MEMORY_BACKEND_CONFIG = {
+    'mem': {
+        'ENGINE': 'track.tests.InMemoryBackend'
+    }
+}
+
+
+class InMemoryBackend(object):
+    """A backend that simply stores all events in memory"""
+
+    def __init__(self):
+        super(InMemoryBackend, self).__init__()
+        self.events = []
+
+    def send(self, event):
+        """Store the event in a list"""
+        self.events.append(event)
+
+
+@freeze_time(FROZEN_TIME)
+@override_settings(
+    EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND_CONFIG
+)
+class EventTrackingTestCase(TestCase):
+    """
+    Supports capturing of emitted events in memory and inspecting them.
+
+    Each test gets a "clean slate" and can retrieve any events emitted during their execution.
+
+    """
+
+    # Make this more robust to the addition of new events that the test doesn't care about.
+
+    def setUp(self):
+        super(EventTrackingTestCase, self).setUp()
+
+        self.tracker = DjangoTracker()
+        tracker.register_tracker(self.tracker)
+
+    @property
+    def backend(self):
+        """A reference to the in-memory backend that stores the events."""
+        return self.tracker.backends['mem']
+
+    def get_event(self, idx=0):
+        """Retrieve an event emitted up to this point in the test."""
+        return self.backend.events[idx]
+
+    def assert_no_events_emitted(self):
+        """Ensure no events were emitted at this point in the test."""
+        self.assertEquals(len(self.backend.events), 0)
+
+    def assert_events_emitted(self):
+        """Ensure at least one event has been emitted at this point in the test."""
+        self.assertGreaterEqual(len(self.backend.events), 1)

common/djangoapps/track/tests/test_shim.py

 """Ensure emitted events contain the fields legacy processors expect to find."""
 
-from datetime import datetime
-
-from freezegun import freeze_time
 from mock import sentinel
-from django.test import TestCase
 from django.test.utils import override_settings
-from pytz import UTC
-
-from eventtracking.django import DjangoTracker
 
+from track.tests import EventTrackingTestCase, FROZEN_TIME
 
-IN_MEMORY_BACKEND = {
-    'mem': {
-        'ENGINE': 'track.tests.test_shim.InMemoryBackend'
-    }
-}
 
 LEGACY_SHIM_PROCESSOR = [
     {
@@ -23,20 +12,14 @@ LEGACY_SHIM_PROCESSOR = [
     }
 ]
 
-FROZEN_TIME = datetime(2013, 10, 3, 8, 24, 55, tzinfo=UTC)
 
-
-@freeze_time(FROZEN_TIME)
-class LegacyFieldMappingProcessorTestCase(TestCase):
+class LegacyFieldMappingProcessorTestCase(EventTrackingTestCase):
     """Ensure emitted events contain the fields legacy processors expect to find."""
 
     @override_settings(
-        EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND,
         EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR,
     )
     def test_event_field_mapping(self):
-        django_tracker = DjangoTracker()
-
         data = {sentinel.key: sentinel.value}
 
         context = {
@@ -49,16 +32,15 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
             'user_id': sentinel.user_id,
             'course_id': sentinel.course_id,
             'org_id': sentinel.org_id,
-            'event_type': sentinel.event_type,
             'client_id': sentinel.client_id,
         }
-        with django_tracker.context('test', context):
-            django_tracker.emit(sentinel.name, data)
+        with self.tracker.context('test', context):
+            self.tracker.emit(sentinel.name, data)
 
-        emitted_event = django_tracker.backends['mem'].get_event()
+        emitted_event = self.get_event()
 
         expected_event = {
-            'event_type': sentinel.event_type,
+            'event_type': sentinel.name,
             'name': sentinel.name,
             'context': {
                 'user_id': sentinel.user_id,
@@ -79,15 +61,12 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
         self.assertEqual(expected_event, emitted_event)
 
     @override_settings(
-        EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND,
         EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR,
     )
     def test_missing_fields(self):
-        django_tracker = DjangoTracker()
+        self.tracker.emit(sentinel.name)
 
-        django_tracker.emit(sentinel.name)
-
-        emitted_event = django_tracker.backends['mem'].get_event()
+        emitted_event = self.get_event()
 
         expected_event = {
             'event_type': sentinel.name,
@@ -104,19 +83,3 @@ class LegacyFieldMappingProcessorTestCase(TestCase):
             'session': '',
         }
         self.assertEqual(expected_event, emitted_event)
-
-
-class InMemoryBackend(object):
-    """A backend that simply stores all events in memory"""
-
-    def __init__(self):
-        super(InMemoryBackend, self).__init__()
-        self.events = []
-
-    def send(self, event):
-        """Store the event in a list"""
-        self.events.append(event)
-
-    def get_event(self):
-        """Return the first event that was emitted."""
-        return self.events[0]

lms/envs/aws.py

@@ -396,9 +396,8 @@ STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUD
 TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))
 EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {}))
 TRACKING_SEGMENTIO_WEBHOOK_SECRET = AUTH_TOKENS.get("TRACKING_SEGMENTIO_WEBHOOK_SECRET", TRACKING_SEGMENTIO_WEBHOOK_SECRET)
-TRACKING_SEGMENTIO_ALLOWED_ACTIONS = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_ACTIONS", TRACKING_SEGMENTIO_ALLOWED_ACTIONS)
-TRACKING_SEGMENTIO_ALLOWED_CHANNELS = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_CHANNELS", TRACKING_SEGMENTIO_ALLOWED_CHANNELS)
-
+TRACKING_SEGMENTIO_ALLOWED_TYPES = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_TYPES", TRACKING_SEGMENTIO_ALLOWED_TYPES)
+TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP)
 
 # Student identity verification settings
 VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT)

lms/envs/common.py

@@ -506,6 +506,9 @@ EVENT_TRACKING_BACKENDS = {
 EVENT_TRACKING_PROCESSORS = [
     {
         'ENGINE': 'track.shim.LegacyFieldMappingProcessor'
+    },
+    {
+        'ENGINE': 'track.shim.VideoEventProcessor'
     }
 ]
 
@@ -524,8 +527,11 @@ if FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
     })
 
 TRACKING_SEGMENTIO_WEBHOOK_SECRET = None
-TRACKING_SEGMENTIO_ALLOWED_ACTIONS = ['Track', 'Screen']
-TRACKING_SEGMENTIO_ALLOWED_CHANNELS = ['mobile']
+TRACKING_SEGMENTIO_ALLOWED_TYPES = ['track']
+TRACKING_SEGMENTIO_SOURCE_MAP = {
+    'analytics-android': 'mobile',
+    'analytics-ios': 'mobile',
+}
 
 ######################## GOOGLE ANALYTICS ###########################
 GOOGLE_ANALYTICS_ACCOUNT = None

lms/urls.py

@@ -28,7 +28,7 @@ urlpatterns = ('',  # nopep8
     url(r'^reject_name_change$', 'student.views.reject_name_change'),
     url(r'^pending_name_changes$', 'student.views.pending_name_changes'),
     url(r'^event$', 'track.views.user_track'),
-    url(r'^segmentio/event$', 'track.views.segmentio.track_segmentio_event'),
+    url(r'^segmentio/event$', 'track.views.segmentio.segmentio_event'),
     url(r'^t/(?P<template>[^/]*)$', 'static_template_view.views.index'),   # TODO: Is this used anymore? What is STATIC_GRAB?
 
     url(r'^accounts/login$', 'student.views.accounts_login', name="accounts_login"),