Revert "Revert "Merge pull request #4545 from edx/renzo/bi-analytics-overhaul""

This reverts commit 079808ee.

common/djangoapps/student/models.py

@@ -47,6 +47,8 @@ from course_modes.models import CourseMode
 
 from ratelimitbackend import admin
 
+import analytics
+
 unenroll_done = Signal(providing_args=["course_enrollment"])
 log = logging.getLogger(__name__)
 AUDIT_LOG = logging.getLogger("audit")
@@ -706,6 +708,7 @@ class CourseEnrollment(models.Model):
 
         if activation_changed or mode_changed:
             self.save()
+
         if activation_changed:
             if self.is_active:
                 self.emit_event(EVENT_NAME_ENROLLMENT_ACTIVATED)
@@ -719,7 +722,7 @@ class CourseEnrollment(models.Model):
 
             else:
                 unenroll_done.send(sender=None, course_enrollment=self)
-
+                
                 self.emit_event(EVENT_NAME_ENROLLMENT_DEACTIVATED)
 
                 dog_stats_api.increment(
@@ -749,6 +752,16 @@ class CourseEnrollment(models.Model):
 
             with tracker.get_tracker().context(event_name, context):
                 tracker.emit(event_name, data)
+
+                if settings.FEATURES.get('SEGMENT_IO_LMS') and settings.SEGMENT_IO_LMS_KEY:
+                    analytics.track(self.user_id, event_name, {
+                        'category': 'conversion',
+                        'label': self.course_id.to_deprecated_string(),
+                        'org': self.course_id.org,
+                        'course': self.course_id.course,
+                        'run': self.course_id.run,
+                        'mode': self.mode,
+                    })
         except:  # pylint: disable=bare-except
             if event_name and self.course_id:
                 log.exception('Unable to emit event %s for user %s and course %s', event_name, self.user.username, self.course_id)
@@ -773,6 +786,8 @@ class CourseEnrollment(models.Model):
 
         It is expected that this method is called from a method which has already
         verified the user authentication and access.
+
+        Also emits relevant events for analytics purposes.
         """
         enrollment = cls.get_or_create_enrollment(user, course_key)
         enrollment.update_enrollment(is_active=True, mode=mode)

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -300,7 +300,8 @@ class @Problem
     Logger.log 'problem_check', @answers
 
     # Segment.io
-    analytics.track "Problem Checked",
+    analytics.track "edx.bi.course.problem.checked",
+      category: "courseware"
       problem_id: @id
       answers: @answers
 

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -128,7 +128,8 @@ class @Sequence
       analytics.pageview @id
 
       # navigation by clicking the tab directly
-      analytics.track "Accessed Sequential Directly",
+      analytics.track "edx.bi.course.sequential.direct.clicked",
+        category: "courseware"
         sequence_id: @id
         current_sequential: @position
         target_sequential: new_position
@@ -167,9 +168,10 @@ class @Sequence
 
     # navigation using the next or previous arrow button.
     tracking_messages =
-      seq_prev: "Accessed Previous Sequential"
-      seq_next: "Accessed Next Sequential"
+      seq_prev: "edx.bi.course.sequential.previous.clicked"
+      seq_next: "edx.bi.course.sequential.next.clicked"
     analytics.track tracking_messages[direction],
+      category: "courseware"
       sequence_id: @id
       current_sequential: @position
       target_sequential: new_position

common/static/js/spec/utility_spec.js

@@ -16,3 +16,26 @@ describe('utility.rewriteStaticLinks', function () {
         ).toBe('<img src="http://www.mysite.org/static/foo.x"/>')
     });
 });
+
+describe('utility.appendParameter', function() {
+    it('creates and populates query string with provided parameter', function() {
+        expect(appendParameter('/cambridge', 'season', 'fall')).toBe('/cambridge?season=fall')
+    });
+    it('appends provided parameter to existing query string parameters', function() {
+        expect(appendParameter('/cambridge?season=fall', 'color', 'red')).toBe('/cambridge?season=fall&color=red')
+    });
+    it('appends provided parameter to existing query string with a trailing ampersand', function() {
+        expect(appendParameter('/cambridge?season=fall&', 'color', 'red')).toBe('/cambridge?season=fall&color=red')
+    });
+    it('overwrites existing parameter with provided value', function() {
+        expect(appendParameter('/cambridge?season=fall', 'season', 'winter')).toBe('/cambridge?season=winter');
+        expect(appendParameter('/cambridge?season=fall&color=red', 'color', 'orange')).toBe('/cambridge?season=fall&color=orange');
+    });
+});
+
+describe('utility.parseQueryString', function() {
+    it('converts a non-empty query string into a key/value object', function() {
+        expect(JSON.stringify(parseQueryString('season=fall'))).toBe(JSON.stringify({season:'fall'}));
+        expect(JSON.stringify(parseQueryString('season=fall&color=red'))).toBe(JSON.stringify({season:'fall', color:'red'}));
+    });
+});

common/static/js/src/utility.js

@@ -38,4 +38,129 @@ window.rewriteStaticLinks = function(content, from, to) {
     // note: add other protocols here
     var regex = new RegExp("(https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}([-a-zA-Z0-9@:%_\+.~#?&//=]*))?"+from, 'g');
     return content.replace(regex, replacer);
-};
\ No newline at end of file
+};
+
+// Appends a parameter to a path; useful for indicating initial or return signin, for example
+window.appendParameter = function(path, key, value) {
+    // Check if the given path already contains a query string by looking for the ampersand separator
+    if (path.indexOf("?") > -1) {
+        var splitPath = path.split("?");
+        var parameters = window.parseQueryString(splitPath[1]);
+        // Check if the provided key already exists in the query string
+        if (key in parameters) {
+            // Overwrite the existing key's value with the provided value
+            parameters[key] = value;
+
+            // Reconstruct the path, including the overwritten key/value pair
+            var reconstructedPath = splitPath[0] + "?";
+            for (var k in parameters) {
+                reconstructedPath = reconstructedPath + k + "=" + parameters[k] + "&";
+            }
+            // Strip the trailing ampersand
+            return reconstructedPath.slice(0, -1);
+        } else {
+            // Check for a trailing ampersand
+            if (path[path.length - 1] != "&") {
+                // Append signin parameter to the existing query string
+                return path + "&" + key + "=" + value;
+            } else {
+                // Append signin parameter to the existing query string, excluding the ampersand
+                return path + key + "=" + value;
+            }
+        }
+    } else {
+        // Append new query string containing the provided parameter
+        return path + "?" + key + "=" + value;
+    }
+};
+
+// Convert a query string to a key/value object
+window.parseQueryString = function(queryString) {
+    var parameters = {}, queries, pair, i, l;
+
+    // Split the query string into key/value pairs
+    queries = queryString.split("&");
+
+    // Break the array of strings into an object
+    for (i = 0, l = queries.length; i < l; i++) {
+        pair = queries[i].split('=');
+        parameters[pair[0]] = pair[1];
+    }
+
+    return parameters
+};
+
+// Check if the user recently enrolled in a course by looking at a referral URL
+window.checkRecentEnrollment = function(referrer) {
+    var enrolledIn = null;
+    
+    // Check if the referrer URL contains a query string
+    if (referrer.indexOf("?") > -1) {
+        referrerQueryString = referrer.split("?")[1];
+    } else {
+        referrerQueryString = "";
+    }
+
+    if (referrerQueryString != "") {
+        // Convert a non-empty query string into a key/value object
+        var referrerParameters = window.parseQueryString(referrerQueryString);
+        if ("course_id" in referrerParameters && "enrollment_action" in referrerParameters) {
+            if (referrerParameters.enrollment_action == "enroll") {
+                enrolledIn = referrerParameters.course_id;
+            }
+        }
+    }
+
+    return enrolledIn
+};
+
+window.assessUserSignIn = function(parameters, userID, email, username) {
+    // Check if the user has logged in to enroll in a course - designed for when "Register" button registers users on click (currently, this could indicate a course registration when there may not have yet been one)
+    var enrolledIn = window.checkRecentEnrollment(document.referrer);
+
+    // Check if the user has just registered
+    if (parameters.signin == "initial") {
+        window.trackAccountRegistration(enrolledIn, userID, email, username);
+    } else {
+        window.trackReturningUserSignIn(enrolledIn, userID, email, username);
+    }
+};
+
+window.trackAccountRegistration = function(enrolledIn, userID, email, username) {
+    // Alias the user's anonymous history with the user's new identity (for Mixpanel)
+    analytics.alias(userID);
+
+    // Map the user's activity to their newly assigned ID
+    analytics.identify(userID, {
+        email: email,
+        username: username
+    });
+
+    // Track the user's account creation
+    analytics.track("edx.bi.user.account.registered", {
+        category: "conversion",
+        label: enrolledIn != null ? enrolledIn : "none"
+    });
+};
+
+window.trackReturningUserSignIn = function(enrolledIn, userID, email, username) {
+    // Map the user's activity to their assigned ID
+    analytics.identify(userID, {
+        email: email,
+        username: username
+    });
+
+    // Track the user's sign in
+    analytics.track("edx.bi.user.account.authenticated", {
+        category: "conversion",
+        label: enrolledIn != null ? enrolledIn : "none"
+    });
+};
+
+window.identifyUser = function(userID, email, username) {
+    // If the signin parameter isn't present but the query string is non-empty, map the user's activity to their assigned ID
+    analytics.identify(userID, {
+        email: email,
+        username: username
+    });    
+};

lms/djangoapps/class_dashboard/dashboard_data.py

@@ -10,7 +10,7 @@ from django.utils.translation import ugettext as _
 
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.inheritance import own_metadata
-from analytics.csvs import create_csv_response
+from instructor_analytics.csvs import create_csv_response
 
 from opaque_keys.edx.locations import Location
 

lms/djangoapps/instructor/views/api.py

@@ -47,9 +47,9 @@ from instructor.enrollment import (
 )
 from instructor.access import list_with_level, allow_access, revoke_access, update_forum_role
 from instructor.offline_gradecalc import student_grades
-import analytics.basic
-import analytics.distributions
-import analytics.csvs
+import instructor_analytics.basic
+import instructor_analytics.distributions
+import instructor_analytics.csvs
 import csv
 
 from submissions import api as sub_api # installed from the edx-submissions repository
@@ -538,7 +538,7 @@ def get_grading_config(request, course_id):
     course = get_course_with_access(
         request.user, 'staff', course_id, depth=None
     )
-    grading_config_summary = analytics.basic.dump_grading_context(course)
+    grading_config_summary = instructor_analytics.basic.dump_grading_context(course)
 
     response_payload = {
         'course_id': course_id.to_deprecated_string(),
@@ -561,14 +561,14 @@ def get_students_features(request, course_id, csv=False):  # pylint: disable=W06
     """
     course_id = SlashSeparatedCourseKey.from_deprecated_string(course_id)
 
-    available_features = analytics.basic.AVAILABLE_FEATURES
+    available_features = instructor_analytics.basic.AVAILABLE_FEATURES
     query_features = [
         'id', 'username', 'name', 'email', 'language', 'location',
         'year_of_birth', 'gender', 'level_of_education', 'mailing_address',
         'goals',
     ]
 
-    student_data = analytics.basic.enrolled_students_features(course_id, query_features)
+    student_data = instructor_analytics.basic.enrolled_students_features(course_id, query_features)
 
     # Provide human-friendly and translatable names for these features. These names
     # will be displayed in the table generated in data_download.coffee. It is not (yet)
@@ -598,8 +598,8 @@ def get_students_features(request, course_id, csv=False):  # pylint: disable=W06
         }
         return JsonResponse(response_payload)
     else:
-        header, datarows = analytics.csvs.format_dictlist(student_data, query_features)
-        return analytics.csvs.create_csv_response("enrolled_profiles.csv", header, datarows)
+        header, datarows = instructor_analytics.csvs.format_dictlist(student_data, query_features)
+        return instructor_analytics.csvs.create_csv_response("enrolled_profiles.csv", header, datarows)
 
 
 @ensure_csrf_cookie
@@ -610,8 +610,8 @@ def get_anon_ids(request, course_id):  # pylint: disable=W0613
     Respond with 2-column CSV output of user-id, anonymized-user-id
     """
     # TODO: the User.objects query and CSV generation here could be
-    # centralized into analytics. Currently analytics has similar functionality
-    # but not quite what's needed.
+    # centralized into instructor_analytics. Currently instructor_analytics 
+    # has similar functionality but not quite what's needed.
     course_id = SlashSeparatedCourseKey.from_deprecated_string(course_id)
     def csv_response(filename, header, rows):
         """Returns a CSV http response for the given header and rows (excel/utf-8)."""
@@ -655,7 +655,7 @@ def get_distribution(request, course_id):
     else:
         feature = str(feature)
 
-    available_features = analytics.distributions.AVAILABLE_PROFILE_FEATURES
+    available_features = instructor_analytics.distributions.AVAILABLE_PROFILE_FEATURES
     # allow None so that requests for no feature can list available features
     if not feature in available_features + (None,):
         return HttpResponseBadRequest(strip_tags(
@@ -666,12 +666,12 @@ def get_distribution(request, course_id):
         'course_id': course_id.to_deprecated_string(),
         'queried_feature': feature,
         'available_features': available_features,
-        'feature_display_names': analytics.distributions.DISPLAY_NAMES,
+        'feature_display_names': instructor_analytics.distributions.DISPLAY_NAMES,
     }
 
     p_dist = None
     if not feature is None:
-        p_dist = analytics.distributions.profile_distribution(course_id, feature)
+        p_dist = instructor_analytics.distributions.profile_distribution(course_id, feature)
         response_payload['feature_results'] = {
             'feature': p_dist.feature,
             'feature_display_name': p_dist.feature_display_name,

lms/djangoapps/analytics/tests/test_basic.py → lms/djangoapps/instructor_analytics/tests/test_basic.py

@@ -7,7 +7,7 @@ from student.models import CourseEnrollment
 from student.tests.factories import UserFactory
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
-from analytics.basic import enrolled_students_features, AVAILABLE_FEATURES, STUDENT_FEATURES, PROFILE_FEATURES
+from instructor_analytics.basic import enrolled_students_features, AVAILABLE_FEATURES, STUDENT_FEATURES, PROFILE_FEATURES
 
 
 class TestAnalyticsBasic(TestCase):

lms/djangoapps/analytics/tests/test_csvs.py → lms/djangoapps/instructor_analytics/tests/test_csvs.py

@@ -3,7 +3,7 @@
 from django.test import TestCase
 from nose.tools import raises
 
-from analytics.csvs import create_csv_response, format_dictlist, format_instances
+from instructor_analytics.csvs import create_csv_response, format_dictlist, format_instances
 
 
 class TestAnalyticsCSVS(TestCase):

lms/djangoapps/analytics/tests/test_distributions.py → lms/djangoapps/instructor_analytics/tests/test_distributions.py

@@ -6,7 +6,7 @@ from student.models import CourseEnrollment
 from student.tests.factories import UserFactory
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
-from analytics.distributions import profile_distribution, AVAILABLE_PROFILE_FEATURES
+from instructor_analytics.distributions import profile_distribution, AVAILABLE_PROFILE_FEATURES
 
 
 class TestAnalyticsDistributions(TestCase):

lms/envs/dev.py

@@ -268,22 +268,21 @@ ANALYTICS_DATA_URL = "http://127.0.0.1:8080"
 ANALYTICS_DATA_TOKEN = ""
 FEATURES['ENABLE_ANALYTICS_ACTIVE_COUNT'] = False
 
-##### segment-io  ######
+##### Segment.io  ######
 
 # If there's an environment variable set, grab it and turn on Segment.io
 SEGMENT_IO_LMS_KEY = os.environ.get('SEGMENT_IO_LMS_KEY')
 if SEGMENT_IO_LMS_KEY:
     FEATURES['SEGMENT_IO_LMS'] = True
 
-###################### Payment ##############################3
+###################### Payment ######################
 
 CC_PROCESSOR['CyberSource']['SHARED_SECRET'] = os.environ.get('CYBERSOURCE_SHARED_SECRET', '')
 CC_PROCESSOR['CyberSource']['MERCHANT_ID'] = os.environ.get('CYBERSOURCE_MERCHANT_ID', '')
 CC_PROCESSOR['CyberSource']['SERIAL_NUMBER'] = os.environ.get('CYBERSOURCE_SERIAL_NUMBER', '')
 CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = os.environ.get('CYBERSOURCE_PURCHASE_ENDPOINT', '')
 
-
-########################## USER API ########################
+########################## USER API ##########################
 EDX_API_KEY = None
 
 ####################### Shoppingcart ###########################

lms/startup.py

@@ -10,6 +10,7 @@ settings.INSTALLED_APPS  # pylint: disable=W0104
 from django_startup import autostartup
 import edxmako
 import logging
+import analytics
 
 log = logging.getLogger(__name__)
 
@@ -31,6 +32,11 @@ def run():
     if settings.FEATURES.get('ENABLE_THIRD_PARTY_AUTH', False):
         enable_third_party_auth()
 
+    # Initialize Segment.io analytics module. Flushes first time a message is received and 
+    # every 50 messages thereafter, or if 10 seconds have passed since last flush
+    if settings.FEATURES.get('SEGMENT_IO_LMS') and settings.SEGMENT_IO_LMS_KEY:
+        analytics.init(settings.SEGMENT_IO_LMS_KEY, flush_at=50)
+
 
 def add_mimetypes():
     """

lms/static/coffee/src/instructor_dashboard_tracking.coffee

 if $('.instructor-dashboard-wrapper').length == 1
-    analytics.track "Loaded a Legacy Instructor Dashboard Page", 
+    analytics.track "edx.bi.course.legacy_instructor_dashboard.loaded", 
+        category: "courseware"
         location: window.location.pathname
         dashboard_page: $('.navbar .selectedmode').text()

lms/templates/login.html

@@ -59,16 +59,16 @@
             next = decodeURIComponent(next);
           }
           if (next && !isExternal(next)) {
-            location.href=next;
+            location.href=appendParameter(next, "signin", "return");
           }  else if(json.redirect_url){
-            location.href=json.redirect_url;
+            location.href=appendParameter(json.redirect_url, "signin", "return");
           } else {
-            location.href="${reverse('dashboard')}";
+            location.href=appendParameter("${reverse('dashboard')}", "signin", "return");
           }
         } else if(json.hasOwnProperty('redirect')) {
           var u=decodeURI(window.location.search);
           if (!isExternal(json.redirect)) { // a paranoid check.  Our server is the one providing json.redirect
-              location.href=json.redirect+u;
+              location.href=appendParameter(json.redirect+u, "signin", "return");
           } // else we just remain on this page, which is fine since this particular path implies a login failure
             // that has been generated via packet tampering (json.redirect has been messed with).
         } else {
@@ -103,7 +103,7 @@
 
     function thirdPartySignin(event, url) {
       event.preventDefault();
-      window.location.href = url;
+      window.location.href = appendParameter(url, "signin", "return");
     }
 
     (function post_form_if_pipeline_running(pipeline_running) {

lms/templates/main.html

@@ -95,8 +95,6 @@
 
   <%include file="${google_analytics_file}" />
 
-  <%include file="widgets/segment-io.html" />
-
 
 % if style_overrides_file:
   <link rel="stylesheet" type="text/css" href="${static.url(style_overrides_file)}" />
@@ -123,6 +121,8 @@
   <%static:js group='module-js'/>
 
   <%block name="js_extra"/>
+
+  <%include file="widgets/segment-io.html" />
 </body>
 </html>
 

lms/templates/register.html

@@ -55,7 +55,7 @@
 
       $('#register-form').on('ajax:success', function(event, json, xhr) {
         var url = json.redirect_url || "${reverse('dashboard')}";
-        location.href = url;
+        location.href = appendParameter(url, "signin", "initial");
       });
 
       $('#register-form').on('ajax:error', function(event, jqXHR, textStatus) {

lms/templates/widgets/segment-io.html

 % if settings.FEATURES.get('SEGMENT_IO_LMS'):
 <!-- begin Segment.io -->
 <%! from django.core.urlresolvers import reverse %>
-<%! import waffle %>
-
-<% active_flags = " + ".join(waffle.get_flags(request)) %>
 
 <script type="text/javascript">
   // Asynchronously load Segment.io's analytics.js library
   window.analytics||(window.analytics=[]),window.analytics.methods=["identify","track","trackLink","trackForm","trackClick","trackSubmit","page","pageview","ab","alias","ready","group","on","once","off"],window.analytics.factory=function(t){return function(){var a=Array.prototype.slice.call(arguments);return a.unshift(t),window.analytics.push(a),window.analytics}};for(var i=0;i<window.analytics.methods.length;i++){var method=window.analytics.methods[i];window.analytics[method]=window.analytics.factory(method)}window.analytics.load=function(t){var a=document.createElement("script");a.type="text/javascript",a.async=!0,a.src=("https:"===document.location.protocol?"https://":"http://")+"d2dq2ahtl5zl1z.cloudfront.net/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(a,n)},window.analytics.SNIPPET_VERSION="2.0.8",
   analytics.load("${ settings.SEGMENT_IO_LMS_KEY }");
+  analytics.page();
 
   % if user.is_authenticated():
+    // Access the query string, stripping the leading "?"
+    var queryString = window.location.search.substring(1);
 
-    analytics.identify("${ user.id }", {
-      "Registered"      : true,
-      email             : "${ user.email }",
-      username          : "${ user.username }",
-      // Count the number of courses in which the user is currently enrolled
-      "Enrollment Count": ${ sum(1 for course in user.courseenrollment_set.values() if course['is_active'] == True) },
-      "Active Flags"    : "${ active_flags }",
-    });
+    if (queryString != "") {
+      // Convert the query string to a key/value object
+      var parameters = window.parseQueryString(queryString);
 
+      if ("signin" in parameters) {
+        window.assessUserSignIn(parameters, "${user.id}", "${user.email}", "${user.username}");
+      } else {
+        window.identifyUser("${user.id}", "${user.email}", "${user.username}");
+      }
+    } else {
+      window.identifyUser("${user.id}", "${user.email}", "${user.username}");
+    }
   % endif
 
-  // Get current page URL and pull out the path
-  path = window.location.href.split("/")[3]
-  // Match on the current path and fire the appropriate pageview event
-  if (path == "register") {
+  // Get current page URL
+  var url = window.location.href
+  // Match on the current url and fire the appropriate pageview event
+  if (url.indexOf("/register") > -1) {
     // Registration page viewed
-    analytics.page("Registration");
-  } else if (path == "login") {
+    analytics.track("edx.bi.page.register.viewed", {
+      category: "pageview"
+    });
+  } else if (url.indexOf("/login") > -1) {
     // Login page viewed
-    analytics.page("Login");
-  } else if (path == "dashboard") {
+    analytics.track("edx.bi.page.login.viewed", {
+      category: "pageview"
+    });
+  } else if (url.indexOf("/dashboard") > -1) {
     // Dashboard viewed
-    analytics.page("Dashboard");
+    analytics.track("edx.bi.page.dashboard.viewed", {
+      category: "pageview"
+    });
   } else {
     // This event serves as a catch-all, firing when any other page is viewed
-    analytics.page("Other");
+    analytics.track("edx.bi.page.other.viewed", {
+      category: "pageview"
+    });
   }
+
 </script>
 <!-- end Segment.io -->
 % else:

lms/urls.py

@@ -4,7 +4,6 @@ from ratelimitbackend import admin
 from django.conf.urls.static import static
 
 import django.contrib.auth.views
-
 from microsite_configuration import microsite
 
 # Uncomment the next two lines to enable the admin:

requirements/edx/base.txt

@@ -128,6 +128,9 @@ splinter==0.5.4
 testtools==0.9.34
 flaky==0.2.0
 
+# Used for Segment.io analytics
+analytics-python==0.4.4
+
 git+https://github.com/mfogel/django-settings-context-processor.git
 
 # django-cas version 2.0.3 with patch to be compatible with django 1.4