Add learner model and collection

AN-6204

analytics_dashboard/core/templatetags/dashboard_extras.py

+import json
+
 from django import template
 from django.conf import settings
 from django.template.defaultfilters import stringfilter
@@ -121,3 +123,16 @@ def format_course_key(course_key, separator=u'/'):
 @stringfilter
 def unicode_slugify(value):
     return slugify(value)
+
+
+@register.filter
+def escape_json(data):
+    """
+    Escape a JSON string (or convert a dict to a JSON string, and then
+    escape it) for being embedded within an HTML template.
+    """
+    json_string = json.dumps(data) if isinstance(data, dict) else data
+    json_string = json_string.replace("&", "\\u0026")
+    json_string = json_string.replace(">", "\\u003e")
+    json_string = json_string.replace("<", "\\u003c")
+    return mark_safe(json_string)

analytics_dashboard/core/tests/test_templatetags.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+import json
 
 from django.template import Template, Context, TemplateSyntaxError
 from django.test import TestCase
@@ -63,3 +64,11 @@ class DashboardExtraTests(TestCase):
     def test_unicode_slugify(self):
         self.assertEqual(dashboard_extras.unicode_slugify('hello world'), 'hello-world')
         self.assertEqual(dashboard_extras.unicode_slugify(u'straße road'), u'strasse-road')
+
+    def test_escape_json(self):
+        data_as_dict = {'user_bio': '</script><script>alert("&hellip;"!)</script>'}
+        data_as_json = json.dumps(data_as_dict)
+        expected_json = \
+            '{"user_bio": "\\u003c/script\\u003e\\u003cscript\\u003ealert(\\"\\u0026hellip;\\"!)\\u003c/script\\u003e"}'
+        self.assertEqual(dashboard_extras.escape_json(data_as_dict), expected_json)
+        self.assertEqual(dashboard_extras.escape_json(data_as_json), expected_json)

analytics_dashboard/courses/templates/courses/learners.html

 {% extends "courses/base-course.html" %}
 
+{% load dashboard_extras %}
 {% load rjs %}
 
 {% comment %}
 View of individual learners within a course.
 {% endcomment %}
 
-{% block javascript %}
-  {{ block.super }}
-  <script src="{% static_rjs 'js/learners-main.js' %}"></script>
-{% endblock javascript %}
+{% block uncompressed_javascript %}
+    {{ block.super }}
+    <script>
+     require(['load/init-page', 'js/learners-app'], function (page, LearnersApp) {
+         var app = new LearnersApp({
+             courseId: '{{ course_id }}',
+             containerSelector: '.learners-app-container',
+             learnerListJson: {{ learner_list_json | escape_json }},
+             learnerListUrl: '{{ learner_list_url }}'
+         });
+         app.start();
+     });
+    </script>
+{% endblock uncompressed_javascript %}
 
 {% block child_content %}
     <div class="learners-app-container">

analytics_dashboard/courses/views/learners.py

+from requests.exceptions import ConnectTimeout
+
+from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from courses.views import CourseTemplateWithNavView
+from learner_analytics_api.v0.clients import LearnerAPIClient
 
 
 class LearnersView(CourseTemplateWithNavView):
@@ -10,5 +14,16 @@ class LearnersView(CourseTemplateWithNavView):
 
     def get_context_data(self, **kwargs):
         context = super(LearnersView, self).get_context_data(**kwargs)
-        context['page_data'] = self.get_page_data(context)
+        context.update({
+            'page_data': self.get_page_data(context),
+            'learner_list_url': reverse('learner_analytics_api:v0:LearnerList')
+        })
+        # Try to grab the first page of learners.  If the analytics
+        # API times out, the front-end will attempt to asynchronously
+        # fetch the first page.
+        client = LearnerAPIClient()
+        try:
+            context['learner_list_json'] = client.learners.get(course_id=self.course_id).json()
+        except ConnectTimeout:
+            context['learner_list_json'] = None
         return context

analytics_dashboard/learner_analytics_api/v0/utils.py → analytics_dashboard/learner_analytics_api/v0/clients.py

@@ -21,6 +21,8 @@ class LearnerApiResource(Resource):
     Learner Analytics API to the browser.
     """
     def _request(self, *args, **kwargs):
+        # Doesn't hide 400s and 500s, however timeouts will still
+        # raise a requests.exceptions.ConnectTimeout.
         try:
             response = super(LearnerApiResource, self)._request(*args, **kwargs)
         except exceptions.SlumberHttpBaseException as e:

analytics_dashboard/learner_analytics_api/v0/tests/test_views.py

@@ -2,6 +2,8 @@ import json
 
 import ddt
 import httpretty
+import mock
+from requests.exceptions import ConnectTimeout
 
 from django.conf import settings
 from django.test import TestCase
@@ -56,6 +58,13 @@ class LearnerAPITestMixin(UserTestCaseMixin, PermissionsTestMixin, SwitchMixin):
         response = self.client.get('/api/learner_analytics/v0' + self.endpoint, self.required_query_params)
         self.assertEqual(response.status_code, 404)
 
+    @mock.patch('learner_analytics_api.v0.clients.LearnerApiResource._request', mock.Mock(side_effect=ConnectTimeout))
+    def test_timeout(self):
+        self.login()
+        self.grant_permission(self.user, 'edX/DemoX/Demo_Course')
+        response = self.client.get('/api/learner_analytics/v0' + self.endpoint, self.required_query_params)
+        self.assertEqual(response.status_code, 504)
+
     @ddt.data((200, {'test': 'value'}), (400, {'a': 'b', 'c': 'd'}), (500, {}))
     @ddt.unpack
     @httpretty.activate

analytics_dashboard/learner_analytics_api/v0/urls.py

@@ -8,8 +8,12 @@ USERNAME_PATTERN = r'(?P<username>.+)'
 
 urlpatterns = patterns(
     '',
-    url(r'^learners/{}/$'.format(USERNAME_PATTERN), views.LearnerDetailView.as_view()),
-    url(r'^learners/$', views.LearnerListView.as_view()),
-    url(r'^engagement_timelines/{}/$'.format(USERNAME_PATTERN), views.EngagementTimelinesView.as_view()),
-    url(r'^course_learner_metadata/{}/$'.format(settings.COURSE_ID_PATTERN), views.CourseLearnerMetadataView.as_view()),
+    url(r'^learners/{}/$'.format(USERNAME_PATTERN), views.LearnerDetailView.as_view(), name='LearnerDetail'),
+    url(r'^learners/$', views.LearnerListView.as_view(), name='LearnerList'),
+    url(r'^engagement_timelines/{}/$'.format(USERNAME_PATTERN),
+        views.EngagementTimelinesView.as_view(),
+        name='EngagementTimeline'),
+    url(r'^course_learner_metadata/{}/$'.format(settings.COURSE_ID_PATTERN),
+        views.CourseLearnerMetadataView.as_view(),
+        name='CourseMetadata'),
 )

analytics_dashboard/learner_analytics_api/v0/views.py

 import json
 
+from requests.exceptions import ConnectTimeout
+
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.generics import RetrieveAPIView
 from rest_framework.permissions import IsAuthenticated
@@ -46,6 +48,18 @@ class BaseLearnerApiView(RetrieveAPIView):
             headers=api_response.headers,
         )
 
+    def handle_exception(self, exc):
+        """
+        Handles timeouts raised by the API client by returning an HTTP
+        504.
+        """
+        if isinstance(exc, ConnectTimeout):
+            return Response(
+                data={'developer_message': 'Learner Analytics API timed out.', 'error_code': 'analytics_api_timeout'},
+                status=504
+            )
+        return super(BaseLearnerApiView, self).handle_exception(exc)
+
 
 class NotFoundLearnerApiViewMixin(object):
     """

analytics_dashboard/static/js/collections/learner-collection.js

+define([
+    'components/pagination/collections/paging_collection',
+    'models/learner-model'
+], function (PagingCollection, LearnerModel) {
+    'use strict';
+
+    var LearnerCollection = PagingCollection.extend({
+        model: LearnerModel,
+
+        initialize: function (models, options) {
+            PagingCollection.prototype.initialize.call(this, options);
+
+            this.url = options.url;
+            this.courseId = options.courseId;
+
+            this.registerSortableField('username', gettext('Username'));
+            this.registerSortableField('problems_attempted', gettext('Problems Attempted'));
+            this.registerSortableField('problems_completed', gettext('Problems Completed'));
+            this.registerSortableField('videos_viewed', gettext('Videos Viewed'));
+            this.registerSortableField('problems_attempted_per_completed', gettext('Problems Attempted per Completed'));
+            this.registerSortableField('discussion_contributions', gettext('Discussion Contributions'));
+
+            this.registerFilterableField('segments', gettext('Segments'));
+            this.registerFilterableField('ignore_segments', gettext('Segments to Ignore'));
+            this.registerFilterableField('cohort', gettext('Cohort'));
+            this.registerFilterableField('enrollment_mode', gettext('Enrollment Mode'));
+        },
+
+        fetch: function (options) {
+            // Handle gateway timeouts
+            return PagingCollection.prototype.fetch.call(this, options).fail(function (jqXHR) {
+                // Note that we're currently only handling gateway
+                // timeouts here, but we can eventually check against
+                // other expected errors and trigger events
+                // accordingly.
+                if (jqXHR.status === 504) {
+                    this.trigger('gatewayTimeout');
+                }
+            }.bind(this));
+        },
+
+        state: {
+            pageSize: 25
+        },
+
+        queryParams: {
+            course_id: function () { return this.courseId; }
+        }
+    });
+
+    return LearnerCollection;
+});

analytics_dashboard/static/js/config.js

@@ -10,9 +10,13 @@ require.config({
         jquery: 'bower_components/jquery/dist/jquery',
         underscore: 'bower_components/underscore/underscore',
         backbone: 'bower_components/backbone/backbone',
+        'backbone.paginator': 'bower_components/backbone.paginator/lib/backbone.paginator.min',
+        'backbone.wreqr': 'bower_components/backbone.wreqr/lib/backbone.wreqr.min',
+        'backbone.babysitter': 'bower_components/backbone.babysitter/lib/backbone.babysitter.min',
         bootstrap: 'bower_components/bootstrap-sass-official/assets/javascripts/bootstrap',
         bootstrap_accessibility: 'bower_components/bootstrapaccessibilityplugin/plugins/js/bootstrap-accessibility',
         models: 'js/models',
+        collections: 'js/collections',
         views: 'js/views',
         utils: 'js/utils',
         load: 'js/load',
@@ -31,7 +35,13 @@ require.config({
         globalize: 'bower_components/globalize/dist/globalize',
         globalization: 'js/utils/globalization',
         collapsible: 'bower_components/edx-ui-toolkit/components/views/collapsible-view',
-        marionette: 'bower_components/marionette/lib/core/backbone.marionette.min'
+        marionette: 'bower_components/marionette/lib/core/backbone.marionette.min',
+        components: 'bower_components/edx-ui-toolkit/components',
+        // URI and its dependencies
+        URI: 'bower_components/uri.js/src/URI',
+        IPv6: 'bower_components/uri.js/src/IPv6',
+        punycode: 'bower_components/uri.js/src/punycode',
+        SecondLevelDomains: 'bower_components/uri.js/src/SecondLevelDomains'
     },
     wrapShim: true,
     shim: {

analytics_dashboard/static/js/learners-app.js

+define([
+    'collections/learner-collection',
+    'jquery',
+    'marionette',
+    'models/learner-model',
+    'underscore'
+], function (LearnerCollection, $, Marionette, LearnerModel, _) {
+    'use strict';
+
+    var LearnersApp = Marionette.Application.extend({
+        /**
+         * Initializes the learner analytics app.
+         *
+         * @param options specifies the following values:
+         * - courseId (string) required - the course id for this
+         *   learner app
+         * - containerSelector (string) required - the CSS selector
+         *   for the HTML element that this app should attach to
+         * - learnerListJson (Object) optional - an Object
+         *   representing an initial server response from the Learner
+         *   List endpoint used for pre-populating the app's
+         *   LearnerCollection.  If not provided, the data is fetched
+         *   asynchronously before app initialization.
+         */
+        initialize: function (options) {
+            this.options = options || {};
+        },
+
+        onBeforeStart: function () {
+            this.learnerCollection = new LearnerCollection(this.options.learnerListJson, {
+                url: this.options.learnerListUrl,
+                courseId: this.options.courseId,
+                parse: this.options.learnerListJson ? true : false
+            });
+            if (!this.options.learnerListJson) {
+                this.learnerCollection.setPage(1);
+            }
+        },
+
+        onStart: function () {
+            // TODO: remove this temporary UI with AN-6205.
+            var LearnerView = Marionette.ItemView.extend({
+                template: _.template(
+                    '<div>' +
+                        '| <%- name %> | ' +
+                        '<%- username %> |' +
+                        '</div>'
+                )
+            }), LearnersView = Marionette.CollectionView.extend({
+                childView: LearnerView
+            });
+
+            new LearnersView({
+                collection: this.learnerCollection,
+                el: $(this.options.containerSelector)
+            }).render();
+        }
+    });
+
+    return LearnersApp;
+});

analytics_dashboard/static/js/learners-main.js

-require([], function () {
-    'use strict';
-});

analytics_dashboard/static/js/models/learner-model.js

+define(['backbone'], function (Backbone) {
+    'use strict';
+
+    var LearnerModel = Backbone.Model.extend({
+        defaults: {
+            name: '',
+            username: '',
+            email: '',
+            account_url: '',
+            enrollment_mode: '',
+            enrollment_date: null,
+            cohort: null,
+            segments: [],
+            engagements: {},
+            last_updated: null,
+            course_id: ''
+        },
+
+        idAttribute: 'username',
+
+        url: function () {
+            return Backbone.Model.prototype.url.call(this) + '?course_id=' + this.get('course_id');
+        },
+
+        /**
+         * Converts the ISO 8601 date strings to JavaScript Date
+         * objects.
+         */
+        parse: function (response) {
+            var parsedResponse = response;
+            parsedResponse.enrollment_date = response.enrollment_date ? new Date(response.enrollment_date) : null;
+            parsedResponse.last_updated = response.last_updated ? new Date(response.last_updated) : null;
+            return parsedResponse;
+        }
+    });
+
+    return LearnerModel;
+});

analytics_dashboard/static/js/test/specs/learner-collection-spec.js

+require(['collections/learner-collection', 'URI'], function (LearnerCollection, URI) {
+    'use strict';
+
+    describe('LearnerCollection', function () {
+        var courseId = 'org/course/run',
+            learners,
+            server,
+            url,
+            lastRequest,
+            getUriForLastRequest;
+
+        lastRequest = function () {
+            return server.requests[server.requests.length - 1];
+        };
+
+        getUriForLastRequest = function () {
+            return new URI(lastRequest().url);
+        };
+
+        beforeEach(function () {
+            server = sinon.fakeServer.create(); // jshint ignore:line
+            learners = new LearnerCollection(null, {url: '/endpoint/', courseId: courseId});
+        });
+
+        afterEach(function () {
+            server.restore();
+        });
+
+        it('passes the required course_id querystring parameter', function () {
+            learners.fetch();
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual(jasmine.objectContaining({course_id: courseId}));
+        });
+
+        it('passes the expected pagination querystring parameters', function () {
+            learners.setPage(1);
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual({page: '1', page_size: '25', course_id: courseId});
+        });
+
+        it('can add and remove filters', function () {
+            learners.setFilterField('segments', ['inactive', 'unenrolled']);
+            learners.setFilterField('cohort', 'Cool Cohort');
+            learners.refresh();
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual({
+                page: '1',
+                page_size: '25',
+                course_id: courseId,
+                segments: 'inactive,unenrolled',
+                cohort: 'Cool Cohort'
+            });
+            learners.unsetAllFilterFields();
+            learners.refresh();
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual({
+                page: '1',
+                page_size: '25',
+                course_id: courseId
+            });
+        });
+
+        describe('Sorting', function () {
+            var testSorting = function (sortField) {
+                learners.setSortField(sortField);
+                learners.refresh();
+                url = getUriForLastRequest(server);
+                expect(url.path()).toEqual('/endpoint/');
+                expect(url.query(true)).toEqual({
+                    page: '1',
+                    page_size: '25',
+                    course_id: courseId,
+                    order_by: sortField,
+                    sort_order: 'asc'
+                });
+                learners.flipSortDirection();
+                learners.refresh();
+                url = getUriForLastRequest(server);
+                expect(url.query(true)).toEqual({
+                    page: '1',
+                    page_size: '25',
+                    course_id: courseId,
+                    order_by: sortField,
+                    sort_order: 'desc'
+                });
+            };
+
+            it('can sort by username', function () {
+                testSorting('username');
+            });
+
+            it('can sort by problems_attempted', function () {
+                testSorting('problems_attempted');
+            });
+
+            it('can sort by problems_completed', function () {
+                testSorting('problems_completed');
+            });
+
+            it('can sort by videos_viewed', function () {
+                testSorting('videos_viewed');
+            });
+
+            it('can sort by problems_attempted_per_completed', function () {
+                testSorting('problems_attempted_per_completed');
+            });
+
+            it('can sort by discussion_contributions', function () {
+                testSorting('discussion_contributions');
+            });
+        });
+
+        it('can do a full text search', function () {
+            learners.setSearchString('search example');
+            learners.refresh();
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual({
+                page: '1',
+                page_size: '25',
+                course_id: courseId,
+                text_search: 'search example'
+            });
+            learners.unsetSearchString();
+            learners.refresh();
+            url = getUriForLastRequest(server);
+            expect(url.query(true)).toEqual({
+                page: '1',
+                page_size: '25',
+                course_id: courseId
+            });
+        });
+
+        it('can filter, sort, and search all at once', function () {
+            learners.setFilterField('ignore_segments', ['highly_engaged', 'unenrolled']);
+            learners.setSortField('videos_viewed');
+            learners.setSearchString('search example');
+            learners.refresh();
+            url = getUriForLastRequest(server);
+            expect(url.path()).toEqual('/endpoint/');
+            expect(url.query(true)).toEqual({
+                page: '1',
+                page_size: '25',
+                course_id: courseId,
+                text_search: 'search example',
+                ignore_segments: 'highly_engaged,unenrolled',
+                order_by: 'videos_viewed',
+                sort_order: 'asc'
+            });
+        });
+
+        it('triggers an event when server gateway timeouts occur', function () {
+            var spy = {eventCallback: function () {}};
+            spyOn(spy, 'eventCallback');
+            learners.on('gatewayTimeout', spy.eventCallback);
+            learners.fetch();
+            lastRequest().respond(504, {}, '');
+            expect(spy.eventCallback).toHaveBeenCalled();
+        });
+    });
+});

analytics_dashboard/static/js/test/specs/learner-model-spec.js

+require(['backbone', 'models/learner-model'], function (Backbone, LearnerModel) {
+    'use strict';
+
+    describe('LearnerModel', function () {
+        it('should have all the expected fields', function () {
+            var learner = new LearnerModel();
+            expect(learner.attributes).toEqual({
+                name: '',
+                username: '',
+                email: '',
+                account_url: '',
+                enrollment_mode: '',
+                enrollment_date: null,
+                cohort: null,
+                segments: [],
+                engagements: {},
+                last_updated: null,
+                course_id: ''
+            });
+        });
+
+        it('should parse ISO 8601 dates', function () {
+            var dateString = '2016-01-11',
+                dateObj = new Date(dateString),
+                learner = new LearnerModel(
+                    {enrollment_date: dateString, last_updated: dateString},
+                    {parse: true}
+                );
+            expect(learner.get('enrollment_date')).toEqual(dateObj);
+            expect(learner.get('last_updated')).toEqual(dateObj);
+        });
+
+        it('should treat the username as its id', function () {
+            var learner = new LearnerModel({username: 'daniel', course_id: 'edX/DemoX/Demo_Course'});
+            new (Backbone.Collection.extend({url: '/endpoint/'}))([learner]);
+            expect(learner.url()).toEqual('/endpoint/daniel?course_id=edX/DemoX/Demo_Course');
+            learner = new (LearnerModel.extend({
+                urlRoot: '/other-endpoint/'
+            }))({username: 'daniel', course_id: 'edX/DemoX/Demo_Course'});
+            expect(learner.url()).toEqual('/other-endpoint/daniel?course_id=edX/DemoX/Demo_Course');
+        });
+    });
+});

analytics_dashboard/templates/base_dashboard.html

@@ -84,8 +84,11 @@
     {% endcompress %}
 
 
+    {# Note: These blocks are purposely separated from the one above so that browsers cache the common JS instead of downloading a single, large file for each page. #}
+    {# The 'uncompressed_javascript' block should be used for small scripts that do not need to block the page load.  It's useful for factory javascript used to dynamically pass server data to the front via django templating, which does not work with offline compression. #}
+    {% block uncompressed_javascript %}
+    {% endblock uncompressed_javascript %}
     {% compress js %}
-      {# Note: This block is purposely separated from the one above so that browsers cache the common JS instead of downloading a single, large file for each page. #}
       {% block javascript %}
       {% endblock javascript %}
     {% endcompress %}

analytics_dashboard/urls.py

@@ -39,7 +39,7 @@ urlpatterns = patterns(
 
 urlpatterns += patterns(
     '',
-    url(r'^api/learner_analytics/', include('learner_analytics_api.urls', namespace='learner_analytics'))
+    url(r'^api/learner_analytics/', include('learner_analytics_api.urls', namespace='learner_analytics_api'))
 )
 
 if settings.DEBUG:  # pragma: no cover

bower.json

@@ -28,7 +28,8 @@
     "font-awesome": "~4.2.0",
     "natural-sort": "overset/javascript-natural-sort#dbf4ca259b327a488bd1d7897fd46d80c414a7e0",
     "cldr-data": "26.0.3",
-    "edx-ui-toolkit": "edx/edx-ui-toolkit#1e025d169f28632cf903274f3ef8aaf6e2fd6825",
-    "marionette": "~2.4.4"
+    "edx-ui-toolkit": "edx/edx-ui-toolkit#172a6d3698de8d49560d68a98eaa0f044f8a16c9",
+    "marionette": "~2.4.4",
+    "uri.js": "1.17"
   }
 }

build.js

@@ -69,7 +69,7 @@
             exclude: ['js/common']
         },
         {
-            name: 'js/learners-main',
+            name: 'js/learners-app',
             exclude: ['js/common']
         }
     ]

karma.conf.js

@@ -19,11 +19,13 @@ module.exports = function (config) {
             {pattern: 'analytics_dashboard/static/bower_components/**/*.js', included: false},
             {pattern: 'analytics_dashboard/static/bower_components/**/*.json', included: false},
             {pattern: 'analytics_dashboard/static/js/models/**/*.js', included: false},
+            {pattern: 'analytics_dashboard/static/js/collections/**/*.js', included: false},
             {pattern: 'analytics_dashboard/static/js/views/**/*.js', included: false},
             {pattern: 'analytics_dashboard/static/js/utils/**/*.js', included: false},
             {pattern: 'analytics_dashboard/static/js/test/specs/*.js', included: false},
             'analytics_dashboard/static/js/config.js',
-            'analytics_dashboard/static/js/test/spec-runner.js'
+            'analytics_dashboard/static/js/test/spec-runner.js',
+            './node_modules/phantomjs-polyfill/bind-polyfill.js' // Implements Function.prototype.bind for PhantomJS
         ],
 
         exclude: [

package.json

@@ -24,6 +24,7 @@
     "karma-requirejs": "^0.2.2",
     "karma-sinon": "^1.0.3",
     "phantomjs": "^1.9.11",
+    "phantomjs-polyfill": "^0.0.1",
     "sinon": "^1.10.3"
   }
 }