Merge pull request #464 from CredoReference/learning_outcomes_reporting

Learning outcomes reporting dashboards

AUTHORS

@@ -6,3 +6,4 @@ Braden MacDonald <braden@opencraft.com>
 Daniel Friedman <dfriedman@edx.org>
 Yihua Lou <supermouselyh@hotmail.com>
 Tyler Hallada <thallada@edx.org>
+Dmitry Viskov <dmitry.viskov@webenterprise.ru>

README.md

@@ -52,15 +52,16 @@ details on utilizing features in code and templates.
 
 The following switches are available:
 
-| Switch                         | Purpose                                               |
-|--------------------------------|-------------------------------------------------------|
-| show_engagement_forum_activity | Show the forum activity on the course engagement page |
-| enable_course_api              | Retrieve course details from the course API           |
-| enable_ccx_courses             | Display CCX Courses in the course listing page.       |
-| enable_engagement_videos_pages | Enable engagement video pages.                        |
-| enable_video_preview           | Enable video preview.                                 |
-| display_names_for_course_index | Display course names on course index page.            |
-| display_course_name_in_nav     | Display course name in navigation bar.                |
+| Switch                               | Purpose                                               |
+|--------------------------------------|-------------------------------------------------------|
+| show_engagement_forum_activity       | Show the forum activity on the course engagement page |
+| enable_course_api                    | Retrieve course details from the course API           |
+| enable_ccx_courses                   | Display CCX Courses in the course listing page.       |
+| enable_engagement_videos_pages       | Enable engagement video pages.                        |
+| enable_video_preview                 | Enable video preview.                                 |
+| display_names_for_course_index       | Display course names on course index page.            |
+| display_course_name_in_nav           | Display course name in navigation bar.                |
+| enable_performance_learning_outcome  | Enable performance section with learning outcome breakdown (functionality based on tagging questions in Studio) | 
 
 [Waffle](http://waffle.readthedocs.org/en/latest/) flags are used to disable/enable
 functionality on request (e.g. turning on beta functionality for superusers). Create a

analytics_dashboard/courses/presenters/__init__.py

 import abc
+from collections import OrderedDict
 import datetime
 import logging
 
@@ -66,7 +67,6 @@ class CourseAPIPresenterMixin(object):
         """ Retrieves course structure from the course API. """
         key = self.get_cache_key('structure')
         structure = cache.get(key)
-
         if not structure:
             logger.debug('Retrieving structure for course: %s', self.course_id)
             structure = self.course_api_client.course_structures(self.course_id).get()
@@ -188,7 +188,7 @@ class CourseAPIPresenterMixin(object):
             module_data = self.fetch_course_module_data()
 
             # Create a lookup table so that submission data can be quickly retrieved by downstream consumers.
-            table = {}
+            table = OrderedDict()
             last_updated = datetime.datetime.min
 
             for datum in module_data:
@@ -221,7 +221,7 @@ class CourseAPIPresenterMixin(object):
             module_data = self._course_module_data()
         except BaseCourseError as e:
             logger.warning(e)
-            module_data = {}
+            module_data = OrderedDict()
 
         for parent_block in parent_blocks:
             parent_block['num_modules'] = len(parent_block['children'])

analytics_dashboard/courses/templates/courses/_tags_content_nav.html

+{% extends "courses/_base_content_nav.html" %}
+
+{% load i18n %}
+{% load dashboard_extras %}
+
+{% block home_content_url %}
+    {% url "courses:performance:learning_outcomes" course_id=course_id %}
+{% endblock %}
+
+{% block select_first_level %}
+    {% trans "Select Outcome" %}
+{% endblock %}
+
+{% block select_first_level_all %}
+  {% trans "All Outcomes" %}
+{% endblock %}
+
+{% block select_second_level %}
+    {% trans "Select Problem" %}
+{% endblock %}

analytics_dashboard/courses/templates/courses/base_performance_learning_outcomes.html

+{% extends "courses/base-course.html" %}
+{% load i18n %}
+{% load dashboard_extras %}
+{% load staticfiles %}
+{% load rjs %}
+
+{% block view-name %}view-course-enrollment view-dashboard{% endblock view-name %}
+
+{% block child_content %}
+
+<section class="view-section" data-section="performance-tags-distribution" aria-hidden="true">
+
+    {% block content_nav %}
+    {% endblock %}
+
+    {% block module_meta %}
+    {% endblock %}
+
+    <div class="section-content section-data-graph">
+      <div class="section-content section-data-viz">
+        <div class="analytics-chart-container">
+          {% if js_data.course.hasData %}
+            {% block chart_info %}
+             {% include "courses/submissions_chart_info.html" %}
+            {% endblock chart_info %}
+            {% captureas chart_tip_text %}{% block chart_tip_text %}{% endblock %}{% endcaptureas %}
+            {% include "chart_tooltip.html" with tip_text=chart_tip_text track_category="bar" %}
+          {% endif %}
+          <div id="chart-view" class="analytics-chart {% if not js_data.course.hasData%}message-only-chart{% endif %}">
+            {% if js_data.course.hasData %}
+              {% include "loading.html" %}
+            {% else %}
+              <div class="clearfix"></div>
+              <div class="chart-message-container">
+                <p class="text-center">
+                  {{ no_data_message }}
+                </p>
+              </div>
+            {% endif %}
+          </div>
+        </div>
+        {% block module_controls %}{% endblock %}
+      </div>
+    </div>
+</section>
+
+{% if js_data.course.hasData %}
+<section class="view-section">
+
+    <div class="section-heading section-data-table-title">
+      <h4 class="section-title">{% block table_title %}{% endblock %}</h4>
+      {% block table_download %}{% endblock %}
+    </div>
+
+    {% if js_data.course.tagsDistribution %}
+      <div class="section-content section-data-table" data-role="data-table">
+        {% include "loading.html" %}
+      </div>
+    {% else %}
+      {% show_table_error %}
+    {% endif %}
+</section>
+{% endif %}
+
+{% endblock %}

analytics_dashboard/courses/templates/courses/performance_learning_outcomes_answer_distribution.html

+{% extends "courses/performance_answer_distribution.html" %}
+{% load i18n %}
+
+{% block content_nav %}
+    {% trans "How did students answer this problem?" as heading_note %}
+    {% include "courses/_tags_content_nav.html" with active="second_level" first_levels=js_data.first_level_content_nav selected_first_level=js_data.first_level_selected second_levels=js_data.second_level_content_nav selected_second_level=js_data.second_level_selected heading_note=heading_note %}
+{% endblock %}
+
+{% block problem_part_url %}
+    {% url 'courses:performance:learning_outcomes_answers_distribution_with_part' tag_value=selected_tag_value course_id=course_id problem_id=problem_id problem_part_id=question.part_id %}
+{% endblock %}

analytics_dashboard/courses/templates/courses/performance_learning_outcomes_content.html

+{% extends "courses/base_performance_learning_outcomes.html" %}
+{% load i18n %}
+{% load rjs %}
+
+{% block content_nav %}
+    {% trans "Measured by learning outcome, how are students performing?" as heading_note %}
+    {% include "courses/_tags_content_nav.html" with active="first_level" first_levels=js_data.first_level_content_nav heading_note=heading_note %}
+{% endblock %}
+
+{% block javascript %}
+    {{ block.super }}
+    <script src="{% static_rjs 'js/performance-learning-outcomes-content-main.js' %}"></script>
+{% endblock javascript %}
+
+{% block chart_tip_text %}
+    {% trans "Each bar shows the average number of correct and incorrect submissions for each outcome." %}
+{% endblock %}
+
+{% block table_title %}
+    {% trans "Outcome Submissions" %}
+{% endblock %}

analytics_dashboard/courses/templates/courses/performance_learning_outcomes_section.html

+{% extends "courses/base_performance_learning_outcomes.html" %}
+{% load i18n %}
+{% load rjs %}
+
+{% block content_nav %}
+    {% trans "Measured by learning outcome, how are students performing?" as heading_note %}
+    {% include "courses/_tags_content_nav.html" with active="second_level" first_levels=js_data.first_level_content_nav selected_first_level=js_data.first_level_selected second_levels=js_data.second_level_content_nav heading_note=heading_note %}
+{% endblock %}
+
+{% block javascript %}
+    {{ block.super }}
+    <script src="{% static_rjs 'js/performance-learning-outcomes-section-main.js' %}"></script>
+{% endblock javascript %}
+
+{% block chart_tip_text %}
+    {% trans "Each bar shows the correct and incorrect number of submissions for each problem for the current outcome." %}
+{% endblock %}
+
+{% block table_title %}
+    {% trans "Problem Submissions" %}
+{% endblock %}

analytics_dashboard/courses/tests/factories.py

+import hashlib
+import re
 import uuid
 
+from collections import OrderedDict
 from slugify import slugify
 
 from common.tests.factories import CourseStructureFactory
 from courses.tests.utils import CREATED_DATETIME_STRING
+from courses import utils
 
 
 class CoursePerformanceDataFactory(CourseStructureFactory):
@@ -335,3 +339,152 @@ class CourseEngagementDataFactory(CourseStructureFactory):
             }
         ]
         return videos
+
+
+class TagsDistributionDataFactory(CourseStructureFactory):
+
+    _count_of_homework_assignments = 0
+    _tags_structure = None
+
+    def __init__(self, tags_data_per_homework_assigment):
+        self._count_of_problems = len(tags_data_per_homework_assigment) + 1
+        self.tags_data_per_homework_assigment = tags_data_per_homework_assigment
+
+        super(TagsDistributionDataFactory, self).__init__()
+        for policy in self.grading_policy:
+            if policy['assignment_type'].startswith('Homework'):
+                self._count_of_homework_assignments = policy['count']
+                break
+        self._generate_tags_structure()
+
+    def _get_block_id(self, block_type, block_format=None, display_name=None, graded=True, children=None):
+        if display_name:
+            return hashlib.md5(display_name).hexdigest()
+        else:
+            return super(TagsDistributionDataFactory, self)._get_block_id(block_type, block_format, display_name,
+                                                                          graded, children)
+
+    def _generate_tags_structure(self):
+        reg = re.compile(r'Homework (\d) Problem (\d)')
+        tags_data = {}
+
+        for k, item in self._structure['blocks'].iteritems():
+            if item['type'] == 'problem' and item['display_name'].startswith('Homework'):
+                m = reg.match(item['display_name'])
+                assig_num = int(m.group(1))
+                problem_num = int(m.group(2)) - 1
+                key = assig_num * 10 + problem_num
+                tags_data[key] = {
+                    "display_name": item['display_name'],
+                    "module_id": item['id'],
+                    "total_submissions": self.tags_data_per_homework_assigment[problem_num]['total_submissions'],
+                    "correct_submissions": self.tags_data_per_homework_assigment[problem_num]['correct_submissions'],
+                    "tags": self.tags_data_per_homework_assigment[problem_num]['tags'],
+                }
+        self._tags_structure = [tags_data[k] for k in sorted(tags_data)]
+
+    @property
+    def problems_and_tags(self):
+        """
+        Mock tags distribution data.
+        """
+        return self._tags_structure
+
+    def get_expected_available_tags(self):
+        tags = {}
+        for item in self.tags_data_per_homework_assigment:
+            for key, val in item['tags'].iteritems():
+                if key not in tags:
+                    tags[key] = set()
+                tags[key].add(val)
+        return tags
+
+    def get_expected_learning_outcome_tags_content_nav(self, key):
+        url_template = '/courses/{}/performance/learning_outcomes/{}/'
+        expected_available_tags = self.get_expected_available_tags()
+        if key in expected_available_tags:
+            return [{'id': v, 'name': v, 'url': url_template.format(self.course_id, slugify(v))}
+                    for v in expected_available_tags[key]]
+        else:
+            return []
+
+    def get_expected_tags_distribution(self, tag_key):
+        index = 0
+        expected = OrderedDict()
+        k = self._count_of_homework_assignments
+
+        for val in self.tags_data_per_homework_assigment:
+            if tag_key in val['tags']:
+                tag_value = val['tags'][tag_key]
+                if tag_value not in expected:
+                    index += 1
+                    expected[tag_value] = {
+                        "id": tag_value,
+                        "index": index,
+                        "name": tag_value,
+                        "total_submissions": 0,
+                        "correct_submissions": 0,
+                        "incorrect_submissions": 0,
+                        "num_modules": 0
+                    }
+
+                incorrect_submissions = val["total_submissions"] - val["correct_submissions"]
+
+                expected[tag_value]["total_submissions"] += val["total_submissions"] * k
+                expected[tag_value]["correct_submissions"] += val["correct_submissions"] * k
+                expected[tag_value]["incorrect_submissions"] += incorrect_submissions * k
+                expected[tag_value]["num_modules"] += k
+
+        url_template = '/courses/{}/performance/learning_outcomes/{}/'
+
+        for tag_val, item in expected.iteritems():
+            item.update({
+                'average_submissions': (item['total_submissions'] * 1.0) / item['num_modules'],
+                'average_correct_submissions': (item['correct_submissions'] * 1.0) / item['num_modules'],
+                'average_incorrect_submissions': (item['incorrect_submissions'] * 1.0) / item['num_modules'],
+                'correct_percent': utils.math.calculate_percent(item['correct_submissions'],
+                                                                item['total_submissions']),
+                'incorrect_percent': utils.math.calculate_percent(item['incorrect_submissions'],
+                                                                  item['total_submissions']),
+                'url': url_template.format(self.course_id, slugify(tag_val))
+            })
+
+        return expected.values()
+
+    def get_expected_modules_marked_with_tag(self, tag_key, tag_value):
+        index = 0
+        expected = []
+        available_tags = self.get_expected_available_tags()
+
+        url_template = '/courses/{}/performance/learning_outcomes/{}/problems/{}/'
+
+        for i in xrange(1, self._count_of_homework_assignments + 1):
+            num = 0
+            for val in self.tags_data_per_homework_assigment:
+                num += 1
+                if tag_key in val['tags'] and val['tags'][tag_key] == tag_value:
+                    display_name = 'Homework %d Problem %d' % (i, num)
+                    incorrect_submissions = val["total_submissions"] - val["correct_submissions"]
+                    new_item_id = 'i4x://edX/DemoX/problem/%s' % hashlib.md5(display_name).hexdigest()
+                    index += 1
+                    new_item = {
+                        'id': new_item_id,
+                        'index': index,
+                        'name': ', '.join(['Demo Course', 'Homework %d' % i, display_name]),
+                        'total_submissions': val['total_submissions'],
+                        'correct_submissions': val['correct_submissions'],
+                        'incorrect_submissions': incorrect_submissions,
+                        'correct_percent': utils.math.calculate_percent(val['correct_submissions'],
+                                                                        val['total_submissions']),
+                        'incorrect_percent': utils.math.calculate_percent(incorrect_submissions,
+                                                                          val['total_submissions']),
+                        'url': url_template.format(self.course_id, slugify(tag_value), new_item_id)
+                    }
+                    if available_tags:
+                        for av_tag_key in available_tags:
+                            if av_tag_key in val['tags']:
+                                new_item[av_tag_key] = val['tags'][av_tag_key]
+                            else:
+                                new_item[av_tag_key] = None
+                    expected.append(new_item)
+        return expected

analytics_dashboard/courses/tests/test_presenters.py

@@ -8,6 +8,7 @@ from django.core.urlresolvers import reverse
 from django.test import (override_settings, TestCase)
 
 from ddt import ddt, data, unpack
+from slugify import slugify
 import mock
 from waffle.testutils import override_switch
 
@@ -26,9 +27,10 @@ from courses.exceptions import NoVideosError
 from courses.presenters import BasePresenter
 from courses.presenters.engagement import (CourseEngagementActivityPresenter, CourseEngagementVideoPresenter)
 from courses.presenters.enrollment import (CourseEnrollmentPresenter, CourseEnrollmentDemographicsPresenter)
-from courses.presenters.performance import CoursePerformancePresenter
+from courses.presenters.performance import CoursePerformancePresenter, TagsDistributionPresenter
 from courses.tests import utils
-from courses.tests.factories import (CourseEngagementDataFactory, CoursePerformanceDataFactory)
+from courses.tests.factories import (CourseEngagementDataFactory, CoursePerformanceDataFactory,
+                                     TagsDistributionDataFactory)
 
 
 class BasePresenterTests(TestCase):
@@ -973,3 +975,133 @@ class CoursePerformancePresenterTests(TestCase):
                 expected_problems = subsection['children']
                 self.assertListEqual(
                     self.presenter.subsection_children(section['id'], subsection['id']), expected_problems)
+
+
+@ddt
+class TagsDistributionPresenterTests(TestCase):
+
+    def setUp(self):
+        cache.clear()
+        self.course_id = PERFORMER_PRESENTER_COURSE_ID
+        self.presenter = TagsDistributionPresenter(settings.COURSE_API_KEY, self.course_id)
+
+    @data(annotated([{"total_submissions": 21, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 11, "correct_submissions": 10,
+                      "tags": {"difficulty": "Easy"}},
+                     {"total_submissions": 15, "correct_submissions": 9,
+                      "tags": {"difficulty": "Medium"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}}], 'only_difficulty_tag'),
+          annotated([{"total_submissions": 21, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard", "learning_outcome": "Learned a few things"}},
+                     {"total_submissions": 11, "correct_submissions": 10,
+                      "tags": {"difficulty": "Easy", "learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 15, "correct_submissions": 9,
+                      "tags": {"difficulty": "Medium", "learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}}], 'learning_outcome_and_difficulty_tags'),
+          annotated([], 'empty_data'),)
+    def test_available_tags(self, init_tags_data):
+        factory = TagsDistributionDataFactory(init_tags_data)
+
+        with mock.patch('slumber.Resource.get', mock.Mock(return_value=factory.structure)):
+            with mock.patch('analyticsclient.course.Course.problems_and_tags',
+                            mock.Mock(return_value=factory.problems_and_tags)):
+                available_tags = self.presenter.get_available_tags()
+                expected_available_tags = factory.get_expected_available_tags()
+                self.assertEqual(available_tags, expected_available_tags)
+
+                expected_tags_content_nav = factory.get_expected_learning_outcome_tags_content_nav('learning_outcome')
+                tags_content_nav, selected = self.presenter.get_tags_content_nav('learning_outcome')
+
+                self.assertEqual(selected, None)
+                self.assertEqual(tags_content_nav, expected_tags_content_nav)
+
+                tags_content_nav, selected = self.presenter.get_tags_content_nav('learning_outcome',
+                                                                                 slugify('Learned a few things'))
+                expected_selected = None
+                for v in tags_content_nav:
+                    if v['name'] == 'Learned a few things':
+                        expected_selected = v
+                        break
+
+                self.assertEqual(selected, expected_selected)
+                self.assertEqual(tags_content_nav, expected_tags_content_nav)
+
+    @data(annotated([{"total_submissions": 21, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 11, "correct_submissions": 10,
+                      "tags": {"difficulty": "Easy"}},
+                     {"total_submissions": 15, "correct_submissions": 9,
+                      "tags": {"difficulty": "Medium"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}}], 'only_difficulty_tag'),
+          annotated([{"total_submissions": 41, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard", "learning_outcome": "Learned a few things"}},
+                     {"total_submissions": 25, "correct_submissions": 25,
+                      "tags": {"difficulty": "Easy", "learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 17, "correct_submissions": 16,
+                      "tags": {"learning_outcome": "Learned everything"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 35, "correct_submissions": 31,
+                      "tags": {"learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 105, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard",
+                               "learning_outcome": "Learned everything"}}], 'learning_outcome_and_difficult'),
+          annotated([], 'empty_data'),)
+    def test_tags_distribution(self, init_tags_data):
+        factory = TagsDistributionDataFactory(init_tags_data)
+
+        with mock.patch('slumber.Resource.get', mock.Mock(return_value=factory.structure)):
+            with mock.patch('analyticsclient.course.Course.problems_and_tags',
+                            mock.Mock(return_value=factory.problems_and_tags)):
+                tags_distribution = self.presenter.get_tags_distribution('learning_outcome')
+                expected_tags_distribution = factory.get_expected_tags_distribution('learning_outcome')
+                self.assertEqual(tags_distribution, expected_tags_distribution)
+
+    @data(annotated([{"total_submissions": 21, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 11, "correct_submissions": 10,
+                      "tags": {"difficulty": "Easy"}},
+                     {"total_submissions": 15, "correct_submissions": 9,
+                      "tags": {"difficulty": "Medium"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}}], 'only_difficulty_tag'),
+          annotated([{"total_submissions": 41, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard", "learning_outcome": "Learned a few things"}},
+                     {"total_submissions": 25, "correct_submissions": 25,
+                      "tags": {"difficulty": "Easy", "learning_outcome": "Learned a few things"}},
+                     {"total_submissions": 17, "correct_submissions": 16,
+                      "tags": {"learning_outcome": "Learned everything"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 35, "correct_submissions": 31,
+                      "tags": {"learning_outcome": "Learned everything"}},
+                     {"total_submissions": 105, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard",
+                               "learning_outcome": "Learned everything"}}], 'learning_outcome_without_learned_nothing'),
+          annotated([{"total_submissions": 41, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard", "learning_outcome": "Learned a few things"}},
+                     {"total_submissions": 25, "correct_submissions": 25,
+                      "tags": {"difficulty": "Easy", "learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 17, "correct_submissions": 16,
+                      "tags": {"learning_outcome": "Learned everything"}},
+                     {"total_submissions": 10, "correct_submissions": 5,
+                      "tags": {"difficulty": "Hard"}},
+                     {"total_submissions": 35, "correct_submissions": 31,
+                      "tags": {"learning_outcome": "Learned nothing"}},
+                     {"total_submissions": 105, "correct_submissions": 10,
+                      "tags": {"difficulty": "Hard",
+                               "learning_outcome": "Learned everything"}}], 'learning_outcome_and_difficult'),
+          annotated([], 'empty_data'),)
+    def test_modules_marked_with_tag(self, init_tags_data):
+        factory = TagsDistributionDataFactory(init_tags_data)
+
+        with mock.patch('slumber.Resource.get', mock.Mock(return_value=factory.structure)):
+            with mock.patch('analyticsclient.course.Course.problems_and_tags',
+                            mock.Mock(return_value=factory.problems_and_tags)):
+                modules = self.presenter.get_modules_marked_with_tag('learning_outcome', slugify('Learned nothing'))
+                expected_modules = factory.get_expected_modules_marked_with_tag('learning_outcome', 'Learned nothing')
+                self.assertEqual(modules, expected_modules)

analytics_dashboard/courses/urls.py

@@ -14,6 +14,7 @@ SUBSECTION_ID_PATTERN = CONTENT_ID_PATTERN.replace('content_id', 'subsection_id'
 VIDEO_ID_PATTERN = CONTENT_ID_PATTERN.replace('content_id', 'video_id')
 PIPELINE_VIDEO_ID = r'(?P<pipeline_video_id>([^/+]+[/+][^/+]+[/+][^/]+)+[|]((?:i4x://?[^/]+/[^/]+/[^/]+' \
                     r'/[^@]+(?:@[^/]+)?)|(?:[^/]+)+))'
+TAG_VALUE_ID_PATTERN = r'(?P<tag_value>[\w-]+)'
 
 answer_distribution_regex = \
     r'^graded_content/assignments/{assignment_id}/problems/{problem_id}/parts/{part_id}/answer_distribution/$'.format(
@@ -70,6 +71,19 @@ PERFORMANCE_URLS = ([
     url(r'^graded_content/assignments/{}/$'.format(ASSIGNMENT_ID_PATTERN),
         performance.PerformanceAssignment.as_view(),
         name='assignment'),
+    url(r'^learning_outcomes/$',
+        performance.PerformanceLearningOutcomesContent.as_view(),
+        name='learning_outcomes'),
+    url(r'^learning_outcomes/{}/$'.format(TAG_VALUE_ID_PATTERN),
+        performance.PerformanceLearningOutcomesSection.as_view(),
+        name='learning_outcomes_section'),
+    url(r'^learning_outcomes/{}/problems/{}/$'.format(TAG_VALUE_ID_PATTERN, PROBLEM_ID_PATTERN),
+        performance.PerformanceLearningOutcomesAnswersDistribution.as_view(),
+        name='learning_outcomes_answers_distribution'),
+    url(r'^learning_outcomes/{}/problems/{}/{}/$'.format(TAG_VALUE_ID_PATTERN, PROBLEM_ID_PATTERN,
+                                                         PROBLEM_PART_ID_PATTERN),
+        performance.PerformanceLearningOutcomesAnswersDistribution.as_view(),
+        name='learning_outcomes_answers_distribution_with_part'),
 ], 'performance')
 
 CSV_URLS = ([

analytics_dashboard/courses/views/__init__.py

@@ -518,24 +518,31 @@ class CourseHome(CourseTemplateWithNavView):
         items.append(engagement_items)
 
         if self.course_api_enabled:
+            subitems = [{
+                'title': _('How are students doing on graded course assignments?'),
+                'view': 'courses:performance:graded_content',
+                'breadcrumbs': [_('Graded Content')],
+                'fragment': ''
+            }, {
+                'title': _('How are students doing on ungraded exercises?'),
+                'view': 'courses:performance:ungraded_content',
+                'breadcrumbs': [_('Ungraded Problems')],
+                'fragment': ''
+            }]
+
+            if switch_is_active('enable_performance_learning_outcome'):
+                subitems.append({
+                    'title': _('What is the breakdown for course learning outcomes?'),
+                    'view': 'courses:performance:learning_outcomes',
+                    'breadcrumbs': [_('Learning Outcomes')],
+                    'fragment': ''
+                })
+
             items.append({
                 'name': _('Performance'),
                 'icon': 'fa-check-square-o',
                 'heading': _('How are students doing on course assignments?'),
-                'items': [
-                    {
-                        'title': _('How are students doing on graded course assignments?'),
-                        'view': 'courses:performance:graded_content',
-                        'breadcrumbs': [_('Graded Content')],
-                        'fragment': ''
-                    },
-                    {
-                        'title': _('How are students doing on ungraded exercises?'),
-                        'view': 'courses:performance:ungraded_content',
-                        'breadcrumbs': [_('Ungraded Problems')],
-                        'fragment': ''
-                    }
-                ]
+                'items': subitems
             })
 
         if flag_is_active(request, 'display_learner_analytics'):

analytics_dashboard/courses/views/performance.py

+import copy
 import logging
 
 from django.conf import settings
 from django.http import Http404
 from django.utils.translation import ugettext_lazy as _
 from slugify import slugify
+from waffle import switch_is_active
 
-from courses.presenters.performance import CoursePerformancePresenter
+from courses.presenters.performance import CoursePerformancePresenter, TagsDistributionPresenter
 from courses.views import (
     CourseTemplateWithNavView,
     CourseAPIMixin,
@@ -28,14 +30,24 @@ class PerformanceTemplateView(CourseStructureExceptionMixin, CourseTemplateWithN
     # Translators: Do not translate UTC.
     update_message = _('Problem submission data was last updated %(update_date)s at %(update_time)s UTC.')
 
-    secondary_nav_items = [
+    secondary_nav_items_base = [
         {'name': 'graded_content', 'label': _('Graded Content'), 'view': 'courses:performance:graded_content'},
-        {'name': 'ungraded_content', 'label': _('Ungraded Problems'), 'view': 'courses:performance:ungraded_content'}
+        {'name': 'ungraded_content', 'label': _('Ungraded Problems'), 'view': 'courses:performance:ungraded_content'},
     ]
+    secondary_nav_items = None
 
     active_primary_nav_item = 'performance'
 
     def get_context_data(self, **kwargs):
+        self.secondary_nav_items = copy.deepcopy(self.secondary_nav_items_base)
+        if switch_is_active('enable_performance_learning_outcome'):
+            if not any(d['name'] == 'learning_outcomes' for d in self.secondary_nav_items):
+                self.secondary_nav_items.append({
+                    'name': 'learning_outcomes',
+                    'label': _('Learning Outcomes'),
+                    'view': 'courses:performance:learning_outcomes'
+                })
+
         context_data = super(PerformanceTemplateView, self).get_context_data(**kwargs)
         self.presenter = CoursePerformancePresenter(self.access_token, self.course_id)
 
@@ -285,3 +297,108 @@ class PerformanceUngradedAnswerDistribution(PerformanceAnswerDistributionMixin,
     template_name = 'courses/performance_ungraded_answer_distribution.html'
     page_name = 'performance_ungraded_answer_distribution'
     page_title = _('Performance: Problem Submissions')
+
+
+class PerformanceLearningOutcomesMixin(PerformanceTemplateView):
+    active_secondary_nav_item = 'learning_outcomes'
+    tags_presenter = None
+    selected_tag_value = None
+    update_message = _('Tags distribution data was last updated %(update_date)s at %(update_time)s UTC.')
+    no_data_message = _('No submissions received for these exercises.')
+
+    def get_context_data(self, **kwargs):
+        context = super(PerformanceLearningOutcomesMixin, self).get_context_data(**kwargs)
+
+        self.selected_tag_value = kwargs.get('tag_value', None)
+        self.tags_presenter = TagsDistributionPresenter(self.access_token, self.course_id)
+
+        first_level_content_nav, first_selected_item = self.tags_presenter.get_tags_content_nav(
+            'learning_outcome', self.selected_tag_value)
+
+        context['selected_tag_value'] = self.selected_tag_value
+        context['update_message'] = self.get_last_updated_message(self.tags_presenter.last_updated)
+        context['js_data'] = {
+            'first_level_content_nav': first_level_content_nav,
+            'first_level_selected': first_selected_item
+        }
+        return context
+
+
+class PerformanceLearningOutcomesContent(PerformanceLearningOutcomesMixin):
+    template_name = 'courses/performance_learning_outcomes_content.html'
+    page_name = 'performance_learning_outcomes_content'
+    page_title = _('Performance: Learning Outcomes')
+
+    def get_context_data(self, **kwargs):
+        context = super(PerformanceLearningOutcomesContent, self).get_context_data(**kwargs)
+
+        tags_distribution = self.tags_presenter.get_tags_distribution('learning_outcome')
+
+        course_data = {'tagsDistribution': tags_distribution,
+                       'hasData': bool(tags_distribution),
+                       'courseId': self.course_id,
+                       'contentTableHeading': "Outcome Name"}
+
+        context['js_data'].update({
+            'course': course_data,
+        })
+        context.update({
+            'page_data': self.get_page_data(context),
+        })
+
+        return context
+
+
+class PerformanceLearningOutcomesSection(PerformanceLearningOutcomesMixin):
+    template_name = 'courses/performance_learning_outcomes_section.html'
+    page_name = 'performance_learning_outcomes_section'
+    page_title = _('Performance: Learning Outcomes')
+    has_part_id_param = False
+
+    def get_context_data(self, **kwargs):
+        context = super(PerformanceLearningOutcomesSection, self).get_context_data(**kwargs)
+
+        if self.has_part_id_param and self.part_id is None and self.problem_id:
+            assignments = self.presenter.course_module_data()
+            if self.problem_id in assignments and len(assignments[self.problem_id]['part_ids']) > 0:
+                self.part_id = assignments[self.problem_id]['part_ids'][0]
+
+        modules_marked_with_tag = self.tags_presenter.get_modules_marked_with_tag('learning_outcome',
+                                                                                  self.selected_tag_value)
+        course_data = {'tagsDistribution': modules_marked_with_tag,
+                       'hasData': bool(modules_marked_with_tag),
+                       'courseId': self.course_id,
+                       'contentTableHeading': "Problem Name"}
+
+        context['js_data'].update({
+            'course': course_data,
+            'second_level_content_nav': modules_marked_with_tag
+        })
+        context.update({
+            'page_data': self.get_page_data(context),
+        })
+
+        return context
+
+
+class PerformanceLearningOutcomesAnswersDistribution(PerformanceAnswerDistributionMixin,
+                                                     PerformanceLearningOutcomesSection):
+    template_name = 'courses/performance_learning_outcomes_answer_distribution.html'
+    page_title = _('Performance: Problem Submissions')
+    page_name = 'performance_learning_outcomes_answer_distribution'
+    has_part_id_param = True
+
+    def get_context_data(self, **kwargs):
+        context = super(PerformanceLearningOutcomesAnswersDistribution, self).get_context_data(**kwargs)
+
+        second_level_selected_item = None
+        for nav_item in context['js_data']['second_level_content_nav']:
+            if nav_item['id'] == self.problem_id:
+                second_level_selected_item = nav_item
+                break
+
+        context['js_data'].update({
+            'second_level_selected': second_level_selected_item
+        })
+
+        return context

analytics_dashboard/static/js/performance-learning-outcomes-content-main.js

+require(['vendor/domReady!', 'load/init-page'], function(doc, page) {
+    'use strict';
+
+    require(['d3', 'underscore', 'views/data-table-view', 'views/stacked-bar-view'],
+        function(d3, _, DataTableView, StackedBarView) {
+            var model = page.models.courseModel,
+                graphSubmissionColumns = [
+                    {
+                        key: 'average_correct_submissions',
+                        percent_key: 'correct_percent',
+                        title: gettext('Average Correct'),
+                        className: 'text-right',
+                        type: 'number',
+                        fractionDigits: 1,
+                        color: '#4BB4FB'
+                    },
+                    {
+                        key: 'average_incorrect_submissions',
+                        percent_key: 'incorrect_percent',
+                        title: gettext('Average Incorrect'),
+                        className: 'text-right',
+                        type: 'number',
+                        fractionDigits: 1,
+                        color: '#CA0061'
+                    }
+                ],
+                tableColumns = [
+                    {key: 'index', title: gettext('Order'), type: 'number', className: 'text-right'},
+                    {key: 'name', title: model.get('contentTableHeading'), type: 'hasNull'}
+                ],
+                performanceLoContentChart,
+                performanceLoContentTable;
+
+            tableColumns = tableColumns.concat(graphSubmissionColumns);
+
+            tableColumns.push({
+                key: 'average_submissions',
+                title: gettext('Average Submissions per Problem'),
+                className: 'text-right',
+                type: 'number',
+                fractionDigits: 1
+            });
+
+            tableColumns.push({
+                key: 'correct_percent',
+                title: gettext('Percentage Correct'),
+                className: 'text-right',
+                type: 'percent'
+            });
+
+            if (model.get('hasData')) {
+                performanceLoContentChart = new StackedBarView({
+                    el: '#chart-view',
+                    model: model,
+                    modelAttribute: 'tagsDistribution',
+                    trends: graphSubmissionColumns
+                });
+                performanceLoContentChart.renderIfDataAvailable();
+            }
+
+            performanceLoContentTable = new DataTableView({
+                el: '[data-role=data-table]',
+                model: model,
+                modelAttribute: 'tagsDistribution',
+                columns: tableColumns,
+                sorting: ['index'],
+                replaceZero: '-'
+            });
+            performanceLoContentTable.renderIfDataAvailable();
+        });
+});

analytics_dashboard/static/js/performance-learning-outcomes-section-main.js

+require(['vendor/domReady!', 'load/init-page'], function(doc, page) {
+    'use strict';
+
+    require(['d3', 'underscore', 'views/data-table-view', 'views/stacked-bar-view'],
+        function(d3, _, DataTableView, StackedBarView) {
+            var model = page.models.courseModel,
+                graphSubmissionColumns = [
+                    {
+                        key: 'correct_submissions',
+                        percent_key: 'correct_percent',
+                        title: gettext('Correct'),
+                        className: 'text-right',
+                        type: 'number',
+                        fractionDigits: 1,
+                        color: '#4BB4FB'
+                    },
+                    {
+                        key: 'incorrect_submissions',
+                        percent_key: 'incorrect_percent',
+                        title: gettext('Incorrect'),
+                        className: 'text-right',
+                        type: 'number',
+                        fractionDigits: 1,
+                        color: '#CA0061'
+                    }
+                ],
+                tableColumns = [
+                    {key: 'index', title: gettext('Order'), type: 'number', className: 'text-right'},
+                    {key: 'name', title: model.get('contentTableHeading'), type: 'hasNull'},
+                    {key: 'difficulty', title: gettext('Difficulty'), type: 'hasNull'}
+                ],
+                performanceLoSectionChart,
+                performanceLoSectionTable;
+
+            tableColumns.push({
+                key: 'correct_submissions',
+                title: gettext('Correct'),
+                className: 'text-right',
+                type: 'number',
+                fractionDigits: 1
+            });
+
+            tableColumns.push({
+                key: 'incorrect_submissions',
+                title: gettext('Incorrect'),
+                className: 'text-right',
+                type: 'number',
+                fractionDigits: 1
+            });
+
+            tableColumns.push({
+                key: 'total_submissions',
+                title: gettext('Total'),
+                className: 'text-right',
+                type: 'number',
+                fractionDigits: 1
+            });
+
+            tableColumns.push({
+                key: 'correct_percent',
+                title: gettext('Percentage Correct'),
+                className: 'text-right',
+                type: 'percent'
+            });
+
+            if (model.get('hasData')) {
+                performanceLoSectionChart = new StackedBarView({
+                    el: '#chart-view',
+                    model: model,
+                    modelAttribute: 'tagsDistribution',
+                    trends: graphSubmissionColumns
+                });
+                performanceLoSectionChart.renderIfDataAvailable();
+            }
+
+            performanceLoSectionTable = new DataTableView({
+                el: '[data-role=data-table]',
+                model: model,
+                modelAttribute: 'tagsDistribution',
+                columns: tableColumns,
+                sorting: ['index'],
+                replaceZero: '-'
+            });
+            performanceLoSectionTable.renderIfDataAvailable();
+        });
+});

build.js

@@ -71,6 +71,14 @@
         {
             name: 'apps/learners/app/app',
             exclude: ['js/common']
+        },
+        {
+            name: 'js/performance-learning-outcomes-content-main',
+            exclude: ['js/common']
+        },
+        {
+            name: 'js/performance-learning-outcomes-section-main',
+            exclude: ['js/common']
         }
     ]
 });

common/tests/factories.py

@@ -34,6 +34,8 @@ class CourseStructureFactory(object):
         }
     ]
 
+    _count_of_problems = 4
+
     def __init__(self):
         self._structure = {}
         self._assignments = []
@@ -51,9 +53,14 @@ class CourseStructureFactory(object):
     def course_id(self, course_id):
         self._course_id = course_id
 
+    def _get_block_id(self, block_type, block_format=None, display_name=None,  # pylint: disable=unused-argument
+                      graded=True, children=None):                             # pylint: disable=unused-argument
+        return uuid.uuid4().hex
+
     def _generate_block(self, block_type, block_format=None, display_name=None, graded=True, children=None):
         return {
-            'id': 'i4x://edX/DemoX/{}/{}'.format(block_type, uuid.uuid4().hex),
+            'id': 'i4x://edX/DemoX/{}/{}'.format(block_type, self._get_block_id(block_type, block_format, display_name,
+                                                                                graded, children)),
             'display_name': display_name,
             'graded': graded,
             'format': block_format,
@@ -97,7 +104,7 @@ class CourseStructureFactory(object):
                 graded_children = []
 
                 # Generate the graded children
-                for problem_index in range(1, 4):
+                for problem_index in range(1, self._count_of_problems):
                     problem = self._generate_subsection_children(assignment_type, display_name, problem_index, True)
                     graded_children.append(problem['id'])