Merge pull request #7804 from edx/diana/progress-summary-refactor

Implement the Weighted Problem Grade Report

common/djangoapps/util/model_utils.py

@@ -4,8 +4,6 @@ Utilities for django models.
 from eventtracking import tracker
 
 from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
-from django.db.models.fields.related import RelatedField
 
 from django_countries.fields import Country
 

common/lib/xmodule/xmodule/graders.py

@@ -10,7 +10,7 @@ log = logging.getLogger("edx.courseware")
 
 # This is a tuple for holding scores, either from problems or sections.
 # Section either indicates the name of the problem or the name of the section
-Score = namedtuple("Score", "earned possible graded section")
+Score = namedtuple("Score", "earned possible graded section module_id")
 
 
 def aggregate_scores(scores, section_name="summary"):
@@ -27,15 +27,21 @@ def aggregate_scores(scores, section_name="summary"):
     total_possible = sum(score.possible for score in scores)
 
     #regardless of whether or not it is graded
-    all_total = Score(total_correct,
-                      total_possible,
-                      False,
-                      section_name)
+    all_total = Score(
+        total_correct,
+        total_possible,
+        False,
+        section_name,
+        None
+    )
     #selecting only graded things
-    graded_total = Score(total_correct_graded,
-                         total_possible_graded,
-                         True,
-                         section_name)
+    graded_total = Score(
+        total_correct_graded,
+        total_possible_graded,
+        True,
+        section_name,
+        None
+    )
 
     return all_total, graded_total
 

common/lib/xmodule/xmodule/tests/test_graders.py

@@ -13,23 +13,27 @@ class GradesheetTest(unittest.TestCase):
         Score.__sub__ = lambda me, other: (me.earned - other.earned) + (me.possible - other.possible)
 
         all_total, graded_total = aggregate_scores(scores)
-        self.assertEqual(all_total, Score(earned=0, possible=0, graded=False, section="summary"))
-        self.assertEqual(graded_total, Score(earned=0, possible=0, graded=True, section="summary"))
+        self.assertEqual(all_total, Score(earned=0, possible=0, graded=False, section="summary", module_id=None))
+        self.assertEqual(graded_total, Score(earned=0, possible=0, graded=True, section="summary", module_id=None))
 
-        scores.append(Score(earned=0, possible=5, graded=False, section="summary"))
+        scores.append(Score(earned=0, possible=5, graded=False, section="summary", module_id=None))
         all_total, graded_total = aggregate_scores(scores)
-        self.assertEqual(all_total, Score(earned=0, possible=5, graded=False, section="summary"))
-        self.assertEqual(graded_total, Score(earned=0, possible=0, graded=True, section="summary"))
+        self.assertEqual(all_total, Score(earned=0, possible=5, graded=False, section="summary", module_id=None))
+        self.assertEqual(graded_total, Score(earned=0, possible=0, graded=True, section="summary", module_id=None))
 
-        scores.append(Score(earned=3, possible=5, graded=True, section="summary"))
+        scores.append(Score(earned=3, possible=5, graded=True, section="summary", module_id=None))
         all_total, graded_total = aggregate_scores(scores)
-        self.assertAlmostEqual(all_total, Score(earned=3, possible=10, graded=False, section="summary"))
-        self.assertAlmostEqual(graded_total, Score(earned=3, possible=5, graded=True, section="summary"))
+        self.assertAlmostEqual(all_total, Score(earned=3, possible=10, graded=False, section="summary", module_id=None))
+        self.assertAlmostEqual(
+            graded_total, Score(earned=3, possible=5, graded=True, section="summary", module_id=None)
+        )
 
-        scores.append(Score(earned=2, possible=5, graded=True, section="summary"))
+        scores.append(Score(earned=2, possible=5, graded=True, section="summary", module_id=None))
         all_total, graded_total = aggregate_scores(scores)
-        self.assertAlmostEqual(all_total, Score(earned=5, possible=15, graded=False, section="summary"))
-        self.assertAlmostEqual(graded_total, Score(earned=5, possible=10, graded=True, section="summary"))
+        self.assertAlmostEqual(all_total, Score(earned=5, possible=15, graded=False, section="summary", module_id=None))
+        self.assertAlmostEqual(
+            graded_total, Score(earned=5, possible=10, graded=True, section="summary", module_id=None)
+        )
 
 
 class GraderTest(unittest.TestCase):
@@ -45,19 +49,19 @@ class GraderTest(unittest.TestCase):
     }
 
     test_gradesheet = {
-        'Homework': [Score(earned=2, possible=20.0, graded=True, section='hw1'),
-                     Score(earned=16, possible=16.0, graded=True, section='hw2')],
+        'Homework': [Score(earned=2, possible=20.0, graded=True, section='hw1', module_id=None),
+                     Score(earned=16, possible=16.0, graded=True, section='hw2', module_id=None)],
         # The dropped scores should be from the assignments that don't exist yet
 
-        'Lab': [Score(earned=1, possible=2.0, graded=True, section='lab1'),  # Dropped
-                Score(earned=1, possible=1.0, graded=True, section='lab2'),
-                Score(earned=1, possible=1.0, graded=True, section='lab3'),
-                Score(earned=5, possible=25.0, graded=True, section='lab4'),  # Dropped
-                Score(earned=3, possible=4.0, graded=True, section='lab5'),  # Dropped
-                Score(earned=6, possible=7.0, graded=True, section='lab6'),
-                Score(earned=5, possible=6.0, graded=True, section='lab7')],
+        'Lab': [Score(earned=1, possible=2.0, graded=True, section='lab1', module_id=None),  # Dropped
+                Score(earned=1, possible=1.0, graded=True, section='lab2', module_id=None),
+                Score(earned=1, possible=1.0, graded=True, section='lab3', module_id=None),
+                Score(earned=5, possible=25.0, graded=True, section='lab4', module_id=None),  # Dropped
+                Score(earned=3, possible=4.0, graded=True, section='lab5', module_id=None),  # Dropped
+                Score(earned=6, possible=7.0, graded=True, section='lab6', module_id=None),
+                Score(earned=5, possible=6.0, graded=True, section='lab7', module_id=None)],
 
-        'Midterm': [Score(earned=50.5, possible=100, graded=True, section="Midterm Exam"), ],
+        'Midterm': [Score(earned=50.5, possible=100, graded=True, section="Midterm Exam", module_id=None), ],
     }
 
     def test_single_section_grader(self):

common/test/acceptance/pages/lms/instructor_dashboard.py

@@ -702,12 +702,47 @@ class DataDownloadPage(PageObject):
     def is_browser_on_page(self):
         return self.q(css='a[data-section=data_download].active-section').present
 
+    @property
+    def generate_student_report_button(self):
+        """
+        Returns the "Download profile information as a CSV" button.
+        """
+        return self.q(css='input[name=list-profiles-csv]')
+
+    @property
+    def generate_grade_report_button(self):
+        """
+        Returns the "Generate Grade Report" button.
+        """
+        return self.q(css='input[name=calculate-grades-csv]')
+
+    @property
+    def generate_problem_report_button(self):
+        """
+        Returns the "Generate Problem Grade Report" button.
+        """
+        return self.q(css='input[name=problem-grade-report]')
+
+    @property
+    def report_download_links(self):
+        """
+        Returns the download links for the current page.
+        """
+        return self.q(css="#report-downloads-table .file-download-link>a")
+
+    def wait_for_available_report(self):
+        """
+        Waits for a downloadable report to be available.
+        """
+        EmptyPromise(
+            lambda: len(self.report_download_links) >= 1, 'Waiting for downloadable report'
+        ).fulfill()
+
     def get_available_reports_for_download(self):
         """
         Returns a list of all the available reports for download.
         """
-        reports = self.q(css="#report-downloads-table .file-download-link>a").map(lambda el: el.text)
-        return reports.results
+        return self.report_download_links.map(lambda el: el.text)
 
 
 class StudentAdminPage(PageObject):

common/test/acceptance/tests/discussion/test_cohort_management.py

@@ -14,7 +14,6 @@ from xmodule.partitions.partitions import Group
 from ...fixtures.course import CourseFixture, XBlockFixtureDesc
 from ...pages.lms.auto_auth import AutoAuthPage
 from ...pages.lms.instructor_dashboard import InstructorDashboardPage, DataDownloadPage
-from ...pages.studio.settings_advanced import AdvancedSettingsPage
 from ...pages.studio.settings_group_configurations import GroupConfigurationsPage
 
 import uuid
@@ -555,9 +554,7 @@ class CohortConfigurationTest(EventsTestMixin, UniqueCourseTest, CohortTestMixin
 
         # Verify the results can be downloaded.
         data_download = self.instructor_dashboard_page.select_data_download()
-        EmptyPromise(
-            lambda: 1 == len(data_download.get_available_reports_for_download()), 'Waiting for downloadable report'
-        ).fulfill()
+        data_download.wait_for_available_report()
         report = data_download.get_available_reports_for_download()[0]
         base_file_name = "cohort_results_"
         self.assertIn("{}_{}".format(

common/test/acceptance/tests/lms/test_lms_instructor_dashboard.py

@@ -5,15 +5,36 @@ End-to-end tests for the LMS Instructor Dashboard.
 
 from nose.plugins.attrib import attr
 
-from ..helpers import UniqueCourseTest, get_modal_alert
+from ..helpers import UniqueCourseTest, get_modal_alert, EventsTestMixin
 from ...pages.common.logout import LogoutPage
 from ...pages.lms.auto_auth import AutoAuthPage
 from ...pages.lms.instructor_dashboard import InstructorDashboardPage
 from ...fixtures.course import CourseFixture
 
 
+class BaseInstructorDashboardTest(EventsTestMixin, UniqueCourseTest):
+    """
+    Mixin class for testing the instructor dashboard.
+    """
+    def log_in_as_instructor(self):
+        """
+        Logs in as an instructor and returns the id.
+        """
+        username = "test_instructor_{uuid}".format(uuid=self.unique_id[0:6])
+        auto_auth_page = AutoAuthPage(self.browser, username=username, course_id=self.course_id, staff=True)
+        return username, auto_auth_page.visit().get_user_id()
+
+    def visit_instructor_dashboard(self):
+        """
+        Visits the instructor dashboard.
+        """
+        instructor_dashboard_page = InstructorDashboardPage(self.browser, self.course_id)
+        instructor_dashboard_page.visit()
+        return instructor_dashboard_page
+
+
 @attr('shard_5')
-class AutoEnrollmentWithCSVTest(UniqueCourseTest):
+class AutoEnrollmentWithCSVTest(BaseInstructorDashboardTest):
     """
     End-to-end tests for Auto-Registration and enrollment functionality via CSV file.
     """
@@ -21,13 +42,8 @@ class AutoEnrollmentWithCSVTest(UniqueCourseTest):
     def setUp(self):
         super(AutoEnrollmentWithCSVTest, self).setUp()
         self.course_fixture = CourseFixture(**self.course_info).install()
-
-        # login as an instructor
-        AutoAuthPage(self.browser, course_id=self.course_id, staff=True).visit()
-
-        # go to the membership page on the instructor dashboard
-        instructor_dashboard_page = InstructorDashboardPage(self.browser, self.course_id)
-        instructor_dashboard_page.visit()
+        self.log_in_as_instructor()
+        instructor_dashboard_page = self.visit_instructor_dashboard()
         self.auto_enroll_section = instructor_dashboard_page.select_membership().select_auto_enroll_section()
 
     def test_browse_and_upload_buttons_are_visible(self):
@@ -91,7 +107,7 @@ class AutoEnrollmentWithCSVTest(UniqueCourseTest):
 
 
 @attr('shard_5')
-class EntranceExamGradeTest(UniqueCourseTest):
+class EntranceExamGradeTest(BaseInstructorDashboardTest):
     """
     Tests for Entrance exam specific student grading tasks.
     """
@@ -112,13 +128,9 @@ class EntranceExamGradeTest(UniqueCourseTest):
 
         LogoutPage(self.browser).visit()
 
-        # login as an instructor
-        AutoAuthPage(self.browser, course_id=self.course_id, staff=True).visit()
-
         # go to the student admin page on the instructor dashboard
-        instructor_dashboard_page = InstructorDashboardPage(self.browser, self.course_id)
-        instructor_dashboard_page.visit()
-        self.student_admin_section = instructor_dashboard_page.select_student_admin()
+        self.log_in_as_instructor()
+        self.student_admin_section = self.visit_instructor_dashboard().select_student_admin()
 
     def test_input_text_and_buttons_are_visible(self):
         """
@@ -291,3 +303,96 @@ class EntranceExamGradeTest(UniqueCourseTest):
         self.student_admin_section.set_student_email(self.student_identifier)
         self.student_admin_section.click_task_history_button()
         self.assertTrue(self.student_admin_section.is_background_task_history_table_visible())
+
+
+class DataDownloadsTest(BaseInstructorDashboardTest):
+    """
+    Bok Choy tests for the "Data Downloads" tab.
+    """
+    def setUp(self):
+        super(DataDownloadsTest, self).setUp()
+        self.course_fixture = CourseFixture(**self.course_info).install()
+        self.instructor_username, self.instructor_id = self.log_in_as_instructor()
+        instructor_dashboard_page = self.visit_instructor_dashboard()
+        self.data_download_section = instructor_dashboard_page.select_data_download()
+
+    def verify_report_requested_event(self, report_type):
+        """
+        Verifies that the correct event is emitted when a report is requested.
+        """
+        self.assert_matching_events_were_emitted(
+            event_filter={'name': u'edx.instructor.report.requested', 'report_type': report_type}
+        )
+
+    def verify_report_downloaded_event(self, report_url):
+        """
+        Verifies that the correct event is emitted when a report is downloaded.
+        """
+        self.assert_matching_events_were_emitted(
+            event_filter={'name': u'edx.instructor.report.downloaded', 'report_url': report_url}
+        )
+
+    def verify_report_download(self, report_name):
+        """
+        Verifies that a report can be downloaded and an event fired.
+        """
+        download_links = self.data_download_section.report_download_links
+        self.assertEquals(len(download_links), 1)
+        download_links[0].click()
+        expected_url = download_links.attrs('href')[0]
+        self.assertIn(report_name, expected_url)
+        self.verify_report_downloaded_event(expected_url)
+
+    def test_student_profiles_report_download(self):
+        """
+        Scenario: Verify that an instructor can download a student profiles report
+
+        Given that I am an instructor
+        And I visit the instructor dashboard's "Data Downloads" tab
+        And I click on the "Download profile information as a CSV" button
+        Then a report should be generated
+        And a report requested event should be emitted
+        When I click on the report
+        Then a report downloaded event should be emitted
+        """
+        report_name = u"student_profile_info"
+        self.data_download_section.generate_student_report_button.click()
+        self.data_download_section.wait_for_available_report()
+        self.verify_report_requested_event(report_name)
+        self.verify_report_download(report_name)
+
+    def test_grade_report_download(self):
+        """
+        Scenario: Verify that an instructor can download a grade report
+
+        Given that I am an instructor
+        And I visit the instructor dashboard's "Data Downloads" tab
+        And I click on the "Generate Grade Report" button
+        Then a report should be generated
+        And a report requested event should be emitted
+        When I click on the report
+        Then a report downloaded event should be emitted
+        """
+        report_name = u"grade_report"
+        self.data_download_section.generate_grade_report_button.click()
+        self.data_download_section.wait_for_available_report()
+        self.verify_report_requested_event(report_name)
+        self.verify_report_download(report_name)
+
+    def test_problem_grade_report_download(self):
+        """
+        Scenario: Verify that an instructor can download a problem grade report
+
+        Given that I am an instructor
+        And I visit the instructor dashboard's "Data Downloads" tab
+        And I click on the "Generate Problem Grade Report" button
+        Then a report should be generated
+        And a report requested event should be emitted
+        When I click on the report
+        Then a report downloaded event should be emitted
+        """
+        report_name = u"problem_grade_report"
+        self.data_download_section.generate_problem_report_button.click()
+        self.data_download_section.wait_for_available_report()
+        self.verify_report_requested_event(report_name)
+        self.verify_report_download(report_name)

lms/djangoapps/course_structure_api/v0/api.py

+"""
+API implementation of the Course Structure API for Python code.
+
+Note: The course list and course detail functionality isn't currently supported here because
+of the tricky interactions between DRF and the code.
+Most of that information is available by accessing the course objects directly.
+"""
+
+from course_structure_api.v0 import serializers
+from course_structure_api.v0.errors import CourseNotFoundError, CourseStructureNotAvailableError
+from openedx.core.djangoapps.content.course_structures import models, tasks
+from courseware import courses
+
+
+def _retrieve_course(course_key):
+    """Retrieves the course for the given course key.
+
+    Args:
+        course_key: The CourseKey for the course we'd like to retrieve.
+    Returns:
+        the course that matches the CourseKey
+    Raises:
+        CourseNotFoundError
+
+    """
+    try:
+        return courses.get_course(course_key)
+    except ValueError:
+        raise CourseNotFoundError
+
+
+def course_structure(course_key):
+    """
+    Retrieves the entire course structure, including information about all the blocks used in the course.
+
+    Args:
+        course_key: the CourseKey of the course we'd like to retrieve.
+    Returns:
+        The serialized output of the course structure:
+            * root: The ID of the root node of the course structure.
+
+            * blocks: A dictionary that maps block IDs to a collection of
+            information about each block. Each block contains the following
+            fields.
+
+                * id: The ID of the block.
+
+                * type: The type of block. Possible values include sequential,
+                    vertical, html, problem, video, and discussion. The type can also be
+                    the name of a custom type of block used for the course.
+
+                * display_name: The display name configured for the block.
+
+                * graded: Whether or not the sequential or problem is graded. The
+                    value is true or false.
+
+                * format: The assignment type.
+
+                * children: If the block has child blocks, a list of IDs of the child
+                blocks.
+    Raises:
+        CourseStructureNotAvailableError, CourseNotFoundError
+    """
+    course = _retrieve_course(course_key)
+    try:
+        requested_course_structure = models.CourseStructure.objects.get(course_id=course.id)
+        return serializers.CourseStructureSerializer(requested_course_structure.structure).data
+    except models.CourseStructure.DoesNotExist:
+        # If we don't have data stored, generate it and return an error.
+        tasks.update_course_structure.delay(unicode(course_key))
+        raise CourseStructureNotAvailableError
+
+
+def course_grading_policy(course_key):
+    """
+    Retrieves the course grading policy.
+
+    Args:
+        course_key: CourseKey the corresponds to the course we'd like to know grading policy information about.
+    Returns:
+        The serialized version of the course grading policy containing the following information:
+            * assignment_type: The type of the assignment, as configured by course
+                staff. For example, course staff might make the assignment types Homework,
+                Quiz, and Exam.
+
+            * count: The number of assignments of the type.
+
+            * dropped: Number of assignments of the type that are dropped.
+
+            * weight: The weight, or effect, of the assignment type on the learner's
+                final grade.
+    """
+    course = _retrieve_course(course_key)
+    return serializers.GradingPolicySerializer(course.raw_grader).data

lms/djangoapps/course_structure_api/v0/errors.py

+""" Errors used by the Course Structure API. """
+
+
+class CourseNotFoundError(Exception):
+    """ The course was not found. """
+    pass
+
+
+class CourseStructureNotAvailableError(Exception):
+    """ The course structure still needs to be generated. """
+    pass

lms/djangoapps/course_structure_api/v0/views.py

@@ -11,10 +11,10 @@ from rest_framework.response import Response
 from xmodule.modulestore.django import modulestore
 from opaque_keys.edx.keys import CourseKey
 
-from course_structure_api.v0 import serializers
+from course_structure_api.v0 import api, serializers
+from course_structure_api.v0.errors import CourseNotFoundError, CourseStructureNotAvailableError
 from courseware import courses
 from courseware.access import has_access
-from openedx.core.djangoapps.content.course_structures import models, tasks
 from openedx.core.lib.api.permissions import IsAuthenticatedOrDebug
 from openedx.core.lib.api.serializers import PaginationSerializer
 from student.roles import CourseInstructorRole, CourseStaffRole
@@ -40,13 +40,37 @@ class CourseViewMixin(object):
             course_id = self.kwargs.get('course_id')
             course_key = CourseKey.from_string(course_id)
             course = courses.get_course(course_key)
-
-            self.check_course_permissions(self.request.user, course)
+            self.check_course_permissions(self.request.user, course_key)
 
             return course
         except ValueError:
             raise Http404
 
+    @staticmethod
+    def course_check(func):
+        """Decorator responsible for catching errors finding and returning a 404 if the user does not have access
+        to the API function.
+
+        :param func: function to be wrapped
+        :returns: the wrapped function
+        """
+        def func_wrapper(self, *args, **kwargs):
+            """Wrapper function for this decorator.
+
+            :param *args: the arguments passed into the function
+            :param **kwargs: the keyword arguments passed into the function
+            :returns: the result of the wrapped function
+            """
+            try:
+                course_id = self.kwargs.get('course_id')
+                self.course_key = CourseKey.from_string(course_id)
+                self.check_course_permissions(self.request.user, self.course_key)
+                return func(self, *args, **kwargs)
+            except CourseNotFoundError:
+                raise Http404
+
+        return func_wrapper
+
     def user_can_access_course(self, user, course):
         """
         Determines if the user is staff or an instructor for the course.
@@ -185,7 +209,6 @@ class CourseDetail(CourseViewMixin, RetrieveAPIView):
         * end: The course end date. If course end date is not specified, the
           value is null.
     """
-
     serializer_class = serializers.CourseSerializer
 
     def get_object(self, queryset=None):
@@ -227,23 +250,16 @@ class CourseStructure(CourseViewMixin, RetrieveAPIView):
           * children: If the block has child blocks, a list of IDs of the child
             blocks.
     """
-    serializer_class = serializers.CourseStructureSerializer
-    course = None
 
-    def retrieve(self, request, *args, **kwargs):
+    @CourseViewMixin.course_check
+    def get(self, request, **kwargs):
         try:
-            return super(CourseStructure, self).retrieve(request, *args, **kwargs)
-        except models.CourseStructure.DoesNotExist:
-            # If we don't have data stored, generate it and return a 503.
-            tasks.update_course_structure.delay(unicode(self.course.id))
+            return Response(api.course_structure(self.course_key))
+        except CourseStructureNotAvailableError:
+            # If we don't have data stored, we will try to regenerate it, so
+            # return a 503 and as them to retry in 2 minutes.
             return Response(status=503, headers={'Retry-After': '120'})
 
-    def get_object(self, queryset=None):
-        # Make sure the course exists and the user has permissions to view it.
-        self.course = self.get_course_or_404()
-        course_structure = models.CourseStructure.objects.get(course_id=self.course.id)
-        return course_structure.structure
-
 
 class CourseGradingPolicy(CourseViewMixin, ListAPIView):
     """
@@ -269,11 +285,8 @@ class CourseGradingPolicy(CourseViewMixin, ListAPIView):
           final grade.
     """
 
-    serializer_class = serializers.GradingPolicySerializer
     allow_empty = False
 
-    def get_queryset(self):
-        course = self.get_course_or_404()
-
-        # Return the raw data. The serializer will handle the field mappings.
-        return course.raw_grader
+    @CourseViewMixin.course_check
+    def get(self, request, **kwargs):
+        return Response(api.course_grading_policy(self.course_key))

lms/djangoapps/courseware/grades.py

@@ -225,16 +225,24 @@ def _grade(student, request, course, keep_raw_scores):
 
                     graded = module_descriptor.graded
                     if not total > 0:
-                        #We simply cannot grade a problem that is 12/0, because we might need it as a percentage
+                        # We simply cannot grade a problem that is 12/0, because we might need it as a percentage
                         graded = False
 
-                    scores.append(Score(correct, total, graded, module_descriptor.display_name_with_default))
+                    scores.append(
+                        Score(
+                            correct,
+                            total,
+                            graded,
+                            module_descriptor.display_name_with_default,
+                            module_descriptor.location
+                        )
+                    )
 
                 _, graded_total = graders.aggregate_scores(scores, section_name)
                 if keep_raw_scores:
                     raw_scores += scores
             else:
-                graded_total = Score(0.0, 1.0, True, section_name)
+                graded_total = Score(0.0, 1.0, True, section_name, None)
 
             #Add the graded total to totaled_scores
             if graded_total.possible > 0:
@@ -364,7 +372,15 @@ def _progress_summary(student, request, course):
                     if correct is None and total is None:
                         continue
 
-                    scores.append(Score(correct, total, graded, module_descriptor.display_name_with_default))
+                    scores.append(
+                        Score(
+                            correct,
+                            total,
+                            graded,
+                            module_descriptor.display_name_with_default,
+                            module_descriptor.location
+                        )
+                    )
 
                 scores.reverse()
                 section_total, _ = graders.aggregate_scores(
@@ -484,7 +500,7 @@ def manual_transaction():
         transaction.commit()
 
 
-def iterate_grades_for(course_or_id, students):
+def iterate_grades_for(course_or_id, students, keep_raw_scores=False):
     """Given a course_id and an iterable of students (User), yield a tuple of:
 
     (student, gradeset, err_msg) for every student enrolled in the course.
@@ -521,7 +537,7 @@ def iterate_grades_for(course_or_id, students):
                 # It's not pretty, but untangling that is currently beyond the
                 # scope of this feature.
                 request.session = {}
-                gradeset = grade(student, request, course)
+                gradeset = grade(student, request, course, keep_raw_scores)
                 yield student, gradeset, ""
             except Exception as exc:  # pylint: disable=broad-except
                 # Keep marching on even if this student couldn't be graded for

lms/djangoapps/django_comment_client/forum/tests.py

@@ -15,10 +15,11 @@ from django_comment_client.tests.group_id import (
     NonCohortedTopicGroupIdTestMixin
 )
 from django_comment_client.tests.unicode import UnicodeTestMixin
-from django_comment_client.tests.utils import CohortedTestCase, ContentGroupTestCase
+from django_comment_client.tests.utils import CohortedTestCase
 from django_comment_client.utils import strip_none
 from student.tests.factories import UserFactory, CourseEnrollmentFactory
 from util.testing import UrlResetMixin
+from openedx.core.djangoapps.util.testing import ContentGroupTestCase
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import (

lms/djangoapps/django_comment_client/tests/test_utils.py

@@ -15,12 +15,12 @@ from edxmako import add_lookup
 
 from django_comment_client.tests.factories import RoleFactory
 from django_comment_client.tests.unicode import UnicodeTestMixin
-from django_comment_client.tests.utils import ContentGroupTestCase
 import django_comment_client.utils as utils
 
 from courseware.tests.factories import InstructorFactory
 from openedx.core.djangoapps.course_groups.cohorts import set_course_cohort_settings
 from student.tests.factories import UserFactory, CourseEnrollmentFactory
+from openedx.core.djangoapps.util.testing import ContentGroupTestCase
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 

lms/djangoapps/django_comment_client/tests/utils.py

 """
 Utilities for tests within the django_comment_client module.
 """
-from datetime import datetime
 from mock import patch
-from pytz import UTC
 
-from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup
 from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory
 from django_comment_common.models import Role
 from django_comment_common.utils import seed_permissions_roles
 from student.tests.factories import CourseEnrollmentFactory, UserFactory
-from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.partitions.partitions import UserPartition, Group
 
 
 class CohortedTestCase(ModuleStoreTestCase):
@@ -49,82 +45,3 @@ class CohortedTestCase(ModuleStoreTestCase):
         self.moderator.roles.add(Role.objects.get(name="Moderator", course_id=self.course.id))
         self.student_cohort.users.add(self.student)
         self.moderator_cohort.users.add(self.moderator)
-
-
-class ContentGroupTestCase(ModuleStoreTestCase):
-    """
-    Sets up discussion modules visible to content groups 'Alpha' and
-    'Beta', as well as a module visible to all students.  Creates a
-    staff user, users with access to Alpha/Beta (by way of cohorts),
-    and a non-cohorted user with no special access.
-    """
-    def setUp(self):
-        super(ContentGroupTestCase, self).setUp()
-
-        self.course = CourseFactory.create(
-            org='org', number='number', run='run',
-            # This test needs to use a course that has already started --
-            # discussion topics only show up if the course has already started,
-            # and the default start date for courses is Jan 1, 2030.
-            start=datetime(2012, 2, 3, tzinfo=UTC),
-            user_partitions=[
-                UserPartition(
-                    0,
-                    'Content Group Configuration',
-                    '',
-                    [Group(1, 'Alpha'), Group(2, 'Beta')],
-                    scheme_id='cohort'
-                )
-            ],
-            cohort_config={'cohorted': True},
-            discussion_topics={}
-        )
-
-        self.staff_user = UserFactory.create(is_staff=True)
-        self.alpha_user = UserFactory.create()
-        self.beta_user = UserFactory.create()
-        self.non_cohorted_user = UserFactory.create()
-        for user in [self.staff_user, self.alpha_user, self.beta_user, self.non_cohorted_user]:
-            CourseEnrollmentFactory.create(user=user, course_id=self.course.id)
-
-        alpha_cohort = CohortFactory(
-            course_id=self.course.id,
-            name='Cohort Alpha',
-            users=[self.alpha_user]
-        )
-        beta_cohort = CohortFactory(
-            course_id=self.course.id,
-            name='Cohort Beta',
-            users=[self.beta_user]
-        )
-        CourseUserGroupPartitionGroup.objects.create(
-            course_user_group=alpha_cohort,
-            partition_id=self.course.user_partitions[0].id,
-            group_id=self.course.user_partitions[0].groups[0].id
-        )
-        CourseUserGroupPartitionGroup.objects.create(
-            course_user_group=beta_cohort,
-            partition_id=self.course.user_partitions[0].id,
-            group_id=self.course.user_partitions[0].groups[1].id
-        )
-        self.alpha_module = ItemFactory.create(
-            parent_location=self.course.location,
-            category='discussion',
-            discussion_id='alpha_group_discussion',
-            discussion_target='Visible to Alpha',
-            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[0].id]}
-        )
-        self.beta_module = ItemFactory.create(
-            parent_location=self.course.location,
-            category='discussion',
-            discussion_id='beta_group_discussion',
-            discussion_target='Visible to Beta',
-            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[1].id]}
-        )
-        self.global_module = ItemFactory.create(
-            parent_location=self.course.location,
-            category='discussion',
-            discussion_id='global_group_discussion',
-            discussion_target='Visible to Everyone'
-        )
-        self.course = self.store.get_item(self.course.location)

lms/djangoapps/instructor/views/api.py

@@ -1954,6 +1954,32 @@ def calculate_grades_csv(request, course_id):
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 @require_level('staff')
+def problem_grade_report(request, course_id):
+    """
+    Request a CSV showing students' grades for all problems in the
+    course.
+
+    AlreadyRunningError is raised if the course's grades are already being
+    updated.
+    """
+    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
+    try:
+        instructor_task.api.submit_problem_grade_report(request, course_key)
+        success_status = _("Your problem grade report is being generated! "
+                           "You can view the status of the generation task in the 'Pending Instructor Tasks' section.")
+        return JsonResponse({"status": success_status})
+    except AlreadyRunningError:
+        already_running_status = _("A problem grade report is already being generated. "
+                                   "Check the 'Pending Instructor Tasks' table for the status of the task. "
+                                   "When completed, the report will be available for download in the table below.")
+        return JsonResponse({
+            "status": already_running_status
+        })
+
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+@require_level('staff')
 @require_query_params('rolename')
 def list_forum_members(request, course_id):
     """

lms/djangoapps/instructor/views/api_urls.py

@@ -87,6 +87,8 @@ urlpatterns = patterns(
         'instructor.views.api.list_report_downloads', name="list_report_downloads"),
     url(r'calculate_grades_csv$',
         'instructor.views.api.calculate_grades_csv', name="calculate_grades_csv"),
+    url(r'problem_grade_report$',
+        'instructor.views.api.problem_grade_report', name="problem_grade_report"),
 
     # Registration Codes..
     url(r'get_registration_codes$',

lms/djangoapps/instructor/views/instructor_dashboard.py

@@ -420,6 +420,7 @@ def _section_data_download(course, access):
         'list_instructor_tasks_url': reverse('list_instructor_tasks', kwargs={'course_id': unicode(course_key)}),
         'list_report_downloads_url': reverse('list_report_downloads', kwargs={'course_id': unicode(course_key)}),
         'calculate_grades_csv_url': reverse('calculate_grades_csv', kwargs={'course_id': unicode(course_key)}),
+        'problem_grade_report_url': reverse('problem_grade_report', kwargs={'course_id': unicode(course_key)}),
     }
     return section_data
 

lms/djangoapps/instructor_task/api.py

@@ -19,6 +19,7 @@ from instructor_task.tasks import (
     delete_problem_state,
     send_bulk_course_email,
     calculate_grades_csv,
+    calculate_problem_grade_report,
     calculate_students_features_csv,
     cohort_students,
 )
@@ -334,6 +335,18 @@ def submit_calculate_grades_csv(request, course_key):
     return submit_task(request, task_type, task_class, course_key, task_input, task_key)
 
 
+def submit_problem_grade_report(request, course_key):
+    """
+    Submits a task to generate a CSV grade report containing problem
+    values.
+    """
+    task_type = 'grade_problems'
+    task_class = calculate_problem_grade_report
+    task_input = {}
+    task_key = ""
+    return submit_task(request, task_type, task_class, course_key, task_input, task_key)
+
+
 def submit_calculate_students_features_csv(request, course_key, features):
     """
     Submits a task to generate a CSV containing student profile info.

lms/djangoapps/instructor_task/tasks.py

@@ -35,6 +35,7 @@ from instructor_task.tasks_helper import (
     reset_attempts_module_state,
     delete_problem_module_state,
     upload_grades_csv,
+    upload_problem_grade_report,
     upload_students_csv,
     cohort_students_and_upload
 )
@@ -156,6 +157,23 @@ def calculate_grades_csv(entry_id, xmodule_instance_args):
 
 
 @task(base=BaseInstructorTask, routing_key=settings.GRADES_DOWNLOAD_ROUTING_KEY)  # pylint: disable=not-callable
+def calculate_problem_grade_report(entry_id, xmodule_instance_args):
+    """
+    Generate a CSV for a course containing all students' problem
+    grades and push the results to an S3 bucket for download.
+    """
+    # Translators: This is a past-tense phrase that is inserted into task progress messages as {action}.
+    action_name = ugettext_noop('problem distribution graded')
+    TASK_LOG.info(
+        u"Task: %s, InstructorTask ID: %s, Task type: %s, Preparing for task execution",
+        xmodule_instance_args.get('task_id'), entry_id, action_name
+    )
+
+    task_fn = partial(upload_problem_grade_report, xmodule_instance_args)
+    return run_main_task(entry_id, task_fn, action_name)
+
+
+@task(base=BaseInstructorTask, routing_key=settings.GRADES_DOWNLOAD_ROUTING_KEY)  # pylint: disable=not-callable
 def calculate_students_features_csv(entry_id, xmodule_instance_args):
     """
     Compute student profile information for a course and upload the

lms/djangoapps/instructor_task/tasks_helper.py

@@ -4,7 +4,10 @@ running state of a course.
 
 """
 import json
+from collections import OrderedDict
 from datetime import datetime
+from eventtracking import tracker
+from itertools import chain
 from time import time
 import unicodecsv
 import logging
@@ -34,6 +37,7 @@ from instructor_task.models import ReportStore, InstructorTask, PROGRESS
 from lms.djangoapps.lms_xblock.runtime import LmsPartitionService
 from openedx.core.djangoapps.course_groups.cohorts import get_cohort
 from openedx.core.djangoapps.course_groups.models import CourseUserGroup
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
 from opaque_keys.edx.keys import UsageKey
 from openedx.core.djangoapps.course_groups.cohorts import add_user_to_cohort, is_course_cohorted
 from student.models import CourseEnrollment
@@ -51,6 +55,9 @@ UPDATE_STATUS_SUCCEEDED = 'succeeded'
 UPDATE_STATUS_FAILED = 'failed'
 UPDATE_STATUS_SKIPPED = 'skipped'
 
+# The setting name used for events when "settings" (account settings, preferences, profile information) change.
+REPORT_REQUESTED_EVENT_NAME = u'edx.instructor.report.requested'
+
 
 class BaseInstructorTask(Task):
     """
@@ -549,6 +556,7 @@ def upload_csv_to_report_store(rows, csv_name, course_id, timestamp):
         ),
         rows
     )
+    tracker.emit(REPORT_REQUESTED_EVENT_NAME, {"report_type": csv_name, })
 
 
 def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):  # pylint: disable=too-many-statements
@@ -705,6 +713,127 @@ def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input,
     return task_progress.update_task_state(extra_meta=current_step)
 
 
+def _order_problems(blocks):
+    """
+    Sort the problems by the assignment type and assignment that it belongs to.
+
+    Args:
+        blocks (OrderedDict) - A course structure containing blocks that have been ordered
+                              (i.e. when we iterate over them, we will see them in the order
+                              that they appear in the course).
+
+    Returns:
+        an OrderedDict that maps a problem id to its headers in the final report.
+    """
+    problems = OrderedDict()
+    assignments = dict()
+    # First, sort out all the blocks into their correct assignments and all the
+    # assignments into their correct types.
+    for block in blocks:
+        # Put the assignments in order into the assignments list.
+        if blocks[block]['block_type'] == 'sequential':
+            block_format = blocks[block]['format']
+            if block_format not in assignments:
+                assignments[block_format] = OrderedDict()
+            assignments[block_format][block] = list()
+
+        # Put the problems into the correct order within their assignment.
+        if blocks[block]['block_type'] == 'problem' and blocks[block]['graded'] is True:
+            current = blocks[block]['parent']
+            # crawl up the tree for the sequential block
+            while blocks[current]['block_type'] != 'sequential':
+                current = blocks[current]['parent']
+
+            current_format = blocks[current]['format']
+            assignments[current_format][current].append(block)
+
+    # Now that we have a sorting and an order for the assignments and problems,
+    # iterate through them in order to generate the header row.
+    for assignment_type in assignments:
+        for assignment_index, assignment in enumerate(assignments[assignment_type].keys(), start=1):
+            for problem in assignments[assignment_type][assignment]:
+                header_name = u"{assignment_type} {assignment_index}: {assignment_name} - {block}".format(
+                    block=blocks[problem]['display_name'],
+                    assignment_type=assignment_type,
+                    assignment_index=assignment_index,
+                    assignment_name=blocks[assignment]['display_name']
+                )
+                problems[problem] = [header_name + " (Earned)", header_name + " (Possible)"]
+
+    return problems
+
+
+def upload_problem_grade_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
+    """
+    Generate a CSV containing all students' problem grades within a given
+    `course_id`.
+    """
+    start_time = time()
+    start_date = datetime.now(UTC)
+    status_interval = 100
+    enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
+    task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
+
+    # This struct encapsulates both the display names of each static item in the
+    # header row as values as well as the django User field names of those items
+    # as the keys.  It is structured in this way to keep the values related.
+    header_row = OrderedDict([('id', 'Student ID'), ('email', 'Email'), ('username', 'Username')])
+
+    try:
+        course_structure = CourseStructure.objects.get(course_id=course_id)
+        blocks = course_structure.ordered_blocks
+        problems = _order_problems(blocks)
+    except CourseStructure.DoesNotExist:
+        return task_progress.update_task_state(
+            extra_meta={'step': 'Generating course structure. Please refresh and try again.'}
+        )
+
+    # Just generate the static fields for now.
+    rows = [list(header_row.values()) + ['Final Grade'] + list(chain.from_iterable(problems.values()))]
+    error_rows = [list(header_row.values()) + ['error_msg']]
+    current_step = {'step': 'Calculating Grades'}
+
+    for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students, keep_raw_scores=True):
+        student_fields = [getattr(student, field_name) for field_name in header_row]
+        task_progress.attempted += 1
+
+        if err_msg:
+            # There was an error grading this student.
+            error_rows.append(student_fields + [err_msg])
+            task_progress.failed += 1
+            continue
+
+        final_grade = gradeset['percent']
+        # Only consider graded problems
+        problem_scores = {unicode(score.module_id): score for score in gradeset['raw_scores'] if score.graded}
+        earned_possible_values = list()
+        for problem_id in problems:
+            try:
+                problem_score = problem_scores[problem_id]
+                earned_possible_values.append([problem_score.earned, problem_score.possible])
+            except KeyError:
+                # The student has not been graded on this problem.  For example,
+                # iterate_grades_for skips problems that students have never
+                # seen in order to speed up report generation.  It could also be
+                # the case that the student does not have access to it (e.g. A/B
+                # test or cohorted courseware).
+                earned_possible_values.append(['N/A', 'N/A'])
+        rows.append(student_fields + [final_grade] + list(chain.from_iterable(earned_possible_values)))
+
+        task_progress.succeeded += 1
+        if task_progress.attempted % status_interval == 0:
+            task_progress.update_task_state(extra_meta=current_step)
+
+    # Perform the upload if any students have been successfully graded
+    if len(rows) > 1:
+        upload_csv_to_report_store(rows, 'problem_grade_report', course_id, start_date)
+    # If there are any error rows, write them out as well
+    if len(error_rows) > 1:
+        upload_csv_to_report_store(error_rows, 'problem_grade_report_err', course_id, start_date)
+
+    return task_progress.update_task_state(extra_meta={'step': 'Uploading CSV'})
+
+
 def upload_students_csv(_xmodule_instance_args, _entry_id, course_id, task_input, action_name):
     """
     For a given `course_id`, generate a CSV file containing profile

lms/djangoapps/instructor_task/tests/test_base.py

@@ -127,7 +127,12 @@ class InstructorTaskCourseTestCase(LoginEnrollmentTestCase, ModuleStoreTestCase)
         if course_factory_kwargs is not None:
             course_args.update(course_factory_kwargs)
         self.course = CourseFactory.create(**course_args)
+        self.add_course_content()
 
+    def add_course_content(self):
+        """
+        Add a chapter and a sequential to the current course.
+        """
         # Add a chapter to the course
         chapter = ItemFactory.create(parent_location=self.course.location,
                                      display_name=TEST_SECTION_NAME)
@@ -141,12 +146,13 @@ class InstructorTaskCourseTestCase(LoginEnrollmentTestCase, ModuleStoreTestCase)
     @staticmethod
     def get_user_email(username):
         """Generate email address based on username"""
-        return '{0}@test.com'.format(username)
+        return u'{0}@test.com'.format(username)
 
     def login_username(self, username):
         """Login the user, given the `username`."""
         if self.current_user != username:
-            self.login(InstructorTaskCourseTestCase.get_user_email(username), "test")
+            user_email = User.objects.get(username=username).email
+            self.login(user_email, "test")
             self.current_user = username
 
     def _create_user(self, username, email=None, is_staff=False, mode='honor'):
@@ -190,16 +196,18 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
     the setup of a course and problem in order to access StudentModule state.
     """
     @staticmethod
-    def problem_location(problem_url_name):
+    def problem_location(problem_url_name, course_key=None):
         """
         Create an internal location for a test problem.
         """
         if "i4x:" in problem_url_name:
             return Location.from_deprecated_string(problem_url_name)
+        elif course_key:
+            return course_key.make_usage_key('problem', problem_url_name)
         else:
             return TEST_COURSE_KEY.make_usage_key('problem', problem_url_name)
 
-    def define_option_problem(self, problem_url_name, parent=None):
+    def define_option_problem(self, problem_url_name, parent=None, **kwargs):
         """Create the problem definition so the answer is Option 1"""
         if parent is None:
             parent = self.problem_section
@@ -212,8 +220,9 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
         ItemFactory.create(parent_location=parent.location,
                            parent=parent,
                            category="problem",
-                           display_name=str(problem_url_name),
-                           data=problem_xml)
+                           display_name=problem_url_name,
+                           data=problem_xml,
+                           **kwargs)
 
     def redefine_option_problem(self, problem_url_name):
         """Change the problem definition so the answer is Option 2"""
@@ -249,9 +258,13 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
             # Note that this is a capa-specific convention.  The form is a version of the problem's
             # URL, modified so that it can be easily stored in html, prepended with "input-" and
             # appended with a sequence identifier for the particular response the input goes to.
-            return 'input_i4x-{0}-{1}-problem-{2}_{3}'.format(TEST_COURSE_ORG.lower(),
-                                                              TEST_COURSE_NUMBER.replace('.', '_'),
-                                                              problem_url_name, response_id)
+            course_key = self.course.id
+            return u'input_i4x-{0}-{1}-problem-{2}_{3}'.format(
+                course_key.org.replace(u'.', u'_'),
+                course_key.course.replace(u'.', u'_'),
+                problem_url_name,
+                response_id
+            )
 
         # make sure that the requested user is logged in, so that the ajax call works
         # on the right problem:
@@ -260,7 +273,7 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
         modx_url = reverse('xblock_handler', kwargs={
             'course_id': self.course.id.to_deprecated_string(),
             'usage_id': quote_slashes(
-                InstructorTaskModuleTestCase.problem_location(problem_url_name).to_deprecated_string()
+                InstructorTaskModuleTestCase.problem_location(problem_url_name, self.course.id).to_deprecated_string()
             ),
             'handler': 'xmodule_handler',
             'suffix': 'problem_check',
@@ -268,7 +281,7 @@ class InstructorTaskModuleTestCase(InstructorTaskCourseTestCase):
 
         # assign correct identifier to each response.
         resp = self.client.post(modx_url, {
-            get_input_id('{}_1').format(index): response for index, response in enumerate(responses, 2)
+            get_input_id(u'{}_1').format(index): response for index, response in enumerate(responses, 2)
         })
         return resp
 
@@ -282,7 +295,7 @@ class TestReportMixin(object):
         if os.path.exists(reports_download_path):
             shutil.rmtree(reports_download_path)
 
-    def verify_rows_in_csv(self, expected_rows, verify_order=True, ignore_other_columns=False):
+    def verify_rows_in_csv(self, expected_rows, file_index=0, verify_order=True, ignore_other_columns=False):
         """
         Verify that the last ReportStore CSV contains the expected content.
 
@@ -291,6 +304,9 @@ class TestReportMixin(object):
                 where each dict represents a row of data in the last
                 ReportStore CSV.  Each dict maps keys from the CSV
                 header to values in that row's corresponding cell.
+            file_index (int): Describes which report store file to
+                open.  Files are ordered by last modified date, and 0
+                corresponds to the most recently modified file.
             verify_order (boolean): When True, we verify that both the
                 content and order of `expected_rows` matches the
                 actual csv rows.  When False (default), we only verify
@@ -299,7 +315,7 @@ class TestReportMixin(object):
                 contain data which is the subset of actual csv rows.
         """
         report_store = ReportStore.from_config()
-        report_csv_filename = report_store.links_for(self.course.id)[0][0]
+        report_csv_filename = report_store.links_for(self.course.id)[file_index][0]
         with open(report_store.path_to(self.course.id, report_csv_filename)) as csv_file:
             # Expand the dict reader generator so we don't lose it's content
             csv_rows = [row for row in unicodecsv.DictReader(csv_file)]

lms/djangoapps/instructor_task/tests/test_integration.py

@@ -14,9 +14,9 @@ from celery.states import SUCCESS, FAILURE
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 
+from openedx.core.djangoapps.util.testing import TestConditionalContent
 from capa.tests.response_xml_factory import (CodeResponseXMLFactory,
                                              CustomResponseXMLFactory)
-from openedx.core.djangoapps.user_api.tests.factories import UserCourseTagFactory
 from xmodule.modulestore.tests.factories import ItemFactory
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.partitions.partitions import Group, UserPartition
@@ -465,103 +465,10 @@ class TestDeleteProblemTask(TestIntegrationTask):
         self.assertEqual(instructor_task.task_state, SUCCESS)
 
 
-class TestGradeReportConditionalContent(TestReportMixin, TestIntegrationTask):
+class TestGradeReportConditionalContent(TestReportMixin, TestConditionalContent, TestIntegrationTask):
     """
-    Check that grade export works when graded content exists within
-    split modules.
+    Test grade report in cases where there are problems contained within split tests.
     """
-    def setUp(self):
-        """
-        Set up a course with graded problems within a split test.
-
-        Course hierarchy is as follows (modeled after how split tests
-        are created in studio):
-        -> course
-            -> chapter
-                -> sequential (graded)
-                    -> vertical
-                        -> split_test
-                            -> vertical (Group A)
-                                -> problem
-                            -> vertical (Group B)
-                                -> problem
-        """
-        super(TestGradeReportConditionalContent, self).setUp()
-
-        # Create user partitions
-        self.user_partition_group_a = 0
-        self.user_partition_group_b = 1
-        self.partition = UserPartition(
-            0,
-            'first_partition',
-            'First Partition',
-            [
-                Group(self.user_partition_group_a, 'Group A'),
-                Group(self.user_partition_group_b, 'Group B')
-            ]
-        )
-
-        # Create course with group configurations and grading policy
-        self.initialize_course(
-            course_factory_kwargs={
-                'user_partitions': [self.partition],
-                'grading_policy': {
-                    "GRADER": [{
-                        "type": "Homework",
-                        "min_count": 1,
-                        "drop_count": 0,
-                        "short_label": "HW",
-                        "weight": 1.0
-                    }]
-                }
-            }
-        )
-
-        # Create users and partition them
-        self.student_a = self.create_student('student_a')
-        self.student_b = self.create_student('student_b')
-        UserCourseTagFactory(
-            user=self.student_a,
-            course_id=self.course.id,
-            key='xblock.partition_service.partition_{0}'.format(self.partition.id),  # pylint: disable=no-member
-            value=str(self.user_partition_group_a)
-        )
-        UserCourseTagFactory(
-            user=self.student_b,
-            course_id=self.course.id,
-            key='xblock.partition_service.partition_{0}'.format(self.partition.id),  # pylint: disable=no-member
-            value=str(self.user_partition_group_b)
-        )
-
-        # Create a vertical to contain our split test
-        problem_vertical = ItemFactory.create(
-            parent_location=self.problem_section.location,
-            category='vertical',
-            display_name='Problem Unit'
-        )
-
-        # Create the split test and child vertical containers
-        vertical_a_url = self.course.id.make_usage_key('vertical', 'split_test_vertical_a')
-        vertical_b_url = self.course.id.make_usage_key('vertical', 'split_test_vertical_b')
-        self.split_test = ItemFactory.create(
-            parent_location=problem_vertical.location,
-            category='split_test',
-            display_name='Split Test',
-            user_partition_id=self.partition.id,  # pylint: disable=no-member
-            group_id_to_child={str(index): url for index, url in enumerate([vertical_a_url, vertical_b_url])}
-        )
-        self.vertical_a = ItemFactory.create(
-            parent_location=self.split_test.location,
-            category='vertical',
-            display_name='Group A problem container',
-            location=vertical_a_url
-        )
-        self.vertical_b = ItemFactory.create(
-            parent_location=self.split_test.location,
-            category='vertical',
-            display_name='Group B problem container',
-            location=vertical_b_url
-        )
 
     def verify_csv_task_success(self, task_result):
         """

lms/static/coffee/src/instructor_dashboard/data_download.coffee

@@ -22,6 +22,7 @@ class DataDownload
     @$list_anon_btn = @$section.find("input[name='list-anon-ids']'")
     @$grade_config_btn = @$section.find("input[name='dump-gradeconf']'")
     @$calculate_grades_csv_btn = @$section.find("input[name='calculate-grades-csv']'")
+    @$problem_grade_report_csv_btn = @$section.find("input[name='problem-grade-report']'")
 
     # response areas
     @$download                        = @$section.find '.data-download-container'
@@ -108,16 +109,22 @@ class DataDownload
           @$download_display_text.html data['grading_config_summary']
 
     @$calculate_grades_csv_btn.click (e) =>
+      @onClickGradeDownload @$calculate_grades_csv_btn, gettext("Error generating grades. Please try again.")
+
+    @$problem_grade_report_csv_btn.click (e) =>
+      @onClickGradeDownload @$problem_grade_report_csv_btn, gettext("Error generating problem grade report. Please try again.")
+
+  onClickGradeDownload: (button, errorMessage) ->
       # Clear any CSS styling from the request-response areas
       #$(".msg-confirm").css({"display":"none"})
       #$(".msg-error").css({"display":"none"})
       @clear_display()
-      url = @$calculate_grades_csv_btn.data 'endpoint'
+      url = button.data 'endpoint'
       $.ajax
         dataType: 'json'
         url: url
         error: (std_ajax_err) =>
-          @$reports_request_response_error.text gettext("Error generating grades. Please try again.")
+          @$reports_request_response_error.text errorMessage
           $(".msg-error").css({"display":"block"})
         success: (data) =>
           @$reports_request_response.text data['status']
@@ -194,6 +201,15 @@ class ReportDownloads
     $table_placeholder = $ '<div/>', class: 'slickgrid'
     @$report_downloads_table.append $table_placeholder
     grid = new Slick.Grid($table_placeholder, report_downloads_data, columns, options)
+    grid.onClick.subscribe(
+        (event) =>
+            report_url = event.target.href
+            if report_url
+                # Record that the user requested to download a report
+                Logger.log('edx.instructor.report.downloaded', {
+                    report_url: report_url
+                })
+    )
     grid.autosizeColumns()
 
 

lms/templates/instructor/instructor_dashboard_2/data_download.html

@@ -41,6 +41,8 @@
     <p>${_("Click to generate a CSV grade report for all currently enrolled students.")}</p>
 
     <p><input type="button" name="calculate-grades-csv" value="${_("Generate Grade Report")}" data-endpoint="${ section_data['calculate_grades_csv_url'] }"/></p>
+
+    <p><input type="button" name="problem-grade-report" value="${_("Generate Problem Grade Report")}" data-endpoint="${ section_data['problem_grade_report_url'] }"/></p>
   %endif
 
     <div class="request-response msg msg-confirm copy" id="report-request-response"></div>

openedx/core/djangoapps/content/course_structures/models.py

 import json
 import logging
 
+from collections import OrderedDict
 from model_utils.models import TimeStampedModel
 
 from util.models import CompressedTextField
@@ -26,6 +27,32 @@ class CourseStructure(TimeStampedModel):
             return json.loads(self.structure_json)
         return None
 
+    @property
+    def ordered_blocks(self):
+        """
+        Return the blocks in the order with which they're seen in the courseware. Parents are ordered before children.
+        """
+        if self.structure:
+            ordered_blocks = OrderedDict()
+            self._traverse_tree(self.structure['root'], self.structure['blocks'], ordered_blocks)
+            return ordered_blocks
+
+    def _traverse_tree(self, block, unordered_structure, ordered_blocks, parent=None):
+        """
+        Traverses the tree and fills in the ordered_blocks OrderedDict with the blocks in
+        the order that they appear in the course.
+        """
+        # find the dictionary entry for the current node
+        cur_block = unordered_structure[block]
+
+        if parent:
+            cur_block['parent'] = parent
+
+        ordered_blocks[block] = cur_block
+
+        for child_node in cur_block['children']:
+            self._traverse_tree(child_node, unordered_structure, ordered_blocks, parent=block)
+
 # Signals must be imported in a file that is automatically loaded at app startup (e.g. models.py). We import them
 # at the end of this file to avoid circular dependencies.
 import signals  # pylint: disable=unused-import

openedx/core/djangoapps/content/course_structures/tests.py

@@ -68,8 +68,8 @@ class CourseStructureTaskTests(ModuleStoreTestCase):
             }
         }
         structure_json = json.dumps(structure)
-        cs = CourseStructure.objects.create(course_id=self.course.id, structure_json=structure_json)
-        self.assertEqual(cs.structure_json, structure_json)
+        structure = CourseStructure.objects.create(course_id=self.course.id, structure_json=structure_json)
+        self.assertEqual(structure.structure_json, structure_json)
 
         # Reload the data to ensure the init signal is fired to decompress the data.
         cs = CourseStructure.objects.get(course_id=self.course.id)
@@ -91,6 +91,41 @@ class CourseStructureTaskTests(ModuleStoreTestCase):
         cs = CourseStructure.objects.create(course_id=self.course.id, structure_json=structure_json)
         self.assertDictEqual(cs.structure, structure)
 
+    def test_ordered_blocks(self):
+        structure = {
+            'root': 'a/b/c',
+            'blocks': {
+                'a/b/c': {
+                    'id': 'a/b/c',
+                    'children': [
+                        'g/h/i'
+                    ]
+                },
+                'd/e/f': {
+                    'id': 'd/e/f',
+                    'children': []
+                },
+                'g/h/i': {
+                    'id': 'h/j/k',
+                    'children': [
+                        'j/k/l',
+                        'd/e/f'
+                    ]
+                },
+                'j/k/l': {
+                    'id': 'j/k/l',
+                    'children': []
+                }
+            }
+        }
+        in_order_blocks = ['a/b/c', 'g/h/i', 'j/k/l', 'd/e/f']
+        structure_json = json.dumps(structure)
+        retrieved_course_structure = CourseStructure.objects.create(
+            course_id=self.course.id, structure_json=structure_json
+        )
+
+        self.assertEqual(retrieved_course_structure.ordered_blocks.keys(), in_order_blocks)
+
     def test_block_with_missing_fields(self):
         """
         The generator should continue to operate on blocks/XModule that do not have graded or format fields.

openedx/core/djangoapps/util/testing.py

+""" Mixins for setting up particular course structures (such as split tests or cohorted content) """
+
+from datetime import datetime
+from pytz import UTC
+
+from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup
+from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory
+from openedx.core.djangoapps.user_api.tests.factories import UserCourseTagFactory
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.partitions.partitions import UserPartition, Group
+from student.tests.factories import CourseEnrollmentFactory, UserFactory
+
+
+class ContentGroupTestCase(ModuleStoreTestCase):
+    """
+    Sets up discussion modules visible to content groups 'Alpha' and
+    'Beta', as well as a module visible to all students.  Creates a
+    staff user, users with access to Alpha/Beta (by way of cohorts),
+    and a non-cohorted user with no special access.
+    """
+    def setUp(self):
+        super(ContentGroupTestCase, self).setUp()
+
+        self.course = CourseFactory.create(
+            org='org', number='number', run='run',
+            # This test needs to use a course that has already started --
+            # discussion topics only show up if the course has already started,
+            # and the default start date for courses is Jan 1, 2030.
+            start=datetime(2012, 2, 3, tzinfo=UTC),
+            user_partitions=[
+                UserPartition(
+                    0,
+                    'Content Group Configuration',
+                    '',
+                    [Group(1, 'Alpha'), Group(2, 'Beta')],
+                    scheme_id='cohort'
+                )
+            ],
+            grading_policy={
+                "GRADER": [{
+                    "type": "Homework",
+                    "min_count": 1,
+                    "drop_count": 0,
+                    "short_label": "HW",
+                    "weight": 1.0
+                }]
+            },
+            cohort_config={'cohorted': True},
+            discussion_topics={}
+        )
+
+        self.staff_user = UserFactory.create(is_staff=True)
+        self.alpha_user = UserFactory.create()
+        self.beta_user = UserFactory.create()
+        self.non_cohorted_user = UserFactory.create()
+        for user in [self.staff_user, self.alpha_user, self.beta_user, self.non_cohorted_user]:
+            CourseEnrollmentFactory.create(user=user, course_id=self.course.id)
+
+        alpha_cohort = CohortFactory(
+            course_id=self.course.id,
+            name='Cohort Alpha',
+            users=[self.alpha_user]
+        )
+        beta_cohort = CohortFactory(
+            course_id=self.course.id,
+            name='Cohort Beta',
+            users=[self.beta_user]
+        )
+        CourseUserGroupPartitionGroup.objects.create(
+            course_user_group=alpha_cohort,
+            partition_id=self.course.user_partitions[0].id,
+            group_id=self.course.user_partitions[0].groups[0].id
+        )
+        CourseUserGroupPartitionGroup.objects.create(
+            course_user_group=beta_cohort,
+            partition_id=self.course.user_partitions[0].id,
+            group_id=self.course.user_partitions[0].groups[1].id
+        )
+        self.alpha_module = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='alpha_group_discussion',
+            discussion_target='Visible to Alpha',
+            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[0].id]}
+        )
+        self.beta_module = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='beta_group_discussion',
+            discussion_target='Visible to Beta',
+            group_access={self.course.user_partitions[0].id: [self.course.user_partitions[0].groups[1].id]}
+        )
+        self.global_module = ItemFactory.create(
+            parent_location=self.course.location,
+            category='discussion',
+            discussion_id='global_group_discussion',
+            discussion_target='Visible to Everyone'
+        )
+        self.course = self.store.get_item(self.course.location)
+
+
+class TestConditionalContent(ModuleStoreTestCase):
+    """
+    Construct a course with graded problems that exist within a split test.
+    """
+    TEST_SECTION_NAME = 'Problem'
+
+    def setUp(self):
+        """
+        Set up a course with graded problems within a split test.
+
+        Course hierarchy is as follows (modeled after how split tests
+        are created in studio):
+        -> course
+            -> chapter
+                -> sequential (graded)
+                    -> vertical
+                        -> split_test
+                            -> vertical (Group A)
+                                -> problem
+                            -> vertical (Group B)
+                                -> problem
+        """
+        super(TestConditionalContent, self).setUp()
+
+        # Create user partitions
+        self.user_partition_group_a = 0
+        self.user_partition_group_b = 1
+        self.partition = UserPartition(
+            0,
+            'first_partition',
+            'First Partition',
+            [
+                Group(self.user_partition_group_a, 'Group A'),
+                Group(self.user_partition_group_b, 'Group B')
+            ]
+        )
+
+        # Create course with group configurations and grading policy
+        self.course = CourseFactory.create(
+            user_partitions=[self.partition],
+            grading_policy={
+                "GRADER": [{
+                    "type": "Homework",
+                    "min_count": 1,
+                    "drop_count": 0,
+                    "short_label": "HW",
+                    "weight": 1.0
+                }]
+            }
+        )
+        chapter = ItemFactory.create(parent_location=self.course.location,
+                                     display_name='Chapter')
+
+        # add a sequence to the course to which the problems can be added
+        self.problem_section = ItemFactory.create(parent_location=chapter.location,
+                                                  category='sequential',
+                                                  metadata={'graded': True, 'format': 'Homework'},
+                                                  display_name=self.TEST_SECTION_NAME)
+
+        # Create users and partition them
+        self.student_a = UserFactory.create(username='student_a', email='student_a@example.com')
+        CourseEnrollmentFactory.create(user=self.student_a, course_id=self.course.id)
+        self.student_b = UserFactory.create(username='student_b', email='student_b@example.com')
+        CourseEnrollmentFactory.create(user=self.student_b, course_id=self.course.id)
+
+        UserCourseTagFactory(
+            user=self.student_a,
+            course_id=self.course.id,
+            key='xblock.partition_service.partition_{0}'.format(self.partition.id),  # pylint: disable=no-member
+            value=str(self.user_partition_group_a)
+        )
+        UserCourseTagFactory(
+            user=self.student_b,
+            course_id=self.course.id,
+            key='xblock.partition_service.partition_{0}'.format(self.partition.id),  # pylint: disable=no-member
+            value=str(self.user_partition_group_b)
+        )
+
+        # Create a vertical to contain our split test
+        problem_vertical = ItemFactory.create(
+            parent_location=self.problem_section.location,
+            category='vertical',
+            display_name='Problem Unit'
+        )
+
+        # Create the split test and child vertical containers
+        vertical_a_url = self.course.id.make_usage_key('vertical', 'split_test_vertical_a')
+        vertical_b_url = self.course.id.make_usage_key('vertical', 'split_test_vertical_b')
+        self.split_test = ItemFactory.create(
+            parent_location=problem_vertical.location,
+            category='split_test',
+            display_name='Split Test',
+            user_partition_id=self.partition.id,  # pylint: disable=no-member
+            group_id_to_child={str(index): url for index, url in enumerate([vertical_a_url, vertical_b_url])}
+        )
+        self.vertical_a = ItemFactory.create(
+            parent_location=self.split_test.location,
+            category='vertical',
+            display_name='Group A problem container',
+            location=vertical_a_url
+        )
+        self.vertical_b = ItemFactory.create(
+            parent_location=self.split_test.location,
+            category='vertical',
+            display_name='Group B problem container',
+            location=vertical_b_url
+        )