Queryable student module app

Makes new tables that are queryable to get aggregate data on student per
problem information and student grades at three levels: overall course,
assignment type, and assignment.

Provides UI in instructor dashboard to view student metrics using
class_dashboard app.

Remove primary keys from queryable tables
    De-normalize user info of id, username and name into queryable tables.

Make all tests pass

* use default parameter	in get_or_create for non-unique-together
  values
* properly set module_state_key

Conflicts:
	lms/envs/test.py

lms/djangoapps/class_dashboard/dashboard_data.py

+"""
+Computes the data to display on the Instructor Dashboard
+"""
+
+from courseware import models
+from django.db.models import Count
+from queryable_student_module.models import StudentModuleExpand, Log
+
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+
+
+def get_problem_grade_distribution(course_id):
+    """
+    Returns the grade distribution per problem for the course
+
+    `course_id` the course ID for the course interested in
+
+    Output is a dict, where the key is the problem 'module_id' and the value is a dict with:
+        'max_grade' - max grade for this problem
+        'grade_distrib' - array of tuples (`grade`,`count`).
+    """
+    db_query = models.StudentModule.objects.filter(
+        course_id__exact=course_id,
+        grade__isnull=False,
+        module_type__exact="problem",
+    ).values('module_state_key', 'grade', 'max_grade').annotate(count_grade=Count('grade'))
+
+    prob_grade_distrib = {}
+    for row in db_query:
+        curr_problem = row['module_state_key']
+
+        if curr_problem in prob_grade_distrib:
+            prob_grade_distrib[curr_problem]['grade_distrib'].append((row['grade'], row['count_grade']))
+
+            if (prob_grade_distrib[curr_problem]['max_grade'] != row['max_grade']) and \
+                    (prob_grade_distrib[curr_problem]['max_grade'] < row['max_grade']):
+                prob_grade_distrib[curr_problem]['max_grade'] = row['max_grade']
+
+        else:
+            prob_grade_distrib[curr_problem] = {
+                'max_grade': row['max_grade'],
+                'grade_distrib': [(row['grade'], row['count_grade'])]
+            }
+
+    return prob_grade_distrib
+
+
+def get_problem_attempt_distrib(course_id, max_attempts=10):
+    """
+    Returns the attempt distribution per problem for the course.
+
+    `course_id` the course ID for the course interested in
+
+    `max_attempts` any students with more attempts than this are grouped together (default 10)
+    Output is a dicts, where the key is the problem `module_id` and the value is an array where the first index is
+    the number of students that only attempted once, second is two times, etc. The last index is all students that
+    attempted more than `max_attempts` times.
+    """
+
+    db_query = StudentModuleExpand.objects.filter(
+        course_id__exact=course_id,
+        attempts__isnull=False,
+        module_type__exact="problem",
+    ).values('module_state_key', 'attempts').annotate(count_attempts=Count('attempts'))
+
+    prob_attempts_distrib = {}
+    for row in db_query:
+        curr_problem = row['module_state_key']
+        if curr_problem not in prob_attempts_distrib:
+            prob_attempts_distrib[curr_problem] = [0] * (max_attempts + 1)
+
+        if row['attempts'] > max_attempts:
+            prob_attempts_distrib[curr_problem][max_attempts] += row['count_attempts']
+        else:
+            prob_attempts_distrib[curr_problem][row['attempts'] - 1] = row['count_attempts']
+
+    return prob_attempts_distrib
+
+
+def get_sequential_open_distrib(course_id):
+    """
+    Returns the number of students that opened each subsection/sequential of the course
+
+    `course_id` the course ID for the course interested in
+
+    Outputs a dict mapping the 'module_id' to the number of students that have opened that subsection/sequential.
+    """
+
+    db_query = models.StudentModule.objects.filter(
+        course_id__exact=course_id,
+        module_type__exact="sequential",
+    ).values('module_state_key').annotate(count_sequential=Count('module_state_key'))
+
+    sequential_open_distrib = {}
+    for row in db_query:
+        sequential_open_distrib[row['module_state_key']] = row['count_sequential']
+
+    return sequential_open_distrib
+
+
+def get_last_populate(course_id, script_id):
+    """
+    Returns the timestamp when a script was last run for a course.
+
+    `course_id` the course ID for the course interested in
+
+    `script_id` string identifying the populate script interested in
+
+    Returns None if there is no known time the script was last run for that course.
+    """
+
+    db_query = Log.objects.filter(course_id__exact=course_id, script_id__exact=script_id)
+
+    if len(db_query) > 0:
+        return db_query[0].created  # Model is sorted last first
+    else:
+        return None
+
+
+def get_problem_set_grade_distribution(course_id, problem_set):
+    """
+    Returns the grade distribution for the problems specified in `problem_set`.
+
+    `course_id` the course ID for the course interested in
+
+    `problem_set` an array of strings representing problem module_id's.
+
+    Requests from the database the a count of each grade for each problem in the `problem_set`.
+
+    Returns a dict, where the key is the problem 'module_id' and the value is a dict with two parts:
+      'max_grade' - the maximum grade possible for the course
+      'grade_distrib' - array of tuples (`grade`,`count`) ordered by `grade`
+    """
+
+    db_query = models.StudentModule.objects.filter(
+        course_id__exact=course_id,
+        grade__isnull=False,
+        module_type__exact="problem",
+        module_state_key__in=problem_set,
+    ).values(
+        'module_state_key',
+        'grade',
+        'max_grade',
+    ).annotate(count_grade=Count('grade')).order_by('module_state_key', 'grade')
+
+    prob_grade_distrib = {}
+    for row in db_query:
+        if row['module_state_key'] not in prob_grade_distrib:
+            prob_grade_distrib[row['module_state_key']] = {
+                'max_grade': 0,
+                'grade_distrib': [],
+            }
+
+        curr_grade_distrib = prob_grade_distrib[row['module_state_key']]
+        curr_grade_distrib['grade_distrib'].append((row['grade'], row['count_grade']))
+
+        if curr_grade_distrib['max_grade'] < row['max_grade']:
+            curr_grade_distrib['max_grade'] = row['max_grade']
+
+    return prob_grade_distrib
+
+
+def get_d3_problem_grade_distribution(course_id):
+    """
+    Returns problem grade distribution information for each section, data already in format for d3 function.
+
+    `course_id` the course ID for the course interested in
+
+    Returns an array of dicts in the order of the sections. Each dict has:
+      'display_name' - display name for the section
+      'data' - data for the d3_stacked_bar_graph function of the grade distribution for that problem
+    """
+
+    prob_grade_distrib = get_problem_grade_distribution(course_id)
+    d3_data = []
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+    for section in course.get_children():
+        curr_section = {}
+        curr_section['display_name'] = own_metadata(section)['display_name']
+        data = []
+        c_subsection = 0
+        for subsection in section.get_children():
+            c_subsection += 1
+            c_unit = 0
+            for unit in subsection.get_children():
+                c_unit += 1
+                c_problem = 0
+                for child in unit.get_children():
+                    if (child.location.category == 'problem'):
+                        c_problem += 1
+                        stack_data = []
+                        label = "P{0}.{1}.{2}".format(c_subsection, c_unit, c_problem)
+
+                        # Some problems have no data because students have not tried them yet
+                        if child.location.url() in prob_grade_distrib:
+                            problem_info = prob_grade_distrib[child.location.url()]
+                            problem_name = own_metadata(child)['display_name']
+                            max_grade = float(problem_info['max_grade'])
+                            for (grade, count_grade) in problem_info['grade_distrib']:
+                                percent = 0.0
+                                if max_grade > 0:
+                                    percent = (grade * 100.0) / max_grade
+
+                                tooltip = "{0} {3} - {1} students ({2:.0f}%: {4:.0f}/{5:.0f} questions)".format(
+                                    label, count_grade, percent, problem_name, grade, max_grade
+                                )
+
+                                stack_data.append({
+                                    'color': percent,
+                                    'value': count_grade,
+                                    'tooltip': tooltip,
+                                })
+
+                        problem = {
+                            'xValue': label,
+                            'stackData': stack_data,
+                        }
+                        data.append(problem)
+        curr_section['data'] = data
+
+        d3_data.append(curr_section)
+
+    return d3_data
+
+
+def get_d3_problem_attempt_distribution(course_id, max_attempts=10):
+    """
+    Returns problem attempt distribution information for each section, data already in format for d3 function.
+
+    `course_id` the course ID for the course interested in
+
+    `max_attempts` any students with more attempts than this are grouped together (default: 10)
+
+    Returns an array of dicts in the order of the sections. Each dict has:
+      'display_name' - display name for the section
+      'data' - data for the attempt distribution of problems in this section for d3_stacked_bar_graph
+    """
+
+    prob_attempts_distrib = get_problem_attempt_distrib(course_id, max_attempts)
+
+    d3_data = []
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+    for section in course.get_children():
+        curr_section = {}
+        curr_section['display_name'] = own_metadata(section)['display_name']
+        data = []
+        c_subsection = 0
+        for subsection in section.get_children():
+            c_subsection += 1
+            c_unit = 0
+            for unit in subsection.get_children():
+                c_unit += 1
+                c_problem = 0
+                for child in unit.get_children():
+                    if (child.location.category == 'problem'):
+                        c_problem += 1
+                        stack_data = []
+                        label = "P{0}.{1}.{2}".format(c_subsection, c_unit, c_problem)
+
+                        if child.location.url() in prob_attempts_distrib:
+                            attempts_distrib = prob_attempts_distrib[child.location.url()]
+                            problem_name = own_metadata(child)['display_name']
+
+                            for i in range(0, max_attempts + 1):
+                                color = (i + 1 if i != max_attempts else "{0}+".format(max_attempts))
+                                tooltip = "{0} {3} - {1} Student(s) had {2} attempt(s)".format(
+                                    label, attempts_distrib[i], color, problem_name
+                                )
+
+                                stack_data.append({
+                                    'color': color,
+                                    'value': attempts_distrib[i],
+                                    'tooltip': tooltip,
+                                })
+
+                        problem = {
+                            'xValue': label,
+                            'stackData': stack_data,
+                        }
+                        data.append(problem)
+        curr_section['data'] = data
+
+        d3_data.append(curr_section)
+
+    return d3_data
+
+
+def get_d3_sequential_open_distribution(course_id):
+    """
+    Returns how many students opened a sequential/subsection for each section, data already in format for d3 function.
+
+    `course_id` the course ID for the course interested in
+
+    Returns an array in the order of the sections and each dict has:
+      'display_name' - display name for the section
+      'data' - data for the d3_stacked_bar_graph function of how many students opened each sequential/subsection
+    """
+    sequential_open_distrib = get_sequential_open_distrib(course_id)
+
+    d3_data = []
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+    for section in course.get_children():
+        curr_section = {}
+        curr_section['display_name'] = own_metadata(section)['display_name']
+        data = []
+        c_subsection = 0
+        for subsection in section.get_children():
+            c_subsection += 1
+            subsection_name = own_metadata(subsection)['display_name']
+
+            num_students = 0
+            if subsection.location.url() in sequential_open_distrib:
+                num_students = sequential_open_distrib[subsection.location.url()]
+
+            stack_data = []
+            tooltip = "{0} student(s) opened Subsection {1}: {2}".format(
+                num_students, c_subsection, subsection_name
+            )
+            stack_data.append({
+                'color': 0,
+                'value': num_students,
+                'tooltip': tooltip,
+            })
+            subsection = {
+                'xValue': "SS {0}".format(c_subsection),
+                'stackData': stack_data,
+            }
+            data.append(subsection)
+
+        curr_section['data'] = data
+        d3_data.append(curr_section)
+
+    return d3_data
+
+
+def get_d3_section_grade_distribution(course_id, section):
+    """
+    Returns the grade distribution for the problems in the `section` section in a format for the d3 code.
+
+    `course_id` a string that is the course's ID.
+
+    `section` an int that is a zero-based index into the course's list of sections.
+
+    Navigates to the section specified to find all the problems associated with that section and then finds the grade
+    distribution for those problems. Finally returns an object formated the way the d3_stacked_bar_graph.js expects its
+    data object to be in.
+
+    If this is requested multiple times quickly for the same course, it is better to call
+    get_d3_problem_grade_distribution and pick out the sections of interest.
+
+    Returns an array of dicts with the following keys (taken from d3_stacked_bar_graph.js's documentation)
+      'xValue' - Corresponding value for the x-axis
+      'stackData' - Array of objects with key, value pairs that represent a bar:
+        'color' - Defines what "color" the bar will map to
+        'value' - Maps to the height of the bar, along the y-axis
+        'tooltip' - (Optional) Text to display on mouse hover
+    """
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+
+    problem_set = []
+    problem_info = {}
+    c_subsection = 0
+    for subsection in course.get_children()[section].get_children():
+        c_subsection += 1
+        c_unit = 0
+        for unit in subsection.get_children():
+            c_unit += 1
+            c_problem = 0
+            for child in unit.get_children():
+                if (child.location.category == 'problem'):
+                    c_problem += 1
+                    problem_set.append(child.location.url())
+                    problem_info[child.location.url()] = {
+                        'id': child.location.url(),
+                        'x_value': "P{0}.{1}.{2}".format(c_subsection, c_unit, c_problem),
+                        'display_name': own_metadata(child)['display_name'],
+                    }
+
+    grade_distrib = get_problem_set_grade_distribution(course_id, problem_set)
+
+    d3_data = []
+    for problem in problem_set:
+        stack_data = []
+
+        if problem in grade_distrib:  # Some problems have no data because students have not tried them yet.
+            max_grade = float(grade_distrib[problem]['max_grade'])
+            for (grade, count_grade) in grade_distrib[problem]['grade_distrib']:
+                percent = 0.0
+                if max_grade > 0:
+                    percent = (grade * 100.0) / max_grade
+
+                tooltip = "{0} {3} - {1} students ({2:.0f}%: {4:.0f}/{5:.0f} questions)".format(
+                    problem_info[problem]['x_value'],
+                    count_grade,
+                    percent,
+                    problem_info[problem]['display_name'],
+                    grade,
+                    max_grade,
+                )
+
+                stack_data.append({
+                    'color': percent,
+                    'value': count_grade,
+                    'tooltip': tooltip,
+                })
+
+        d3_data.append({
+            'xValue': problem_info[problem]['x_value'],
+            'stackData': stack_data,
+        })
+
+    return d3_data
+
+
+def get_section_display_name(course_id):
+    """
+    Returns an array of the display names for each section in the course.
+
+    `course_id` the course ID for the course interested in
+
+    The ith string in the array is the display name of the ith section in the course.
+    """
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+
+    section_display_name = [""] * len(course.get_children())
+    i = 0
+    for section in course.get_children():
+        section_display_name[i] = own_metadata(section)['display_name']
+        i += 1
+
+    return section_display_name
+
+
+def get_array_section_has_problem(course_id):
+    """
+    Returns an array of true/false whether each section has problems.
+
+    `course_id` the course ID for the course interested in
+
+    The ith value in the array is true if the ith section in the course contains problems and false otherwise.
+    """
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+
+    b_section_has_problem = [False] * len(course.get_children())
+    i = 0
+    for section in course.get_children():
+        for subsection in section.get_children():
+            for unit in subsection.get_children():
+                for child in unit.get_children():
+                    if child.location.category == 'problem':
+                        b_section_has_problem[i] = True
+                        break  # out of child loop
+                if b_section_has_problem[i]:
+                    break  # out of unit loop
+            if b_section_has_problem[i]:
+                break  # out of subsection loop
+
+        i += 1
+
+    return b_section_has_problem

lms/djangoapps/class_dashboard/test/test_dashboard_data.py

+
+
+from mock import Mock, patch
+import json
+
+from django.test.utils import override_settings
+from django.test import TestCase
+from django.core import management
+
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from courseware.tests.factories import StudentModuleFactory
+from student.tests.factories import UserFactory, CourseEnrollmentFactory
+from courseware.models import StudentModule
+from capa.tests.response_xml_factory import StringResponseXMLFactory
+from xmodule.modulestore import Location
+from queryable_student_module.management.commands import populate_studentmoduleexpand
+
+from xmodule.course_module import CourseDescriptor
+
+from class_dashboard.dashboard_data import get_problem_grade_distribution, get_problem_attempt_distrib, get_sequential_open_distrib, \
+                                           get_last_populate, get_problem_set_grade_distribution, get_d3_problem_grade_distribution, \
+                                           get_d3_problem_attempt_distribution, get_d3_sequential_open_distribution, \
+                                           get_d3_section_grade_distribution, get_section_display_name, get_array_section_has_problem
+
+USER_COUNT = 11
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestGetProblemGradeDistribution(ModuleStoreTestCase):
+    """
+    Tests needed:
+      - simple test, make sure output correct
+      - test when a problem has two max_grade's, should just take the larger value
+    """
+    
+    def setUp(self):
+        
+        self.command = 'populate_studentmoduleexpand'
+        self.script_id = "studentmoduleexpand"
+        self.attempts = 3
+        
+        self.course = CourseFactory.create()
+        
+        section = ItemFactory.create(
+            parent_location=self.course.location,
+            category="chapter",
+            display_name="test factory section",
+        )
+        sub_section = ItemFactory.create(
+            parent_location=section.location,
+            category="sequential",
+           # metadata={'graded': True, 'format': 'Homework'}
+        )
+    
+        unit = ItemFactory.create(
+            parent_location=sub_section.location,
+            category="vertical",
+            metadata={'graded': True, 'format': 'Homework'}
+        )
+        
+        self.users = [UserFactory.create() for _ in xrange(USER_COUNT)]
+    
+        for user in self.users:
+            CourseEnrollmentFactory.create(user=user, course_id=self.course.id)
+    
+        for i in xrange(USER_COUNT - 1):
+            category = "problem"
+            item = ItemFactory.create(
+                parent_location=unit.location,
+                category=category,
+                data=StringResponseXMLFactory().build_xml(answer='foo'),
+                metadata={'rerandomize': 'always'}
+            )
+    
+            for j, user in enumerate(self.users):
+                StudentModuleFactory.create(
+                    grade=1 if i < j else 0,
+                    max_grade=1 if i< j else 0.5,
+                    student=user,
+                    course_id=self.course.id,
+                    module_state_key=Location(item.location).url(),
+                    state=json.dumps({'attempts': self.attempts}),
+                )
+
+            for j, user in enumerate(self.users):
+                StudentModuleFactory.create(
+                    course_id=self.course.id,
+                    module_type='sequential',
+                    module_state_key=Location(item.location).url(),
+                )
+
+    def test_get_problem_grade_distribution(self):
+
+        prob_grade_distrib = get_problem_grade_distribution(self.course.id)
+
+        for problem in prob_grade_distrib:
+            max_grade = prob_grade_distrib[problem]['max_grade']
+            self.assertEquals(1, max_grade)
+
+
+    def test_get_problem_attempt_distribution(self):
+ 
+        # Call command
+        management.call_command(self.command, self.course.id)
+        prob_attempts_distrib = get_problem_attempt_distrib(self.course.id)
+        
+        for problem in prob_attempts_distrib:
+            num_attempts = prob_attempts_distrib[problem][self.attempts -1]
+            self.assertEquals(USER_COUNT, num_attempts)
+
+
+    def test_get_sequential_open_distibution(self):
+        
+        sequential_open_distrib = get_sequential_open_distrib(self.course.id)
+        
+        for problem in sequential_open_distrib:
+            num_students = sequential_open_distrib[problem]
+            self.assertEquals(USER_COUNT, num_students)
+ 
+
+    def test_get_last_populate(self):
+        
+        timestamp = get_last_populate(self.course.id, self.script_id)
+        self.assertEquals(timestamp, None)
+        
+        management.call_command(self.command, self.course.id)
+        timestamp = get_last_populate(self.course.id, self.script_id)
+        self.assertNotEquals(timestamp, None)
+
+
+    def test_get_problemset_grade_distrib(self):
+
+        prob_grade_distrib = get_problem_grade_distribution(self.course.id)
+        probset_grade_distrib = get_problem_set_grade_distribution(self.course.id, prob_grade_distrib)
+
+        for problem in probset_grade_distrib:
+            max_grade = probset_grade_distrib[problem]['max_grade']
+            self.assertEquals(1, max_grade)
+
+            grade_distrib = probset_grade_distrib[problem]['grade_distrib']
+            sum_attempts = 0
+            for item in grade_distrib:
+                sum_attempts += item[1]
+            self.assertEquals(USER_COUNT, sum_attempts)
+
+
+  #  @patch('class_dashboard.dashboard_data.get_problem_grade_distribution')
+    def test_get_d3_problem_grade_distrib(self): #, mock_get_data):
+        
+        d3_data = get_d3_problem_grade_distribution(self.course.id)
+        for data in d3_data:
+            for stack_data in data['data']:
+                sum_values = 0
+                for problem in stack_data['stackData']:
+                    sum_values += problem['value']
+                self.assertEquals(USER_COUNT, sum_values)
+
+
+    def test_get_d3_problem_attempt_distrib(self):
+        
+        # Call command
+        management.call_command(self.command, self.course.id)
+        d3_data = get_d3_problem_attempt_distribution(self.course.id)
+        
+        for data in d3_data:
+            for stack_data in data['data']:
+                sum_values = 0
+                for problem in stack_data['stackData']:
+                    sum_values += problem['value']
+                self.assertEquals(USER_COUNT, sum_values)   
+
+
+    def test_get_d3_sequential_open_distrib(self):
+        
+        d3_data = get_d3_sequential_open_distribution(self.course.id)
+        
+        for data in d3_data:
+            for stack_data in data['data']:
+                for problem in stack_data['stackData']:
+                    value = problem['value']
+                self.assertEquals(0, value)   
+
+
+    def test_get_d3_section_grade_distrib(self):
+
+
+        d3_data = get_d3_section_grade_distribution(self.course.id, 0)
+        
+        for stack_data in d3_data:
+            sum_values = 0
+            for problem in stack_data['stackData']:
+                sum_values += problem['value']
+            self.assertEquals(USER_COUNT, sum_values)
+
+
+    def test_get_section_display_name(self):
+        
+        section_display_name = get_section_display_name(self.course.id)
+        
+        self.assertMultiLineEqual(section_display_name[0], 'test factory section')
+
+
+    def test_get_array_section_has_problem(self):
+        
+        b_section_has_problem = get_array_section_has_problem(self.course.id)
+        
+        print b_section_has_problem
+        self.assertEquals(b_section_has_problem[0], True)

lms/djangoapps/class_dashboard/test/test_views.py

+from mock import Mock, patch
+
+from django.test import TestCase
+from django.test.client import RequestFactory
+from django.utils import simplejson
+
+from class_dashboard import views
+
+
+class TestViews(TestCase):
+
+    def setUp(self):
+        self.request_factory = RequestFactory()
+        self.request = self.request_factory.get('')
+        self.request.user = None
+        self.simple_data = {'test': 'test'}
+
+    @patch('class_dashboard.dashboard_data.get_d3_problem_attempt_distribution')
+    @patch('class_dashboard.views.has_instructor_access_for_class')
+    def test_all_problem_attempt_distribution_has_access(self, has_access, data_method):
+        """
+        Test returns proper value when have proper access
+        """
+        has_access.return_value = True
+
+        data_method.return_value = self.simple_data
+
+        response = views.all_problem_attempt_distribution(self.request, 'test/test/test')
+
+        self.assertEqual(simplejson.dumps(self.simple_data), response.content)
+
+    @patch('class_dashboard.dashboard_data.get_d3_problem_grade_distribution')
+    @patch('class_dashboard.views.has_instructor_access_for_class')
+    def test_all_problem_grade_distribution_has_access(self, has_access, data_method):
+        """
+        Test returns proper value when have proper access
+        """
+        has_access.return_value = True
+
+        data_method.return_value = self.simple_data
+
+        response = views.all_problem_grade_distribution(self.request, 'test/test/test')
+
+        self.assertEqual(simplejson.dumps(self.simple_data), response.content)
+
+    @patch('class_dashboard.dashboard_data.get_d3_sequential_open_distribution')
+    @patch('class_dashboard.views.has_instructor_access_for_class')
+    def test_all_sequential_open_distribution_has_access(self, has_access, data_method):
+        """
+        Test returns proper value when have proper access
+        """
+        has_access.return_value = True
+
+        data_method.return_value = self.simple_data
+
+        response = views.all_sequential_open_distribution(self.request, 'test/test/test')
+
+        self.assertEqual(simplejson.dumps(self.simple_data), response.content)
+
+    @patch('class_dashboard.dashboard_data.get_d3_section_grade_distribution')
+    @patch('class_dashboard.views.has_instructor_access_for_class')
+    def test_section_problem_grade_distribution_has_access(self, has_access, data_method):
+        """
+        Test returns proper value when have proper access
+        """
+        has_access.return_value = True
+
+        data_method.return_value = self.simple_data
+
+        response = views.section_problem_grade_distribution(self.request, 'test/test/test', '1')
+
+        self.assertEqual(simplejson.dumps(self.simple_data), response.content)

lms/djangoapps/class_dashboard/views.py

+"""
+Handles requests for data, returning a json
+"""
+
+from django.utils import simplejson
+from django.http import HttpResponse
+
+from courseware.courses import get_course_with_access
+from courseware.access import has_access
+from class_dashboard import dashboard_data
+
+
+def has_instructor_access_for_class(user, course_id):
+    """
+    Returns true if the `user` is an instructor for the course.
+    """
+
+    course = get_course_with_access(user, course_id, 'staff', depth=None)
+    return has_access(user, course, 'instructor')
+
+
+def all_problem_attempt_distribution(request, course_id):
+    """
+    Creates a json with the attempt distribution for all the problems in the course.
+
+    `request` django request
+
+    `course_id` the course ID for the course interested in
+
+    Returns the format in dashboard_data.get_d3_problem_attempt_distribution
+    """
+    json = {}
+
+    # Only instructor for this particular course can request this information
+    if has_instructor_access_for_class(request.user, course_id):
+        json = dashboard_data.get_d3_problem_attempt_distribution(course_id)
+    else:
+        json = {'error': "Access Denied: User does not have access to this course's data"}
+
+    return HttpResponse(simplejson.dumps(json), mimetype="application/json")
+
+
+def all_sequential_open_distribution(request, course_id):
+    """
+    Creates a json with the open distribution for all the subsections in the course.
+
+    `request` django request
+
+    `course_id` the course ID for the course interested in
+
+    Returns the format in dashboard_data.get_d3_sequential_open_distribution
+    """
+    json = {}
+
+    # Only instructor for this particular course can request this information
+    if has_instructor_access_for_class(request.user, course_id):
+        json = dashboard_data.get_d3_sequential_open_distribution(course_id)
+    else:
+        json = {'error': "Access Denied: User does not have access to this course's data"}
+
+    return HttpResponse(simplejson.dumps(json), mimetype="application/json")
+
+
+def all_problem_grade_distribution(request, course_id):
+    """
+    Creates a json with the grade distribution for all the problems in the course.
+
+    `Request` django request
+
+    `course_id` the course ID for the course interested in
+
+    Returns the format in dashboard_data.get_d3_problem_grade_distribution
+    """
+    json = {}
+
+    # Only instructor for this particular course can request this information
+    if has_instructor_access_for_class(request.user, course_id):
+        json = dashboard_data.get_d3_problem_grade_distribution(course_id)
+    else:
+        json = {'error': "Access Denied: User does not have access to this course's data"}
+
+    return HttpResponse(simplejson.dumps(json), mimetype="application/json")
+
+
+def section_problem_grade_distribution(request, course_id, section):
+    """
+    Creates a json with the grade distribution for the problems in the specified section.
+
+    `request` django request
+
+    `course_id` the course ID for the course interested in
+
+    `section` The zero-based index of the section for the course
+
+    Returns the format in dashboard_data.get_d3_section_grade_distribution.
+
+    If this is requested multiple times quickly for the same course, it is better to call all_problem_grade_distribution
+    and pick out the sections of interest.
+    """
+    json = {}
+
+    # Only instructor for this particular course can request this information
+    if has_instructor_access_for_class(request.user, course_id):
+        json = dashboard_data.get_d3_section_grade_distribution(course_id, int(section))
+    else:
+        json = {'error': "Access Denied: User does not have access to this course's data"}
+
+    return HttpResponse(simplejson.dumps(json), mimetype="application/json")

lms/djangoapps/instructor/views/legacy.py

@@ -47,6 +47,7 @@ from instructor_task.api import (get_running_instructor_tasks,
                                  submit_rescore_problem_for_student,
                                  submit_reset_problem_attempts_for_all_students)
 from instructor_task.views import get_task_completion_info
+from class_dashboard import dashboard_data
 from mitxmako.shortcuts import render_to_response
 from psychometrics import psychoanalyze
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
@@ -779,6 +780,15 @@ def instructor_dashboard(request, course_id):
             analytics_results[analytic_name] = get_analytics_result(analytic_name)
 
     #----------------------------------------
+    # Metrics
+
+    metrics_results = {}
+    if settings.MITX_FEATURES.get('CLASS_DASHBOARD') and idash_mode == 'Metrics':
+        metrics_results['attempts_timestamp'] = dashboard_data.get_last_populate(course_id, "studentmoduleexpand")
+        metrics_results['section_display_name'] = dashboard_data.get_section_display_name(course_id)
+        metrics_results['section_has_problem'] = dashboard_data.get_array_section_has_problem(course_id)
+
+    #----------------------------------------
     # offline grades?
 
     if use_offline:
@@ -842,6 +852,7 @@ def instructor_dashboard(request, course_id):
                'cohorts_ajax_url': reverse('cohorts', kwargs={'course_id': course_id}),
 
                'analytics_results': analytics_results,
+               'metrics_results': metrics_results,
                }
 
     if settings.MITX_FEATURES.get('ENABLE_INSTRUCTOR_BETA_DASHBOARD'):

lms/djangoapps/queryable_student_module/__init__.py

+"""
+queryable app allows 2 commands to be run from rake:
+    populate_studentgrade
+    populate_studentmoduleexpand
+
+These commands populate table in the SQL database and allow for the class_dashboard app
+to render course metrics under the instructor dashboard.
+"""

lms/djangoapps/queryable_student_module/management/__init__.py

+"""
+Commands for queryable_student_module app:
+populate_studentgrades
+populate_studentmoduleexpand
+"""

lms/djangoapps/queryable_student_module/management/commands/__init__.py

+"""
+Commands for queryable app:
+populate_studentgrades
+populate_studentmoduleexpand
+"""

lms/djangoapps/queryable_student_module/management/commands/populate_studentgrades.py

+"""
+======== Populate Student Grades  ====================================================================================
+
+Populates the student grade tables of the queryable_table model (CourseGrade, AssignmentTypeGrade, AssignmentGrade).
+
+For the provided course_id, it will find all students that may have changed their grade since the last populate. Of
+these students rows for the course grade and assignment type are created only if the student has submitted at
+least one answer to any problem in the course. Rows for assignments are only created if the student has submitted an
+answer to one of the problems in that assignment. Updates only occur if there is a change in the values the row should
+be storing.
+"""
+
+import re
+
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+
+from courseware import grades
+from courseware.courses import get_course_by_id
+from courseware.models import StudentModule
+
+from queryable_student_module.models import Log, CourseGrade, AssignmentTypeGrade, AssignmentGrade
+from queryable_student_module import util
+
+
+################## Helper Functions ##################
+def update_course_grade(course_grade, gradeset):
+    """
+    Returns true if the course grade needs to be updated.
+    """
+    return (not util.approx_equal(course_grade.percent, gradeset['percent'])) or (course_grade.grade != gradeset['grade'])
+
+
+def get_assignment_index(assignment):
+    """
+    Returns the assignment's index, -1 if an index can't be found.
+
+    `assignment` is a string formatted like this "HW 02" and this function returns 2 in this case.
+
+    The string is the 'label' for each section in the 'section_breakdown' of the dictionary returned by the grades.grade
+    function.
+    """
+
+    match = re.search(r'.* (\d+)', assignment)
+    index = -1
+    if match:
+        index = int(match.group(1)) - 1
+
+    return index
+
+
+def assignment_exists_and_has_prob(assignment_problems_map, category, index):
+    """
+    Returns True if the assignment for the category and index exists and has problems
+
+    `assignment_problems_map` a dictionary returned by get_assignment_to_problem_map(course_id)
+
+    `category` string specifying the category or assignment type for this assignment
+
+    `index` zero-based indexing into the array of assignments for that category
+    """
+
+    if index < 0:
+        return False
+
+    if category not in assignment_problems_map:
+        return False
+
+    if index >= len(assignment_problems_map[category]):
+        return False
+
+    return len(assignment_problems_map[category][index]) > 0
+
+
+def get_student_problems(course_id, student):
+    """
+    Returns an array of problem ids that the student has answered for this course.
+
+    `course_id` the course ID for the course interested in
+
+    `student` the student want to get his/her problems
+
+    Queries the database to get the problems the student has submitted an answer to for the course specified.
+    """
+
+    query = StudentModule.objects.filter(
+        course_id__exact=course_id,
+        student=student,
+        grade__isnull=False,
+        module_type__exact='problem',
+    ).values('module_state_key').distinct()
+
+    student_problems = []
+    for problem in query:
+        student_problems.append(problem['module_state_key'])
+
+    return student_problems
+
+
+def student_did_problems(student_problems, problem_set):
+    """
+    Returns true if `student_problems` and `problem_set` share problems.
+
+    `student_problems` array of problem ids the student has done
+
+    `problem_set` array of problem ids
+    """
+
+    return (set(student_problems) & set(problem_set))
+
+
+def store_course_grade_if_need(student, course_id, gradeset):
+    """
+    Stores the course grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `gradeset` the values returned by grades.grade
+
+    The course grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    update_course_grade is true.
+    """
+
+    course_grade, created = CourseGrade.objects.get_or_create(user_id=student.id,
+                                                              course_id=course_id,
+                                                              defaults={'username': student.username,
+                                                                        'name': student.profile.name})
+
+    if created or update_course_grade(course_grade, gradeset):
+        course_grade.percent = gradeset['percent']
+        course_grade.grade = gradeset['grade']
+        course_grade.save()
+        return True
+
+    return False
+
+
+def store_assignment_type_grade(student, course_id, category, percent):
+    """
+    Stores the assignment type grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `category` the category for the assignment type, found in the return value of the grades.grade function
+
+    `percent` the percent grade the student received for this assignment
+
+    The assignment type grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    if the percent value is different than what is currently in the database.
+    """
+
+    assign_type_grade, created = AssignmentTypeGrade.objects.get_or_create(
+        user_id=student.id,
+        course_id=course_id,
+        category=category,
+        defaults={'username': student.username,
+                  'name': student.profile.name})
+
+    if created or not util.approx_equal(assign_type_grade.percent, percent):
+        assign_type_grade.percent = percent
+        assign_type_grade.save()
+        return True
+
+    return False
+
+
+def store_assignment_grade_if_need(student, course_id, label, percent):
+    """
+    Stores the assignment grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `label` the label for the assignment, found in the return value of the grades.grade function
+
+    `percent` the percent grade the student received for this assignment
+
+    The assignment grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    if the percent value is different than what is currently in the database.
+    """
+
+    assign_grade, created = AssignmentGrade.objects.get_or_create(
+        user_id=student.id,
+        course_id=course_id,
+        label=label,
+        defaults={'username':student.username,
+                  'name':student.profile.name})
+
+    if created or not util.approx_equal(assign_grade.percent, percent):
+        assign_grade.percent = percent
+        assign_grade.save()
+        return True
+
+    return False
+
+
+class Command(BaseCommand):
+    """
+    populate_studentgrades command
+    """
+
+    help = "Populates the queryable.StudentGrades table.\n"
+    help += "Usage: populate_studentgrades course_id\n"
+    help += "   course_id: course's ID, such as Medicine/HRP258/Statistics_in_Medicine\n"
+
+    option_list = BaseCommand.option_list + (util.more_options(),)
+
+    def handle(self, *args, **options):
+        script_id = "studentgrades"
+
+        print "args = ", args
+
+        if len(args) > 0:
+            course_id = args[0]
+        else:
+            print self.help
+            return
+
+        assignment_problems_map = util.get_assignment_to_problem_map(course_id)
+
+        iterative_populate, tstart, last_log_run = util.pre_run_command(script_id, options, course_id)
+
+        # If iterative populate get all students since last populate, otherwise get all students that fit the criteria.
+        # Criteria: match course_id, module_type is 'problem', grade is not null because it means they have submitted an
+        # answer to a problem that might effect their grade.
+        if iterative_populate:
+            students = User.objects.select_related('profile').filter(studentmodule__course_id=course_id,
+                                           studentmodule__module_type='problem',
+                                           studentmodule__grade__isnull=False,
+                                           studentmodule__modified__gte=last_log_run[0].created).distinct()
+        else:
+            students = User.objects.select_related('profile').filter(studentmodule__course_id=course_id,
+                                           studentmodule__module_type='problem',
+                                           studentmodule__grade__isnull=False).distinct()
+#.select_related('profile')
+
+        # Create a dummy request to pass to the grade function.
+        # Code originally from lms/djangoapps/instructor/offline_gradecalc.py
+        # Copying instead of using that code so everything is self contained in this django app.
+        class DummyRequest(object):
+            """
+            Create a dummy request to pass to the grade function.
+            Code originally from lms/djangoapps/instructor/offline_gradecalc.py
+            Copying instead of using that code so everything is self contained in this django app.
+            """
+
+            META = {}
+
+            def __init__(self):
+                self.user = None
+                return
+
+            def is_secure(self):
+                return False
+            
+            def get_host(self):
+                return 'edx.mit.edu'
+
+        # Get course using the id, to pass to the grade function
+        course = get_course_by_id(course_id)
+
+        c_updated_students = 0
+        for student in students:
+            updated = False
+            student_problems = None
+
+            # Create dummy request and set its user
+            request = DummyRequest()
+            request.user = student
+            request.session = {}
+
+            # Call grade to get the gradeset
+
+            gradeset = grades.grade(student, request, course, keep_raw_scores=False)
+
+            updated = store_course_grade_if_need(student, course_id, gradeset)
+
+            # Iterate through the section_breakdown
+            for section in gradeset['section_breakdown']:
+                # If the dict has 'prominent' and it's True this is at the assignment type level, store it if need
+                if ('prominent' in section) and section['prominent']:
+                    updated = store_assignment_type_grade(
+                        student, course_id, section['category'], section['percent']
+                    )
+
+                else:  # If no 'prominent' or it's False this is at the assignment level
+                    store = False
+
+                    # If the percent is 0, there are three possibilities:
+                    # 1. There are no problems for that assignment yet -> skip section
+                    # 2. The student hasn't submitted an answer to any problem for that assignment -> skip section
+                    # 3. The student has submitted answers and got zero -> record
+                    # Only store for #3
+                    if section['percent'] > 0:
+                        store = True
+                    else:
+                        # Find which assignment this is for this type/category
+                        index = get_assignment_index(section['label'])
+                        if index < 0:
+                            print "WARNING: Can't find index for the following section, skipping"
+                            print section
+                        else:
+                            if assignment_exists_and_has_prob(assignment_problems_map, section['category'], index):
+
+                                # Get problems student has done, only do this database call if needed
+                                if student_problems is None:
+                                    student_problems = get_student_problems(course_id, student)
+
+                                curr_assignment_problems = assignment_problems_map[section['category']][index]
+
+                                if student_did_problems(student_problems, curr_assignment_problems):
+                                    store = True
+
+                    if store:
+                        updated = store_assignment_grade_if_need(
+                            student, course_id, section['label'], section['percent']
+                        )
+
+            if updated:
+                c_updated_students += 1
+
+        c_all_students = len(students)
+        print "--------------------------------------------------------------------------------"
+        print "Done! Updated {0} students' grades out of {1}".format(c_updated_students, c_all_students)
+        print "--------------------------------------------------------------------------------"
+
+        # Save since everything finished successfully, log latest run.
+        q_log = Log(script_id=script_id, course_id=course_id, created=tstart)
+        q_log.save()

lms/djangoapps/queryable_student_module/management/commands/populate_studentmoduleexpand.py

+"""
+======== Populate StudentModuleExpand  ===============================================================================
+
+Populates the StudentModuleExpand table of the queryable_table model.
+
+For the provided course_id, it will find all rows in the StudentModule table of the courseware model that have
+module_type 'problem' and the grade is not null. Then for any rows that have changed since the last populate or do not
+have a corresponding row, update the attempts value.
+"""
+
+import json
+
+from django.core.management.base import BaseCommand
+
+from courseware.models import StudentModule
+from queryable_student_module.models import Log, StudentModuleExpand
+from queryable_student_module.util import pre_run_command, more_options
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+
+
+class Command(BaseCommand):
+    """
+    populate_studentmoduleexpand command
+    """
+
+    help = "Populates the queryable.StudentModuleExpand table.\n"
+    help += "Usage: populate_studentmoduleexpand course_id\n"
+    help += "   course_id: course's ID, such as Medicine/HRP258/Statistics_in_Medicine\n"
+
+    option_list = BaseCommand.option_list + (more_options(),)
+
+    def handle(self, *args, **options):
+        script_id = "studentmoduleexpand"
+
+        print "args = ", args
+
+        if len(args) > 0:
+            course_id = args[0]
+        else:
+            print self.help
+            return
+
+        iterative_populate, tstart, last_log_run = pre_run_command(script_id, options, course_id)
+
+        # If iterative populate, get all the problems that students have submitted an answer to for this course,
+        # since the last run
+        if iterative_populate:
+            sm_rows = StudentModule.objects.select_related('student', 'student__profile').filter(course_id__exact=course_id, grade__isnull=False,
+                                                   module_type__exact="problem", modified__gte=last_log_run[0].created)
+        else:
+            sm_rows = StudentModule.objects.select_related('student', 'student__profile').filter(course_id__exact=course_id, grade__isnull=False,
+                                                   module_type__exact="problem")
+
+        c_updated_rows = 0
+        # For each problem, get or create the corresponding StudentModuleExpand row
+        for sm_row in sm_rows:
+            
+            # Get the display name for the problem
+            module_state_key = sm_row.module_state_key
+            problem = modulestore().get_instance(course_id, module_state_key, 0)
+            problem_name = own_metadata(problem)['display_name']
+
+            sme, created = StudentModuleExpand.objects.get_or_create(student_id=sm_row.student_id, course_id=course_id,
+                                                                     module_state_key=module_state_key,
+                                                                     student_module_id=sm_row.id)
+
+            # If the StudentModuleExpand row is new or the StudentModule row was
+            # more recently updated than the StudentModuleExpand row, fill in/update
+            # everything and save
+            if created or (sme.modified < sm_row.modified):
+                c_updated_rows += 1
+                sme.grade = sm_row.grade
+                sme.max_grade = sm_row.max_grade
+                state = json.loads(sm_row.state)
+                sme.attempts = state["attempts"]
+                sme.label = problem_name
+                sme.username = sm_row.student.username
+                sme.name = sm_row.student.profile.name
+                sme.save()
+
+        c_all_rows = len(sm_rows)
+        print "--------------------------------------------------------------------------------"
+        print "Done! Updated/Created {0} queryable rows out of {1} from courseware_studentmodule".format(
+            c_updated_rows, c_all_rows)
+        print "--------------------------------------------------------------------------------"
+
+        # Save since everything finished successfully, log latest run.
+        q_log = Log(script_id=script_id, course_id=course_id, created=tstart)
+        q_log.save()

lms/djangoapps/queryable_student_module/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'StudentModuleExpand'
+        db.create_table('queryable_studentmoduleexpand', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('student_module_id', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('attempts', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('module_type', self.gf('django.db.models.fields.CharField')(default='problem', max_length=32, db_index=True)),
+            ('module_state_key', self.gf('django.db.models.fields.CharField')(max_length=255, db_column='module_id', db_index=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('label', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)),
+            ('student_id', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=30, null=True, blank=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
+            ('grade', self.gf('django.db.models.fields.FloatField')(db_index=True, null=True, blank=True)),
+            ('max_grade', self.gf('django.db.models.fields.FloatField')(null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable_student_module', ['StudentModuleExpand'])
+
+        # Adding unique constraint on 'StudentModuleExpand', fields ['student_id', 'module_state_key', 'course_id']
+        db.create_unique('queryable_studentmoduleexpand', ['student_id', 'module_id', 'course_id'])
+
+        # Adding model 'CourseGrade'
+        db.create_table('queryable_coursegrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('grade', self.gf('django.db.models.fields.CharField')(max_length=32, null=True, db_index=True)),
+            ('user_id', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=30, null=True, blank=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable_student_module', ['CourseGrade'])
+
+        # Adding unique constraint on 'CourseGrade', fields ['user_id', 'course_id']
+        db.create_unique('queryable_coursegrade', ['user_id', 'course_id'])
+
+        # Adding model 'AssignmentTypeGrade'
+        db.create_table('queryable_assignmenttypegrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('user_id', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=30, null=True, blank=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable_student_module', ['AssignmentTypeGrade'])
+
+        # Adding unique constraint on 'AssignmentTypeGrade', fields ['user_id', 'course_id', 'category']
+        db.create_unique('queryable_assignmenttypegrade', ['user_id', 'course_id', 'category'])
+
+        # Adding model 'AssignmentGrade'
+        db.create_table('queryable_assignmentgrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('label', self.gf('django.db.models.fields.CharField')(max_length=32, db_index=True)),
+            ('detail', self.gf('django.db.models.fields.CharField')(max_length=255, null=True, blank=True)),
+            ('user_id', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=30, null=True, blank=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable_student_module', ['AssignmentGrade'])
+
+        # Adding unique constraint on 'AssignmentGrade', fields ['user_id', 'course_id', 'label']
+        db.create_unique('queryable_assignmentgrade', ['user_id', 'course_id', 'label'])
+
+        # Adding model 'Log'
+        db.create_table('queryable_log', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('script_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+        ))
+        db.send_create_signal('queryable_student_module', ['Log'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'AssignmentGrade', fields ['user_id', 'course_id', 'label']
+        db.delete_unique('queryable_assignmentgrade', ['user_id', 'course_id', 'label'])
+
+        # Removing unique constraint on 'AssignmentTypeGrade', fields ['user_id', 'course_id', 'category']
+        db.delete_unique('queryable_assignmenttypegrade', ['user_id', 'course_id', 'category'])
+
+        # Removing unique constraint on 'CourseGrade', fields ['user_id', 'course_id']
+        db.delete_unique('queryable_coursegrade', ['user_id', 'course_id'])
+
+        # Removing unique constraint on 'StudentModuleExpand', fields ['student_id', 'module_state_key', 'course_id']
+        db.delete_unique('queryable_studentmoduleexpand', ['student_id', 'module_id', 'course_id'])
+
+        # Deleting model 'StudentModuleExpand'
+        db.delete_table('queryable_studentmoduleexpand')
+
+        # Deleting model 'CourseGrade'
+        db.delete_table('queryable_coursegrade')
+
+        # Deleting model 'AssignmentTypeGrade'
+        db.delete_table('queryable_assignmenttypegrade')
+
+        # Deleting model 'AssignmentGrade'
+        db.delete_table('queryable_assignmentgrade')
+
+        # Deleting model 'Log'
+        db.delete_table('queryable_log')
+
+
+    models = {
+        'queryable_student_module.assignmentgrade': {
+            'Meta': {'unique_together': "(('user_id', 'course_id', 'label'),)", 'object_name': 'AssignmentGrade', 'db_table': "'queryable_assignmentgrade'"},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'detail': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'queryable_student_module.assignmenttypegrade': {
+            'Meta': {'unique_together': "(('user_id', 'course_id', 'category'),)", 'object_name': 'AssignmentTypeGrade', 'db_table': "'queryable_assignmenttypegrade'"},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'queryable_student_module.coursegrade': {
+            'Meta': {'unique_together': "(('user_id', 'course_id'),)", 'object_name': 'CourseGrade', 'db_table': "'queryable_coursegrade'"},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'queryable_student_module.log': {
+            'Meta': {'ordering': "['-created']", 'object_name': 'Log', 'db_table': "'queryable_log'"},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'script_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'})
+        },
+        'queryable_student_module.studentmoduleexpand': {
+            'Meta': {'unique_together': "(('student_id', 'module_state_key', 'course_id'),)", 'object_name': 'StudentModuleExpand', 'db_table': "'queryable_studentmoduleexpand'"},
+            'attempts': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_state_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_column': "'module_id'", 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'student_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'student_module_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['queryable_student_module']
\ No newline at end of file

lms/djangoapps/queryable_student_module/models.py

+"""
+model file for queryable app
+"""
+
+from django.contrib.auth.models import User
+from django.db import models
+from courseware.models import StudentModule
+
+
+class StudentModuleExpand(models.Model):
+    """
+    Expanded version of courseware's model StudentModule. This is only for
+    instances of module type 'problem'. Adds attribute 'attempts' that is pulled
+    out of the json in the state attribute.
+    """
+
+    EXPAND_TYPES = {'problem'}
+
+    student_module_id = models.IntegerField(blank=True, null=True, db_index=True)
+
+    # The value mapped to 'attempts' in the json in state
+    attempts = models.IntegerField(null=True, blank=True, db_index=True)
+
+    # Values from StudentModule
+    module_type = models.CharField(max_length=32, default='problem', db_index=True)
+    module_state_key = models.CharField(max_length=255, db_index=True, db_column='module_id')
+    course_id = models.CharField(max_length=255, db_index=True)
+    label = models.CharField(max_length=50, null=True, blank=True)
+    student_id = models.IntegerField(blank=True, null=True, db_index=True)
+    username = models.CharField(max_length=30, blank=True, null=True, db_index=True)
+    name = models.CharField(max_length=255, blank=True, null=True, db_index=True)
+
+    class Meta:
+        """
+        Meta definitions
+        """
+
+        db_table = "queryable_studentmoduleexpand"
+        unique_together = (('student_id', 'module_state_key', 'course_id'),)
+
+    grade = models.FloatField(null=True, blank=True, db_index=True)
+    max_grade = models.FloatField(null=True, blank=True)
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    modified = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class CourseGrade(models.Model):
+    """
+    Holds student's overall course grade as a percentage and letter grade (if letter grade present).
+    """
+
+    course_id = models.CharField(max_length=255, db_index=True)
+    percent = models.FloatField(db_index=True, null=True)
+    grade = models.CharField(max_length=32, db_index=True, null=True)
+    user_id = models.IntegerField(blank=True, null=True, db_index=True)
+    username = models.CharField(max_length=30, blank=True, null=True, db_index=True)
+    name = models.CharField(max_length=255, blank=True, null=True, db_index=True)
+
+    class Meta:
+        """
+        Meta definitions
+        """
+
+        db_table = "queryable_coursegrade"
+        unique_together = (('user_id', 'course_id'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class AssignmentTypeGrade(models.Model):
+    """
+    Holds student's average grade for each assignment type per course.
+    """
+
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    category = models.CharField(max_length=255, db_index=True)
+    percent = models.FloatField(db_index=True, null=True)
+    user_id = models.IntegerField(blank=True, null=True, db_index=True)
+    username = models.CharField(max_length=30, blank=True, null=True, db_index=True)
+    name = models.CharField(max_length=255, blank=True, null=True, db_index=True)
+
+    class Meta:
+        """
+        Meta definitions
+        """
+
+        db_table = "queryable_assignmenttypegrade"
+        unique_together = (('user_id', 'course_id', 'category'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class AssignmentGrade(models.Model):
+    """
+    Holds student's assignment grades per course.
+    """
+
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    category = models.CharField(max_length=255, db_index=True)
+    percent = models.FloatField(db_index=True, null=True)
+    label = models.CharField(max_length=32, db_index=True)
+    detail = models.CharField(max_length=255, blank=True, null=True)
+    user_id = models.IntegerField(blank=True, null=True, db_index=True)
+    username = models.CharField(max_length=30, blank=True, null=True, db_index=True)
+    name = models.CharField(max_length=255, blank=True, null=True, db_index=True)
+
+    class Meta:
+        """
+        Meta definitions
+        """
+
+        db_table = "queryable_assignmentgrade"
+        unique_together = (('user_id', 'course_id', 'label'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class Log(models.Model):
+    """
+    Log of when a script in this django app was last run. Use to filter out students or rows that don't need to be
+    processed in the populate scripts and show instructors how fresh the data is.
+    """
+
+    script_id = models.CharField(max_length=255, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+    created = models.DateTimeField(null=True, db_index=True)
+
+    class Meta:
+        """
+        Meta definitions
+        """
+
+        db_table = "queryable_log"
+        ordering = ["-created"]
+        get_latest_by = "created"

lms/djangoapps/queryable_student_module/tests/test_populate_studentgrades.py

+import json
+from datetime import datetime
+from pytz import UTC
+from mock import Mock, patch
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from django.core.management import call_command
+
+from courseware import grades
+from courseware.tests.factories import StudentModuleFactory
+from courseware.tests.modulestore_config import TEST_DATA_MONGO_MODULESTORE
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from student.tests.factories import UserFactory as StudentUserFactory
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+
+from queryable_student_module.models import Log, CourseGrade, AssignmentTypeGrade, AssignmentGrade
+from queryable_student_module.management.commands import populate_studentgrades
+
+
+class TestPopulateStudentGradesUpdateCourseGrade(TestCase):
+    """
+    Tests the helper fuction update_course_grade in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.course_grade = CourseGrade(percent=0.9, grade='A')
+        self.gradeset = {'percent': 0.9, 'grade': 'A'}
+
+    def test_no_update(self):
+        """
+        Values are the same, so no update
+        """
+        self.assertFalse(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+    def test_percents_not_equal(self):
+        """
+        Update because the percents don't equal
+        """
+        self.course_grade.percent = 1.0
+
+        self.assertTrue(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+    def test_different_grade(self):
+        """
+        Update because the grade is different
+        """
+        self.course_grade.grade = 'Foo'
+
+        self.assertTrue(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+    def test_grade_as_null(self):
+        """
+        Percent is the same and grade are both null, so no update
+        """
+        self.course_grade.grade = None
+        self.gradeset['grade'] = None
+
+        self.assertFalse(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+
+class TestPopulateStudentGradesGetAssignmentIndex(TestCase):
+    """
+    Tests the helper fuction get_assignment_index in the populate_studentgrades custom command
+    """
+
+    def test_simple(self):
+        """
+        Simple test if returns correct index.
+        """
+
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 3"), 2)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 02"), 1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 11"), 10)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 001"), 0)
+
+    def test_no_index(self):
+        """
+        Test if returns -1 for badly formed input
+        """
+
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW Avg"), -1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW"), -1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW "), -1)
+
+
+class TestPopulateStudentGradesGetStudentProblems(TestCase):
+    """
+    Tests the helper fuction get_student_problems in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student_module = StudentModuleFactory(
+            module_type='problem',
+            module_state_key='one',
+            grade=1,
+            max_grade=1,
+        )
+
+    def test_single_problem(self):
+        """
+        Test returns a single problem
+        """
+
+        problem_set = populate_studentgrades.get_student_problems(
+            self.student_module.course_id,
+            self.student_module.student,
+        )
+
+        self.assertEquals(len(problem_set), 1)
+        self.assertEquals(problem_set[0], self.student_module.module_state_key)
+
+    def test_problem_with_no_submission(self):
+        """
+        Test to make sure only returns the problems with a submission.
+        """
+
+        student_module_no_submission = StudentModuleFactory(
+            course_id=self.student_module.course_id,
+            student=self.student_module.student,
+            module_type='problem',
+            module_state_key='no_submission',
+            grade=None,
+            max_grade=None,
+        )
+
+        problem_set = populate_studentgrades.get_student_problems(
+            self.student_module.course_id,
+            self.student_module.student,
+        )
+
+        self.assertEquals(len(problem_set), 1)
+        self.assertEquals(problem_set[0], self.student_module.module_state_key)
+
+
+class TestPopulateStudentGradesAssignmentExistsAndHasProblems(TestCase):
+    """
+    Tests the helper fuction assignment_exists_and_has_prob in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.category = 'HW'
+        self.assignment_problems_map = {
+            self.category: [
+                ['cat_1_problem_id_1'],
+            ]
+        }
+
+    def test_simple(self):
+        """
+        Test where assignment does exist and has problems
+        """
+
+        self.assertTrue(populate_studentgrades.assignment_exists_and_has_prob(
+                        self.assignment_problems_map,
+                        self.category,
+                        len(self.assignment_problems_map[self.category]) - 1, )
+                        )
+
+    def test_assignment_exist_no_problems(self):
+        """
+        Test where assignment exists but has no problems
+        """
+
+        self.assignment_problems_map['Final'] = [[]]
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob(
+                         self.assignment_problems_map, 'Final', 0)
+                         )
+
+    def test_negative_index(self):
+        """
+        Test handles negative indexes well by returning False
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob({}, "", -1))
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob({}, "", -5))
+
+    def test_non_existing_category(self):
+        """
+        Test handled a category that doesn't actually exist by returning False
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob({}, "Foo", 0))
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob(self.assignment_problems_map, "Foo", 0))
+
+    def test_index_too_high(self):
+        """
+        Test that if the index is higher than the actual number of assignments
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_prob(
+                         self.assignment_problems_map, self.category, len(self.assignment_problems_map[self.category])))
+
+
+class TestPopulateStudentGradesStudentDidProblems(TestCase):
+    """
+    Tests the helper fuction student_did_problems in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student_problems = ['cat_1_problem_1']
+
+    def test_student_did_do_problems(self):
+        """
+        Test where student did do some of the problems
+        """
+
+        self.assertTrue(populate_studentgrades.student_did_problems(self.student_problems, self.student_problems))
+
+        problem_set = list(self.student_problems)
+        problem_set.append('cat_2_problem_1')
+        self.assertTrue(populate_studentgrades.student_did_problems(self.student_problems, problem_set))
+
+    def test_student_did_not_do_problems(self):
+        """
+        Test where student didn't do any problems in the list
+        """
+
+        self.assertFalse(populate_studentgrades.student_did_problems(self.student_problems, []))
+        self.assertFalse(populate_studentgrades.student_did_problems([], self.student_problems))
+
+        problem_set = ['cat_1_problem_2']
+        self.assertFalse(populate_studentgrades.student_did_problems(self.student_problems, problem_set))
+
+
+class TestPopulateStudentGradesStoreCourseGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_course_grade_if_need in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.gradeset = {
+            'percent': 1.0,
+            'grade': 'A',
+        }
+        self.course_grade = CourseGrade(
+            user_id=self.student.id,
+            course_id=self.course_id,
+            percent=self.gradeset['percent'],
+            grade=self.gradeset['grade'],
+        )
+        self.course_grade.save()
+
+    def test_new_course_grade_store(self):
+        """
+        Test stores because it's a new CourseGrade
+        """
+
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course_id)), 1)
+        student = StudentUserFactory()
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            student, self.course_id, self.gradeset
+        )
+
+        self.assertTrue(return_value)
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course_id)), 2)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.update_course_grade')
+    def test_update_store(self, mock_update_course_grade):
+        """
+        Test stores because update_course_grade returns True
+        """
+        mock_update_course_grade.return_value = True
+
+        updated_time = self.course_grade.updated
+
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            self.student, self.course_id, self.gradeset
+        )
+
+        self.assertTrue(return_value)
+
+        course_grades = CourseGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+        )
+        self.assertEqual(len(course_grades), 1)
+        self.assertNotEqual(updated_time, course_grades[0].updated)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.update_course_grade')
+    def test_no_update_no_store(self, mock_update_course_grade):
+        """
+        Test doesn't touch the row because it is not newly created and update_course_grade returns False
+        """
+        mock_update_course_grade.return_value = False
+
+        updated_time = self.course_grade.updated
+
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            self.student, self.course_id, self.gradeset
+        )
+
+        self.assertFalse(return_value)
+
+        course_grades = CourseGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+        )
+        self.assertEqual(len(course_grades), 1)
+        self.assertEqual(updated_time, course_grades[0].updated)
+
+
+class TestPopulateStudentGradesStoreAssignmentTypeGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_assignment_type_grade in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.category = 'Homework'
+        self.percent = 1.0
+        self.assignment_type_grade = AssignmentTypeGrade(
+            user_id=self.student.id,
+            username=self.student.username,
+            name=self.student.profile.name,
+            course_id=self.course_id,
+            category=self.category,
+            percent=self.percent,
+        )
+        self.assignment_type_grade.save()
+
+    def test_new_assignment_type_grade_store(self):
+        """
+        Test the function both stores the new assignment type grade and returns True meaning that it had
+        """
+
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course_id)), 1)
+        return_value = populate_studentgrades.store_assignment_type_grade(
+            self.student, self.course_id, 'Foo 01', 1.0
+        )
+
+        self.assertTrue(return_value)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course_id)), 2)
+
+    def test_difference_percent_store(self):
+        """
+        Test updates the percent value when it is different
+        """
+
+        new_percent = self.percent - 0.1
+        return_value = populate_studentgrades.store_assignment_type_grade(
+            self.student, self.course_id, self.category, new_percent
+        )
+
+        self.assertTrue(return_value)
+
+        assignment_type_grades = AssignmentTypeGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+            category=self.category,
+        )
+        self.assertEqual(len(assignment_type_grades), 1)
+        self.assertEqual(assignment_type_grades[0].percent, new_percent)
+
+    def test_same_percent_no_store(self):
+        """
+        Test does not touch row if the row exists and the precent is not different
+        """
+        updated_time = self.assignment_type_grade.updated
+
+        return_value = populate_studentgrades.store_assignment_type_grade(
+            self.student, self.course_id, self.category, self.percent
+        )
+
+        self.assertFalse(return_value)
+
+        assignment_type_grades = AssignmentTypeGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+            category=self.category,
+        )
+        self.assertEqual(len(assignment_type_grades), 1)
+        self.assertEqual(assignment_type_grades[0].percent, self.percent)
+        self.assertEqual(assignment_type_grades[0].updated, updated_time)
+
+
+class TestPopulateStudentGradesStoreAssignmentGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_assignment_grade_if_need in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.label = 'HW 01'
+        self.percent = 1.0
+        self.assignment_grade = AssignmentGrade(
+            user_id=self.student.id,
+            username=self.student.username,
+            name=self.student.profile.name,
+            course_id=self.course_id,
+            label=self.label,
+            percent=self.percent,
+        )
+        self.assignment_grade.save()
+
+    def test_new_assignment_grade_store(self):
+        """
+        Test the function both stores the new assignment grade and returns True meaning that it had
+        """
+
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course_id)), 1)
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, 'Foo 01', 1.0
+        )
+
+        self.assertTrue(return_value)
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course_id)), 2)
+
+    def test_difference_percent_store(self):
+        """
+        Test updates the percent value when it is different
+        """
+
+        new_percent = self.percent - 0.1
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, self.label, new_percent
+        )
+
+        self.assertTrue(return_value)
+
+        assignment_grades = AssignmentGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+            label=self.label,
+        )
+        self.assertEqual(len(assignment_grades), 1)
+        self.assertEqual(assignment_grades[0].percent, new_percent)
+
+    def test_same_percent_no_store(self):
+        """
+        Test does not touch row if the row exists and the precent is not different
+        """
+        updated_time = self.assignment_grade.updated
+
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, self.label, self.percent
+        )
+
+        self.assertFalse(return_value)
+
+        assignment_grades = AssignmentGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user_id=self.student.id,
+            label=self.label,
+        )
+        self.assertEqual(len(assignment_grades), 1)
+        self.assertEqual(assignment_grades[0].percent, self.percent)
+        self.assertEqual(assignment_grades[0].updated, updated_time)
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestPopulateStudentGradesCommand(ModuleStoreTestCase):
+
+    def create_studentmodule(self):
+        """
+        Creates a StudentModule. This can't be in setUp because some functions can't have one in the database.
+        """
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+        )
+
+    def create_log_entry(self):
+        """
+        Adds a queryable log entry to the database
+        """
+        log = Log(script_id=self.script_id, course_id=self.course.id, created=datetime.now(UTC))
+        log.save()
+
+    def setUp(self):
+        self.command = 'populate_studentgrades'
+        self.script_id = 'studentgrades'
+        self.course = CourseFactory.create()
+        self.category = 'Homework'
+        self.gradeset = {
+            'percent': 1.0,
+            'grade': 'A',
+            'section_breakdown': [
+                {'category': self.category, 'label': 'HW Avg', 'percent': 1.0, 'prominent': True},
+                {'category': self.category, 'label': 'HW 01', 'percent': 1.0},
+            ],
+        }
+        # Make sure these are correct with the above gradeset
+        self.assignment_type_index = 0
+        self.assignment_index = 1
+
+    def test_missing_input(self):
+        """
+        Fails safely when not given enough input
+        """
+        try:
+            call_command(self.command)
+            self.assertTrue(True)
+        except:
+            self.assertTrue(False)
+
+    def test_just_logs_if_empty_course(self):
+        """
+        If the course has nothing in it, just logs the run in the log table.
+        """
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 1)
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course.id)), 0)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course.id)), 0)
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course.id)), 0)
+
+    @patch('courseware.grades.grade')
+    def test_force_update(self, mock_grade):
+        """
+        Even if there is a log entry for incremental update, force a full update
+
+        This may be done because something happened in the last update.
+        """
+        mock_grade.return_value = self.gradeset
+
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+        )
+
+        self.create_log_entry()
+
+        call_command(self.command, self.course.id, force=True)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 2)
+        self.assertEqual(len(CourseGrade.objects.filter(user_id=sm.student.id, course_id__exact=self.course.id)), 1)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(
+                             user_id=sm.student.id, course_id__exact=self.course.id, category=self.category)), 1)
+        self.assertEqual(len(AssignmentGrade.objects.filter(
+                             user_id=sm.student.id,
+                             course_id__exact=self.course.id,
+                             label=self.gradeset['section_breakdown'][self.assignment_index]['label'], )), 1)
+
+    @patch('courseware.grades.grade')
+    def test_incremental_update_if_log_exists(self, mock_grade):
+        """
+        Make sure it uses the log entry if it exists and we aren't forcing a full update
+        """
+        mock_grade.return_value = self.gradeset
+
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+        )
+        sm.student.last_name = "Student1"
+        sm.student.save()
+
+        self.create_log_entry()
+
+        # Create a StudentModule that is after the log entry, different name
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+        )
+        sm.student.last_name = "Student2"
+        sm.student.save()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_grade.call_count, 1)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_course_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_store_course_grade(self, mock_grade, mock_method):
+        """
+        Calls store_course_grade_if_need for all students
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_method.call_count, 1)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_assignment_type_grade')
+    @patch('courseware.grades.grade')
+    def test_store_assignment_type_grade(self, mock_grade, mock_method):
+        """
+        Calls store_assignment_type_grade when such a section exists
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_method.call_count, 1)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_store_assignment_grade_percent_not_zero(self, mock_grade, mock_method):
+        """
+        Calls store_assignment_grade_if_need when the percent for that assignment is not zero
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_method.call_count, 1)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.get_assignment_index')
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_bad_index(self, mock_grade, mock_method, mock_assign_index):
+        """
+        Does not call store_assignment_grade_if_need when the percent is zero because get_assignment_index returns a
+        negative number.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_index.return_value = -1
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_grade.call_count, 1)
+        self.assertEqual(mock_method.call_count, 0)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.get_student_problems')
+    @patch('queryable_student_module.management.commands.populate_studentgrades.assignment_exists_and_has_prob')
+    @patch('queryable_student_module.util.get_assignment_to_problem_map')
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_no_student_problems(self, mock_grade, mock_method, mock_assign_problem_map,
+                                                               mock_assign_exists, mock_student_problems):
+        """
+        Does not call store_assignment_grade_if_need when the percent is zero because the student did not submit
+        answers to any problems in that assignment.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_problem_map.return_value = {
+            self.gradeset['section_breakdown'][self.assignment_index]['category']: [[]]
+        }
+
+        mock_assign_exists.return_value = True
+
+        mock_student_problems.return_value = []
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_method.call_count, 0)
+
+    @patch('queryable_student_module.management.commands.populate_studentgrades.get_student_problems')
+    @patch('queryable_student_module.management.commands.populate_studentgrades.assignment_exists_and_has_prob')
+    @patch('queryable_student_module.util.get_assignment_to_problem_map')
+    @patch('queryable_student_module.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_has_student_problems(self, mock_grade, mock_method, mock_assign_problem_map,
+                                                                mock_assign_exists, mock_student_problems):
+        """
+        Calls store_assignment_grade_if_need when the percent is zero because the student did submit answers to
+        problems in that assignment.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_problem_map.return_value = {
+            self.gradeset['section_breakdown'][self.assignment_index]['category']: [['problem_1']]
+        }
+
+        mock_assign_exists.return_value = True
+
+        mock_student_problems.return_value = ['problem_1']
+
+        self.create_studentmodule()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_method.call_count, 1)

lms/djangoapps/queryable_student_module/tests/test_populate_studentmoduleexpand.py

+import json
+from datetime import datetime
+from pytz import UTC
+from StringIO import StringIO
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from django.core import management
+
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from courseware.tests.factories import StudentModuleFactory
+from queryable_student_module.models import Log, StudentModuleExpand
+from capa.tests.response_xml_factory import StringResponseXMLFactory
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestPopulateStudentModuleExpand(TestCase):
+
+    def setUp(self):
+        self.command = 'populate_studentmoduleexpand'
+        self.script_id = "studentmoduleexpand"
+        #self.course_id = 'test/test/test'
+        
+        self.course = CourseFactory.create()
+        
+        section = ItemFactory.create(
+            parent_location=self.course.location,
+            category="chapter",
+            display_name="test factory section",
+        )
+        sub_section = ItemFactory.create(
+            parent_location=section.location,
+            category="sequential",
+           # metadata={'graded': True, 'format': 'Homework'}
+        )
+    
+        unit = ItemFactory.create(
+            parent_location=sub_section.location,
+            category="vertical",
+            metadata={'graded': True, 'format': 'Homework'}
+        )
+        
+        category = "problem"
+        self.item = ItemFactory.create(
+            parent_location=unit.location,
+            category=category,
+            data=StringResponseXMLFactory().build_xml(answer='foo'),
+            metadata={'rerandomize': 'always'}
+        )
+
+        self.item2 = ItemFactory.create(
+            parent_location=unit.location,
+            category=category,
+            data=StringResponseXMLFactory().build_xml(answer='foo'),
+            metadata={'rerandomize': 'always'}
+        )
+
+    def test_missing_input(self):
+        """
+        Fails safely when not given enough input
+        """
+        try:
+            management.call_command(self.command)
+            self.assertTrue(True)
+        except:
+            self.assertTrue(False)
+
+    def test_just_logs_if_empty_course(self):
+        """
+        If the course has nothing in it, just logs the run in the log table
+        """
+        management.call_command(self.command, self.course.id)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 1)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)), 0)
+
+    def test_force_update(self):
+        """
+        Even if there is a log entry for incremental update, force a full update
+
+        This may be done because something happened in the last update.
+        """
+
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+            module_state_key=self.item.location
+        )
+
+        # Create the log entry
+        log = Log(script_id=self.script_id, course_id=self.course.id, created=datetime.now(UTC))
+        log.save()
+
+        # Create a StudentModuleExpand that is after the log entry and has a different attempts value
+        sme = StudentModuleExpand(
+            course_id=self.course.id,
+            module_state_key=sm.module_state_key,
+            student_module_id=sm.id,
+            attempts=0,
+        )
+
+        # Call command with the -f flag
+        management.call_command(self.command, self.course.id, force=True)
+
+        # Check to see if new rows have been added
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 2)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)), 1)
+        self.assertEqual(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)[0].attempts, 1)
+
+    def test_incremental_update_if_log_exists(self):
+        """
+        Make sure it uses the log entry if it exists and we aren't forcing a full update
+        """
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+            module_state_key=self.item.location
+        )
+
+        # Create the log entry
+        log = Log(script_id=self.script_id, course_id=self.course.id, created=datetime.now(UTC))
+        log.save()
+
+        # Create a StudentModule that is after the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 1}),
+            module_state_key=self.item.location
+        )
+
+        # Call command
+        management.call_command(self.command, self.course.id)
+
+        # Check to see if new row has been added to log
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 2)
+
+        # Even though there are two studentmodules only one row should be created
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)), 1)
+
+    def test_update_only_if_row_modified(self):
+        """
+        Test populate does not update a row if it is not necessary
+        For example the problem may have a more recent modified date but the attempts value has not changed.
+        """
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)), 0)
+
+        # Create a StudentModule
+        sm1 = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            module_state_key=self.item.location
+        )
+        # Create a StudentModuleExpand
+        sme1 = StudentModuleExpand(
+            course_id=self.course.id,
+            student_module_id=sm1.id,
+            module_state_key=sm1.module_state_key,
+            student_id=sm1.student.id,
+            attempts=0,
+        )
+        sme1.save()
+
+        # Touch the StudentModule row so it has a later modified time
+        sm1.state = json.dumps({'attempts': 1})
+        sm1.save()
+
+        # Create a StudentModule
+        sm2 = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            module_state_key=self.item2.location,
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts': 2}),
+        )
+        # Create a StudentModuleExpand that has the same attempts value
+        sme2 = StudentModuleExpand(
+            course_id=self.course.id,
+            student_module_id=sm2.id,
+            module_state_key=sm2.module_state_key,
+            student_id=sm2.student.id,
+            attempts=2,
+        )
+        sme2.save()
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course.id)), 2)
+
+        # Call command
+        management.call_command(self.command, self.course.id)
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(
+                         course_id__exact=self.course.id, module_state_key__exact=sme1.module_state_key)), 1)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(
+                         course_id__exact=self.course.id, module_state_key__exact=sme2.module_state_key)), 1)
+
+        self.assertEqual(StudentModuleExpand.objects.filter(
+                         course_id__exact=self.course.id, module_state_key__exact=sme1.module_state_key)[0].attempts, 1)
+        self.assertEqual(StudentModuleExpand.objects.filter(
+                         course_id__exact=self.course.id, module_state_key__exact=sme2.module_state_key)[0].attempts, 2)

lms/djangoapps/queryable_student_module/tests/test_util.py

+from django.test import TestCase
+from django.test.utils import override_settings
+
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+
+from queryable_student_module import util
+
+
+class TestUtilApproxEqual(TestCase):
+    """
+    Check the approx_equal function
+    """
+
+    def test_default_tolerance(self):
+        """
+        Check that function with default tolerance
+        """
+        self.assertTrue(util.approx_equal(1.00001, 1.0))
+        self.assertTrue(util.approx_equal(1.0, 1.00001))
+
+        self.assertFalse(util.approx_equal(1.0, 2.0))
+        self.assertFalse(util.approx_equal(1.0, 1.0002))
+
+    def test_smaller_default_tolerance(self):
+        """
+        Set tolerance smaller than default and check if still correct
+        """
+
+        self.assertTrue(util.approx_equal(1.0, 1.0, 1))
+        self.assertTrue(util.approx_equal(1.0, 1.000001, 0.000001))
+
+    def test_bigger_default_tolerance(self):
+        """
+        Set tolerance bigger than default and check if still correct
+        """
+
+        self.assertFalse(util.approx_equal(1.0, 2.0, 0.75))
+        self.assertFalse(util.approx_equal(2.0, 1.0, 0.75))
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestUtilGetAssignmentToProblemMap(TestCase):
+    """
+    Tests the get_assignemnt_to_problem_map
+    """
+
+    def setUp(self):
+        self.course = CourseFactory.create()
+
+    def test_empty_course(self):
+        """
+        Test for course with nothing in it
+        """
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        self.assertEqual(problems_map, {})
+
+    def test_single_assignment(self):
+        """
+        Test returns the problems for a course with a single assignment
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential",
+        )
+        subsection_metadata = own_metadata(subsection)
+        subsection_metadata['graded'] = True
+        subsection_metadata['format'] = "Homework"
+        modulestore().update_metadata(subsection.location, subsection_metadata)
+
+        unit = ItemFactory.create(
+            parent_location=subsection.location.url(),
+            category="vertical",
+        )
+
+        problem1 = ItemFactory.create(
+            parent_location=unit.location.url(),
+            category="problem",
+        )
+        problem2 = ItemFactory.create(
+            parent_location=unit.location.url(),
+            category="problem",
+        )
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {'Homework': [[problem1.location.url(), problem2.location.url()], ],
+                  }
+
+        self.assertEqual(problems_map, answer)
+
+    def test_two_assignments_same_type(self):
+        """
+        Test if has two assignments
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['graded'] = True
+        subsection_metadata2['format'] = "Homework"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {'Homework': [[problem1.location.url()], [problem2.location.url()], ],
+                  }
+
+        self.assertEqual(problems_map, answer)
+
+    def test_two_assignments_different_types(self):
+        """
+        Creates two assignments of different types
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['graded'] = True
+        subsection_metadata2['format'] = "Quiz"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {'Homework': [[problem1.location.url()], ],
+                  'Quiz': [[problem2.location.url()], ],
+                  }
+
+        self.assertEqual(problems_map, answer)
+
+    def test_return_only_graded_subsections(self):
+        """
+        Make sure only returns problems and assignments that are graded
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['format'] = "Quiz"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {'Homework': [[problem1.location.url()], ],
+                  }
+
+        self.assertEqual(problems_map, answer)

lms/djangoapps/queryable_student_module/util.py

+"""
+Utility functions to help with population
+"""
+
+from datetime import datetime
+from pytz import UTC
+import logging
+from optparse import make_option
+
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+from queryable_student_module.models import Log
+
+log = logging.getLogger("mitx.queryable")
+
+
+def get_assignment_to_problem_map(course_id):
+    """
+    Returns a dictionary with assignment types/categories as keys and the value is an array of arrays. Each inner array
+    holds problem ids for an assignment. The arrays are ordered in the outer array as they are seen in the course, which
+    is how they are numbered in a student's progress page.
+    """
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+
+    assignment_problems_map = {}
+    for section in course.get_children():
+        for subsection in section.get_children():
+            subsection_metadata = own_metadata(subsection)
+            if ('graded' in subsection_metadata) and subsection_metadata['graded']:
+                category = subsection_metadata['format']
+                if category not in assignment_problems_map:
+                    assignment_problems_map[category] = []
+
+                problems = []
+                for unit in subsection.get_children():
+                    for child in unit.get_children():
+                        if child.location.category == 'problem':
+                            problems.append(child.location.url())
+
+                assignment_problems_map[category].append(problems)
+
+    return assignment_problems_map
+
+
+def approx_equal(first, second, tolerance=0.0001):
+    """
+    Checks if first and second are at most the specified tolerance away from each other.
+    """
+    return abs(first - second) <= tolerance
+
+
+def pre_run_command(script_id, options, course_id):
+    """
+    Common pre-run method for both populate_studentgrades and populate_studentmoduleexpand commands.
+    """
+
+    log.info("--------------------------------------------------------------------------------")
+    log.info("Populating queryable.{0} table for course {1}".format(script_id, course_id))
+    log.info("--------------------------------------------------------------------------------")
+
+    # Grab when we start, to log later
+    tstart = datetime.now(UTC)
+
+    iterative_populate = True
+    last_log_run = {}
+    if options['force']:
+        log.info("--------------------------------------------------------------------------------")
+        log.info("Full populate: Forced full populate")
+        log.info("--------------------------------------------------------------------------------")
+        iterative_populate = False
+
+    if iterative_populate:
+        # Get when this script was last run for this course
+        last_log_run = Log.objects.filter(script_id__exact=script_id, course_id__exact=course_id)
+
+        length = len(last_log_run)
+        log.info("--------------------------------------------------------------------------------")
+        if length > 0:
+            log.info("Iterative populate: Last log run %s", str(last_log_run[0].created))
+        else:
+            log.info("Full populate: Can't find log of last run")
+            iterative_populate = False
+        log.info("--------------------------------------------------------------------------------")
+
+    return iterative_populate, tstart, last_log_run
+
+
+def more_options():
+    """
+    Appends common options to options list
+    """
+
+    option_list =  make_option('-f', '--force',
+                               action='store_true',
+                               dest='force',
+                               default=False,
+                               help='Forces a full populate for all students and rows, rather than iterative.')
+    
+    return option_list
+
+        
\ No newline at end of file

lms/envs/common.py

@@ -912,6 +912,13 @@ VERIFY_STUDENT = {
     "DAYS_GOOD_FOR" : 365, # How many days is a verficiation good for?
 }
 
+########################## QUERYABLE TABLES ########################
+INSTALLED_APPS += ('queryable_student_module',)
+
+########################## CLASS DASHBOARD ########################
+INSTALLED_APPS += ('class_dashboard',)
+MITX_FEATURES['CLASS_DASHBOARD'] = False
+
 ######################## CAS authentication ###########################
 
 if MITX_FEATURES.get('AUTH_USE_CAS'):

lms/envs/dev.py

@@ -272,10 +272,12 @@ CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = os.environ.get('CYBERSOURCE_P
 ########################## USER API ########################
 EDX_API_KEY = None
 
-
 ####################### Shoppingcart ###########################
 MITX_FEATURES['ENABLE_SHOPPING_CART'] = True
 
+########################## CLASS DASHBOARD ########################
+MITX_FEATURES['CLASS_DASHBOARD'] = True
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

lms/envs/test.py

@@ -230,6 +230,9 @@ DATABASES['jabber'] = {
 
 INSTALLED_APPS += ('jabber',)
 
+########################## CLASS DASHBOARD ########################
+MITX_FEATURES['CLASS_DASHBOARD'] = True
+
 ################### Make tests quieter
 
 # OpenID spews messages like this to stderr, we don't need to see them:
@@ -237,3 +240,6 @@ INSTALLED_APPS += ('jabber',)
 
 import openid.oidutil
 openid.oidutil.log = lambda message, level=0: None
+
+### QUERYABLE APP ###
+INSTALLED_APPS += ('queryable_student_module',)

lms/templates/class_dashboard/all_section_metrics.js

+<%page args="id_opened_prefix, id_grade_prefix, id_attempt_prefix, id_tooltip_prefix, course_id, **kwargs"/>
+<%!
+  import json
+  from django.core.urlresolvers import reverse
+%>
+
+$(function () {
+
+  d3.json("${reverse('all_sequential_open_distribution', kwargs=dict(course_id=course_id))}", function(error, json) {
+    var section, paramOpened, barGraphOpened;
+    var i, curr_id;
+
+    i = 0;
+    for (section in json) {
+      curr_id = "#${id_opened_prefix}"+i;
+      paramOpened = {
+        data: json[section].data,
+        width: $(curr_id).width(),
+        height: $(curr_id).height()-25, // Account for header
+        tag: "opened"+i,
+        bVerticalXAxisLabel : true,
+        bLegend : false,
+        margin: {left:0},
+      };
+      
+      if (paramOpened.data.length > 0) {
+        barGraphOpened = edx_d3CreateStackedBarGraph(paramOpened, d3.select(curr_id).append("svg"),
+                                                     d3.select("#${id_tooltip_prefix}"+i));
+        barGraphOpened.scale.stackColor.range(["#555555","#555555"]);
+        
+        barGraphOpened.drawGraph();
+      }
+
+      i+=1;
+    }
+  });
+
+  d3.json("${reverse('all_problem_grade_distribution', kwargs=dict(course_id=course_id))}", function(error, json) {
+    var section, paramGrade, barGraphGrade;
+    var i, curr_id;
+
+    i = 0;
+    for (section in json) {
+      curr_id = "#${id_grade_prefix}"+i;
+      paramGrade = {
+        data: json[section].data,
+        width: $(curr_id).width(),
+        height: $(curr_id).height()-25, // Account for header
+        tag: "grade"+i,
+        bVerticalXAxisLabel : true,
+      };
+      
+      if ( paramGrade.data.length > 0 ) {
+        barGraphGrade = edx_d3CreateStackedBarGraph(paramGrade, d3.select(curr_id).append("svg"),
+                                                    d3.select("#${id_tooltip_prefix}"+i));
+        barGraphGrade.scale.stackColor.domain([0,50,100]).range(["#e13f29","#cccccc","#17a74d"]);
+        barGraphGrade.legend.width += 2;
+        
+        barGraphGrade.drawGraph();
+      }
+
+      i+=1;
+    }
+  });
+
+  d3.json("${reverse('all_problem_attempt_distribution', kwargs=dict(course_id=course_id))}", function(error, json) {
+    var section, paramAttempt, barGraphAttempt;
+    var i, curr_id;
+
+    i = 0;
+    for (section in json) {
+      curr_id = "#${id_attempt_prefix}"+i;
+      paramAttempt = {
+        data: json[section].data,
+        width: $(curr_id).width(),
+        height: $(curr_id).height()-25, // Account for header
+        tag: "attempt"+i,
+        bVerticalXAxisLabel : true,
+      };
+      
+      if ( paramAttempt.data.length > 0 ) {
+        barGraphAttempt = edx_d3CreateStackedBarGraph(paramAttempt, d3.select(curr_id).append("svg"),
+                                                       d3.select("#${id_tooltip_prefix}"+i));
+        barGraphAttempt.scale.stackColor
+          .range(["#c3c4cd","#b0b4d1","#9ca3d6","#8993da","#7682de","#6372e3",
+                  "#4f61e7","#3c50eb","#2940ef","#1530f4","#021ff8"]);
+        barGraphAttempt.legend.width += 2;
+        
+        barGraphAttempt.drawGraph();
+      }
+
+      i+=1;
+    }
+  });
+  
+});
\ No newline at end of file

lms/templates/class_dashboard/d3_stacked_bar_graph.js

+/*
+There are three parameters:
+(1) Parameter is of type object. Inside can include (* marks required):
+  data* - Array of objects with key, value pairs that represent a single stack of bars:
+    xValue - Corresponding value for the x-axis
+    stackData - Array of objects with key, value pairs that represent a bar:
+      color - Defines what "color" the bar will map to
+      value - Maps to the height of the bar, along the y-axis
+      tooltip - (Optional) Text to display on mouse hover
+  
+  height - Height of the SVG the graph will be displayed in (default: 500)
+
+  width - Width of the SVG the graph will be displayed in (default: 500)
+
+  margin - Object with key, value pairs for the graph's margins within the SVG (default for all: 10)
+    top - Top margin
+    bottom - Bottom margin
+    right - Right margin
+    left - Left margin
+
+  yRange - Array of two values, representing the min and max respectively. (default: [0, <calculated max>])
+
+  xRange - Array of either the min and max or ordered ordinals (default: calculated min and max or ordered ordinals given in data)
+
+  colorRange - Array of either the min and max or ordered ordinals (default: calculated min and max or ordered ordinals given in data)
+
+  bVerticalXAxisLabel - Boolean whether to make the labels in the x-axis veritcal (default: false)
+
+  bLegend - Boolean if false does not create the graph with a legend (default: true)
+
+(2) Parameter is a d3 pointer to the SVG the graph will draw itself in.
+
+(3) Parameter is a d3 pointer to a div that will be used for the graph's tooltip.
+
+****Does not actually draw graph.**** Returns an object that includes a function
+  drawGraph, for when ready to draw graph. Reason for this is, because of all
+  the defaults, some changes may be needed before drawing the graph
+
+returns an object with the following:
+  state - All information that can be put in parameters and adding:
+    margin.axisX - margin to accomodate the x-axis
+    margin.axisY - margin to acommodate the y-axis
+
+  drawGraph - function to call when ready to draw graph
+
+  scale - Object containing three d3 scales
+    x - d3 scale for the x-axis
+    y - d3 scale for the y-axis
+    stackColor - d3 scale for the stack color
+
+  axis - Object containg the graph's two d3 axis
+    x - d3 axis for the x-axis
+    y - d3 axis for the y-axis
+
+  svg - d3 pointer to the svg holding the graph
+
+  svgGroup - object holding the svg groups
+    main - svg group holding all other groups
+    xAxis - svg group holding the x-axis
+    yAxis - svg group holding the x-axis
+    bars - svg groups holding the bars
+
+  yAxisLabel - d3 pointer to the text component that holds the y axis label
+
+  divTooltip - d3 pointer to the div that is used as the tooltip for the graph
+
+  rects - d3 collection of the rects used in the bars
+
+  legend - object containing information for the legend
+    height - height of the legend
+    width - width of the legend (if change, need to update state.margin.axisY also)
+    range - array of values that appears in the legend
+    barHeight - height of a bar in the legend, based on height and length of range
+*/
+
+edx_d3CreateStackedBarGraph = function(parameters, svg, divTooltip) {
+  var graph = {
+    svg : svg,
+    state : {
+      data : undefined,
+      height : 500,
+      width : 500,
+      margin: {top: 10, bottom: 10, right: 10, left: 10},
+      yRange: [0],
+      xRange : undefined,
+      colorRange : undefined,
+      tag : "",
+      bVerticalXAxisLabel : false,
+      bLegend : true,
+    },
+    divTooltip : divTooltip,
+  };
+
+  var state = graph.state;
+
+  // Handle parameters
+  state.data = parameters.data;
+
+  if (parameters.margin != undefined) {
+    for (var key in state.margin) {
+      if ((state.margin.hasOwnProperty(key) &&
+           (parameters.margin[key] != undefined))) {
+        state.margin[key] = parameters.margin[key];
+      }
+    }
+  }
+
+  for (var key in state) {
+    if ((key != "data") && (key != "margin")) {
+      if (state.hasOwnProperty(key) && (parameters[key] != undefined)) {
+        state[key] = parameters[key];
+      }
+    }
+  }
+
+  if (state.tag != "")
+    state.tag = state.tag+"-";
+
+  if ((state.xRange == undefined) || (state.yRange.length < 2 || 
+                                      state.colorRange == undefined)) {
+    var aryXRange = [];
+    var bXIsOrdinal = false;
+    var maxYRange = 0;
+    var aryColorRange = [];
+    var bColorIsOrdinal = false;
+
+    for (var stackKey in state.data) {
+      var stack = state.data[stackKey];
+      aryXRange.push(stack.xValue);
+      if (isNaN(stack.xValue))
+        bXIsOrdinal = true;
+      
+      var valueTotal = 0;
+      for (var barKey in stack.stackData) {
+        var bar = stack.stackData[barKey];
+        valueTotal += bar.value;
+
+        if (isNaN(bar.color))
+          bColorIsOrdinal = true;
+
+        if (aryColorRange.indexOf(bar.color) < 0)
+          aryColorRange.push(bar.color);
+      }
+      if (maxYRange < valueTotal)
+        maxYRange = valueTotal;
+    }
+
+    if (state.xRange == undefined){
+      if (bXIsOrdinal)
+        state.xRange = aryXRange;
+      else
+        state.xRange = [
+          Math.min.apply(null,aryXRange),
+          Math.max.apply(null,aryXRange)
+        ];
+    }
+
+    if (state.yRange.length < 2)
+      state.yRange[1] = maxYRange;
+
+    if (state.colorRange == undefined){
+      if (bColorIsOrdinal)
+        state.colorRange = aryColorRange;
+      else
+        state.colorRange = [
+          Math.min.apply(null,aryColorRange),
+          Math.max.apply(null,aryColorRange)
+        ];
+    }
+  }
+
+  // Find needed spacing for axes
+  var tmpEl = graph.svg.append("text").text(state.yRange[1]+"1234")
+    .attr("id",state.tag+"stacked-bar-graph-long-str");
+  state.margin.axisY = document.getElementById(state.tag+"stacked-bar-graph-long-str")
+    .getComputedTextLength()+state.margin.left;
+
+  var longestXAxisStr = "";
+  if (isNaN(state.xRange[0])) {
+    for (var i in state.xRange) {
+      if (longestXAxisStr.length < state.xRange[i].length)
+        longestXAxisStr = state.xRange[i]+"1234";
+    }
+  } else {
+    longestXAxisStr = state.xRange[1]+"1234";
+  }
+
+  tmpEl.text(longestXAxisStr);
+  if (state.bVerticalXAxisLabel) {
+    state.margin.axisX = document.getElementById(state.tag+"stacked-bar-graph-long-str")
+      .getComputedTextLength()+state.margin.bottom;
+  } else {
+    state.margin.axisX = document.getElementById(state.tag+"stacked-bar-graph-long-str")
+      .clientHeight+state.margin.bottom;
+  }
+
+  tmpEl.remove();
+
+  // Add y0 and y1 of the y-axis based on the count and order of the colorRange.
+  // First, case if color is a number range
+  if ((state.colorRange.length == 2) && !(isNaN(state.colorRange[0])) &&
+    !(isNaN(state.colorRange[1]))) {
+    for (var stackKey in state.data) {
+      var stack = state.data[stackKey];
+      stack.stackData.sort(function(a,b) { return a.color - b.color; });
+
+      var currTotal = 0;
+      for (var barKey in stack.stackData) {
+        var bar = stack.stackData[barKey];
+        bar.y0 = currTotal;
+        currTotal += bar.value;
+        bar.y1 = currTotal;
+      }
+    }
+  } else {
+    for (var stackKey in state.data) {
+      var stack = state.data[stackKey];
+      
+      var tmpStackData = [];
+      for (var barKey in stack.stackData) {
+        var bar = stack.stackData[barKey];
+        tmpStackData[state.colorRange.indexOf(bar.color)] = bar;
+      }
+      stack.stackData = tmpStackData;
+
+      var currTotal = 0;
+      for (var barKey in stack.stackData) {
+        var bar = stack.stackData[barKey];
+        bar.y0 = currTotal;
+        currTotal += bar.value;
+        bar.y1 = currTotal;
+      }
+    }
+  }
+
+  // Add information to create legend
+  if (state.bLegend) {
+    graph.legend = {
+      height : (state.height-state.margin.top-state.margin.axisX),
+      width : 30,
+      range : state.colorRange,
+    };
+    if ((state.colorRange.length == 2) && !(isNaN(state.colorRange[0])) &&
+        !(isNaN(state.colorRange[1]))) {
+      graph.legend.range = [];
+      
+      var i = 0;
+      var min = state.colorRange[0];
+      var max = state.colorRange[1];
+      while (i <= 10) {
+        graph.legend.range[i] = min+((max-min)/10)*i;
+        i += 1;
+      }
+    }
+    graph.legend.barHeight = graph.legend.height/graph.legend.range.length;
+    
+    // Shifting the axis over to make room
+    graph.state.margin.axisY += graph.legend.width;
+  }
+
+  // Make the scales
+  graph.scale = {
+    x: d3.scale.ordinal()
+      .domain(graph.state.xRange)
+      .rangeRoundBands([
+        (graph.state.margin.axisY),
+        (graph.state.width-graph.state.margin.right)],
+                       .3),
+
+    y: d3.scale.linear()
+      .domain(graph.state.yRange) // yRange is the range of the y-axis values
+      .range([
+        (graph.state.height-graph.state.margin.axisX),
+        graph.state.margin.top
+      ]),
+    
+    stackColor: d3.scale.ordinal()
+      .domain(graph.state.colorRange)
+      .range(["#ffeeee","#ffebeb","#ffd8d8","#ffc4c4","#ffb1b1","#ff9d9d","#ff8989","#ff7676","#ff6262","#ff4e4e","#ff3b3b"])
+  };
+
+  if ((state.colorRange.length == 2) && !(isNaN(state.colorRange[0])) &&
+    !(isNaN(state.colorRange[1]))) {
+    graph.scale.stackColor = d3.scale.linear()
+      .domain(state.colorRange)
+      .range(["#e13f29","#17a74d"]);
+  }
+
+  // Setup axes
+  graph.axis = {
+    x: d3.svg.axis()
+      .scale(graph.scale.x),
+    y: d3.svg.axis()
+      .scale(graph.scale.y),
+  }
+
+  graph.axis.x.orient("bottom");
+  graph.axis.y.orient("left");
+
+  // Draw graph function, to call when ready.
+  graph.drawGraph = function() {
+    var graph = this;
+    
+    // Steup SVG
+    graph.svg.attr("id", graph.state.tag+"stacked-bar-graph")
+      .attr("class", "stacked-bar-graph")
+      .attr("width", graph.state.width)
+      .attr("height", graph.state.height);
+    graph.svgGroup = {};
+
+    graph.svgGroup.main = graph.svg.append("g");
+
+    // Draw Bars
+    graph.svgGroup.bars = graph.svgGroup.main.selectAll(".stacked-bar")
+      .data(graph.state.data)
+      .enter().append("g")
+      .attr("class", "stacked-bar")
+      .attr("transform", function(d) {
+          return "translate("+graph.scale.x(d.xValue)+",0)";
+      });
+
+    graph.rects = graph.svgGroup.bars.selectAll("rect")
+      .data(function(d) { return d.stackData; })
+      .enter().append("rect")
+      .attr("width", function(d) {
+          return graph.scale.x.rangeBand()
+      })
+      .attr("y", function(d) { return graph.scale.y(d.y1); })
+      .attr("height", function(d) {
+        return graph.scale.y(d.y0) - graph.scale.y(d.y1);
+      })
+      .style("fill", function(d) { return graph.scale.stackColor(d.color); })
+      .style("stroke", "white")
+      .style("stroke-width", "0.5px");
+
+    // Setup tooltip
+    if (graph.divTooltip != undefined) {
+      graph.divTooltip
+        .style("position", "absolute")
+        .style("z-index", "10")
+        .style("visibility", "hidden");
+    }
+
+    graph.rects
+      .on("mouseover", function(d) {
+        var pos = d3.mouse(graph.divTooltip.node().parentNode);
+        var left = pos[0]+10;
+        var top = pos[1]-10;
+        var width = $('#'+graph.divTooltip.attr("id")).width();
+
+        graph.divTooltip.style("visibility", "visible")
+          .text(d.tooltip);
+
+        if ((left+width+30) > $("#"+graph.divTooltip.node().parentNode.id).width())
+          left -= (width+30);
+        
+        graph.divTooltip.style("top", top+"px")
+          .style("left", left+"px");
+      })
+      .on("mouseout", function(d){
+        graph.divTooltip.style("visibility", "hidden")
+      });
+
+    // Add legend
+    if (graph.state.bLegend) {
+      graph.svgGroup.legendG = graph.svgGroup.main.append("g")
+        .attr("class","stacked-bar-graph-legend")
+        .attr("transform","translate("+graph.state.margin.left+","+
+              graph.state.margin.top+")");
+      graph.svgGroup.legendGs = graph.svgGroup.legendG.selectAll(".stacked-bar-graph-legend-g")
+        .data(graph.legend.range)
+        .enter().append("g")
+        .attr("class","stacked-bar-graph-legend-g")
+        .attr("id",function(d,i) { return graph.state.tag+"legend-"+i; })
+        .attr("transform", function(d,i) {
+          return "translate(0,"+
+            (graph.state.height-graph.state.margin.axisX-((i+1)*(graph.legend.barHeight))) + ")";
+        });
+      
+      graph.svgGroup.legendGs.append("rect")
+        .attr("class","stacked-bar-graph-legend-rect")
+        .attr("height", graph.legend.barHeight)
+        .attr("width", graph.legend.width)
+        .style("fill", graph.scale.stackColor)
+        .style("stroke", "white");
+
+      graph.svgGroup.legendGs.append("text")
+        .attr("class","axis-label")
+        .attr("transform", function(d) {
+          var str = "translate("+(graph.legend.width/2)+","+
+            (graph.legend.barHeight/2)+")";
+          return str;
+        })
+        .attr("dy", ".35em")
+        .attr("dx", "-1px")
+        .style("text-anchor", "middle")
+        .text(function(d,i) { return d; });
+    }
+
+
+    // Draw Axes
+    graph.svgGroup.xAxis = graph.svgGroup.main.append("g")
+      .attr("class","stacked-bar-graph-axis")
+      .attr("id",graph.state.tag+"x-axis");
+
+    var tmpS = "translate(0,"+(graph.state.height-graph.state.margin.axisX)+")";
+    if (graph.state.bVerticalXAxisLabel) {
+      graph.axis.x.orient("left");
+      tmpS = "rotate(270), translate(-"+(graph.state.height-graph.state.margin.axisX)+",0)";
+    }
+    graph.svgGroup.xAxis.attr("transform", tmpS)
+      .call(graph.axis.x);
+
+    graph.svgGroup.yAxis = graph.svgGroup.main.append("g")
+      .attr("class","stacked-bar-graph-axis")
+      .attr("id",graph.state.tag+"y-axis")
+      .attr("transform","translate("+
+            (graph.state.margin.axisY)+",0)")
+      .call(graph.axis.y);
+    graph.yAxisLabel = graph.svgGroup.yAxis.append("text")
+      .attr("dy","1em")
+      .attr("transform","rotate(-90)")
+      .style("text-anchor","end")
+      .text("Number of Students");
+  };
+
+  return graph;
+};
\ No newline at end of file

lms/templates/courseware/instructor_dashboard.html

@@ -95,6 +95,38 @@ textarea {
     top: 30px;
 }
 
+.metrics-container {
+    position: relative;
+    width: 100%;
+    float: left;
+    clear: both;
+}
+.metrics-left {
+    position: relative;
+    width: 30%;
+    height: 640px;
+    float: left;
+    margin-right: 2.5%;
+}
+.metrics-right {
+    position: relative;
+    width: 65%;
+    height: 295px;
+    float: left;
+    margin-left: 2.5%;
+    margin-bottom: 25px;
+}
+
+.metrics-tooltip {
+    width: 250px;
+    background-color: lightgray;
+    padding: 3px;
+}
+
+.stacked-bar-graph-legend {
+    fill: white;
+}
+
 </style>
 
 <script language="JavaScript" type="text/javascript">
@@ -135,6 +167,9 @@ function goto( mode)
           %if settings.MITX_FEATURES.get('ENABLE_INSTRUCTOR_ANALYTICS'):
              | <a href="#" onclick="goto('Analytics');" class="${modeflag.get('Analytics')}">${_("Analytics")}</a>
           %endif
+          %if settings.MITX_FEATURES.get('CLASS_DASHBOARD'):
+             | <a href="#" onclick="goto('Metrics');" class="${modeflag.get('Metrics')}">${_("Metrics")}</a>
+          %endif
 	  ]
     </h2>
 
@@ -591,6 +626,47 @@ function goto( mode)
   %endif
 %endif
 
+%if modeflag.get('Metrics'):
+  %if not any (metrics_results.values()):
+    <p>${_("There is no data available to display at this time.")}</p>
+  %else:
+    <%namespace name="d3_stacked_bar_graph" file="/class_dashboard/d3_stacked_bar_graph.js"/>
+    <%namespace name="all_section_metrics" file="/class_dashboard/all_section_metrics.js"/>
+
+    <script>
+      ${d3_stacked_bar_graph.body()}
+    </script>
+
+    <div id="metrics"></div>
+
+    %for i in range(0,len(metrics_results['section_display_name'])):
+      <div class="metrics-container" id="metrics_section_${i}">
+        <h2>Section: ${metrics_results['section_display_name'][i]}</h2>
+        <div class="metrics-tooltip" id="metric_tooltip_${i}"></div>
+        <div class="metrics-left" id="metric_opened_${i}">
+          <h3>Count of Students Opened a Subsection</h3>
+        </div>
+        <div class="metrics-right" id="metric_grade_${i}">
+          <h3>Grade Distribution per Problem</h3>
+          %if not metrics_results['section_has_problem'][i]:
+            <p>${_("There are no problems in this section.")}</p>
+          %endif
+        </div>
+        <div class="metrics-right" id="metric_attempts_${i}">
+          <h3>Attempt Distribution per Problem (Last update: ${metrics_results['attempts_timestamp']})</h3>
+          %if not metrics_results['section_has_problem'][i]:
+            <p>${_("There are no problems in this section.")}</p>
+          %endif
+        </div>
+      </div>
+    %endfor
+      <script>
+        ${all_section_metrics.body("metric_opened_","metric_grade_","metric_attempts_","metric_tooltip_",course.id)}
+      </script>
+
+  %endif
+%endif
+
 %if modeflag.get('Analytics In Progress'):
 
   ##This is not as helpful as it could be -- let's give full point distribution

lms/urls.py

@@ -365,6 +365,25 @@ if settings.COURSEWARE_ENABLED and settings.MITX_FEATURES.get('ENABLE_INSTRUCTOR
             include('instructor.views.api_urls'))
     )
 
+if settings.MITX_FEATURES.get('CLASS_DASHBOARD'):
+    urlpatterns += (
+        # Json request data for metrics for entire course
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/all_problem_attempt_distribution$',
+            'class_dashboard.views.all_problem_attempt_distribution', name="all_problem_attempt_distribution"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/all_sequential_open_distribution$',
+            'class_dashboard.views.all_sequential_open_distribution', name="all_sequential_open_distribution"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/all_problem_grade_distribution$',
+            'class_dashboard.views.all_problem_grade_distribution', name="all_problem_grade_distribution"),
+
+        #Json request data for metrics for particular section
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/problem_grade_distribution/(?P<section>\d+)$',
+            'class_dashboard.views.section_problem_grade_distribution', name="section_problem_grade_distribution"),
+    )
+
+
+if settings.ENABLE_JASMINE:
+    urlpatterns += (url(r'^_jasmine/', include('django_jasmine.urls')),)
+
 if settings.DEBUG or settings.MITX_FEATURES.get('ENABLE_DJANGO_ADMIN_SITE'):
     ## Jasmine and admin
     urlpatterns += (url(r'^admin/', include(admin.site.urls)),)