(YONK-8) Performance Improvement - UsersCoursesGradesDetail

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -1390,6 +1390,8 @@ class EdxJSONEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, (CourseKey, UsageKey)):
             return unicode(obj)
+        elif isinstance(obj, Location):
+            return obj.dict()
         elif isinstance(obj, datetime.datetime):
             if obj.tzinfo is not None:
                 if obj.utcoffset() is None:

lms/djangoapps/api_manager/users/tests.py

@@ -1444,6 +1444,12 @@ class UsersApiTests(ModuleStoreTestCase):
             metadata={'rerandomize': 'always', 'graded': True, 'format': "Homework"},
             due=self.course_end_date.replace(tzinfo=timezone.utc)
         )
+        points_scored = 5
+        points_possible = 10
+        user = self.user
+        module = self.get_module_for_user(user, course, item5)
+        grade_dict = {'value': points_scored, 'max_value': points_possible, 'user_id': user.id}
+        module.system.publish(module, 'grade', grade_dict)
 
         test_uri = '{}/{}/courses/{}/grades'.format(self.users_base_uri, user_id, unicode(course.id))
 
@@ -1471,16 +1477,16 @@ class UsersApiTests(ModuleStoreTestCase):
         self.assertGreater(len(grading_policy['GRADER']), 0)
         self.assertIsNotNone(grading_policy['GRADE_CUTOFFS'])
 
-        self.assertEqual(response.data['current_grade'], 0.73)
-        self.assertEqual(response.data['proforma_grade'], 0.9375)
+        self.assertEqual(response.data['current_grade'], 0.74)
+        self.assertEqual(response.data['proforma_grade'], 0.9174999999999999)
 
         test_uri = '{}/{}/courses/grades'.format(self.users_base_uri, user_id)
 
         response = self.do_get(test_uri)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data[0]['course_id'], unicode(course.id))
-        self.assertEqual(response.data[0]['current_grade'], 0.73)
-        self.assertEqual(response.data[0]['proforma_grade'], 0.9375)
+        self.assertEqual(response.data[0]['current_grade'], 0.74)
+        self.assertEqual(response.data[0]['proforma_grade'], 0.9174999999999999)
         self.assertEqual(response.data[0]['complete_status'], False)
 
     def is_user_profile_created_updated(self, response, data):

lms/djangoapps/api_manager/users/views.py

 """ API implementation for user-oriented interactions. """
 
+import json
 import logging
 from edx_notifications.lib.consumer import mark_notification_read
 from requests.exceptions import ConnectionError
@@ -1000,19 +1001,21 @@ class UsersCoursesGradesDetail(SecureAPIView):
         if not course_descriptor:
             return Response({}, status=status.HTTP_404_NOT_FOUND)
 
-        progress_summary = grades.progress_summary(student, request, course_descriptor)  # pylint: disable=W0612
-        grade_summary = grades.grade(student, request, course_descriptor)
-        grading_policy = course_descriptor.grading_policy
-
         queryset = StudentGradebook.objects.filter(
             user=student,
             course_id__exact=course_key,
         )
         current_grade = 0
         proforma_grade = 0
+        progress_summary = {}
+        grade_summary = {}
+        grading_policy = {}
         if len(queryset):
             current_grade = queryset[0].grade
             proforma_grade = queryset[0].proforma_grade
+            progress_summary = json.loads(queryset[0].progress_summary)
+            grade_summary = json.loads(queryset[0].grade_summary)
+            grading_policy = json.loads(queryset[0].grading_policy)
 
         response_data = {
             'courseware_summary': progress_summary,

lms/djangoapps/gradebook/management/commands/generate_gradebook_entries.py

 """
-One-time data migration script -- shoulen't need to run it again
+One-time data migration script -- shouldn't need to run it again
 """
+import json
 import logging
 from optparse import make_option
 
@@ -10,6 +11,7 @@ from courseware import grades
 from gradebook.models import StudentGradebook
 from student.models import CourseEnrollment
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import EdxJSONEncoder
 from util.request import RequestMockWithoutMiddleware
 
 log = logging.getLogger(__name__)
@@ -21,7 +23,7 @@ class Command(BaseCommand):
     """
 
     def handle(self, *args, **options):
-        help = "Command to creaete or update gradebook entries"
+        help = "Command to create or update gradebook entries"
         option_list = BaseCommand.option_list + (
             make_option(
                 "-c",
@@ -69,15 +71,28 @@ class Command(BaseCommand):
                 request.user = user
                 grade_data = grades.grade(user, request, course)
                 grade = grade_data['percent']
-                proforma_grade = grades.calculate_proforma_grade(grade_data, course.grading_policy)
+                grading_policy = course.grading_policy
+                proforma_grade = grades.calculate_proforma_grade(grade_data, grading_policy)
+                progress_summary = grades.progress_summary(user, request, course)
                 try:
                     gradebook_entry = StudentGradebook.objects.get(user=user, course_id=course.id)
                     if gradebook_entry.grade != grade or gradebook_entry.proforma_grade != proforma_grade:
                         gradebook_entry.grade = grade
                         gradebook_entry.proforma_grade = proforma_grade
+                        gradebook_entry.progress_summary = json.dumps(progress_summary, cls=EdxJSONEncoder)
+                        gradebook_entry.grade_summary = json.dumps(grade_data, cls=EdxJSONEncoder)
+                        gradebook_entry.grading_policy = json.dumps(grading_policy, cls=EdxJSONEncoder)
                         gradebook_entry.save()
                 except StudentGradebook.DoesNotExist:
-                    StudentGradebook.objects.create(user=user, course_id=course.id, grade=grade, proforma_grade=proforma_grade)
+                    StudentGradebook.objects.create(
+                        user=user,
+                        course_id=course.id,
+                        grade=grade,
+                        proforma_grade=proforma_grade,
+                        progress_summary=json.dumps(progress_summary, cls=EdxJSONEncoder),
+                        grade_summary=json.dumps(grade_data, cls=EdxJSONEncoder),
+                        grading_policy=json.dumps(grading_policy, cls=EdxJSONEncoder)
+                    )
                 log_msg = 'Gradebook entry created -- Course: {}, User: {}  (grade: {}, proforma_grade: {})'.format(course.id, user.id, grade, proforma_grade)
                 print log_msg
                 log.info(log_msg)

lms/djangoapps/gradebook/migrations/0003_add_fields_studentgradebookhistory_progress_summary_grade_summary_grading_policy.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'StudentGradebookHistory.progress_summary'
+        db.add_column('gradebook_studentgradebookhistory', 'progress_summary',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+        # Adding field 'StudentGradebookHistory.grade_summary'
+        db.add_column('gradebook_studentgradebookhistory', 'grade_summary',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+        # Adding field 'StudentGradebookHistory.grading_policy'
+        db.add_column('gradebook_studentgradebookhistory', 'grading_policy',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+        # Adding field 'StudentGradebook.progress_summary'
+        db.add_column('gradebook_studentgradebook', 'progress_summary',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+        # Adding field 'StudentGradebook.grade_summary'
+        db.add_column('gradebook_studentgradebook', 'grade_summary',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+        # Adding field 'StudentGradebook.grading_policy'
+        db.add_column('gradebook_studentgradebook', 'grading_policy',
+                      self.gf('django.db.models.fields.TextField')(default='', blank=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'StudentGradebookHistory.progress_summary'
+        db.delete_column('gradebook_studentgradebookhistory', 'progress_summary')
+
+        # Deleting field 'StudentGradebookHistory.grade_summary'
+        db.delete_column('gradebook_studentgradebookhistory', 'grade_summary')
+
+        # Deleting field 'StudentGradebookHistory.grading_policy'
+        db.delete_column('gradebook_studentgradebookhistory', 'grading_policy')
+
+        # Deleting field 'StudentGradebook.progress_summary'
+        db.delete_column('gradebook_studentgradebook', 'progress_summary')
+
+        # Deleting field 'StudentGradebook.grade_summary'
+        db.delete_column('gradebook_studentgradebook', 'grade_summary')
+
+        # Deleting field 'StudentGradebook.grading_policy'
+        db.delete_column('gradebook_studentgradebook', 'grading_policy')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'gradebook.studentgradebook': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'StudentGradebook'},
+            'course_id': ('xmodule_django.models.CourseKeyField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
+            'grade': ('django.db.models.fields.FloatField', [], {}),
+            'grade_summary': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'grading_policy': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
+            'proforma_grade': ('django.db.models.fields.FloatField', [], {}),
+            'progress_summary': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'gradebook.studentgradebookhistory': {
+            'Meta': {'object_name': 'StudentGradebookHistory'},
+            'course_id': ('xmodule_django.models.CourseKeyField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
+            'grade': ('django.db.models.fields.FloatField', [], {}),
+            'grade_summary': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'grading_policy': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
+            'proforma_grade': ('django.db.models.fields.FloatField', [], {}),
+            'progress_summary': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['gradebook']
\ No newline at end of file

lms/djangoapps/gradebook/models.py

@@ -13,15 +13,19 @@ from model_utils.models import TimeStampedModel
 from student.models import CourseEnrollment
 from xmodule_django.models import CourseKeyField
 
+
 class StudentGradebook(TimeStampedModel):
     """
-    StudentGradebook is essentiall a container used to cache calculated
+    StudentGradebook is essentially a container used to cache calculated
     grades (see courseware.grades.grade), which can be an expensive operation.
     """
     user = models.ForeignKey(User, db_index=True)
     course_id = CourseKeyField(db_index=True, max_length=255, blank=True)
     grade = models.FloatField()
     proforma_grade = models.FloatField()
+    progress_summary = models.TextField(blank=True)
+    grade_summary = models.TextField(blank=True)
+    grading_policy = models.TextField(blank=True)
 
     class Meta:
         """
@@ -147,6 +151,7 @@ class StudentGradebook(TimeStampedModel):
 
         return data
 
+
 class StudentGradebookHistory(TimeStampedModel):
     """
     A running audit trail for the StudentGradebook model.  Listens for
@@ -156,6 +161,9 @@ class StudentGradebookHistory(TimeStampedModel):
     course_id = CourseKeyField(db_index=True, max_length=255, blank=True)
     grade = models.FloatField()
     proforma_grade = models.FloatField()
+    progress_summary = models.TextField(blank=True)
+    grade_summary = models.TextField(blank=True)
+    grading_policy = models.TextField(blank=True)
 
     @receiver(post_save, sender=StudentGradebook)
     def save_history(sender, instance, **kwargs):  # pylint: disable=no-self-argument, unused-argument
@@ -169,7 +177,13 @@ class StudentGradebookHistory(TimeStampedModel):
 
         create_history_entry = False
         if latest_history_entry is not None:
-            if (latest_history_entry.grade != instance.grade) or (latest_history_entry.proforma_grade != instance.proforma_grade):
+            if (
+                latest_history_entry.grade != instance.grade or
+                latest_history_entry.proforma_grade != instance.proforma_grade or
+                latest_history_entry.progress_summary != instance.progress_summary or
+                latest_history_entry.grade_summary != instance.grade_summary or
+                latest_history_entry.grading_policy != instance.grading_policy
+            ):
                 create_history_entry = True
         else:
             create_history_entry = True
@@ -179,6 +193,9 @@ class StudentGradebookHistory(TimeStampedModel):
                 user=instance.user,
                 course_id=instance.course_id,
                 grade=instance.grade,
-                proforma_grade=instance.proforma_grade
+                proforma_grade=instance.proforma_grade,
+                progress_summary=instance.progress_summary,
+                grade_summary=instance.grade_summary,
+                grading_policy=instance.grading_policy
             )
             new_history_entry.save()

lms/djangoapps/gradebook/receivers.py

@@ -3,12 +3,14 @@ Signal handlers supporting various gradebook use cases
 """
 import logging
 import sys
+import json
 from django.dispatch import receiver
 from django.conf import settings
 from django.db.models.signals import post_save, pre_save
 
 from courseware import grades
 from courseware.signals import score_changed
+from xmodule.modulestore import EdxJSONEncoder
 from util.request import RequestMockWithoutMiddleware
 from util.signals import course_deleted
 from student.roles import get_aggregate_exclusion_user_ids
@@ -36,17 +38,31 @@ def on_score_changed(sender, **kwargs):
     course_descriptor = get_course(course_key, depth=None)
     request = RequestMockWithoutMiddleware().get('/')
     request.user = user
-    grade_data = grades.grade(user, request, course_descriptor)
-    grade = grade_data['percent']
-    proforma_grade = grades.calculate_proforma_grade(grade_data, course_descriptor.grading_policy)
+    progress_summary = grades.progress_summary(user, request, course_descriptor)
+    grade_summary = grades.grade(user, request, course_descriptor)
+    grading_policy = course_descriptor.grading_policy
+    grade = grade_summary['percent']
+    proforma_grade = grades.calculate_proforma_grade(grade_summary, grading_policy)
+
     try:
         gradebook_entry = StudentGradebook.objects.get(user=user, course_id=course_key)
         if gradebook_entry.grade != grade:
             gradebook_entry.grade = grade
             gradebook_entry.proforma_grade = proforma_grade
+            gradebook_entry.progress_summary = json.dumps(progress_summary, cls=EdxJSONEncoder)
+            gradebook_entry.grade_summary = json.dumps(grade_summary, cls=EdxJSONEncoder)
+            gradebook_entry.grading_policy = json.dumps(grading_policy, cls=EdxJSONEncoder)
             gradebook_entry.save()
     except StudentGradebook.DoesNotExist:
-        StudentGradebook.objects.create(user=user, course_id=course_key, grade=grade, proforma_grade=proforma_grade)
+        StudentGradebook.objects.create(
+            user=user,
+            course_id=course_key,
+            grade=grade,
+            proforma_grade=proforma_grade,
+            progress_summary=json.dumps(progress_summary, cls=EdxJSONEncoder),
+            grade_summary=json.dumps(grade_summary, cls=EdxJSONEncoder),
+            grading_policy=json.dumps(grading_policy, cls=EdxJSONEncoder)
+        )
 
 
 @receiver(course_deleted)