More response to CR

common/djangoapps/student/models.py

@@ -10,10 +10,12 @@ file and check it in at the same time as your model changes. To do that,
 2. ./manage.py lms schemamigration student --auto description_of_your_change
 3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/
 """
+import crum
 from datetime import datetime
 import hashlib
 import json
 import logging
+from pytz import UTC
 import uuid
 
 from django.conf import settings
@@ -25,16 +27,13 @@ from django.dispatch import receiver, Signal
 import django.dispatch
 from django.forms import ModelForm, forms
 from django.core.exceptions import ObjectDoesNotExist
-
-from course_modes.models import CourseMode
-import lms.lib.comment_client as cc
-from pytz import UTC
-import crum
-
 from track import contexts
 from track.views import server_track
 from eventtracking import tracker
 
+from course_modes.models import CourseMode
+import lms.lib.comment_client as cc
+
 unenroll_done = Signal(providing_args=["course_enrollment"])
 log = logging.getLogger(__name__)
 AUDIT_LOG = logging.getLogger("audit")
@@ -581,15 +580,17 @@ class CourseEnrollment(models.Model):
         )
 
     @classmethod
-    def enrollments_in(cls, course_id, mode=None):
+    def enrollment_counts(cls, course_id):
         """
-        Return a queryset of CourseEnrollment for every active enrollment in the course course_id.
-        Returns only CourseEnrollments with the given mode, if a mode is supplied by the caller.
+        Returns a dictionary that stores the total enrollment count for a course, as well as the
+        enrollment count for each individual mode.
         """
-        if mode is None:
-            return cls.objects.filter(course_id=course_id, is_active=True,)
-        else:
-            return cls.objects.filter(course_id=course_id, is_active=True, mode=mode)
+        d = {}
+        d['total'] = cls.objects.filter(course_id=course_id, is_active=True).count()
+        d['honor'] = cls.objects.filter(course_id=course_id, is_active=True, mode='honor').count()
+        d['audit'] = cls.objects.filter(course_id=course_id, is_active=True, mode='audit').count()
+        d['verified'] = cls.objects.filter(course_id=course_id, is_active=True, mode='verified').count()
+        return d
 
     def activate(self):
         """Makes this `CourseEnrollment` record active. Saves immediately."""

common/djangoapps/util/query.py

+from django.conf import settings
+
+def use_read_replica_if_available(queryset):
+    return queryset.using("read_replica") if "read_replica" in settings.DATABASES else queryset
\ No newline at end of file

lms/djangoapps/shoppingcart/migrations/0007_auto__add_refundreport__add_report__add_certificatestatusreport__add_i.py → lms/djangoapps/shoppingcart/migrations/0007_auto__add_field_orderitem_service_fee.py

@@ -8,57 +8,30 @@ from django.db import models
 class Migration(SchemaMigration):
 
     def forwards(self, orm):
-        # Adding model 'RefundReport'
-        db.create_table('shoppingcart_refundreport', (
-            ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)),
-        ))
-        db.send_create_signal('shoppingcart', ['RefundReport'])
-
-        # Adding model 'Report'
-        db.create_table('shoppingcart_report', (
-            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-        ))
-        db.send_create_signal('shoppingcart', ['Report'])
-
-        # Adding model 'CertificateStatusReport'
-        db.create_table('shoppingcart_certificatestatusreport', (
-            ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)),
-        ))
-        db.send_create_signal('shoppingcart', ['CertificateStatusReport'])
-
-        # Adding model 'ItemizedPurchaseReport'
-        db.create_table('shoppingcart_itemizedpurchasereport', (
-            ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)),
-        ))
-        db.send_create_signal('shoppingcart', ['ItemizedPurchaseReport'])
-
-        # Adding model 'UniversityRevenueShareReport'
-        db.create_table('shoppingcart_universityrevenuesharereport', (
-            ('report_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['shoppingcart.Report'], unique=True, primary_key=True)),
-        ))
-        db.send_create_signal('shoppingcart', ['UniversityRevenueShareReport'])
-
         # Adding field 'OrderItem.service_fee'
         db.add_column('shoppingcart_orderitem', 'service_fee',
                       self.gf('django.db.models.fields.DecimalField')(default=0.0, max_digits=30, decimal_places=2),
                       keep_default=False)
 
+        # Adding index on 'OrderItem', fields ['status']
+        db.create_index('shoppingcart_orderitem', ['status'])
 
-    def backwards(self, orm):
-        # Deleting model 'RefundReport'
-        db.delete_table('shoppingcart_refundreport')
+        # Adding index on 'OrderItem', fields ['fulfilled_time']
+        db.create_index('shoppingcart_orderitem', ['fulfilled_time'])
 
-        # Deleting model 'Report'
-        db.delete_table('shoppingcart_report')
+        # Adding index on 'OrderItem', fields ['refund_requested_time']
+        db.create_index('shoppingcart_orderitem', ['refund_requested_time'])
 
-        # Deleting model 'CertificateStatusReport'
-        db.delete_table('shoppingcart_certificatestatusreport')
 
-        # Deleting model 'ItemizedPurchaseReport'
-        db.delete_table('shoppingcart_itemizedpurchasereport')
+    def backwards(self, orm):
+        # Removing index on 'OrderItem', fields ['refund_requested_time']
+        db.delete_index('shoppingcart_orderitem', ['refund_requested_time'])
+
+        # Removing index on 'OrderItem', fields ['fulfilled_time']
+        db.delete_index('shoppingcart_orderitem', ['fulfilled_time'])
 
-        # Deleting model 'UniversityRevenueShareReport'
-        db.delete_table('shoppingcart_universityrevenuesharereport')
+        # Removing index on 'OrderItem', fields ['status']
+        db.delete_index('shoppingcart_orderitem', ['status'])
 
         # Deleting field 'OrderItem.service_fee'
         db.delete_column('shoppingcart_orderitem', 'service_fee')
@@ -108,14 +81,6 @@ class Migration(SchemaMigration):
             'mode': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
             'orderitem_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.OrderItem']", 'unique': 'True', 'primary_key': 'True'})
         },
-        'shoppingcart.certificatestatusreport': {
-            'Meta': {'object_name': 'CertificateStatusReport', '_ormbases': ['shoppingcart.Report']},
-            'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'})
-        },
-        'shoppingcart.itemizedpurchasereport': {
-            'Meta': {'object_name': 'ItemizedPurchaseReport', '_ormbases': ['shoppingcart.Report']},
-            'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'})
-        },
         'shoppingcart.order': {
             'Meta': {'object_name': 'Order'},
             'bill_to_cardtype': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
@@ -139,15 +104,15 @@ class Migration(SchemaMigration):
         'shoppingcart.orderitem': {
             'Meta': {'object_name': 'OrderItem'},
             'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
-            'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'fulfilled_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'line_desc': ('django.db.models.fields.CharField', [], {'default': "'Misc. Item'", 'max_length': '1024'}),
             'order': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['shoppingcart.Order']"}),
             'qty': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
-            'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'refund_requested_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
             'report_comments': ('django.db.models.fields.TextField', [], {'default': "''"}),
             'service_fee': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}),
-            'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'cart'", 'max_length': '32', 'db_index': 'True'}),
             'unit_cost': ('django.db.models.fields.DecimalField', [], {'default': '0.0', 'max_digits': '30', 'decimal_places': '2'}),
             'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
         },
@@ -163,18 +128,6 @@ class Migration(SchemaMigration):
             'course_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
         },
-        'shoppingcart.refundreport': {
-            'Meta': {'object_name': 'RefundReport', '_ormbases': ['shoppingcart.Report']},
-            'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'})
-        },
-        'shoppingcart.report': {
-            'Meta': {'object_name': 'Report'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
-        },
-        'shoppingcart.universityrevenuesharereport': {
-            'Meta': {'object_name': 'UniversityRevenueShareReport', '_ormbases': ['shoppingcart.Report']},
-            'report_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['shoppingcart.Report']", 'unique': 'True', 'primary_key': 'True'})
-        },
         'student.courseenrollment': {
             'Meta': {'ordering': "('user', 'course_id')", 'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseEnrollment'},
             'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),

lms/djangoapps/shoppingcart/models.py

+from collections import namedtuple
 from datetime import datetime
+from decimal import Decimal
 import pytz
 import logging
 import smtplib
 import unicodecsv
 
-from model_utils.managers import InheritanceManager
-from collections import namedtuple
 from boto.exception import BotoServerError  # this is a super-class of SESError and catches connection errors
-
 from django.dispatch import receiver
 from django.db import models
 from django.conf import settings
@@ -16,7 +15,9 @@ from django.core.mail import send_mail
 from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _
 from django.db import transaction
+from django.db.models import Sum
 from django.core.urlresolvers import reverse
+from model_utils.managers import InheritanceManager
 
 from xmodule.modulestore.django import modulestore
 from xmodule.course_module import CourseDescriptor
@@ -26,6 +27,7 @@ from course_modes.models import CourseMode
 from edxmako.shortcuts import render_to_string
 from student.views import course_from_id
 from student.models import CourseEnrollment, unenroll_done
+from util.query import use_read_replica_if_available
 
 from verify_student.models import SoftwareSecurePhotoVerification
 
@@ -40,8 +42,6 @@ ORDER_STATUSES = (
     ('refunded', 'refunded'),
 )
 
-
-
 # we need a tuple to represent the primary key of various OrderItem subclasses
 OrderItemSubclassPK = namedtuple('OrderItemSubclassPK', ['cls', 'pk'])  # pylint: disable=C0103
 
@@ -570,6 +570,25 @@ class CertificateItem(OrderItem):
                      billing_email=settings.PAYMENT_SUPPORT_EMAIL)
 
     @classmethod
-    def verified_certificates_in(cls, course_id, status):
+    def verified_certificates_count(cls, course_id, status):
         """Return a queryset of CertificateItem for every verified enrollment in course_id with the given status."""
-        return CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status)
+        return use_read_replica_if_available(
+            CertificateItem.objects.filter(course_id=course_id, mode='verified', status=status).count())
+
+    # TODO combine these three methods into one
+    @classmethod
+    def verified_certificates_monetary_field_sum(cls, course_id, status, field_to_aggregate):
+        """
+        Returns a Decimal indicating the total sum of field_to_aggregate for all verified certificates with a particular status.
+
+        Sample usages: 
+        - status 'refunded' and field_to_aggregate 'unit_cost' will give the total amount of money refunded for course_id
+        - status 'purchased' and field_to_aggregate 'service_fees' gives the sum of all service fees for purchased certificates
+        etc
+        """
+        query = use_read_replica_if_available(
+            CertificateItem.objects.filter(course_id=course_id, mode='verified', status='purchased').aggregate(Sum(field_to_aggregate)))[field_to_aggregate + '__sum']
+        if query is None:
+            return Decimal(0.00)
+        else:
+            return query

lms/djangoapps/shoppingcart/tests/test_models.py

@@ -356,26 +356,25 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase):
         self.cart.purchase()
         self.now = datetime.datetime.now(pytz.UTC)
 
-        # We can't modify the values returned by report_row_generator directly, since it's a generator, but
-        # we need the times on CORRECT_CSV and the generated report to match.  So, we extract the times from
-        # the report_row_generator and place them in CORRECT_CSV.
-        self.time_str = {}
-        report = initialize_report("itemized_purchase_report")
-        purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
-        num_of_item = 0
-        for item in purchases:
-            num_of_item += 1
-            self.time_str[num_of_item] = item[0]
+        paid_reg = PaidCourseRegistration.objects.get(course_id=self.course_id, user=self.user)
+        paid_reg.fulfilled_time = self.now
+        paid_reg.refund_requested_time = self.now
+        paid_reg.save()
+
+        cert = CertificateItem.objects.get(course_id=self.course_id, user=self.user)
+        cert.fulfilled_time = self.now
+        cert.refund_requested_time = self.now
+        cert.save()
 
         self.CORRECT_CSV = dedent("""
             Purchase Time,Order ID,Status,Quantity,Unit Cost,Total Cost,Currency,Description,Comments
-            {time_str1},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85
-            {time_str2},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course",
-            """.format(time_str1=str(self.time_str[1]), time_str2=str(self.time_str[2])))
+            {time_str},1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,Ba\xc3\xbc\xe5\x8c\x85
+            {time_str},1,purchased,1,40,40,usd,"Certificate of Achievement, verified cert for course Robot Super Course",
+            """.format(time_str=str(self.now)))
 
     def test_purchased_items_btw_dates(self):
         report = initialize_report("itemized_purchase_report")
-        purchases = report.report_row_generator(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        purchases = report.rows(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
 
         # since there's not many purchases, just run through the generator to make sure we've got the right number
         num_purchases = 0
@@ -385,7 +384,7 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase):
         #self.assertIn(self.reg.orderitem_ptr, purchases)
         #self.assertIn(self.cert_item.orderitem_ptr, purchases)
 
-        no_purchases = report.report_row_generator(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS)
+        no_purchases = report.rows(self.now + self.FIVE_MINS, self.now + self.FIVE_MINS + self.FIVE_MINS)
 
         num_purchases = 0
         for item in no_purchases:
@@ -398,7 +397,7 @@ class ItemizedPurchaseReportTest(ModuleStoreTestCase):
         """
         report = initialize_report("itemized_purchase_report")
         csv_file = StringIO.StringIO()
-        report.make_report(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        report.write_csv(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
         csv = csv_file.getvalue()
         csv_file.close()
         # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n

lms/djangoapps/shoppingcart/tests/test_views.py

@@ -365,26 +365,6 @@ class CSVReportViewsTest(ModuleStoreTestCase):
         self.assertIn(_("There was an error in your date input.  It should be formatted as YYYY-MM-DD"),
                       response.content)
 
-    @patch('shoppingcart.views.render_to_response', render_mock)
-    @override_settings(PAYMENT_REPORT_MAX_ITEMS=0)
-    def test_report_csv_too_long(self):
-        PaidCourseRegistration.add_to_order(self.cart, self.course_id)
-        self.cart.purchase()
-        self.login_user()
-        self.add_to_download_group(self.user)
-        response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01',
-                                                                    'end_date': '2100-01-01',
-                                                                    'requested_report': 'itemized_purchase_report'})
-
-        ((template, context), unused_kwargs) = render_mock.call_args
-        self.assertEqual(template, 'shoppingcart/download_report.html')
-        self.assertTrue(context['total_count_error'])
-        self.assertFalse(context['date_fmt_error'])
-        self.assertIn(_("There are too many results in your report.") + " (>0)", response.content)
-
-    # just going to ignored the date in this test, since we already deal with date testing
-    # in test_models.py
-
     CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,"
 
     def test_report_csv_itemized(self):
@@ -398,7 +378,7 @@ class CSVReportViewsTest(ModuleStoreTestCase):
                                                                     'requested_report': report_type})
         self.assertEqual(response['Content-Type'], 'text/csv')
         report = initialize_report(report_type)
-        self.assertIn(",".join(report.csv_report_header_row()), response.content)
+        self.assertIn(",".join(report.header()), response.content)
         self.assertIn(self.CORRECT_CSV_NO_DATE_ITEMIZED_PURCHASE, response.content)
 
     def test_report_csv_university_revenue_share(self):
@@ -412,7 +392,7 @@ class CSVReportViewsTest(ModuleStoreTestCase):
                                                                     'requested_report': report_type})
         self.assertEqual(response['Content-Type'], 'text/csv')
         report = initialize_report(report_type)
-        self.assertIn(",".join(report.csv_report_header_row()), response.content)
+        self.assertIn(",".join(report.header()), response.content)
         # TODO add another test here
 
 

lms/djangoapps/shoppingcart/views.py

@@ -11,11 +11,11 @@ from django.core.urlresolvers import reverse
 from django.views.decorators.csrf import csrf_exempt
 from django.contrib.auth.decorators import login_required
 from edxmako.shortcuts import render_to_response
-from student.models import CourseEnrollment
 from shoppingcart.reports import RefundReport, ItemizedPurchaseReport, UniversityRevenueShareReport, CertificateStatusReport
+from student.models import CourseEnrollment
+from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException
 from .models import Order, PaidCourseRegistration, OrderItem
 from .processors import process_postpay_callback, render_purchase_form_html
-from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException, ReportTypeDoesNotExistException
 
 log = logging.getLogger("shoppingcart")
 
@@ -213,20 +213,12 @@ def csv_report(request):
             return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, date_fmt_error=True)
 
         report = initialize_report(report_type)
-        items = report.report_row_generator(start_date, end_date, start_letter, end_letter)
-
-        # TODO add this back later as a query-est function or something
-        try:
-            if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS:
-            # Error case: too many items would be generated in the report and we're at risk of timeout
-                return _render_report_form(start_str, end_str, start_letter, end_letter, report_type, total_count_error=True)
-        except:
-            pass
+        items = report.rows(start_date, end_date, start_letter, end_letter)
 
         response = HttpResponse(mimetype='text/csv')
         filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S"))
         response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename)
-        report.make_report(response, start_date, end_date, start_letter, end_letter)
+        report.write_csv(response, start_date, end_date, start_letter, end_letter)
         return response
 
     elif request.method == 'GET':

lms/envs/aws.py

@@ -170,7 +170,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CUR
 
 # Payment Report Settings
 PAYMENT_REPORT_GENERATOR_GROUP = ENV_TOKENS.get('PAYMENT_REPORT_GENERATOR_GROUP', PAYMENT_REPORT_GENERATOR_GROUP)
-PAYMENT_REPORT_MAX_ITEMS = ENV_TOKENS.get('PAYMENT_REPORT_MAX_ITEMS', PAYMENT_REPORT_MAX_ITEMS)
 
 # Bulk Email overrides
 BULK_EMAIL_DEFAULT_FROM_EMAIL = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_FROM_EMAIL', BULK_EMAIL_DEFAULT_FROM_EMAIL)

lms/envs/common.py

@@ -555,8 +555,6 @@ PAID_COURSE_REGISTRATION_CURRENCY = ['usd', '$']
 
 # Members of this group are allowed to generate payment reports
 PAYMENT_REPORT_GENERATOR_GROUP = 'shoppingcart_report_access'
-# Maximum number of rows the report can contain
-PAYMENT_REPORT_MAX_ITEMS = 10000
 
 ################################# open ended grading config  #####################
 

lms/templates/shoppingcart/download_report.html

@@ -12,12 +12,6 @@
       ${_("There was an error in your date input.  It should be formatted as YYYY-MM-DD")}
     </section>
   % endif
-  % if total_count_error:
-    <section class="error_msg">
-      ${_("There are too many results in your report.")} (>${settings.PAYMENT_REPORT_MAX_ITEMS}).
-      ${_("Try making the date range smaller.")}
-    </section>
-  % endif
   <form method="post">
     <p>${_("These reports are delimited by start and end dates.")}</p>
     <label for="start_date">${_("Start Date: ")}</label>