Initial PR for verified cert reporting

Refactored from code in Stanford's itemized revenue reports into a new Report class used for all revenue-related reporting.

Models for all cert types complete; test coverage for half of cert types complete.

If this architecture is deemed solid, the next steps are to add a reasonable UI folks will use to select reports to download, allow them to restrict based on dates/universities, and of course complete test coverage.

lms/djangoapps/shoppingcart/models.py

@@ -23,6 +23,7 @@ from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore.exceptions import ItemNotFoundError
 
 from course_modes.models import CourseMode
+from courseware.courses import get_course_by_id
 from edxmako.shortcuts import render_to_string
 from student.views import course_from_id
 from student.models import CourseEnrollment, unenroll_done
@@ -259,66 +260,6 @@ class OrderItem(models.Model):
         """
         return self.pk_with_subclass, set([])
 
-    @classmethod
-    def purchased_items_btw_dates(cls, start_date, end_date):
-        """
-        Returns a QuerySet of the purchased items between start_date and end_date inclusive.
-        """
-        return cls.objects.filter(
-            status="purchased",
-            fulfilled_time__gte=start_date,
-            fulfilled_time__lt=end_date,
-        )
-
-    @classmethod
-    def csv_purchase_report_btw_dates(cls, filelike, start_date, end_date):
-        """
-        Outputs a CSV report into "filelike" (a file-like python object, such as an actual file, an HttpRequest,
-        or sys.stdout) of purchased items between start_date and end_date inclusive.
-        Opening and closing filelike (if applicable) should be taken care of by the caller
-        """
-        items = cls.purchased_items_btw_dates(start_date, end_date).order_by("fulfilled_time")
-
-        writer = unicodecsv.writer(filelike, encoding="utf-8")
-        writer.writerow(OrderItem.csv_report_header_row())
-
-        for item in items:
-            writer.writerow(item.csv_report_row)
-
-    @classmethod
-    def csv_report_header_row(cls):
-        """
-        Returns the "header" row for a csv report of purchases
-        """
-        return [
-            "Purchase Time",
-            "Order ID",
-            "Status",
-            "Quantity",
-            "Unit Cost",
-            "Total Cost",
-            "Currency",
-            "Description",
-            "Comments"
-        ]
-
-    @property
-    def csv_report_row(self):
-        """
-        Returns an array which can be fed into csv.writer to write out one csv row
-        """
-        return [
-            self.fulfilled_time,
-            self.order_id,  # pylint: disable=no-member
-            self.status,
-            self.qty,
-            self.unit_cost,
-            self.line_cost,
-            self.currency,
-            self.line_desc,
-            self.report_comments,
-        ]
-
     @property
     def pk_with_subclass(self):
         """
@@ -625,3 +566,204 @@ class CertificateItem(OrderItem):
                  "Please include your order number in your e-mail. "
                  "Please do NOT include your credit card information.").format(
                      billing_email=settings.PAYMENT_SUPPORT_EMAIL)
+
+class Report(models.Model):
+    """
+    Base class for making CSV reports related to revenue, enrollments, etc
+
+    To make a different type of report, write a new subclass that implements
+    the methods get_query, csv_report_header_row, and csv_report_row.
+    """
+
+    @classmethod
+    def initialize_report(cls, report_type):
+        if report_type == "refund_report":
+            return RefundReport()
+        elif report_type == "itemized_purchase_report":
+            return ItemizedPurchaseReport()
+        elif report_type == "university_revenue_share":
+            return UniversityRevenueShareReport()
+        elif report_type == "certificate_status":
+            return CertificateStatusReport()
+        else:
+            return  # TODO return an error
+
+    def get_query(self, start_date, end_date):
+        raise NotImplementedError
+
+    def csv_report_header_row(self, start_date, end_date):
+        raise NotImplementedError
+
+    def csv_report_row(self):
+        raise NotImplementedError
+
+    @classmethod
+    def make_report(cls, report_type, filelike, start_date, end_date):
+        report = cls.initialize_report(report_type)
+        items = report.get_query(start_date, end_date)
+        writer = unicodecsv.writer(filelike, encoding="utf-8")
+        writer.writerow(report.csv_report_header_row())
+        for item in items:
+            writer.writerow(report.csv_report_row(item))
+
+
+class RefundReport(Report):
+    def get_query(self, start_date, end_date):
+        return CertificateItem.objects.filter(
+                status="refunded",
+            )
+
+    def csv_report_header_row(self):
+        return [
+            "Order Number",
+            "Customer Name",
+            "Date of Original Transaction",
+            "Date of Refund",
+            "Amount of Refund",
+            "Service Fees (if any)",
+        ]
+
+    def csv_report_row(self, item):
+        return [
+            item.order_id,
+            item.user.get_full_name(),
+            item.fulfilled_time,
+            item.refund_requested_time, # actually may need to use refund_fulfilled here
+            item.line_cost,
+            0, # TODO: determine if service_fees field is necessary; if so, add
+        ]
+
+class ItemizedPurchaseReport(Report):
+    def get_query(self, start_date, end_date):
+        return OrderItem.objects.filter(
+                status="purchased",
+                fulfilled_time__gte=start_date,
+                fulfilled_time__lt=end_date,
+            ).order_by("fulfilled_time")
+
+    def csv_report_header_row(self):
+        return [
+            "Purchase Time",
+            "Order ID",
+            "Status",
+            "Quantity",
+            "Unit Cost",
+            "Total Cost",
+            "Currency",
+            "Description",
+            "Comments"
+        ]
+
+    def csv_report_row(self, item):
+        return [
+            item.fulfilled_time,
+            item.order_id,  # pylint: disable=no-member
+            item.status,
+            item.qty,
+            item.unit_cost,
+            item.line_cost,
+            item.currency,
+            item.line_desc,
+            item.report_comments,
+        ]
+
+
+class CertificateStatusReport(Report):
+    def get_query(self, start_date, send_date):
+        results = []
+        for course_id in settings.COURSE_LISTINGS:
+            cur_course = get_course_by_id(course)
+            university = cur_course.org
+            course = cur_course.number + " " + course.display_name #TODO add term (i.e. Fall 2013)?
+            enrollments =  CourseEnrollment.objects.filter(course_id=course_id)
+            total_enrolled = enrollments.objects.count()
+            audit_enrolled = enrollments.objects.filter(mode="audit").count()
+            honor_enrolled = enrollments.objects.filter(mode="honor").count()
+            verified_enrollments = enrollments.objects.filter(mode="verified")
+            verified_enrolled = verified_enrollments.objects.count()
+            gross_rev = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost'))
+            gross_rev_over_min = gross_rev - (CourseMode.objects.get('course_id').min_price * verified_enrollments)
+            num_verified_over_min = 0 # TODO clarify with billing what exactly this means
+            refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded")
+            number_of_refunds = refunded_enrollments.objects.count()
+            dollars_refunded = refunded_enrollments.objects.aggregate(Sum('unit_cost'))
+
+            result = [
+                university, 
+                course, 
+                total_enrolled, 
+                audit_enrolled, 
+                honor_enrolled,
+                verified_enrolled,
+                gross_rev,
+                gross_rev_over_min,
+                num_verified_over_min,
+                number_of_refunds,
+                dollars_refunded
+            ]
+
+            results.append(result)
+        return results
+
+    def csv_report_header_row(self):
+        return [
+            "University",
+            "Course",
+            "Total Enrolled",
+            "Audit Enrollment",
+            "Honor Code Enrollment",
+            "Verified Enrollment",
+            "Gross Revenue",
+            "Gross Revenue over the Minimum",
+            "Number of Verified over the Minimum",
+            "Number of Refunds",
+            "Dollars Refunded",
+        ]
+
+    def csv_report_row(self, item):
+        return item
+
+
+class UniversityRevenueShareReport(Report):
+    def get_query(self, start_date, end_date):
+        results = []
+        for course_id in settings.COURSE_LISTINGS:
+            cur_course = get_course_by_id(course)
+            university = cur_course.org
+            course = cur_course.number + " " + course.display_name
+            num_transactions = 0 # TODO clarify with building what transactions are included in this (purchases? refunds? etc)
+            total_payments_collected = CertificateItem.objects.filter(course_id=course_id, mode="verified").aggregate(Sum('unit_cost')) 
+                #note: we're assuming certitems are the only way to make money right now
+            service_fees = 0 # TODO add an actual service fees field, in case needed in future
+            refunded_enrollments = CertificateItem.objects.filter(course_id='course_id', mode="refunded")
+            num_refunds = refunded_enrollments.objects.count()
+            amount_refunds = refunded_enrollments.objects.aggregate(Sum('unit_cost'))
+
+            result = [
+                university,
+                course,
+                num_transactions,
+                total_payments_collected,
+                service_fees,
+                refunded_enrollments,
+                num_refunds,
+                amount_refunds
+            ]
+            results.append(result)
+
+        return results
+
+    def csv_report_header_row(self):
+        return [
+            "University",
+            "Course",
+            "Number of Transactions", 
+            "Total Payments Collected",
+            "Service Fees (if any)",
+            "Number of Successful Refunds",
+            "Total Amount of Refunds",
+            # note this is restricted by a date range
+        ]
+
+    def csv_report_row(self, item):
+        return item

lms/djangoapps/shoppingcart/tests/test_models.py

@@ -17,7 +17,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from shoppingcart.models import (Order, OrderItem, CertificateItem, InvalidCartItem, PaidCourseRegistration,
-                                 OrderItemSubclassPK, PaidCourseRegistrationAnnotation)
+                                 OrderItemSubclassPK, PaidCourseRegistrationAnnotation, Report)
 from student.tests.factories import UserFactory
 from student.models import CourseEnrollment
 from course_modes.models import CourseMode
@@ -324,7 +324,82 @@ class PaidCourseRegistrationTest(ModuleStoreTestCase):
 
 
 @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
-class PurchaseReportTest(ModuleStoreTestCase):
+class RefundReportTest(ModuleStoreTestCase):
+    FIVE_MINS = datetime.timedelta(minutes=5)
+
+    def setUp(self):
+        self.user = UserFactory.create()
+        self.user.first_name = "John"
+        self.user.last_name = "Doe"
+        self.user.save()
+        self.course_id = "MITx/999/Robot_Super_Course"
+        self.cost = 40
+        self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course')
+        course_mode = CourseMode(course_id=self.course_id,
+                                 mode_slug="honor",
+                                 mode_display_name="honor cert",
+                                 min_price=self.cost)
+        course_mode.save()
+
+        course_mode2 = CourseMode(course_id=self.course_id,
+                                  mode_slug="verified",
+                                  mode_display_name="verified cert",
+                                  min_price=self.cost)
+        course_mode2.save()
+
+        self.cart = Order.get_cart_for_user(self.user)
+        CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified')
+        self.cart.purchase()
+
+        # should auto-refund the relevant cert
+        CourseEnrollment.unenroll(self.user, self.course_id)
+
+        self.cert_item = CertificateItem.objects.get(user=self.user, course_id=self.course_id)
+
+        self.now = datetime.datetime.now(pytz.UTC)
+
+    def test_get_query(self):
+        report_type = "refund_report"
+        report = Report.initialize_report(report_type)
+        refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        self.assertEqual(len(refunded_certs), 1)
+        self.assertIn(self.cert_item, refunded_certs)
+        # TODO no time restrictions yet
+
+    test_time = datetime.datetime.now(pytz.UTC)
+
+    CORRECT_CSV = dedent("""
+        Order Number,Customer Name,Date of Original Transaction,Date of Refund,Amount of Refund,Service Fees (if any)
+        1,John Doe,{time_str},{time_str},40,0
+        """.format(time_str=str(test_time)))
+
+    def test_purchased_csv(self):
+        """
+        Tests that a generated purchase report CSV is as we expect
+        """
+        # coerce the purchase times to self.test_time so that the test can match.
+        # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we
+        # make the times match this way
+        # TODO test multiple report types
+        report_type = "refund_report"
+        report = Report.initialize_report(report_type)
+        for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS):
+            item.fulfilled_time = self.test_time
+            item.refund_requested_time = self.test_time #hm do we want to make these different
+            item.save()
+
+        # add annotation to the
+        csv_file = StringIO.StringIO()
+        Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        csv = csv_file.getvalue()
+        csv_file.close()
+        # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n
+        self.assertEqual(csv.replace('\r\n', '\n').strip(), self.CORRECT_CSV.strip())
+
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class ItemizedPurchaseReportTest(ModuleStoreTestCase):
 
     FIVE_MINS = datetime.timedelta(minutes=5)
     TEST_ANNOTATION = u'Ba\xfc\u5305'
@@ -353,11 +428,14 @@ class PurchaseReportTest(ModuleStoreTestCase):
         self.now = datetime.datetime.now(pytz.UTC)
 
     def test_purchased_items_btw_dates(self):
-        purchases = OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        # TODO test multiple report types
+        report_type = "itemized_purchase_report"
+        report = Report.initialize_report(report_type)
+        purchases = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
         self.assertEqual(len(purchases), 2)
         self.assertIn(self.reg.orderitem_ptr, purchases)
         self.assertIn(self.cert_item.orderitem_ptr, purchases)
-        no_purchases = OrderItem.purchased_items_btw_dates(self.now + self.FIVE_MINS,
+        no_purchases = report.get_query(self.now + self.FIVE_MINS,
                                                            self.now + self.FIVE_MINS + self.FIVE_MINS)
         self.assertFalse(no_purchases)
 
@@ -376,13 +454,16 @@ class PurchaseReportTest(ModuleStoreTestCase):
         # coerce the purchase times to self.test_time so that the test can match.
         # It's pretty hard to patch datetime.datetime b/c it's a python built-in, which is immutable, so we
         # make the times match this way
-        for item in OrderItem.purchased_items_btw_dates(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS):
+        # TODO test multiple report types
+        report_type = "itemized_purchase_report"
+        report = Report.initialize_report(report_type)
+        for item in report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS):
             item.fulfilled_time = self.test_time
             item.save()
 
         # add annotation to the
         csv_file = StringIO.StringIO()
-        OrderItem.csv_purchase_report_btw_dates(csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        Report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
         csv = csv_file.getvalue()
         csv_file.close()
         # Using excel mode csv, which automatically ends lines with \r\n, so need to convert to \n
@@ -403,6 +484,88 @@ class PurchaseReportTest(ModuleStoreTestCase):
         """
         self.assertEqual(unicode(self.annotation), u'{} : {}'.format(self.course_id, self.TEST_ANNOTATION))
 
+# TODO: finish this test class
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class CertificateStatusReportTest(ModuleStoreTestCase):
+    FIVE_MINS = datetime.timedelta(minutes=5)
+
+    def setUp(self):
+        self.user = UserFactory.create()
+        self.user.first_name = "John"
+        self.user.last_name = "Doe"
+        self.user.save()
+        self.course_id = "MITx/999/Robot_Super_Course"
+        self.cost = 40
+        self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course')
+        course_mode = CourseMode(course_id=self.course_id,
+                                 mode_slug="honor",
+                                 mode_display_name="honor cert",
+                                 min_price=self.cost)
+        course_mode.save()
+
+        course_mode2 = CourseMode(course_id=self.course_id,
+                                  mode_slug="verified",
+                                  mode_display_name="verified cert",
+                                  min_price=self.cost)
+        course_mode2.save()
+
+        self.cart = Order.get_cart_for_user(self.user)
+        CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified')
+        self.cart.purchase()
+
+        self.now = datetime.datetime.now(pytz.UTC)
+
+    # TODO finish these tests. This is just a basic test to start with, making sure the regular
+    # flow doesn't throw any strange errors while running
+    def test_basic(self):
+        report_type = "certificate_status"
+        report = Report.initialize_report(report_type)
+        refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        csv_file = StringIO.StringIO()
+        report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        # TODO no time restrictions yet
+
+# TODO: finish this test class
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class UniversityRevenueShareReportTest(ModuleStoreTestCase):
+    FIVE_MINS = datetime.timedelta(minutes=5)
+
+    def setUp(self):
+        self.user = UserFactory.create()
+        self.user.first_name = "John"
+        self.user.last_name = "Doe"
+        self.user.save()
+        self.course_id = "MITx/999/Robot_Super_Course"
+        self.cost = 40
+        self.course = CourseFactory.create(org='MITx', number='999', display_name=u'Robot Super Course')
+        course_mode = CourseMode(course_id=self.course_id,
+                                 mode_slug="honor",
+                                 mode_display_name="honor cert",
+                                 min_price=self.cost)
+        course_mode.save()
+
+        course_mode2 = CourseMode(course_id=self.course_id,
+                                  mode_slug="verified",
+                                  mode_display_name="verified cert",
+                                  min_price=self.cost)
+        course_mode2.save()
+
+        self.cart = Order.get_cart_for_user(self.user)
+        CertificateItem.add_to_order(self.cart, self.course_id, self.cost, 'verified')
+        self.cart.purchase()
+
+        self.now = datetime.datetime.now(pytz.UTC)
+
+    # TODO finish these tests. This is just a basic test to start with, making sure the regular
+    # flow doesn't throw any strange errors while running
+    def test_basic(self):
+        report_type = "university_revenue_share"
+        report = Report.initialize_report(report_type)
+        refunded_certs = report.get_query(self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        csv_file = StringIO.StringIO()
+        report.make_report(report_type, csv_file, self.now - self.FIVE_MINS, self.now + self.FIVE_MINS)
+        # TODO no time restrictions yet
+
 
 @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
 class CertificateItemTest(ModuleStoreTestCase):

lms/djangoapps/shoppingcart/tests/test_views.py

@@ -14,7 +14,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from shoppingcart.views import _can_download_report, _get_date_from_str
-from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem
+from shoppingcart.models import Order, CertificateItem, PaidCourseRegistration, OrderItem, Report
 from student.tests.factories import UserFactory
 from student.models import CourseEnrollment
 from course_modes.models import CourseMode
@@ -379,6 +379,9 @@ class CSVReportViewsTest(ModuleStoreTestCase):
     CORRECT_CSV_NO_DATE = ",1,purchased,1,40,40,usd,Registration for Course: Robot Super Course,"
 
     def test_report_csv(self):
+        # TODO test multiple types
+        report_type = "itemized_purchase_report"
+
         PaidCourseRegistration.add_to_order(self.cart, self.course_id)
         self.cart.purchase()
         self.login_user()
@@ -386,7 +389,8 @@ class CSVReportViewsTest(ModuleStoreTestCase):
         response = self.client.post(reverse('payment_csv_report'), {'start_date': '1970-01-01',
                                                                     'end_date': '2100-01-01'})
         self.assertEqual(response['Content-Type'], 'text/csv')
-        self.assertIn(",".join(OrderItem.csv_report_header_row()), response.content)
+        report = Report.initialize_report(report_type)
+        self.assertIn(",".join(report.csv_report_header_row()), response.content)
         self.assertIn(self.CORRECT_CSV_NO_DATE, response.content)
 
 

lms/djangoapps/shoppingcart/views.py

@@ -11,8 +11,8 @@ from django.core.urlresolvers import reverse
 from django.views.decorators.csrf import csrf_exempt
 from django.contrib.auth.decorators import login_required
 from edxmako.shortcuts import render_to_response
-from .models import Order, PaidCourseRegistration, OrderItem
 from student.models import CourseEnrollment
+from .models import Order, PaidCourseRegistration, OrderItem, Report
 from .processors import process_postpay_callback, render_purchase_form_html
 from .exceptions import ItemAlreadyInCartException, AlreadyEnrolledInCourseException, CourseDoesNotExistException
 
@@ -174,6 +174,9 @@ def csv_report(request):
     """
     Downloads csv reporting of orderitems
     """
+    # TODO: change this to something modular later
+    report_type = "itemized_purchase_report"
+
     if not _can_download_report(request.user):
         return HttpResponseForbidden(_('You do not have permission to view this page.'))
 
@@ -187,7 +190,8 @@ def csv_report(request):
             # Error case: there was a badly formatted user-input date string
             return _render_report_form(start_str, end_str, date_fmt_error=True)
 
-        items = OrderItem.purchased_items_btw_dates(start_date, end_date)
+        report = Report.initialize_report(report_type)
+        items = report.get_query(start_date, end_date)
         if items.count() > settings.PAYMENT_REPORT_MAX_ITEMS:
             # Error case: too many items would be generated in the report and we're at risk of timeout
             return _render_report_form(start_str, end_str, total_count_error=True)
@@ -195,7 +199,8 @@ def csv_report(request):
         response = HttpResponse(mimetype='text/csv')
         filename = "purchases_report_{}.csv".format(datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d-%H-%M-%S"))
         response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename)
-        OrderItem.csv_purchase_report_btw_dates(response, start_date, end_date)
+        # this flos is a little odd; what's up with report_type being called twice? check later
+        report.make_report(report_type, response, start_date, end_date)
         return response
 
     elif request.method == 'GET':

lms/djangoapps/verify_student/models.py

@@ -477,7 +477,7 @@ class SoftwareSecurePhotoVerification(PhotoVerification):
         # developing and aren't interested in working on student identity
         # verification functionality. If you do want to work on it, you have to
         # explicitly enable these in your private settings.
-        if settings.FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'):
+        if settings.MITX_FEATURES.get('AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'):
             return
 
         aes_key_str = settings.VERIFY_STUDENT["SOFTWARE_SECURE"]["FACE_IMAGE_AES_KEY"]

lms/djangoapps/verify_student/views.py

@@ -132,6 +132,7 @@ def create_order(request):
     """
     Submit PhotoVerification and create a new Order for this verified cert
     """
+    from nose.tools import set_trace; set_trace()
     if not SoftwareSecurePhotoVerification.user_has_valid_or_pending(request.user):
         attempt = SoftwareSecurePhotoVerification(user=request.user)
         b64_face_image = request.POST['face_image'].split(",")[1]