Added course migration and Clean history

Added course migration and clean history command
so we can remove history data before we can remove tables

LEARNER-3526

ecommerce/core/management/commands/clean_history.py

+from __future__ import unicode_literals
+
+import logging
+import time
+
+from dateutil.parser import parse
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from oscar.core.loading import get_model
+
+from ecommerce.courses.models import Course
+from ecommerce.invoice.models import Invoice
+
+logger = logging.getLogger(__name__)
+Order = get_model('order', 'Order')
+OrderLine = get_model('order', 'Line')
+Product = get_model('catalogue', 'Product')
+ProductAttributeValue = get_model('catalogue', 'ProductAttributeValue')
+Refund = get_model('refund', 'Refund')
+RefundLine = get_model('refund', 'RefundLine')
+StockRecord = get_model('partner', 'StockRecord')
+
+
+class Command(BaseCommand):
+    help = 'Clean history data'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--cutoff_date',
+                            action='store',
+                            dest='cutoff_date',
+                            type=str,
+                            required=True,
+                            help='Cutoff date before which the history data should be cleaned. '
+                                 'format is YYYY-MM-DD')
+        parser.add_argument('--batch_size',
+                            action='store',
+                            dest='batch_size',
+                            type=int,
+                            default=1000,
+                            help='Maximum number of database rows to delete per query. '
+                                 'This helps avoid locking the database when deleting large amounts of data.')
+        parser.add_argument('--sleep_time',
+                            action='store',
+                            dest='sleep_time',
+                            type=int,
+                            default=10,
+                            help='Sleep time between deletion of batches')
+
+    def handle(self, *args, **options):
+        cutoff_date = options['cutoff_date']
+        batch_size = options['batch_size']
+        sleep_time = options['sleep_time']
+
+        try:
+            cutoff_date = parse(cutoff_date)
+        except:  # pylint: disable=bare-except
+            msg = 'Failed to parse cutoff date: {}'.format(cutoff_date)
+            logger.exception(msg)
+            raise CommandError(msg)
+
+        models = (
+            Order, OrderLine, Refund, RefundLine, ProductAttributeValue, Product, StockRecord, Course, Invoice,
+        )
+
+        for model in models:
+            qs = model.history.filter(history_date__lte=cutoff_date).order_by('-pk')
+
+            message = 'Cleaning {} rows from {} table'.format(qs.count(), model.__name__)
+            logger.info(message)
+
+            try:
+                # use Primary keys sorting to make sure unique batching as
+                # filtering batch does not work for huge data
+                max_pk = qs[0].pk
+                batch_start = qs.reverse()[0].pk
+                batch_stop = batch_start + batch_size
+            except IndexError:
+                continue
+
+            logger.info(message)
+
+            while batch_start <= max_pk:
+                queryset = model.history.filter(pk__gte=batch_start, pk__lt=batch_stop)
+
+                with transaction.atomic():
+                    queryset.delete()
+                    logger.info(
+                        'Deleted instances of %s with PKs between %d and %d',
+                        model.__name__, batch_start, batch_stop
+                    )
+
+                if batch_stop < max_pk:
+                    time.sleep(sleep_time)
+
+                batch_start = batch_stop
+                batch_stop += batch_size

ecommerce/core/management/commands/tests/test_clean_history.py

+import datetime
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.db.models import QuerySet
+from django.utils.timezone import now
+from oscar.core.loading import get_model
+from oscar.test.factories import OrderFactory
+from testfixtures import LogCapture
+
+from ecommerce.tests.testcases import TestCase
+
+LOGGER_NAME = 'ecommerce.core.management.commands.clean_history'
+Order = get_model('order', 'Order')
+
+
+def counter(fn):
+    """
+    Adds a call counter to the given function.
+    Source: http://code.activestate.com/recipes/577534-counting-decorator/
+    """
+    def _counted(*largs, **kargs):
+        _counted.invocations += 1
+        fn(*largs, **kargs)
+
+    _counted.invocations = 0
+    return _counted
+
+
+class CleanHistoryTests(TestCase):
+
+    def test_invalid_cutoff_date(self):
+        with LogCapture(LOGGER_NAME) as log:
+            with self.assertRaises(CommandError):
+                call_command('clean_history', '--cutoff_date=YYYY-MM-DD')
+                log.check(
+                    (
+                        LOGGER_NAME,
+                        'EXCEPTION',
+                        'Failed to parse cutoff date: YYYY-MM-DD'
+                    )
+                )
+
+    def test_clean_history(self):
+        initial_count = 5
+        OrderFactory.create_batch(initial_count)
+        cutoff_date = now() + datetime.timedelta(days=1)
+        self.assertEqual(Order.history.filter(history_date__lte=cutoff_date).count(), initial_count)
+
+        QuerySet.delete = counter(QuerySet.delete)
+        call_command(
+            'clean_history', '--cutoff_date={}'.format(cutoff_date.strftime('%Y-%m-%d')), batch_size=1, sleep_time=1
+        )
+        self.assertEqual(QuerySet.delete.invocations, initial_count)
+        self.assertEqual(Order.history.filter(history_date__lte=cutoff_date).count(), 0)

ecommerce/courses/migrations/0006_auto_20171204_1036.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-12-04 10:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    def add_created_modified_date(apps, schema_editor):
+        Course = apps.get_model('courses', 'Course')
+        HistoricalCourse = apps.get_model('courses', 'historicalcourse')
+        courses = Course.objects.all()
+
+        for course in courses:
+            history = HistoricalCourse.objects.filter(id=course.id)
+            course.created = history.earliest().history_date
+            course.modified = history.latest().history_date
+            course.save()
+
+    dependencies = [
+        ('courses', '0005_auto_20170525_0131'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='course',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='modified',
+            field=models.DateTimeField(auto_now=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='created',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='modified',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+        migrations.RunPython(add_created_modified_date, migrations.RunPython.noop),
+    ]

ecommerce/courses/models.py

@@ -40,6 +40,8 @@ class Course(models.Model):
         help_text=_('Last date/time on which verification for this product can be submitted.')
     )
     history = HistoricalRecords()
+    created = models.DateTimeField(null=True, auto_now_add=True)
+    modified = models.DateTimeField(null=True, auto_now=True)
     thumbnail_url = models.URLField(null=True, blank=True)
 
     def __unicode__(self):

ecommerce/extensions/api/serializers.py

@@ -7,7 +7,6 @@ from decimal import Decimal
 import waffle
 from dateutil.parser import parse
 from django.contrib.auth import get_user_model
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
@@ -298,10 +297,7 @@ class CourseSerializer(serializers.HyperlinkedModelSerializer):
             self.fields.pop('products', None)
 
     def get_last_edited(self, obj):
-        try:
-            return obj.history.latest().history_date.strftime(ISO_8601_FORMAT)
-        except ObjectDoesNotExist:
-            return None
+        return obj.modified.strftime(ISO_8601_FORMAT) if obj.modified else None
 
     def get_products_url(self, obj):
         return reverse('api:v2:course-product-list', kwargs={'parent_lookup_course_id': obj.id},

ecommerce/extensions/api/v2/tests/views/test_courses.py

@@ -6,7 +6,6 @@ import jwt
 import mock
 from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from oscar.core.loading import get_class, get_model
 
@@ -42,11 +41,7 @@ class CourseViewSetTests(ProductSerializerMixin, DiscoveryTestMixin, TestCase):
         products_url = self.get_full_url(reverse('api:v2:course-product-list',
                                                  kwargs={'parent_lookup_course_id': course.id}))
 
-        last_edited = None
-        try:
-            last_edited = course.history.latest().history_date.strftime(ISO_8601_FORMAT)
-        except ObjectDoesNotExist:
-            pass
+        last_edited = course.modified.strftime(ISO_8601_FORMAT)
         enrollment_code = course.enrollment_code_product
 
         data = {
@@ -114,14 +109,6 @@ class CourseViewSetTests(ProductSerializerMixin, DiscoveryTestMixin, TestCase):
         response = self.client.get(self.list_path)
         self.assertDictEqual(json.loads(response.content), {'count': 0, 'next': None, 'previous': None, 'results': []})
 
-    def test_list_without_history(self):
-        course = Course.objects.all()[0]
-        course.history.all().delete()
-
-        response = self.client.get(self.list_path)
-        self.assertEqual(response.status_code, 200)
-        self.assertListEqual(json.loads(response.content)['results'], [self.serialize_course(self.course)])
-
     def test_create(self):
         """ Verify the view can create a new Course."""
         Course.objects.all().delete()

ecommerce/extensions/catalogue/models.py

@@ -2,7 +2,8 @@ from django.db import models
 from django.db.models.signals import post_init, post_save
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
-from oscar.apps.catalogue.abstract_models import AbstractProduct
+from oscar.apps.catalogue.abstract_models import AbstractProduct, AbstractProductAttributeValue
+from simple_history.models import HistoricalRecords
 
 from ecommerce.core.constants import (
     COUPON_PRODUCT_CLASS_NAME,
@@ -19,6 +20,7 @@ class Product(AbstractProduct):
     )
     expires = models.DateTimeField(null=True, blank=True,
                                    help_text=_('Last date/time on which this product can be purchased.'))
+    history = HistoricalRecords()
     original_expires = None
 
     @property
@@ -80,6 +82,10 @@ def update_enrollment_code(sender, **kwargs):  # pylint: disable=unused-argument
         instance.original_expires = instance.expires
 
 
+class ProductAttributeValue(AbstractProductAttributeValue):
+    history = HistoricalRecords()
+
+
 class Catalog(models.Model):
     name = models.CharField(max_length=255)
     partner = models.ForeignKey('partner.Partner', related_name='catalogs', on_delete=models.CASCADE)

ecommerce/extensions/order/models.py

 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from oscar.apps.order.abstract_models import AbstractOrder, AbstractPaymentEvent
+from oscar.apps.order.abstract_models import AbstractLine, AbstractOrder, AbstractPaymentEvent
+from simple_history.models import HistoricalRecords
 
 from ecommerce.extensions.fulfillment.status import ORDER
 
 
 class Order(AbstractOrder):
+    history = HistoricalRecords()
 
     @property
     def is_fulfillable(self):
@@ -22,6 +24,10 @@ class PaymentEvent(AbstractPaymentEvent):
     processor_name = models.CharField(_('Payment Processor'), max_length=32, blank=True, null=True)
 
 
+class Line(AbstractLine):
+    history = HistoricalRecords()
+
+
 # If two models with the same name are declared within an app, Django will only use the first one.
 # noinspection PyUnresolvedReferences
 from oscar.apps.order.models import *  # noqa isort:skip pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position,wrong-import-order,ungrouped-imports

ecommerce/extensions/partner/models.py

 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from oscar.apps.partner.abstract_models import AbstractPartner
+from oscar.apps.partner.abstract_models import AbstractPartner, AbstractStockRecord
+from simple_history.models import HistoricalRecords
+
+
+class StockRecord(AbstractStockRecord):
+    history = HistoricalRecords()
 
 
 class Partner(AbstractPartner):

ecommerce/extensions/refund/models.py

@@ -10,6 +10,7 @@ from ecommerce_worker.sailthru.v1.tasks import send_course_refund_email
 from oscar.apps.payment.exceptions import PaymentError
 from oscar.core.loading import get_class, get_model
 from oscar.core.utils import get_default_currency
+from simple_history.models import HistoricalRecords
 
 from ecommerce.core.constants import COURSE_ENTITLEMENT_PRODUCT_CLASS_NAME, SEAT_PRODUCT_CLASS_NAME
 from ecommerce.extensions.analytics.utils import audit_log
@@ -80,6 +81,7 @@ class Refund(StatusMixin, TimeStampedModel):
             (REFUND.COMPLETE, REFUND.COMPLETE),
         ]
     )
+    history = HistoricalRecords()
 
     pipeline_setting = 'OSCAR_REFUND_STATUS_PIPELINE'
 
@@ -310,6 +312,7 @@ class RefundLine(StatusMixin, TimeStampedModel):
             (REFUND_LINE.COMPLETE, REFUND_LINE.COMPLETE),
         ]
     )
+    history = HistoricalRecords()
 
     pipeline_setting = 'OSCAR_REFUND_LINE_STATUS_PIPELINE'
 

ecommerce/invoice/models.py

@@ -2,6 +2,7 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from simple_history.models import HistoricalRecords
 
 
 class Invoice(TimeStampedModel):
@@ -45,6 +46,7 @@ class Invoice(TimeStampedModel):
         validators=[MinValueValidator(1), MaxValueValidator(100)],
         null=True, blank=True
     )
+    history = HistoricalRecords()
 
     @property
     def total(self):