Added management command to clean historical data

LEARNER-2697

ecommerce/core/management/commands/clean_history.py

+from __future__ import unicode_literals
+
+import logging
+
+from dateutil.parser import parse
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from oscar.core.loading import get_model
+
+from ecommerce.courses.models import Course
+from ecommerce.invoice.models import Invoice
+
+logger = logging.getLogger(__name__)
+Order = get_model('order', 'Order')
+OrderLine = get_model('order', 'Line')
+Product = get_model('catalogue', 'Product')
+ProductAttributeValue = get_model('catalogue', 'ProductAttributeValue')
+Refund = get_model('refund', 'Refund')
+RefundLine = get_model('refund', 'RefundLine')
+StockRecord = get_model('partner', 'StockRecord')
+
+
+class Command(BaseCommand):
+    help = 'Clean history data'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--cutoff_date',
+                            action='store',
+                            dest='cutoff_date',
+                            type=str,
+                            required=True,
+                            help='Cutoff date before which the history data should be cleaned. '
+                                 'format is YYYY-MM-DD')
+        parser.add_argument('--batch_size',
+                            action='store',
+                            dest='batch_size',
+                            type=int,
+                            default=1000,
+                            help='Maximum number of database rows to delete per query. '
+                                 'This helps avoid locking the database when deleting large amounts of data.')
+
+    def handle(self, *args, **options):
+        cutoff_date = options['cutoff_date']
+        batch_size = options['batch_size']
+
+        try:
+            cutoff_date = parse(cutoff_date)
+        except:  # pylint: disable=bare-except
+            msg = 'Failed to parse cutoff date: {}'.format(cutoff_date)
+            logger.exception(msg)
+            raise CommandError(msg)
+
+        models = (
+            Order, OrderLine, Refund, RefundLine, ProductAttributeValue, Product, StockRecord, Course, Invoice,
+        )
+
+        for model in models:
+            qs = model.history.filter(history_date__lte=cutoff_date)
+
+            message = 'Cleaning {} rows from {} table'.format(qs.count(), model.__name__)
+            logger.info(message)
+
+            qs = qs[:batch_size]
+            while qs.exists():
+                history_batch = qs.values_list('id', flat=True)
+                with transaction.atomic():
+                    model.history.filter(pk__in=list(history_batch)).delete()
+
+                qs = model.history.filter(history_date__lte=cutoff_date)[:batch_size]

ecommerce/core/management/commands/tests/test_clean_history.py

+import datetime
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.db.models import QuerySet
+from django.utils.timezone import now
+from oscar.core.loading import get_model
+from oscar.test.factories import OrderFactory
+from testfixtures import LogCapture
+
+from ecommerce.tests.testcases import TestCase
+
+LOGGER_NAME = 'ecommerce.core.management.commands.clean_history'
+Order = get_model('order', 'Order')
+
+
+def counter(fn):
+    """
+    Adds a call counter to the given function.
+
+    Source: http://code.activestate.com/recipes/577534-counting-decorator/
+    """
+    def _counted(*largs, **kargs):
+        _counted.invocations += 1
+        fn(*largs, **kargs)
+
+    _counted.invocations = 0
+    return _counted
+
+
+class CleanHistoryTests(TestCase):
+
+    def test_invalid_cutoff_date(self):
+        with LogCapture(LOGGER_NAME) as log:
+            with self.assertRaises(CommandError):
+                call_command('clean_history', '--cutoff_date=YYYY-MM-DD')
+                log.check(
+                    (
+                        LOGGER_NAME,
+                        'EXCEPTION',
+                        'Failed to parse cutoff date: YYYY-MM-DD'
+                    )
+                )
+
+    def test_clean_history(self):
+        initial_count = 5
+        OrderFactory.create_batch(initial_count)
+        cutoff_date = now() + datetime.timedelta(days=1)
+        self.assertEqual(Order.history.filter(history_date__lte=cutoff_date).count(), initial_count)
+
+        QuerySet.delete = counter(QuerySet.delete)
+        call_command('clean_history', '--cutoff_date={}'.format(cutoff_date.strftime('%Y-%m-%d')), batch_size=1)
+        self.assertEqual(QuerySet.delete.invocations, initial_count)
+        self.assertEqual(Order.history.filter(history_date__lte=cutoff_date).count(), 0)