update entitlement product data loading from ecommerce

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -311,7 +311,13 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         for body in results:
             body = self.clean_strings(body)
             skus.append(self.update_entitlement(body))
-        CourseEntitlement.objects.exclude(sku__in=skus).delete()
+        entitlements_to_delete = CourseEntitlement.objects.filter(partner=self.partner).exclude(sku__in=skus)
+        for entitlement in entitlements_to_delete:
+            msg = 'Deleting entitlement with sku {sku} for partner {partner}'.format(
+                sku=entitlement.sku, partner=entitlement.partner
+            )
+            logger.info(msg)
+        entitlements_to_delete.delete()
 
     def update_seats(self, body):
         course_run_key = body['id']
@@ -366,12 +372,6 @@ class EcommerceApiDataLoader(AbstractDataLoader):
     def update_entitlement(self, body):
         attributes = {attribute['name']: attribute['value'] for attribute in body['attribute_values']}
         course_uuid = attributes.get('UUID')
-        try:
-            course = Course.objects.get(uuid=course_uuid)
-        except Course.DoesNotExist:
-            msg = 'Could not find course {uuid}'.format(uuid=course_uuid)
-            logger.warning(msg)
-            return None
 
         stock_record = body['stockrecords'][0]
         currency_code = stock_record['price_currency']
@@ -379,26 +379,44 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         sku = stock_record['partner_sku']
 
         try:
+            course = Course.objects.get(uuid=course_uuid)
+        except Course.DoesNotExist:
+            msg = 'Could not find course {uuid} while loading entitlement {entitlement} with sku {sku}'.format(
+                uuid=course_uuid, entitlement=body['title'], sku=sku
+            )
+            logger.warning(msg)
+            return None
+
+        try:
             currency = Currency.objects.get(code=currency_code)
         except Currency.DoesNotExist:
-            msg = 'Could not find currency {code}'.format(code=currency_code)
+            msg = 'Could not find currency {code} while loading entitlement {entitlement} with sku {sku}'.format(
+                code=currency_code, entitlement=body['title'], sku=sku
+            )
             logger.warning(msg)
             return None
 
         mode_name = attributes.get('certificate_type')
         try:
-            mode = SeatType.objects.get(name=mode_name)
+            mode = SeatType.objects.get(slug=mode_name)
         except SeatType.DoesNotExist:
-            msg = 'Could not find course entitlement mode {mode}'.format(mode=mode_name)
+            msg = 'Could not find mode {mode} while loading entitlement {entitlement} with sku {sku}'.format(
+                mode=mode_name, entitlement=body['title'], sku=sku
+            )
             logger.warning(msg)
             return None
 
         defaults = {
+            'partner': self.partner,
             'price': price,
             'currency': currency,
             'sku': sku,
             'expires': self.parse_date(body['expires'])
         }
+        msg = 'Creating entitlement {entitlement} with sku {sku} for partner {partner}'.format(
+            entitlement=body['title'], sku=sku, partner=self.partner
+        )
+        logger.info(msg)
         course.entitlements.update_or_create(mode=mode, defaults=defaults)
         return sku
 

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

@@ -18,7 +18,8 @@ from course_discovery.apps.course_metadata.models import (
     Course, CourseEntitlement, CourseRun, Organization, Program, ProgramType, Seat, SeatType
 )
 from course_discovery.apps.course_metadata.tests.factories import (
-    CourseFactory, CourseRunFactory, ImageFactory, OrganizationFactory, SeatFactory, VideoFactory
+    CourseEntitlementFactory, CourseFactory, CourseRunFactory, ImageFactory, OrganizationFactory, SeatFactory,
+    VideoFactory
 )
 
 LOGGER_PATH = 'course_discovery.apps.course_metadata.data_loaders.api.logger'
@@ -371,6 +372,7 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
             {
                 "structure": "child",
                 "product_class": "Course Entitlement",
+                "title": "Course Intro to Everything",
                 "price": "10.00",
                 "expires": None,
                 "attribute_values": [
@@ -386,9 +388,9 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
                 "is_available_to_buy": True,
                 "stockrecords": [
                     {
+                        "partner_sku": "sku132",
                         "price_currency": alt_currency if alt_currency else "USD",
                         "price_excl_tax": "10.00",
-                        "partner_sku": "sku132",
                     }
                 ]
             }
@@ -402,6 +404,17 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         )
         return bodies
 
+    def compose_warning_log(self, alt_course, alt_currency, alt_mode):
+        msg = 'Could not find '
+        if alt_course:
+            msg += 'course ' + alt_course
+        elif alt_currency:
+            msg += 'currency ' + alt_currency
+        elif alt_mode:
+            msg += 'mode ' + alt_mode
+        msg += ' while loading entitlement Course Intro to Everything with sku sku132'
+        return msg
+
     def assert_seats_loaded(self, body):
         """ Assert a Seat corresponding to the specified data body was properly loaded into the database. """
         course_run = CourseRun.objects.get(key=body['id'])
@@ -463,7 +476,6 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
 
             self.assertEqual(entitlement.expires, expires)
             self.assertEqual(entitlement.course, course)
-            self.assertEqual(entitlement.mode, mode)
             self.assertEqual(entitlement.price, price)
             self.assertEqual(entitlement.currency.code, price_currency)
             self.assertEqual(entitlement.sku, sku)
@@ -474,9 +486,9 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         courses_api_data = self.mock_courses_api()
         loaded_course_run_data = courses_api_data[:-1]
         loaded_seat_data = courses_api_data[:-2]
+        self.assertEqual(CourseRun.objects.count(), len(loaded_course_run_data))
 
         products_api_data = self.mock_products_api()
-        self.assertEqual(CourseRun.objects.count(), len(loaded_course_run_data))
 
         # Verify a seat exists on all courses already
         for course_run in CourseRun.objects.all():
@@ -496,10 +508,27 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         self.loader.ingest()
 
     @responses.activate
+    @mock.patch(LOGGER_PATH)
+    def test_ingest_deletes(self, mock_logger):
+        """ Verifiy the method deletes stale data. """
+        self.mock_courses_api()
+        products_api_data = self.mock_products_api()
+        entitlement = CourseEntitlementFactory(partner=self.partner)
+
+        self.loader.ingest()
+        # Ensure that only entitlements retrieved from the Ecommerce API remain in Discovery,
+        # and that the sku and partner of the deleted entitlement are logged
+        self.assert_entitlements_loaded(products_api_data)
+        msg = 'Deleting entitlement with sku {sku} for partner {partner}'.format(
+            sku=entitlement.sku, partner=entitlement.partner
+        )
+        mock_logger.info.assert_any_call(msg)
+
+    @responses.activate
     @ddt.data(
         ('a01354b1-c0de-4a6b-c5de-ab5c6d869e76', None, None),
         (None, "NRC", None),
-        (None, None, "notamode")
+        (None, None, "notamode"),
     )
     @ddt.unpack
     def test_ingest_fails(self, alt_course, alt_currency, alt_mode):
@@ -508,13 +537,7 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         self.mock_products_api(alt_course=alt_course, alt_currency=alt_currency, alt_mode=alt_mode)
         with mock.patch(LOGGER_PATH) as mock_logger:
             self.loader.ingest()
-            msg = 'Could not find '
-            if alt_course:
-                msg += 'course ' + alt_course
-            elif alt_currency:
-                msg += 'currency ' + alt_currency
-            else:
-                msg += 'course entitlement mode ' + alt_mode
+            msg = self.compose_warning_log(alt_course, alt_currency, alt_mode)
             mock_logger.warning.assert_called_with(msg)
 
     @ddt.unpack

course_discovery/apps/course_metadata/migrations/0072_courseentitlement_partner.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-12-07 19:07
+from __future__ import unicode_literals
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_auto_20171004_1133'),
+        ('course_metadata', '0071_auto_20171128_1945'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='courseentitlement',
+            name='partner',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='core.Partner'),
+        ),
+    ]

course_discovery/apps/course_metadata/models.py

@@ -839,6 +839,7 @@ class CourseEntitlement(TimeStampedModel):
     }
     course = models.ForeignKey(Course, related_name='entitlements')
     mode = models.ForeignKey(SeatType)
+    partner = models.ForeignKey(Partner, null=True, blank=False)
     price = models.DecimalField(**PRICE_FIELD_CONFIG)
     currency = models.ForeignKey(Currency)
     sku = models.CharField(max_length=128, null=True, blank=True)