add enrollment code SKU in Seat model, modify ecommerce data loader to get enrollment codes

course_discovery/apps/api/serializers.py

@@ -387,6 +387,7 @@ class SeatSerializer(serializers.ModelSerializer):
     credit_provider = serializers.CharField()
     credit_hours = serializers.IntegerField()
     sku = serializers.CharField()
+    bulk_sku = serializers.CharField()
 
     @classmethod
     def prefetch_queryset(cls):
@@ -394,7 +395,7 @@ class SeatSerializer(serializers.ModelSerializer):
 
     class Meta(object):
         model = Seat
-        fields = ('type', 'price', 'currency', 'upgrade_deadline', 'credit_provider', 'credit_hours', 'sku',)
+        fields = ('type', 'price', 'currency', 'upgrade_deadline', 'credit_provider', 'credit_hours', 'sku', 'bulk_sku')
 
 
 class CourseEntitlementSerializer(serializers.ModelSerializer):

course_discovery/apps/api/tests/test_serializers.py

@@ -1117,7 +1117,8 @@ class SeatSerializerTests(TestCase):
             'upgrade_deadline': json_date_format(seat.upgrade_deadline),
             'credit_provider': seat.credit_provider,  # pylint: disable=no-member
             'credit_hours': seat.credit_hours,  # pylint: disable=no-member
-            'sku': seat.sku
+            'sku': seat.sku,
+            'bulk_sku': seat.bulk_sku
         }
 
         self.assertDictEqual(serializer.data, expected)

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -257,6 +257,7 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         super(EcommerceApiDataLoader, self).__init__(
             partner, api_url, access_token, token_type, max_workers, is_threadsafe, **kwargs
         )
+        self.enrollment_skus = []
         self.entitlement_skus = []
 
     def ingest(self):
@@ -265,11 +266,14 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         initial_page = 1
         course_runs = self._request_course_runs(initial_page)
         entitlements = self._request_entitlments(initial_page)
+        enrollment_codes = self._request_enrollment_codes(initial_page)
         count = course_runs['count'] + entitlements['count']
         pages = math.ceil(count / self.PAGE_SIZE)
         self.entitlement_skus = []
+        self.enrollment_skus = []
         self._process_course_runs(course_runs)
         self._process_entitlements(entitlements)
+        self._process_enrollment_codes(enrollment_codes)
 
         pagerange = range(initial_page + 1, pages + 1)
 
@@ -284,9 +288,13 @@ class EcommerceApiDataLoader(AbstractDataLoader):
                 for future in [executor.submit(self._request_entitlments, page) for page in pagerange]:
                     response = future.result()
                     self._process_entitlements(response)
+                for future in [executor.submit(self._request_enrollment_codes, page) for page in pagerange]:
+                    response = future.result()
+                    self._process_enrollment_codes(response)
 
-        logger.info('Retrieved %d course seats and %d course entitlements from %s.', course_runs['count'],
-                    entitlements['count'], self.partner.ecommerce_api_url)
+        logger.info('Retrieved %d course seats, %d course entitlements, and %d course enrollment codes from %s.',
+                    course_runs['count'], entitlements['count'],
+                    enrollment_codes['count'], self.partner.ecommerce_api_url)
 
         self.delete_orphans()
         self._delete_entitlements()
@@ -297,6 +305,8 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         self._process_course_runs(course_runs)
         entitlements = self._request_entitlments(page)
         self._process_entitlements(entitlements)
+        enrollment_codes = self._request_enrollment_codes(page)
+        self._process_enrollment_codes(enrollment_codes)
 
     def _request_course_runs(self, page):
         return self.api_client.courses().get(page=page, page_size=self.PAGE_SIZE, include_products=True)
@@ -304,6 +314,9 @@ class EcommerceApiDataLoader(AbstractDataLoader):
     def _request_entitlments(self, page):
         return self.api_client.products().get(page=page, page_size=self.PAGE_SIZE, product_class='Course Entitlement')
 
+    def _request_enrollment_codes(self, page):
+        return self.api_client.products().get(page=page, page_size=self.PAGE_SIZE, product_class='Enrollment Code')
+
     def _process_course_runs(self, response):
         results = response['results']
         logger.info('Retrieved %d course seats...', len(results))
@@ -320,6 +333,14 @@ class EcommerceApiDataLoader(AbstractDataLoader):
             body = self.clean_strings(body)
             self.entitlement_skus.append(self.update_entitlement(body))
 
+    def _process_enrollment_codes(self, response):
+        results = response['results']
+        logger.info('Retrieved %d course enrollment codes...', len(results))
+
+        for body in results:
+            body = self.clean_strings(body)
+            self.enrollment_skus.append(self.update_enrollment_code(body))
+
     def _delete_entitlements(self):
         entitlements_to_delete = CourseEntitlement.objects.filter(
             partner=self.partner
@@ -453,6 +474,73 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         course.entitlements.update_or_create(mode=mode, defaults=defaults)
         return sku
 
+    def update_enrollment_code(self, body):
+        """
+        Argument:
+            body (dict): enrollment code product data from ecommerce
+        Returns:
+            enrollment code product sku if no exceptions, else None
+        """
+        attributes = {attribute['code']: attribute['value'] for attribute in body['attribute_values']}
+        course_key = attributes.get('course_key')
+        title = body['title']
+
+        if body['stockrecords']:
+            stock_record = body['stockrecords'][0]
+        else:
+            msg = 'Enrollment code product {title} has no stockrecords'.format(title=title)
+            logger.warning(msg)
+            return None
+
+        try:
+            currency_code = stock_record['price_currency']
+            bulk_sku = stock_record['partner_sku']
+        except (KeyError, ValueError):
+            msg = 'A necessary stockrecord field is missing or incorrectly set for enrollment code {title}'.format(
+                title=title
+            )
+            logger.warning(msg)
+            return None
+
+        try:
+            course_run = CourseRun.objects.get(key=course_key)
+        except CourseRun.DoesNotExist:
+            msg = 'Could not find course run {key} while loading enrollment code {title} with sku {sku}'.format(
+                key=course_key, title=title, sku=bulk_sku
+            )
+            logger.warning(msg)
+            return None
+
+        try:
+            Currency.objects.get(code=currency_code)
+        except Currency.DoesNotExist:
+            msg = 'Could not find currency {code} while loading enrollment code {title} with sku {sku}'.format(
+                code=currency_code, title=title, sku=bulk_sku
+            )
+            logger.warning(msg)
+            return None
+
+        seat_type = attributes.get('seat_type')
+        try:
+            Seat.objects.get(course_run=course_run, type=seat_type)
+        except SeatType.DoesNotExist:
+            msg = 'Could not find seat type {type} while loading enrollment code {title} with sku {sku}'.format(
+                type=seat_type, title=title, sku=bulk_sku
+            )
+            logger.warning(msg)
+            return None
+
+        defaults = {
+            'bulk_sku': bulk_sku
+        }
+
+        msg = 'Creating enrollment code {title} with sku {sku} for partner {partner}'.format(
+            title=title, sku=bulk_sku, partner=self.partner
+        )
+        logger.info(msg)
+        course_run.seats.update_or_create(type=seat_type, defaults=defaults)
+        return bulk_sku
+
     def get_certificate_type(self, product):
         return next(
             (att['value'] for att in product['attribute_values'] if att['name'] == 'certificate_type'),

course_discovery/apps/course_metadata/data_loaders/tests/mock_data.py

@@ -231,6 +231,23 @@ ECOMMERCE_API_BODIES = [
                         "partner_sku": "sku003",
                     }
                 ]
+            },
+            {
+                "structure": "standalone",
+                "expires": "2017-01-01T12:00:00Z",
+                "attribute_values": [
+                    {
+                        "code": "seat_type",
+                        "value": "verified"
+                    }
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "EUR",
+                        "price_excl_tax": "25.00",
+                        "partner_sku": "sku004"
+                    }
+                ]
             }
         ]
     },
@@ -255,7 +272,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "USD",
                         "price_excl_tax": "0.00",
-                        "partner_sku": "sku004",
+                        "partner_sku": "sku005",
                     }
                 ]
             },
@@ -272,7 +289,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "USD",
                         "price_excl_tax": "25.00",
-                        "partner_sku": "sku005",
+                        "partner_sku": "sku006",
                     }
                 ]
             },
@@ -301,7 +318,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "USD",
                         "price_excl_tax": "250.00",
-                        "partner_sku": "sku006",
+                        "partner_sku": "sku007",
                     }
                 ]
             },
@@ -330,7 +347,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "USD",
                         "price_excl_tax": "250.00",
-                        "partner_sku": "sku007",
+                        "partner_sku": "sku008",
                     }
                 ]
             }
@@ -355,7 +372,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "123",
                         "price_excl_tax": "0.00",
-                        "partner_sku": "sku008",
+                        "partner_sku": "sku009",
                     }
                 ]
             }
@@ -380,7 +397,7 @@ ECOMMERCE_API_BODIES = [
                     {
                         "price_currency": "USD",
                         "price_excl_tax": "0.00",
-                        "partner_sku": "sku009",
+                        "partner_sku": "sku010",
                     }
                 ]
             }

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

 import datetime
+import json
 from decimal import Decimal
 
 import ddt
@@ -365,11 +366,20 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         return bodies
 
     def mock_products_api(self, alt_course=None, alt_currency=None, alt_mode=None, has_stockrecord=True,
-                          valid_stockrecord=True):
-        """ Return a new Course Entitlement to be added by ingest """
+                          valid_stockrecord=True, product_class=None):
+        """ Return a new Course Entitlement and Enrollment Code to be added by ingest """
         course = CourseFactory()
 
-        bodies = [
+        # If product_class is given, make sure it's either entitlement or enrollment_code
+        if product_class:
+            self.assertIn(product_class, ['entitlement', 'enrollment_code'])
+
+        data = {
+            "entitlement": {
+                "count": 1,
+                "num_pages": 1,
+                "current_page": 1,
+                "results": [
                     {
                         "structure": "child",
                         "product_class": "Course Entitlement",
@@ -388,8 +398,43 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
                         ],
                         "is_available_to_buy": True,
                         "stockrecords": []
+                    },
+                ],
+                "next": None,
+                "start": 0,
+                "previous": None
+            },
+            "enrollment_code": {
+                "count": 1,
+                "num_pages": 1,
+                "current_page": 1,
+                "results": [
+                    {
+                        "structure": "standalone",
+                        "product_class": "Enrollment Code",
+                        "title": "Course Intro to Everything",
+                        "price": "10.00",
+                        "expires": None,
+                        "attribute_values": [
+                            {
+                                "code": "seat_type",
+                                "value": "verified"
+                            },
+                            {
+                                "code": "course_key",
+                                "value": 'verified/course/run'
+                            }
+                        ],
+                        "is_available_to_buy": True,
+                        "stockrecords": []
                     }
-        ]
+                ],
+                "next": None,
+                "start": 0,
+                "previous": None
+            }
+        }
+
         stockrecord = {
             "price_currency": alt_currency if alt_currency else "USD",
             "price_excl_tax": "10.00",
@@ -397,18 +442,33 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         if valid_stockrecord:
             stockrecord.update({"partner_sku": "sku132"})
         if has_stockrecord:
-            bodies[0]["stockrecords"].append(stockrecord)
+            data['entitlement']['results'][0]["stockrecords"].append(stockrecord)
+            data['enrollment_code']['results'][0]["stockrecords"].append(stockrecord)
 
         url = '{url}products/'.format(url=self.api_url)
-        responses.add_callback(
+
+        responses.add(
             responses.GET,
-            url,
-            callback=mock_api_callback(url, bodies),
-            content_type=JSON
+            '{url}?product_class={product}&page=1&page_size=50'.format(url=url, product='Course Entitlement'),
+            body=json.dumps(data['entitlement']),
+            content_type='application/json',
+            status=200,
+            match_querystring=True
+        )
+
+        responses.add(
+            responses.GET,
+            '{url}?product_class={product}&page=1&page_size=50'.format(url=url, product='Enrollment Code'),
+            body=json.dumps(data['enrollment_code']),
+            content_type='application/json',
+            status=200,
+            match_querystring=True
         )
-        return bodies
 
-    def compose_warning_log(self, alt_course, alt_currency, alt_mode):
+        all_products = data['entitlement']['results'] + data['enrollment_code']['results']
+        return all_products if product_class is None else data[product_class]['results']
+
+    def compose_warning_log(self, alt_course, alt_currency, alt_mode, product_class):
         msg = 'Could not find '
         if alt_course:
             msg += 'course ' + alt_course
@@ -416,10 +476,23 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
             msg += 'currency ' + alt_currency
         elif alt_mode:
             msg += 'mode ' + alt_mode
-        msg += ' while loading entitlement Course Intro to Everything with sku sku132'
+        msg += ' while loading {product_class}'.format(product_class=product_class)
+        msg += ' Course Intro to Everything with sku sku132'
         return msg
 
-    def assert_seats_loaded(self, body):
+    def get_product_bulk_sku(self, seat_type, course_run, products):
+        bulk_sku = None
+        products = [p for p in products if p['structure'] == 'standalone']
+        course_key = course_run.key
+        for product in products:
+            attributes = {attribute['code']: attribute['value'] for attribute in product['attribute_values']}
+            if attributes['seat_type'] == seat_type and attributes['course_key'] == course_key:
+                stock_record = product['stockrecords'][0]
+                bulk_sku = stock_record['partner_sku']
+
+        return bulk_sku
+
+    def assert_seats_loaded(self, body, mock_products):
         """ Assert a Seat corresponding to the specified data body was properly loaded into the database. """
         course_run = CourseRun.objects.get(key=body['id'])
         products = [p for p in body['products'] if p['structure'] == 'child']
@@ -450,6 +523,7 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
                 elif att['name'] == 'credit_hours':
                     credit_hours = att['value']
 
+            bulk_sku = self.get_product_bulk_sku(certificate_type, course_run, mock_products)
             seat = course_run.seats.get(type=certificate_type, credit_provider=credit_provider, currency=price_currency)
 
             self.assertEqual(seat.course_run, course_run)
@@ -460,9 +534,11 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
             self.assertEqual(seat.credit_hours, credit_hours)
             self.assertEqual(seat.upgrade_deadline, upgrade_deadline)
             self.assertEqual(seat.sku, sku)
+            self.assertEqual(seat.bulk_sku, bulk_sku)
 
     def assert_entitlements_loaded(self, body):
         """ Assert a Course Entitlement was loaded into the database for each entry in the specified data body. """
+        body = [d for d in body if d['product_class'] == 'Course Entitlement']
         self.assertEqual(CourseEntitlement.objects.count(), len(body))
         for datum in body:
             expires = datum['expires']
@@ -474,7 +550,7 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
             sku = stock_record['partner_sku']
 
             mode_name = attributes['certificate_type']
-            mode = SeatType.objects.get(name=mode_name)
+            mode = SeatType.objects.get(slug=mode_name)
 
             entitlement = course.entitlements.get(mode=mode)
 
@@ -484,6 +560,21 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
             self.assertEqual(entitlement.currency.code, price_currency)
             self.assertEqual(entitlement.sku, sku)
 
+    def assert_enrollment_codes_loaded(self, body):
+        """ Assert a Course Enrollment Code was loaded into the database for each entry in the specified data body. """
+        body = [d for d in body if d['product_class'] == 'Enrollment Code']
+        for datum in body:
+            attributes = {attribute['code']: attribute['value'] for attribute in datum['attribute_values']}
+            course_run = CourseRun.objects.get(key=attributes['course_key'])
+            stock_record = datum['stockrecords'][0]
+            bulk_sku = stock_record['partner_sku']
+
+            mode_name = attributes['seat_type']
+            seat = course_run.seats.get(type=mode_name)
+
+            self.assertEqual(seat.course_run, course_run)
+            self.assertEqual(seat.bulk_sku, bulk_sku)
+
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the E-Commerce API. """
@@ -501,12 +592,13 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         self.loader.ingest()
 
         # Verify the API was called with the correct authorization header
-        self.assert_api_called(2)
+        self.assert_api_called(3)
 
         for datum in loaded_seat_data:
-            self.assert_seats_loaded(datum)
+            self.assert_seats_loaded(datum, products_api_data)
 
         self.assert_entitlements_loaded(products_api_data)
+        self.assert_enrollment_codes_loaded(products_api_data)
 
         # Verify multiple calls to ingest data do NOT result in data integrity errors.
         self.loader.ingest()
@@ -535,34 +627,44 @@ class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestC
         products = self.mock_products_api(has_stockrecord=False)
         self.loader.ingest()
         msg = 'Entitlement product {entitlement} has no stockrecords'.format(entitlement=products[0]['title'])
-        mock_logger.warning.assert_called_with(msg)
+        mock_logger.warning.assert_any_call(msg)
 
     @responses.activate
-    @mock.patch(LOGGER_PATH)
-    def test_invalid_stockrecord(self, mock_logger):
+    @ddt.data(
+        ({'key': 'entitlement', 'value': 'entitlement'}),
+        ({'key': 'enrollment_code', 'value': 'enrollment code'})
+    )
+    def test_invalid_stockrecord(self, product_class):
         self.mock_courses_api()
-        products = self.mock_products_api(valid_stockrecord=False)
+        products = self.mock_products_api(valid_stockrecord=False, product_class=product_class['key'])
+        with mock.patch(LOGGER_PATH) as mock_logger:
             self.loader.ingest()
-        msg = 'A necessary stockrecord field is missing or incorrectly set for entitlement {entitlement}'.format(
-            entitlement=products[0]['title']
+            msg = 'A necessary stockrecord field is missing or incorrectly set for {product_class} {title}'.format(
+                product_class=product_class['value'],
+                title=products[0]['title']
             )
-        mock_logger.warning.assert_called_with(msg)
+            mock_logger.warning.assert_any_call(msg)
 
     @responses.activate
     @ddt.data(
-        ('a01354b1-c0de-4a6b-c5de-ab5c6d869e76', None, None),
-        (None, "NRC", None),
-        (None, None, "notamode"),
+        ('a01354b1-c0de-4a6b-c5de-ab5c6d869e76', None, None, {'key': 'entitlement', 'value': 'entitlement'}),
+        (None, "NRC", None, {'key': 'enrollment_code', 'value': 'enrollment code'}),
+        (None, None, "notamode", {'key': 'entitlement', 'value': 'entitlement'})
     )
     @ddt.unpack
-    def test_ingest_fails(self, alt_course, alt_currency, alt_mode):
+    def test_ingest_fails(self, alt_course, alt_currency, alt_mode, product_class):
         """ Verify the proper warnings are logged when data objects are not present. """
         self.mock_courses_api()
-        self.mock_products_api(alt_course=alt_course, alt_currency=alt_currency, alt_mode=alt_mode)
+        self.mock_products_api(
+            alt_course=alt_course,
+            alt_currency=alt_currency,
+            alt_mode=alt_mode,
+            product_class=product_class['key']
+        )
         with mock.patch(LOGGER_PATH) as mock_logger:
             self.loader.ingest()
-            msg = self.compose_warning_log(alt_course, alt_currency, alt_mode)
-            mock_logger.warning.assert_called_with(msg)
+            msg = self.compose_warning_log(alt_course, alt_currency, alt_mode, product_class['value'])
+            mock_logger.warning.assert_any_call(msg)
 
     @ddt.unpack
     @ddt.data(

course_discovery/apps/course_metadata/migrations/0080_seat_bulk_sku.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2018-03-09 19:38
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('course_metadata', '0079_enable_program_default_true'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='seat',
+            name='bulk_sku',
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+    ]

course_discovery/apps/course_metadata/models.py

@@ -832,6 +832,7 @@ class Seat(TimeStampedModel):
     credit_provider = models.CharField(max_length=255, null=True, blank=True)
     credit_hours = models.IntegerField(null=True, blank=True)
     sku = models.CharField(max_length=128, null=True, blank=True)
+    bulk_sku = models.CharField(max_length=128, null=True, blank=True)
 
     class Meta(object):
         unique_together = (

course_discovery/apps/course_metadata/tests/factories.py

@@ -155,6 +155,7 @@ class SeatFactory(factory.DjangoModelFactory):
     currency = factory.Iterator(Currency.objects.all())
     upgrade_deadline = FuzzyDateTime(datetime.datetime(2014, 1, 1, tzinfo=UTC))
     sku = FuzzyText(length=8)
+    bulk_sku = FuzzyText(length=8)
     course_run = factory.SubFactory(CourseRunFactory)
 
     class Meta: