initial attempt at refresh_meta_data

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -14,6 +14,7 @@ from course_discovery.apps.course_metadata.data_loaders import AbstractDataLoade
 from course_discovery.apps.course_metadata.models import (
     Course, CourseEntitlement, CourseRun, Organization, Program, ProgramType, Seat, SeatType, Video
 )
+from course_discovery.apps.journal.data_loaders.helpers import EcommerceJournalDataLoader
 
 logger = logging.getLogger(__name__)
 
@@ -258,6 +259,7 @@ class EcommerceApiDataLoader(AbstractDataLoader):
             partner, api_url, access_token, token_type, max_workers, is_threadsafe, **kwargs
         )
         self.entitlement_skus = []
+        self.journal_loader = EcommerceJournalDataLoader(self)
 
     def ingest(self):
         logger.info('Refreshing course seats from %s...', self.partner.ecommerce_api_url)
@@ -265,11 +267,14 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         initial_page = 1
         course_runs = self._request_course_runs(initial_page)
         entitlements = self._request_entitlments(initial_page)
-        count = course_runs['count'] + entitlements['count']
+        journals = self.journal_loader.request_journals(initial_page)
+        count = course_runs['count'] + entitlements['count'] + journals['count']
         pages = math.ceil(count / self.PAGE_SIZE)
         self.entitlement_skus = []
         self._process_course_runs(course_runs)
         self._process_entitlements(entitlements)
+        self.journal_loader.journal_skus = []
+        self.journal_loader.process_journals(journals)
 
         pagerange = range(initial_page + 1, pages + 1)
 
@@ -284,12 +289,16 @@ class EcommerceApiDataLoader(AbstractDataLoader):
                 for future in [executor.submit(self._request_entitlments, page) for page in pagerange]:
                     response = future.result()
                     self._process_entitlements(response)
+                for future in [executor.submit(self.journal_loader.request_journals, page) for page in pagerange]:
+                    response = future.result()
+                    self.journal_loader.process_journals(response)
 
-        logger.info('Retrieved %d course seats and %d course entitlements from %s.', course_runs['count'],
-                    entitlements['count'], self.partner.ecommerce_api_url)
+        logger.info('Retrieved %d course seats, %d course entitlements and %d journals from %s.', course_runs['count'],
+                    entitlements['count'], journals['count'], self.partner.ecommerce_api_url)
 
         self.delete_orphans()
         self._delete_entitlements()
+        self.journal_loader.delete_journals()
 
     def _load_data(self, page):  # pragma: no cover
         """Make a request for the given page and process the response."""
@@ -297,6 +306,8 @@ class EcommerceApiDataLoader(AbstractDataLoader):
         self._process_course_runs(course_runs)
         entitlements = self._request_entitlments(page)
         self._process_entitlements(entitlements)
+        journals = self.journal_loader.request_journals(page)
+        self.journal_loader.process_journals(journals)
 
     def _request_course_runs(self, page):
         return self.api_client.courses().get(page=page, page_size=self.PAGE_SIZE, include_products=True)

course_discovery/apps/journal/data_loaders/helpers.py

+from decimal import Decimal
+import logging
+
+from course_discovery.apps.core.models import Currency
+from course_discovery.apps.journal.models import Journal
+
+logger = logging.getLogger(__name__)
+
+
+class EcommerceJournalDataLoader():
+    '''
+    Data Loader to fetch Journal information from Ecommerce
+    '''
+    def __init__(self, ecomm_data_loader):
+        self.journal_skus = []
+        self.partner = ecomm_data_loader.partner
+        self.api_client = ecomm_data_loader.api_client
+        self.page_size = ecomm_data_loader.PAGE_SIZE
+        self.ecomm_data_loader = ecomm_data_loader
+
+    def request_journals(self, page):
+        return self.api_client.products().get(page=page, page_size=self.page_size, product_class='Journal')
+
+    def process_journals(self, response):
+        results = response['results']
+        logger.info('Retrieved %d journals...', len(results))
+
+        for body in results:
+            body = self.ecomm_data_loader.clean_strings(body)
+            self.journal_skus.append(self.update_journal(body))
+
+    def update_journal(self, body):
+        """
+        Argument:
+            body (dict): journals data from ecommerce
+        Returns:
+            journal product sku if no exceptions, else None
+        """
+        attributes = {attribute['name']: attribute['value'] for attribute in body['attribute_values']}
+        journal_uuid = attributes.get('UUID')
+        title = body['title']
+        key = journal_uuid  # TODO, either drop this or create another attribute on the ecommerce product class
+
+        if body['stockrecords']:
+            stock_record = body['stockrecords'][0]
+        else:
+            msg = 'journal product {pub} has no stockrecords'.format(pub=title)
+            logger.warning(msg)
+            return None
+
+        try:
+            currency_code = stock_record['price_currency']
+            price = Decimal(stock_record['price_excl_tax'])
+            sku = stock_record['partner_sku']
+        except (KeyError, ValueError):
+            msg = 'A necessary stockrecord field is missing or incorrectly set for journal {journal}'.format(
+                journal=title
+            )
+            logger.warning(msg)
+            return None
+
+        try:
+            currency = Currency.objects.get(code=currency_code)
+        except Currency.DoesNotExist:
+            msg = 'Could not find currency {code} while loading entitlement {entitlement} with sku {sku}'.format(
+                code=currency_code, entitlement=title, sku=sku
+            )
+            logger.warning(msg)
+            return None
+
+        defaults = {
+            'partner': self.partner,
+            'uuid': journal_uuid,
+            'key': key,
+            'title': title,
+            'price': price,
+            'currency': currency,
+            'sku': sku,
+            'expires': self.ecomm_data_loader.parse_date(body['expires'])
+        }
+
+        msg = 'Creating journal {journal} with sku {sku} for partner {partner}'.format(
+            journal=title, sku=sku, partner=self.partner
+        )
+        logger.info(msg)
+        Journal.objects.update_or_create(defaults=defaults)
+        return sku
+
+    def delete_journals(self):
+        '''
+        Delete journal instances
+        '''
+        pubs_to_delete = Journal.objects.filter(
+            partner=self.partner
+        ).exclude(sku__in=self.journal_skus)
+
+        for pub in pubs_to_delete:
+            msg = 'Deleting journal with sku {sku} for partner {partner}'.format(
+                sku=pub.sku, partner=pub.partner
+            )
+            logger.info(msg)
+        pubs_to_delete.delete()