Update refresh_course_metadata to load course information from the published about pages

EDUCATOR-1818

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

@@ -360,28 +360,46 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
         return kwargs
 
     def process_node(self, data):
-        course_run = self.get_course_run(data)
 
-        if course_run:
-            self.update_course_run(course_run, data)
-            try:
-                course = self.update_course(course_run.canonical_for_course, data)
-                self.set_subjects(course, data)
-                self.set_authoring_organizations(course, data)
-                logger.info('Processed course with key [%s].', course.key)
-            except AttributeError:
-                pass
+        if not data.get('field_course_uuid'):
+            course_run = self.get_course_run(data)
+
+            if course_run:
+                self.update_course_run(course_run, data)
+                if self.get_course_run_status(data) == CourseRunStatus.Published:
+                    # Only update the course object with published course about page
+                    try:
+                        course = self.update_course(course_run.course, data)
+                        self.set_subjects(course, data)
+                        self.set_authoring_organizations(course, data)
+                        logger.info('Processed course with key [%s].', course.key)
+                    except AttributeError:
+                        pass
+                else:
+                    logger.info(
+                        'Course_run [%s] is unpublished, so the course [%s] related is not updated.',
+                        data['field_course_id'],
+                        course_run.course.number
+                    )
+            else:
+                created = False
+                # If the page is not generated from discovery service
+                # Do shall then attempt to create a course out of it
+                try:
+                    course, created = self.get_or_create_course(data)
+                    course_run = self.create_course_run(course, data)
+                except InvalidKeyError:
+                    logger.error('Invalid course key [%s].', data['field_course_id'])
+
+                if created:
+                    course.canonical_course_run = course_run
+                    course.save()
         else:
-            created = False
-            try:
-                course, created = self.get_or_create_course(data)
-                course_run = self.create_course_run(course, data)
-            except InvalidKeyError:
-                logger.error('Invalid course key [%s].', data['field_course_id'])
-
-            if created:
-                course.canonical_course_run = course_run
-                course.save()
+            logger.info(
+                'Course_run [%s] has uuid [%s] already on course about page. No need to ingest',
+                data['field_course_id'],
+                data['field_course_uuid']
+            )
 
     def get_course_run(self, data):
         course_run_key = data['field_course_id']
@@ -424,11 +442,6 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
         validated_data = self.format_course_data(data)
         self._update_instance(course, validated_data)
 
-        if self.get_course_run_status(data) != CourseRunStatus.Published:
-            logger.warning(
-                'Updating course [%s] with data from unpublished course_run [%s].', course.uuid, data['field_course_id']
-            )
-
         return course
 
     def _update_instance(self, instance, validated_data, **kwargs):

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -14,6 +14,7 @@ from opaque_keys.edx.keys import CourseKey
 from testfixtures import LogCapture
 
 from course_discovery.apps.course_metadata.choices import CourseRunPacing, CourseRunStatus
+from course_discovery.apps.course_metadata.data_loaders.marketing_site import logger as marketing_site_logger
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
     CourseMarketingSiteDataLoader, PersonMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
     SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
@@ -458,11 +459,11 @@ class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
         {'field_course_id': 'Bio Course ID'}
     )
     def test_process_node(self, data):
-        with LogCapture() as l:
+        with LogCapture(marketing_site_logger.name) as lc:
             self.loader.process_node(data)
-            l.check(
+            lc.check(
                 (
-                    'course_discovery.apps.course_metadata.data_loaders.marketing_site',
+                    marketing_site_logger.name,
                     'ERROR',
                     'Invalid course key [{}].'.format(data['field_course_id'])
                 )
@@ -576,7 +577,7 @@ class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
             self.assert_course_loaded(datum)
 
     @responses.activate
-    def test_canonical(self):
+    def test_course_run_creation(self):
         self.mocked_data = [
             mock_data.ORIGINAL_MARKETING_SITE_API_COURSE_BODY,
             mock_data.NEW_RUN_MARKETING_SITE_API_COURSE_BODY,
@@ -598,3 +599,52 @@ class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
         self.assertNotEqual(course.title, new_run_title)
         with self.assertRaises(AttributeError):
             course_run.canonical_for_course  # pylint: disable=pointless-statement
+
+    @responses.activate
+    def test_discovery_created_course_run(self):
+        self.mocked_data = [
+            mock_data.DISCOVERY_CREATED_MARKETING_SITE_API_COURSE_BODY
+        ]
+
+        self.mock_login_response()
+        self.mock_api()
+
+        with LogCapture(marketing_site_logger.name) as lc:
+            self.loader.ingest()
+            lc.check(
+                (
+                    marketing_site_logger.name,
+                    'INFO',
+                    'Course_run [{}] has uuid [{}] already on course about page. No need to ingest'.format(
+                        mock_data.DISCOVERY_CREATED_MARKETING_SITE_API_COURSE_BODY['field_course_id'],
+                        mock_data.DISCOVERY_CREATED_MARKETING_SITE_API_COURSE_BODY['field_course_uuid'])
+                )
+            )
+
+    @responses.activate
+    def test_discovery_unpublished_course_run(self):
+        self.mocked_data = [
+            mock_data.UPDATED_MARKETING_SITE_API_COURSE_BODY,
+            mock_data.ORIGINAL_MARKETING_SITE_API_COURSE_BODY
+        ]
+
+        self.mock_login_response()
+        self.mock_api()
+
+        with LogCapture(marketing_site_logger.name) as lc:
+            self.loader.ingest()
+            lc.check(
+                (
+                    marketing_site_logger.name,
+                    'INFO',
+                    'Processed course run with UUID [{}].'.format(
+                        mock_data.UPDATED_MARKETING_SITE_API_COURSE_BODY['uuid'])
+                ),
+                (
+                    marketing_site_logger.name,
+                    'INFO',
+                    'Course_run [{}] is unpublished, so the course [{}] related is not updated.'.format(
+                        mock_data.ORIGINAL_MARKETING_SITE_API_COURSE_BODY['field_course_id'],
+                        mock_data.ORIGINAL_MARKETING_SITE_API_COURSE_BODY['field_course_code'])
+                )
+            )