Remove XSeries data loader

XSeries will now be managed using discovery's Django admin like all other program types. Changes to XSeries program may now trigger publication to the marketing site.

LEARNER-1148

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

@@ -15,7 +15,7 @@ from opaque_keys.edx.keys import CourseKey
 from course_discovery.apps.course_metadata.choices import CourseRunPacing, CourseRunStatus
 from course_discovery.apps.course_metadata.data_loaders import AbstractDataLoader
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseRun, LevelType, Organization, Person, Position, Program, Subject
+    Course, CourseRun, LevelType, Organization, Person, Position, Subject
 )
 from course_discovery.apps.course_metadata.utils import MarketingSiteAPIClient
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
@@ -133,44 +133,6 @@ class AbstractMarketingSiteDataLoader(AbstractDataLoader):
         pass
 
 
-class XSeriesMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
-    @property
-    def node_type(self):
-        return 'xseries'
-
-    def process_node(self, data):
-        marketing_slug = data['url'].split('/')[-1]
-
-        try:
-            program = Program.objects.get(marketing_slug=marketing_slug, partner=self.partner)
-        except Program.DoesNotExist:
-            logger.error('Program [%s] exists on the marketing site, but not in the Programs Service!', marketing_slug)
-            return None
-
-        card_image_url = self._get_nested_url(data.get('field_card_image'))
-        video_url = self._get_nested_url(data.get('field_product_video'))
-
-        # NOTE (CCB): Remove the heading at the beginning of the overview. Why this isn't part of the template
-        # is beyond me. It's just silly.
-        overview = self.clean_html(data['body']['value'])
-        overview = overview.lstrip('### XSeries Program Overview').strip()
-
-        data = {
-            'subtitle': data.get('field_xseries_subtitle_short'),
-            'card_image_url': card_image_url,
-            'overview': overview,
-            'video': self.get_or_create_video(video_url),
-            'credit_redemption_overview': data.get('field_cards_section_description')
-        }
-
-        for field, value in data.items():
-            setattr(program, field, value)
-
-        program.save()
-        logger.info('Processed XSeries with marketing_slug [%s].', marketing_slug)
-        return program
-
-
 class SubjectMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
     @property
     def node_type(self):

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -16,11 +16,11 @@ from testfixtures import LogCapture
 from course_discovery.apps.course_metadata.choices import CourseRunPacing, CourseRunStatus
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
     CourseMarketingSiteDataLoader, PersonMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
-    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, XSeriesMarketingSiteDataLoader
+    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
 )
 from course_discovery.apps.course_metadata.data_loaders.tests import JSON, mock_data
 from course_discovery.apps.course_metadata.data_loaders.tests.mixins import DataLoaderTestMixin
-from course_discovery.apps.course_metadata.models import Course, Organization, Person, Program, Subject, Video
+from course_discovery.apps.course_metadata.models import Course, Organization, Person, Subject
 from course_discovery.apps.course_metadata.tests import factories
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
@@ -137,67 +137,6 @@ class AbstractMarketingSiteDataLoaderTestMixin(DataLoaderTestMixin):
             self.loader_class(self.partner, self.api_url)  # pylint: disable=not-callable
 
 
-class XSeriesMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
-    loader_class = XSeriesMarketingSiteDataLoader
-    mocked_data = mock_data.MARKETING_SITE_API_XSERIES_BODIES
-
-    def create_mock_programs(self, programs):
-        for program in programs:
-            marketing_slug = program['url'].split('/')[-1]
-            factories.ProgramFactory(marketing_slug=marketing_slug, partner=self.partner)
-
-    def mock_api(self):
-        bodies = super().mock_api()
-        self.create_mock_programs(bodies)
-        return bodies
-
-    def assert_program_loaded(self, data):
-        marketing_slug = data['url'].split('/')[-1]
-        program = Program.objects.get(marketing_slug=marketing_slug, partner=self.partner)
-
-        overview = self.loader.clean_html(data['body']['value'])
-        overview = overview.lstrip('### XSeries Program Overview').strip()
-        self.assertEqual(program.overview, overview)
-
-        self.assertEqual(program.subtitle, data.get('field_xseries_subtitle_short'))
-
-        card_image_url = data.get('field_card_image', {}).get('url')
-        self.assertEqual(program.card_image_url, card_image_url)
-
-        video_url = data.get('field_product_video', {}).get('url')
-        if video_url:
-            video = Video.objects.get(src=video_url)
-            self.assertEqual(program.video, video)
-
-    @responses.activate
-    def test_ingest(self):
-        self.mock_login_response()
-        api_data = self.mock_api()
-
-        self.loader.ingest()
-
-        for datum in api_data:
-            self.assert_program_loaded(datum)
-
-    @responses.activate
-    def test_ingest_with_missing_programs(self):
-        """ Verify ingestion properly logs issues when programs exist on the marketing site,
-        but not the Programs API. """
-        self.mock_login_response()
-        api_data = self.mock_api()
-
-        Program.objects.all().delete()
-        self.assertEqual(Program.objects.count(), 0)
-
-        with mock.patch(LOGGER_PATH) as mock_logger:
-            self.loader.ingest()
-            self.assertEqual(Program.objects.count(), 0)
-
-            calls = [mock.call('Program [%s] exists on the marketing site, but not in the Programs Service!',
-                               datum['url'].split('/')[-1]) for datum in api_data]
-            mock_logger.error.assert_has_calls(calls)
-
-
 class SubjectMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
     loader_class = SubjectMarketingSiteDataLoader
     mocked_data = mock_data.MARKETING_SITE_API_SUBJECT_BODIES

course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py

@@ -18,7 +18,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
     CourseMarketingSiteDataLoader, PersonMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
-    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, XSeriesMarketingSiteDataLoader
+    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
 )
 from course_discovery.apps.course_metadata.models import Course, DataLoaderConfig
 
@@ -154,9 +154,6 @@ class Command(BaseCommand):
                     (EcommerceApiDataLoader, partner.ecommerce_api_url, 1),
                     (ProgramsApiDataLoader, partner.programs_api_url, max_workers),
                 ),
-                (
-                    (XSeriesMarketingSiteDataLoader, partner.marketing_site_url_root, max_workers),
-                ),
             )
 
             if waffle.switch_is_active('parallel_refresh_pipeline'):

course_discovery/apps/course_metadata/management/commands/tests/test_refresh_course_metadata.py

@@ -14,7 +14,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
     CourseMarketingSiteDataLoader, PersonMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
-    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, XSeriesMarketingSiteDataLoader
+    SponsorMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
 )
 from course_discovery.apps.course_metadata.data_loaders.tests import mock_data
 from course_discovery.apps.course_metadata.management.commands.refresh_course_metadata import execute_parallel_loader
@@ -31,7 +31,8 @@ class RefreshCourseMetadataCommandTests(TransactionTestCase):
         super(RefreshCourseMetadataCommandTests, self).setUp()
         self.partner = PartnerFactory()
         partner = self.partner
-        self.pipeline = [(SubjectMarketingSiteDataLoader, partner.marketing_site_url_root, None),
+        self.pipeline = [
+            (SubjectMarketingSiteDataLoader, partner.marketing_site_url_root, None),
             (SchoolMarketingSiteDataLoader, partner.marketing_site_url_root, None),
             (SponsorMarketingSiteDataLoader, partner.marketing_site_url_root, None),
             (PersonMarketingSiteDataLoader, partner.marketing_site_url_root, None),
@@ -40,7 +41,7 @@ class RefreshCourseMetadataCommandTests(TransactionTestCase):
             (CoursesApiDataLoader, partner.courses_api_url, None),
             (EcommerceApiDataLoader, partner.ecommerce_api_url, 1),
             (ProgramsApiDataLoader, partner.programs_api_url, None),
-                         (XSeriesMarketingSiteDataLoader, partner.marketing_site_url_root, None)]
+        ]
         self.kwargs = {'username': 'bob'}
         self.mock_access_token_api()
 
@@ -210,7 +211,6 @@ class RefreshCourseMetadataCommandTests(TransactionTestCase):
                     CoursesApiDataLoader,
                     EcommerceApiDataLoader,
                     ProgramsApiDataLoader,
-                    XSeriesMarketingSiteDataLoader,
                 )
                 expected_calls = [mock.call('%s failed!', loader_class.__name__) for loader_class in loader_classes]
                 mock_logger.exception.assert_has_calls(expected_calls)

course_discovery/apps/course_metadata/publishers.py

@@ -233,7 +233,13 @@ class ProgramMarketingSitePublisher(BaseMarketingSitePublisher):
                 determine if publication is necessary. May not exist if the program
                 is being saved for the first time.
         """
-        if obj.type.name in {'MicroMasters', 'Professional Certificate'}:
+        types_to_publish = {
+            'XSeries',
+            'MicroMasters',
+            'Professional Certificate',
+        }
+
+        if obj.type.name in types_to_publish:
             node_data = self.serialize_obj(obj)
 
             node_id = None

course_discovery/apps/course_metadata/tests/test_publishers.py

@@ -279,7 +279,13 @@ class ProgramMarketingSitePublisherTests(MarketingSitePublisherTestMixin):
         for mocked_method in mocked_methods:
             assert not mocked_method.called
 
-        for name in ('MicroMasters', 'Professional Certificate'):
+        types_to_publish = (
+            'XSeries',
+            'MicroMasters',
+            'Professional Certificate'
+        )
+
+        for name in types_to_publish:
             for mocked_method in mocked_methods:
                 mocked_method.reset_mock()