Optimized performance of the Catalog CSV endpoint (#321)

ECOM-5534

course_discovery/apps/api/serializers.py

@@ -559,7 +559,7 @@ class FlattenedCourseRunWithCourseSerializer(CourseRunSerializer):
         )
 
     def get_level_type(self, obj):
-        return obj.course.level_type
+        return obj.level_type.name
 
     def get_course_key(self, obj):
         return obj.course.key

course_discovery/apps/api/v1/tests/test_views/test_catalogs.py

@@ -141,7 +141,7 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         CourseRunFactory(enrollment_end=enrollment_end, course__title='ABC Test Course 2')
         CourseRunFactory(enrollment_end=enrollment_end, course=self.course)
 
-        with self.assertNumQueries(40):
+        with self.assertNumQueries(41):
             response = self.client.get(url)
             self.assertEqual(response.status_code, 200)
             self.assertListEqual(response.data['results'], self.serialize_catalog_course(courses, many=True))
@@ -167,7 +167,7 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
 
         url = reverse('api:v1:catalog-csv', kwargs={'id': self.catalog.id})
 
-        with self.assertNumQueries(24):
+        with self.assertNumQueries(21):
             response = self.client.get(url)
 
         course_run = self.serialize_catalog_flat_course_run(self.course_run)

course_discovery/apps/api/v1/views.py

@@ -37,16 +37,21 @@ logger = logging.getLogger(__name__)
 User = get_user_model()
 
 PREFETCH_FIELDS = {
-    'course_run': (
+    'course_run': [
         'course__partner', 'course__level_type', 'course__programs', 'course__programs__type',
         'course__programs__partner', 'seats', 'transcript_languages', 'seats__currency', 'staff',
         'staff__position', 'staff__position__organization', 'language',
-    ),
-    'course': (
+    ],
+    'course': [
         'level_type', 'video', 'programs', 'course_runs', 'subjects', 'prerequisites', 'expected_learning_items',
         'authoring_organizations', 'authoring_organizations__tags', 'authoring_organizations__partner',
         'sponsoring_organizations', 'sponsoring_organizations__tags', 'sponsoring_organizations__partner',
-    ),
+    ],
+}
+
+SELECT_RELATED_FIELDS = {
+    'course': ['level_type', 'video', ],
+    'course_run': ['course', 'language', 'video', ],
 }
 
 
@@ -67,11 +72,11 @@ def prefetch_related_objects_for_courses(queryset):
         QuerySet
     """
     # Prefetch the data for the related course runs
-    course_run_prefetch_fields = PREFETCH_FIELDS['course_run']
+    course_run_prefetch_fields = PREFETCH_FIELDS['course_run'] + SELECT_RELATED_FIELDS['course_run']
     course_run_prefetch_fields = ['course_runs__' + field for field in course_run_prefetch_fields]
     queryset = queryset.prefetch_related(*course_run_prefetch_fields)
 
-    queryset = queryset.select_related('level_type', 'video')
+    queryset = queryset.select_related(*SELECT_RELATED_FIELDS['course'])
     queryset = queryset.prefetch_related(*PREFETCH_FIELDS['course'])
     return queryset
 
@@ -192,11 +197,14 @@ class CatalogViewSet(viewsets.ModelViewSet):
         serializer: serializers.FlattenedCourseRunWithCourseSerializer
         """
         catalog = self.get_object()
-        courses = catalog.courses().active()
-        course_runs = []
+        courses = catalog.courses()
+        course_runs = CourseRun.objects.filter(course__in=courses).active().marketable()
 
-        for course in courses:
-            course_runs += list(course.course_runs.active().marketable())
+        # We use select_related and prefetch_related to decrease our database query count
+        course_runs = course_runs.select_related(*SELECT_RELATED_FIELDS['course_run'])
+        prefetch_fields = ['course__' + field for field in PREFETCH_FIELDS['course']]
+        prefetch_fields += PREFETCH_FIELDS['course_run']
+        course_runs = course_runs.prefetch_related(*prefetch_fields)
 
         serializer = serializers.FlattenedCourseRunWithCourseSerializer(
             course_runs, many=True, context={'request': request}
@@ -289,7 +297,7 @@ class CourseRunViewSet(viewsets.ReadOnlyModelViewSet):
             return qs
         else:
             queryset = super(CourseRunViewSet, self).get_queryset().filter(course__partner=partner)
-            queryset = queryset.select_related('course', 'language', 'video')
+            queryset = queryset.select_related(*SELECT_RELATED_FIELDS['course_run'])
             queryset = queryset.prefetch_related(*PREFETCH_FIELDS['course_run'])
             return queryset