Merge pull request #347 from edx/renzo/query-optimization

Query optimization

course_discovery/apps/api/serializers.py

 # pylint: disable=abstract-method
+import datetime
 import json
 from urllib.parse import urlencode
 
+import pytz
 from django.contrib.auth import get_user_model
+from django.db.models.query import Prefetch
 from django.utils.translation import ugettext_lazy as _
 from drf_haystack.serializers import HaystackSerializer, HaystackFacetSerializer
 from rest_framework import serializers
@@ -67,25 +70,39 @@ PROGRAM_FACET_FIELDS = BASE_PROGRAM_FIELDS + ('organizations',)
 
 PREFETCH_FIELDS = {
     'course_run': [
-        'course__partner', 'course__level_type', 'course__programs', 'course__programs__type',
-        'course__programs__partner', 'seats', 'transcript_languages', 'seats__currency', 'staff',
-        'staff__position', 'staff__position__organization', 'language',
+        'course__level_type',
+        'course__partner',
+        'course__programs',
+        'course__programs__partner',
+        'course__programs__type',
+        'language',
+        'seats',
+        'seats__currency',
+        'staff',
+        'staff__position',
+        'staff__position__organization',
+        'transcript_languages',
     ],
     'course': [
-        'level_type', 'video', 'programs', 'course_runs', 'subjects', 'prerequisites', 'expected_learning_items',
-        'authoring_organizations', 'authoring_organizations__tags', 'authoring_organizations__partner',
-        'sponsoring_organizations', 'sponsoring_organizations__tags', 'sponsoring_organizations__partner',
-    ],
-    'program': [
-        'authoring_organizations', 'authoring_organizations__tags', 'authoring_organizations__partner',
-        'excluded_course_runs', 'courses', 'courses__authoring_organizations', 'courses__course_runs',
+        'authoring_organizations',
+        'authoring_organizations__partner',
+        'authoring_organizations__tags',
+        'course_runs',
+        'expected_learning_items',
+        'level_type',
+        'prerequisites',
+        'programs',
+        'sponsoring_organizations',
+        'sponsoring_organizations__partner',
+        'sponsoring_organizations__tags',
+        'subjects',
+        'video',
     ],
 }
 
 SELECT_RELATED_FIELDS = {
-    'course': ['level_type', 'video', 'partner', ],
-    'course_run': ['course', 'language', 'video', ],
-    'program': ['type', 'video', 'partner', ],
+    'course': ['level_type', 'partner', 'video'],
+    'course_run': ['course', 'language', 'video'],
 }
 
 
@@ -182,6 +199,10 @@ class PersonSerializer(serializers.ModelSerializer):
     """Serializer for the ``Person`` model."""
     position = PositionSerializer()
 
+    @classmethod
+    def prefetch_queryset(cls):
+        return Person.objects.all().select_related('position__organization')
+
     class Meta(object):
         model = Person
         fields = ('uuid', 'given_name', 'family_name', 'bio', 'profile_image_url', 'slug', 'position')
@@ -191,6 +212,10 @@ class EndorsementSerializer(serializers.ModelSerializer):
     """Serializer for the ``Endorsement`` model."""
     endorser = PersonSerializer()
 
+    @classmethod
+    def prefetch_queryset(cls):
+        return Endorsement.objects.all().select_related('endorser')
+
     class Meta(object):
         model = Endorsement
         fields = ('endorser', 'quote',)
@@ -201,6 +226,12 @@ class CorporateEndorsementSerializer(serializers.ModelSerializer):
     image = ImageSerializer()
     individual_endorsements = EndorsementSerializer(many=True)
 
+    @classmethod
+    def prefetch_queryset(cls):
+        return CorporateEndorsement.objects.all().select_related('image').prefetch_related(
+            Prefetch('endorser', queryset=EndorsementSerializer.prefetch_queryset()),
+        )
+
     class Meta(object):
         model = CorporateEndorsement
         fields = ('corporation_name', 'statement', 'image', 'individual_endorsements',)
@@ -220,26 +251,26 @@ class SeatSerializer(serializers.ModelSerializer):
     credit_provider = serializers.CharField()
     credit_hours = serializers.IntegerField()
 
+    @classmethod
+    def prefetch_queryset(cls):
+        return Seat.objects.all().select_related('currency')
+
     class Meta(object):
         model = Seat
         fields = ('type', 'price', 'currency', 'upgrade_deadline', 'credit_provider', 'credit_hours',)
 
 
-class MinimalOrganizationSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = Organization
-        fields = ('uuid', 'key', 'name',)
-
-
-class OrganizationSerializer(TaggitSerializer, MinimalOrganizationSerializer):
+class OrganizationSerializer(TaggitSerializer, serializers.ModelSerializer):
     """Serializer for the ``Organization`` model."""
     tags = TagListSerializerField()
 
-    class Meta(MinimalOrganizationSerializer.Meta):
+    @classmethod
+    def prefetch_queryset(cls):
+        return Organization.objects.all().select_related('partner').prefetch_related('tags')
+
+    class Meta(object):
         model = Organization
-        fields = MinimalOrganizationSerializer.Meta.fields + (
-            'description', 'homepage_url', 'tags', 'logo_image_url', 'marketing_url',
-        )
+        fields = ('key', 'name', 'description', 'homepage_url', 'tags', 'logo_image_url', 'marketing_url')
 
 
 class CatalogSerializer(serializers.ModelSerializer):
@@ -277,13 +308,7 @@ class NestedProgramSerializer(serializers.ModelSerializer):
         read_only_fields = ('uuid', 'marketing_url',)
 
 
-class MinimalCourseRunSerializer(TimestampModelSerializer):
-    class Meta:
-        model = CourseRun
-        fields = ('key', 'uuid', 'title',)
-
-
-class CourseRunSerializer(MinimalCourseRunSerializer):
+class CourseRunSerializer(TimestampModelSerializer):
     """Serializer for the ``CourseRun`` model."""
     course = serializers.SlugRelatedField(read_only=True, slug_field='key')
     content_language = serializers.SlugRelatedField(
@@ -299,13 +324,21 @@ class CourseRunSerializer(MinimalCourseRunSerializer):
     marketing_url = serializers.SerializerMethodField()
     level_type = serializers.SlugRelatedField(read_only=True, slug_field='name')
 
-    class Meta(MinimalCourseRunSerializer.Meta):
+    @classmethod
+    def prefetch_queryset(cls):
+        return CourseRun.objects.all().select_related('course', 'language', 'video').prefetch_related(
+            'transcript_languages',
+            Prefetch('seats', queryset=SeatSerializer.prefetch_queryset()),
+            Prefetch('staff', queryset=PersonSerializer.prefetch_queryset()),
+        )
+
+    class Meta:
         model = CourseRun
-        fields = MinimalCourseRunSerializer.Meta.fields + (
-            'course', 'short_description', 'full_description', 'start', 'end', 'enrollment_start', 'enrollment_end',
-            'announcement', 'image', 'video', 'seats', 'content_language', 'transcript_languages', 'instructors',
-            'staff', 'pacing_type', 'min_effort', 'max_effort', 'modified', 'marketing_url', 'level_type',
-            'availability',
+        fields = (
+            'course', 'key', 'title', 'short_description', 'full_description', 'start', 'end',
+            'enrollment_start', 'enrollment_end', 'announcement', 'image', 'video', 'seats',
+            'content_language', 'transcript_languages', 'instructors', 'staff',
+            'pacing_type', 'min_effort', 'max_effort', 'modified', 'marketing_url', 'level_type', 'availability',
         )
 
     def get_marketing_url(self, obj):
@@ -332,16 +365,7 @@ class ContainedCourseRunsSerializer(serializers.Serializer):
     )
 
 
-class MinimalCourseSerializer(TimestampModelSerializer):
-    course_runs = MinimalCourseRunSerializer(many=True)
-    owners = MinimalOrganizationSerializer(many=True, source='authoring_organizations')
-
-    class Meta:
-        model = Course
-        fields = ('key', 'uuid', 'title', 'course_runs', 'owners',)
-
-
-class CourseSerializer(MinimalCourseSerializer):
+class CourseSerializer(TimestampModelSerializer):
     """Serializer for the ``Course`` model."""
     level_type = serializers.SlugRelatedField(read_only=True, slug_field='name')
     subjects = SubjectSerializer(many=True)
@@ -354,11 +378,23 @@ class CourseSerializer(MinimalCourseSerializer):
     course_runs = CourseRunSerializer(many=True)
     marketing_url = serializers.SerializerMethodField()
 
-    class Meta(MinimalCourseSerializer.Meta):
+    @classmethod
+    def prefetch_queryset(cls):
+        return Course.objects.all().select_related('level_type', 'video', 'partner').prefetch_related(
+            'expected_learning_items',
+            'prerequisites',
+            'subjects',
+            Prefetch('course_runs', queryset=CourseRunSerializer.prefetch_queryset()),
+            Prefetch('authoring_organizations', queryset=OrganizationSerializer.prefetch_queryset()),
+            Prefetch('sponsoring_organizations', queryset=OrganizationSerializer.prefetch_queryset()),
+        )
+
+    class Meta:
         model = Course
-        fields = MinimalCourseSerializer.Meta.fields + (
-            'short_description', 'full_description', 'level_type', 'subjects', 'prerequisites',
-            'expected_learning_items', 'image', 'video', 'sponsors', 'modified', 'marketing_url',
+        fields = (
+            'key', 'title', 'short_description', 'full_description', 'level_type', 'subjects', 'prerequisites',
+            'expected_learning_items', 'image', 'video', 'owners', 'sponsors', 'modified', 'course_runs',
+            'marketing_url',
         )
 
     def get_marketing_url(self, obj):
@@ -390,55 +426,29 @@ class ContainedCoursesSerializer(serializers.Serializer):
     )
 
 
-class ProgramCourseSerializer(MinimalCourseSerializer):
+class ProgramCourseSerializer(CourseSerializer):
     """Serializer used to filter out excluded course runs in a course associated with the program"""
     course_runs = serializers.SerializerMethodField()
 
     def get_course_runs(self, course):
-        program = self.context['program']
-        course_runs = list(course.course_runs.all())
-        excluded_course_runs = list(program.excluded_course_runs.all())
-        course_runs = [course_run for course_run in course_runs if course_run not in excluded_course_runs]
+        course_runs = self.context['course_runs']
+        course_runs = [course_run for course_run in course_runs if course_run.course == course]
 
         if self.context.get('published_course_runs_only'):
             course_runs = [course_run for course_run in course_runs if course_run.status == CourseRunStatus.Published]
 
-        return MinimalCourseRunSerializer(
+        return CourseRunSerializer(
             course_runs,
             many=True,
             context={'request': self.context.get('request')}
         ).data
 
 
-class MinimalProgramSerializer(serializers.ModelSerializer):
-    authoring_organizations = MinimalOrganizationSerializer(many=True)
-    banner_image = StdImageSerializerField()
+class ProgramSerializer(serializers.ModelSerializer):
     courses = serializers.SerializerMethodField()
+    authoring_organizations = OrganizationSerializer(many=True)
     type = serializers.SlugRelatedField(slug_field='name', queryset=ProgramType.objects.all())
-
-    def get_courses(self, program):
-        course_serializer = ProgramCourseSerializer(
-            program.courses.all(),
-            many=True,
-            context={
-                'request': self.context.get('request'),
-                'program': program,
-                'published_course_runs_only': self.context.get('published_course_runs_only'),
-            }
-        )
-
-        return course_serializer.data
-
-    class Meta:
-        model = Program
-        fields = (
-            'uuid', 'title', 'subtitle', 'type', 'status', 'marketing_slug', 'marketing_url', 'banner_image', 'courses',
-            'authoring_organizations', 'card_image_url',
-        )
-        read_only_fields = ('uuid', 'marketing_url', 'banner_image')
-
-
-class ProgramSerializer(MinimalProgramSerializer):
+    banner_image = StdImageSerializerField()
     video = VideoSerializer()
     expected_learning_items = serializers.SlugRelatedField(many=True, read_only=True, slug_field='value')
     faq = FAQSerializer(many=True)
@@ -457,14 +467,113 @@ class ProgramSerializer(MinimalProgramSerializer):
     subjects = SubjectSerializer(many=True)
     staff = PersonSerializer(many=True)
 
-    class Meta(MinimalProgramSerializer.Meta):
+    @classmethod
+    def prefetch_queryset(cls):
+        """
+        Prefetch the related objects that will be serialized with a `Program`.
+
+        We use Pefetch objects so that we can prefetch and select all the way down the
+        chain of related fields from programs to course runs (i.e., we want control over
+        the querysets that we're prefetching).
+        """
+        return Program.objects.all().select_related('type', 'video', 'partner').prefetch_related(
+            'excluded_course_runs',
+            'expected_learning_items',
+            'faq',
+            'job_outlook_items',
+            # `type` is serialized by a third-party serializer. Providing this field name allows us to
+            # prefetch `applicable_seat_types`, a m2m on `ProgramType`, through `type`, a foreign key to
+            # `ProgramType` on `Program`.
+            'type__applicable_seat_types',
+            Prefetch('courses', queryset=ProgramCourseSerializer.prefetch_queryset()),
+            Prefetch('authoring_organizations', queryset=OrganizationSerializer.prefetch_queryset()),
+            Prefetch('credit_backing_organizations', queryset=OrganizationSerializer.prefetch_queryset()),
+            Prefetch('corporate_endorsements', queryset=CorporateEndorsementSerializer.prefetch_queryset()),
+            Prefetch('individual_endorsements', queryset=EndorsementSerializer.prefetch_queryset()),
+        )
+
+    def get_courses(self, program):
+        courses, course_runs = self.sort_courses(program)
+
+        course_serializer = ProgramCourseSerializer(
+            courses,
+            many=True,
+            context={
+                'request': self.context.get('request'),
+                'program': program,
+                'published_course_runs_only': self.context.get('published_course_runs_only'),
+                'course_runs': course_runs,
+            }
+        )
+
+        return course_serializer.data
+
+    def sort_courses(self, program):
+        """
+        Sorting by enrollment start then by course start yields a list ordered by course start, with
+        ties broken by enrollment start. This works because Python sorting is stable: two objects with
+        equal keys appear in the same order in sorted output as they appear in the input.
+
+        Courses are only created if there's at least one course run belonging to that course, so
+        course_runs should never be empty. If it is, key functions in this method attempting to find the
+        min of an empty sequence will raise a ValueError.
+        """
+        course_runs = list(program.course_runs)
+
+        def min_run_enrollment_start(course):
+            # Enrollment starts may be empty. When this is the case, we make the same assumption as
+            # the LMS: no enrollment_start is equivalent to (offset-aware) datetime.datetime.min.
+            min_datetime = datetime.datetime.min.replace(tzinfo=pytz.UTC)
+
+            # Course runs excluded from the program are excluded here, too.
+            #
+            # If this becomes a candidate for optimization in the future, be careful sorting null values
+            # in the database. PostgreSQL and MySQL sort null values as if they are higher than non-null
+            # values, while SQLite does the opposite.
+            #
+            # For more, refer to https://docs.djangoproject.com/en/1.10/ref/models/querysets/#latest.
+            _course_runs = [course_run for course_run in course_runs if course_run.course == course]
+
+            # Return early if we have no course runs since min() will fail.
+            if not _course_runs:
+                return min_datetime
+
+            run = min(_course_runs, key=lambda run: run.enrollment_start or min_datetime)
+
+            return run.enrollment_start or min_datetime
+
+        def min_run_start(course):
+            # Course starts may be empty. Since this means the course can't be started, missing course
+            # start date is equivalent to (offset-aware) datetime.datetime.max.
+            max_datetime = datetime.datetime.max.replace(tzinfo=pytz.UTC)
+
+            _course_runs = [course_run for course_run in course_runs if course_run.course == course]
+
+            # Return early if we have no course runs since min() will fail.
+            if not _course_runs:
+                return max_datetime
+
+            run = min(_course_runs, key=lambda run: run.start or max_datetime)
+
+            return run.start or max_datetime
+
+        courses = list(program.courses.all())
+        courses.sort(key=min_run_enrollment_start)
+        courses.sort(key=min_run_start)
+
+        return courses, course_runs
+
+    class Meta:
         model = Program
-        fields = MinimalProgramSerializer.Meta.fields + (
-            'overview', 'weeks_to_complete', 'min_hours_effort_per_week', 'max_hours_effort_per_week', 'video',
+        fields = (
+            'uuid', 'title', 'subtitle', 'type', 'status', 'marketing_slug', 'marketing_url', 'courses',
+            'overview', 'weeks_to_complete', 'min_hours_effort_per_week', 'max_hours_effort_per_week',
+            'authoring_organizations', 'banner_image', 'banner_image_url', 'card_image_url', 'video',
             'expected_learning_items', 'faq', 'credit_backing_organizations', 'corporate_endorsements',
             'job_outlook_items', 'individual_endorsements', 'languages', 'transcript_languages', 'subjects',
             'price_ranges', 'staff', 'credit_redemption_overview',
         )
+        read_only_fields = ('uuid', 'marketing_url', 'banner_image')
 
 
 class AffiliateWindowSerializer(serializers.ModelSerializer):
@@ -481,7 +590,7 @@ class AffiliateWindowSerializer(serializers.ModelSerializer):
     category = serializers.SerializerMethodField()
     price = serializers.SerializerMethodField()
 
-    class Meta(object):
+    class Meta:
         model = Seat
         fields = (
             'name', 'pid', 'desc', 'category', 'purl', 'imgurl', 'price', 'currency'
@@ -509,7 +618,7 @@ class FlattenedCourseRunWithCourseSerializer(CourseRunSerializer):
     course_key = serializers.SlugRelatedField(read_only=True, source='course', slug_field='key')
     image = ImageField(read_only=True, source='card_image_url')
 
-    class Meta(object):
+    class Meta:
         model = CourseRun
         fields = (
             'key', 'title', 'short_description', 'full_description', 'level_type', 'subjects', 'prerequisites',

course_discovery/apps/api/tests/test_serializers.py

@@ -9,14 +9,12 @@ from rest_framework.test import APIRequestFactory
 
 from course_discovery.apps.api.fields import ImageField, StdImageSerializerField
 from course_discovery.apps.api.serializers import (
-    CatalogSerializer, CourseRunSerializer, ContainedCoursesSerializer, ImageSerializer,
+    CatalogSerializer, CourseSerializer, CourseRunSerializer, ContainedCoursesSerializer, ImageSerializer,
     SubjectSerializer, PrerequisiteSerializer, VideoSerializer, OrganizationSerializer, SeatSerializer,
     PersonSerializer, AffiliateWindowSerializer, ContainedCourseRunsSerializer, CourseRunSearchSerializer,
     ProgramSerializer, ProgramSearchSerializer, ProgramCourseSerializer, NestedProgramSerializer,
     CourseRunWithProgramsSerializer, CourseWithProgramsSerializer, CorporateEndorsementSerializer,
-    FAQSerializer, EndorsementSerializer, PositionSerializer, FlattenedCourseRunWithCourseSerializer,
-    MinimalCourseSerializer, MinimalOrganizationSerializer, MinimalCourseRunSerializer, MinimalProgramSerializer,
-    CourseSerializer
+    FAQSerializer, EndorsementSerializer, PositionSerializer, FlattenedCourseRunWithCourseSerializer
 )
 from course_discovery.apps.catalogs.tests.factories import CatalogFactory
 from course_discovery.apps.core.models import User
@@ -32,7 +30,7 @@ from course_discovery.apps.course_metadata.tests.factories import (
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
 
-# pylint:disable=no-member, test-inherits-tests
+# pylint:disable=no-member
 
 def json_date_format(datetime_obj):
     return datetime.strftime(datetime_obj, "%Y-%m-%dT%H:%M:%S.%fZ")
@@ -94,36 +92,19 @@ class CatalogSerializerTests(TestCase):
         self.assertEqual(User.objects.filter(username=username).count(), 0)  # pylint: disable=no-member
 
 
-class MinimalCourseSerializerTests(TestCase):
-    serializer_class = MinimalCourseSerializer
-
-    def get_expected_data(self, course, request):
-        context = {'request': request}
-
-        return {
-            'key': course.key,
-            'uuid': str(course.uuid),
-            'title': course.title,
-            'course_runs': MinimalCourseRunSerializer(course.course_runs, many=True, context=context).data,
-            'owners': MinimalOrganizationSerializer(course.authoring_organizations, many=True, context=context).data,
-        }
-
+class CourseSerializerTests(TestCase):
     def test_data(self):
-        request = make_request()
-        organizations = OrganizationFactory()
-        course = CourseFactory(authoring_organizations=[organizations])
-        CourseRunFactory.create_batch(2, course=course)
-        serializer = self.serializer_class(course, context={'request': request})
-        expected = self.get_expected_data(course, request)
-        self.assertDictEqual(serializer.data, expected)
+        course = CourseFactory()
+        video = course.video
 
+        request = make_request()
 
-class CourseSerializerTests(MinimalCourseSerializerTests):
-    serializer_class = CourseSerializer
+        CourseRunFactory.create_batch(3, course=course)
+        serializer = CourseWithProgramsSerializer(course, context={'request': request})
 
-    def get_expected_data(self, course, request):
-        expected = super().get_expected_data(course, request)
-        expected.update({
+        expected = {
+            'key': course.key,
+            'title': course.title,
             'short_description': course.short_description,
             'full_description': course.full_description,
             'level_type': course.level_type.name,
@@ -131,9 +112,11 @@ class CourseSerializerTests(MinimalCourseSerializerTests):
             'prerequisites': [],
             'expected_learning_items': [],
             'image': ImageField().to_representation(course.card_image_url),
-            'video': VideoSerializer(course.video).data,
+            'video': VideoSerializer(video).data,
+            'owners': OrganizationSerializer(course.authoring_organizations, many=True).data,
             'sponsors': OrganizationSerializer(course.sponsoring_organizations, many=True).data,
             'modified': json_date_format(course.modified),  # pylint: disable=no-member
+            'course_runs': CourseRunSerializer(course.course_runs, many=True, context={'request': request}).data,
             'marketing_url': '{url}?{params}'.format(
                 url=course.marketing_url,
                 params=urlencode({
@@ -141,47 +124,22 @@ class CourseSerializerTests(MinimalCourseSerializerTests):
                     'utm_medium': request.user.referral_tracking_id,
                 })
             ),
-            'course_runs': CourseRunSerializer(course.course_runs, many=True, context={'request': request}).data,
-            'owners': OrganizationSerializer(course.authoring_organizations, many=True).data,
-        })
-        return expected
-
-
-class CourseWithProgramsSerializerTests(CourseSerializerTests):  # pylint: disable=test-inherits-tests
-    serializer_class = CourseWithProgramsSerializer
-
-    def get_expected_data(self, course, request):
-        expected = super().get_expected_data(course, request)
-        expected.update({
             'programs': NestedProgramSerializer(course.programs, many=True, context={'request': request}).data,
-        })
-        return expected
-
+        }
 
-class MinimalCourseRunSerializerTests(TestCase):
-    serializer_class = MinimalCourseRunSerializer
+        self.assertDictEqual(serializer.data, expected)
 
-    def get_expected_data(self, course_run, request):  # pylint: disable=unused-argument
-        return {
-            'key': course_run.key,
-            'uuid': str(course_run.uuid),
-            'title': course_run.title,
-        }
 
+class CourseRunSerializerTests(TestCase):
     def test_data(self):
         request = make_request()
         course_run = CourseRunFactory()
-        serializer = self.serializer_class(course_run, context={'request': request})
-        expected = self.get_expected_data(course_run, request)
-        self.assertDictEqual(serializer.data, expected)
-
-
-class CourseRunSerializerTests(MinimalCourseRunSerializerTests):  # pylint: disable=test-inherits-tests
-    serializer_class = CourseRunSerializer
+        course = course_run.course
+        video = course_run.video
+        serializer = CourseRunSerializer(course_run, context={'request': request})
+        ProgramFactory(courses=[course])
 
-    def get_expected_data(self, course_run, request):
-        expected = super().get_expected_data(course_run, request)
-        expected.update({
+        expected = {
             'course': course_run.course.key,
             'key': course_run.key,
             'title': course_run.title,  # pylint: disable=no-member
@@ -193,7 +151,7 @@ class CourseRunSerializerTests(MinimalCourseRunSerializerTests):  # pylint: disa
             'enrollment_end': json_date_format(course_run.enrollment_end),
             'announcement': json_date_format(course_run.announcement),
             'image': ImageField().to_representation(course_run.card_image_url),
-            'video': VideoSerializer(course_run.video).data,
+            'video': VideoSerializer(video).data,
             'pacing_type': course_run.pacing_type,
             'content_language': course_run.language.code,
             'transcript_languages': [],
@@ -212,8 +170,9 @@ class CourseRunSerializerTests(MinimalCourseRunSerializerTests):  # pylint: disa
             ),
             'level_type': course_run.level_type.name,
             'availability': course_run.availability,
-        })
-        return expected
+        }
+
+        self.assertDictEqual(serializer.data, expected)
 
 
 class CourseRunWithProgramsSerializerTests(TestCase):
@@ -325,96 +284,92 @@ class FlattenedCourseRunWithCourseSerializerTests(TestCase):  # pragma: no cover
 class ProgramCourseSerializerTests(TestCase):
     def setUp(self):
         super(ProgramCourseSerializerTests, self).setUp()
-        self.request = make_request()
-        self.course_list = CourseFactory.create_batch(3)
-        self.program = ProgramFactory(courses=self.course_list)
+        self.program = ProgramFactory(courses=[CourseFactory()])
+
+    def assert_program_courses_serialized(self, program):
+        request = make_request()
 
-    def test_no_run(self):
-        """
-        Make sure that if a course has no runs, the serializer still works as expected
-        """
         serializer = ProgramCourseSerializer(
-            self.course_list,
+            program.courses,
             many=True,
-            context={'request': self.request, 'program': self.program}
+            context={
+                'request': request,
+                'program': program,
+                'course_runs': program.course_runs
+            }
         )
-
-        expected = MinimalCourseSerializer(self.course_list, many=True, context={'request': self.request}).data
-
+        expected = CourseSerializer(program.courses, many=True, context={'request': request}).data
         self.assertSequenceEqual(serializer.data, expected)
 
-    def test_with_runs(self):
-        for course in self.course_list:
-            CourseRunFactory.create_batch(2, course=course)
-        serializer = ProgramCourseSerializer(
-            self.course_list,
-            many=True,
-            context={'request': self.request, 'program': self.program}
-        )
+    def test_data(self):
+        for course in self.program.courses.all():
+            CourseRunFactory(course=course)
 
-        expected = MinimalCourseSerializer(self.course_list, many=True, context={'request': self.request}).data
+        self.assert_program_courses_serialized(self.program)
 
-        self.assertSequenceEqual(serializer.data, expected)
+    def test_data_without_course_runs(self):
+        """
+        Make sure that if a course has no runs, the serializer still works as expected
+        """
+        self.assert_program_courses_serialized(self.program)
 
     def test_with_exclusions(self):
         """
         Test serializer with course_run exclusions within program
         """
+        request = make_request()
         course = CourseFactory()
         excluded_runs = []
         course_runs = CourseRunFactory.create_batch(2, course=course)
         excluded_runs.append(course_runs[0])
         program = ProgramFactory(courses=[course], excluded_course_runs=excluded_runs)
 
-        serializer_context = {'request': self.request, 'program': program}
+        serializer_context = {'request': request, 'program': program, 'course_runs': program.course_runs}
         serializer = ProgramCourseSerializer(course, context=serializer_context)
 
-        expected = MinimalCourseSerializer(course, context=serializer_context).data
-        expected['course_runs'] = MinimalCourseRunSerializer([course_runs[1]], many=True,
-                                                             context={'request': self.request}).data
+        expected = CourseSerializer(course, context=serializer_context).data
+        expected['course_runs'] = CourseRunSerializer([course_runs[1]], many=True,
+                                                      context={'request': request}).data
         self.assertDictEqual(serializer.data, expected)
 
     def test_with_published_course_runs_only_context(self):
         """ Verify setting the published_course_runs_only context value excludes unpublished course runs. """
         # Create a program and course. The course should have both published and un-published course runs.
+        request = make_request()
         course = CourseFactory()
-        courses = [course]
-        program = ProgramFactory(courses=courses)
+        program = ProgramFactory(courses=[course])
         unpublished_course_run = CourseRunFactory(status=CourseRunStatus.Unpublished, course=course)
         CourseRunFactory(status=CourseRunStatus.Published, course=course)
 
         # We do NOT expect the results to included the unpublished data
-        expected = MinimalCourseSerializer(courses, many=True, context={'request': self.request}).data
-        expected[0]['course_runs'] = [course_run for course_run in expected[0]['course_runs'] if
-                                      course_run['uuid'] != str(unpublished_course_run.uuid)]
-        self.assertEqual(len(expected[0]['course_runs']), 1)
+        expected = CourseSerializer(course, context={'request': request}).data
+        expected['course_runs'] = [course_run for course_run in expected['course_runs'] if
+                                   course_run['key'] != str(unpublished_course_run.key)]
+        self.assertEqual(len(expected['course_runs']), 1)
 
         serializer = ProgramCourseSerializer(
-            courses,
-            many=True,
+            course,
             context={
-                'request': self.request,
+                'request': request,
                 'program': program,
                 'published_course_runs_only': True,
+                'course_runs': program.course_runs,
             }
         )
 
         self.assertSequenceEqual(serializer.data, expected)
 
 
-class MinimalProgramSerializerTests(TestCase):
-    serializer_class = MinimalProgramSerializer
-
+class ProgramSerializerTests(TestCase):
     def create_program(self):
-        organizations = OrganizationFactory.create_batch(2)
+        organizations = [OrganizationFactory()]
         person = PersonFactory()
 
-        courses = CourseFactory.create_batch(3)
-        for course in courses:
-            CourseRunFactory.create_batch(2, course=course, staff=[person])
+        course = CourseFactory()
+        CourseRunFactory(course=course, staff=[person])
 
         program = ProgramFactory(
-            courses=courses,
+            courses=[course],
             authoring_organizations=organizations,
             credit_backing_organizations=organizations,
             corporate_endorsements=CorporateEndorsementFactory.create_batch(1),
@@ -439,28 +394,17 @@ class MinimalProgramSerializerTests(TestCase):
             'marketing_slug': program.marketing_slug,
             'marketing_url': program.marketing_url,
             'banner_image': image_field.to_representation(program.banner_image),
-            'courses': ProgramCourseSerializer(program.courses, many=True,
-                                               context={'request': request, 'program': program}).data,
-            'authoring_organizations': MinimalOrganizationSerializer(program.authoring_organizations, many=True).data,
+            'banner_image_url': program.banner_image_url,
+            'courses': ProgramCourseSerializer(
+                program.courses,
+                many=True,
+                context={
+                    'request': request,
+                    'program': program,
+                    'course_runs': program.course_runs,
+                }).data,
+            'authoring_organizations': OrganizationSerializer(program.authoring_organizations, many=True).data,
             'card_image_url': program.card_image_url,
-        }
-
-    def test_data(self):
-        request = make_request()
-        program = self.create_program()
-        serializer = self.serializer_class(program, context={'request': request})
-        expected = self.get_expected_data(program, request)
-        self.assertDictEqual(serializer.data, expected)
-
-
-class ProgramSerializerTests(MinimalProgramSerializerTests):  # pylint: disable=test-inherits-tests
-    serializer_class = ProgramSerializer
-
-    def get_expected_data(self, program, request):
-        expected = super().get_expected_data(program, request)
-        expected.update({
-            'marketing_slug': program.marketing_slug,
-            'marketing_url': program.marketing_url,
             'video': VideoSerializer(program.video).data,
             'credit_redemption_overview': program.credit_redemption_overview,
             'corporate_endorsements': CorporateEndorsementSerializer(program.corporate_endorsements, many=True).data,
@@ -478,11 +422,17 @@ class ProgramSerializerTests(MinimalProgramSerializerTests):  # pylint: disable=
             'max_hours_effort_per_week': program.max_hours_effort_per_week,
             'min_hours_effort_per_week': program.min_hours_effort_per_week,
             'overview': program.overview,
-            'price_ranges': [],
+            'price_ranges': program.price_ranges,
             'subjects': SubjectSerializer(program.subjects, many=True).data,
             'transcript_languages': [serialize_language_to_code(l) for l in program.transcript_languages],
-        })
-        return expected
+        }
+
+    def test_data(self):
+        request = make_request()
+        program = self.create_program()
+        serializer = ProgramSerializer(program, context={'request': request})
+        expected = self.get_expected_data(program, request)
+        self.assertDictEqual(dict(serializer.data), expected)
 
     def test_data_with_exclusions(self):
         """
@@ -622,45 +572,24 @@ class VideoSerializerTests(TestCase):
         self.assertDictEqual(serializer.data, expected)
 
 
-class MinimalOrganizationSerializerTests(TestCase):
-    serializer_class = MinimalOrganizationSerializer
-
-    def create_organization(self):
-        return OrganizationFactory()
+class OrganizationSerializerTests(TestCase):
+    def test_data(self):
+        organization = OrganizationFactory()
+        TAG = 'test'
+        organization.tags.add(TAG)
+        serializer = OrganizationSerializer(organization)
 
-    def get_expected_data(self, organization):
-        return {
-            'uuid': str(organization.uuid),
+        expected = {
             'key': organization.key,
             'name': organization.name,
-        }
-
-    def test_data(self):
-        organization = self.create_organization()
-        serializer = self.serializer_class(organization)
-        expected = self.get_expected_data(organization)
-        self.assertDictEqual(serializer.data, expected)
-
-
-class OrganizationSerializerTests(MinimalOrganizationSerializerTests):
-    TAG = 'test-tag'
-    serializer_class = OrganizationSerializer
-
-    def create_organization(self):
-        organization = super().create_organization()
-        organization.tags.add(self.TAG)
-        return organization
-
-    def get_expected_data(self, organization):
-        expected = super().get_expected_data(organization)
-        expected.update({
             'description': organization.description,
             'homepage_url': organization.homepage_url,
             'logo_image_url': organization.logo_image_url,
-            'tags': [self.TAG],
+            'tags': [TAG],
             'marketing_url': organization.marketing_url,
-        })
-        return expected
+        }
+
+        self.assertDictEqual(serializer.data, expected)
 
 
 class SeatSerializerTests(TestCase):
@@ -806,7 +735,7 @@ class ProgramSearchSerializerTests(TestCase):
             'partner': program.partner.short_code,
             'authoring_organization_uuids': get_uuids(program.authoring_organizations.all()),
             'subject_uuids': get_uuids([course.subjects for course in program.courses.all()]),
-            'staff_uuids': get_uuids([course.staff for course in program.course_runs.all()])
+            'staff_uuids': get_uuids([course.staff for course in program.course_runs])
         }
 
     def test_data(self):

course_discovery/apps/api/v1/tests/test_views/test_programs.py

@@ -40,14 +40,14 @@ class ProgramViewSetTests(APITestCase):
     def test_retrieve(self):
         """ Verify the endpoint returns the details for a single program. """
         program = ProgramFactory()
-        with self.assertNumQueries(34):
+        with self.assertNumQueries(33):
             self.assert_retrieve_success(program)
 
     def test_retrieve_without_course_runs(self):
         """ Verify the endpoint returns data for a program even if the program's courses have no course runs. """
         course = CourseFactory()
         program = ProgramFactory(courses=[course])
-        with self.assertNumQueries(49):
+        with self.assertNumQueries(55):
             self.assert_retrieve_success(program)
 
     def assert_list_results(self, url, expected, expected_query_count):
@@ -76,14 +76,14 @@ class ProgramViewSetTests(APITestCase):
         """ Verify the endpoint returns a list of all programs. """
         expected = ProgramFactory.create_batch(3)
         expected.reverse()
-        self.assert_list_results(self.list_path, expected, 40)
+        self.assert_list_results(self.list_path, expected, 14)
 
     def test_filter_by_type(self):
         """ Verify that the endpoint filters programs to those of a given type. """
         program_type_name = 'foo'
         program = ProgramFactory(type__name=program_type_name)
         url = self.list_path + '?type=' + program_type_name
-        self.assert_list_results(url, [program], 18)
+        self.assert_list_results(url, [program], 14)
 
         url = self.list_path + '?type=bar'
         self.assert_list_results(url, [], 4)
@@ -98,11 +98,11 @@ class ProgramViewSetTests(APITestCase):
         # Create a third program, which should be filtered out.
         ProgramFactory()
 
-        self.assert_list_results(url, expected, 29)
+        self.assert_list_results(url, expected, 14)
 
     @ddt.data(
         (ProgramStatus.Unpublished, False, 4),
-        (ProgramStatus.Active, True, 40),
+        (ProgramStatus.Active, True, 14),
     )
     @ddt.unpack
     def test_filter_by_marketable(self, status, is_marketable, expected_query_count):

course_discovery/apps/api/v1/views.py

@@ -64,23 +64,6 @@ def prefetch_related_objects_for_courses(queryset):
     return queryset
 
 
-def prefetch_related_objects_for_programs(queryset):
-    """
-    Pre-fetches the related objects that will be serialized with a `Program`.
-    Args:
-        queryset (QuerySet): original query
-    Returns:
-        QuerySet
-    """
-    course = serializers.PREFETCH_FIELDS['course'] + serializers.SELECT_RELATED_FIELDS['course']
-    course = ['courses__' + field for field in course]
-    queryset = queryset.prefetch_related(*course)
-
-    queryset = queryset.select_related(*serializers.SELECT_RELATED_FIELDS['program'])
-    queryset = queryset.prefetch_related(*serializers.PREFETCH_FIELDS['program'])
-    return queryset
-
-
 # pylint: disable=no-member
 class CatalogViewSet(viewsets.ModelViewSet):
     """ Catalog resource. """
@@ -393,12 +376,16 @@ class ProgramViewSet(viewsets.ReadOnlyModelViewSet):
     """ Program resource. """
     lookup_field = 'uuid'
     lookup_value_regex = '[0-9a-f-]+'
-    queryset = prefetch_related_objects_for_programs(Program.objects.all())
     permission_classes = (IsAuthenticated,)
     serializer_class = serializers.ProgramSerializer
     filter_backends = (DjangoFilterBackend,)
     filter_class = filters.ProgramFilter
 
+    def get_queryset(self):
+        # This method prevents prefetches on the program queryset from "stacking,"
+        # which happens when the queryset is stored in a class property.
+        return self.serializer_class.prefetch_queryset()
+
     def get_serializer_context(self, *args, **kwargs):
         context = super().get_serializer_context(*args, **kwargs)
         context['published_course_runs_only'] = int(self.request.GET.get('published_course_runs_only', 0))

course_discovery/apps/course_metadata/models.py

 import datetime
 import itertools
 import logging
+from collections import defaultdict
 from urllib.parse import urljoin
 from uuid import uuid4
 
@@ -635,57 +636,66 @@ class Program(TimeStampedModel):
 
     @property
     def course_runs(self):
+        """
+        Warning! Only call this method after retrieving programs from `ProgramSerializer.prefetch_queryset()`.
+        Otherwise, this method will incur many, many queries when fetching related courses and course runs.
+        """
         excluded_course_run_ids = [course_run.id for course_run in self.excluded_course_runs.all()]
-        return CourseRun.objects.filter(course__programs=self).exclude(id__in=excluded_course_run_ids)
+
+        for course in self.courses.all():
+            for run in course.course_runs.all():
+                if run.id not in excluded_course_run_ids:
+                    yield run
 
     @property
     def languages(self):
-        course_runs = self.course_runs.select_related('language')
-        return set(course_run.language for course_run in course_runs if course_run.language is not None)
+        return set(course_run.language for course_run in self.course_runs if course_run.language is not None)
 
     @property
     def transcript_languages(self):
-        course_runs = self.course_runs.prefetch_related('transcript_languages')
-        languages = [list(course_run.transcript_languages.all()) for course_run in course_runs]
+        languages = [course_run.transcript_languages.all() for course_run in self.course_runs]
         languages = itertools.chain.from_iterable(languages)
         return set(languages)
 
     @property
     def subjects(self):
-        courses = self.courses.prefetch_related('subjects')
-        subjects = [list(course.subjects.all()) for course in courses]
+        subjects = [course.subjects.all() for course in self.courses.all()]
         subjects = itertools.chain.from_iterable(subjects)
         return set(subjects)
 
     @property
     def seats(self):
-        applicable_seat_types = self.type.applicable_seat_types.values_list('slug', flat=True)
-        return Seat.objects.filter(course_run__in=self.course_runs, type__in=applicable_seat_types) \
-            .select_related('currency')
+        applicable_seat_types = set(seat_type.slug for seat_type in self.type.applicable_seat_types.all())
+
+        for run in self.course_runs:
+            for seat in run.seats.all():
+                if seat.type in applicable_seat_types:
+                    yield seat
 
     @property
     def seat_types(self):
-        return set(self.seats.values_list('type', flat=True))
+        return set(seat.type for seat in self.seats)
 
     @property
     def price_ranges(self):
-        seats = self.seats.values('currency').annotate(models.Min('price'), models.Max('price'))
-        price_ranges = []
+        currencies = defaultdict(list)
+        for seat in self.seats:
+            currencies[seat.currency].append(seat.price)
 
-        for seat in seats:
+        price_ranges = []
+        for currency, prices in currencies.items():
             price_ranges.append({
-                'currency': seat['currency'],
-                'min': seat['price__min'],
-                'max': seat['price__max'],
+                'currency': currency.code,
+                'min': min(prices),
+                'max': max(prices),
             })
+
         return price_ranges
 
     @property
     def start(self):
         """ Start datetime, calculated by determining the earliest start datetime of all related course runs. """
-        course_runs = self.course_runs
-
-        if course_runs:
+        if self.course_runs:
             start_dates = [course_run.start for course_run in self.course_runs if course_run.start]
 
             if start_dates:
@@ -695,7 +705,7 @@ class Program(TimeStampedModel):
 
     @property
     def staff(self):
-        staff = [list(course_run.staff.all()) for course_run in self.course_runs]
+        staff = [course_run.staff.all() for course_run in self.course_runs]
         staff = itertools.chain.from_iterable(staff)
         return set(staff)
 

course_discovery/apps/course_metadata/search_indexes.py

@@ -201,7 +201,7 @@ class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
         return [str(subject.uuid) for course in obj.courses.all() for subject in course.subjects.all()]
 
     def prepare_staff_uuids(self, obj):
-        return [str(staff.uuid) for course_run in obj.course_runs.all() for staff in course_run.staff.all()]
+        return [str(staff.uuid) for course_run in obj.course_runs for staff in course_run.staff.all()]
 
     def prepare_credit_backing_organizations(self, obj):
         return self._prepare_organizations(obj.credit_backing_organizations.all())

course_discovery/apps/course_metadata/tests/test_admin.py

@@ -100,7 +100,7 @@ class AdminTests(TestCase):
         """ Verify that course selection page with posting the data. """
 
         self.assertEqual(1, self.program.excluded_course_runs.all().count())
-        self.assertEqual(3, len(self.program.course_runs.all()))
+        self.assertEqual(3, sum(1 for _ in self.program.course_runs))
 
         params = {
             'excluded_course_runs': [self.excluded_course_run.id, self.course_runs[0].id],
@@ -114,7 +114,7 @@ class AdminTests(TestCase):
             target_status_code=200
         )
         self.assertEqual(2, self.program.excluded_course_runs.all().count())
-        self.assertEqual(2, len(self.program.course_runs.all()))
+        self.assertEqual(2, sum(1 for _ in self.program.course_runs))
 
     def test_page_with_post_without_course_run(self):
         """ Verify that course selection page without posting any selected excluded check run. """
@@ -132,7 +132,7 @@ class AdminTests(TestCase):
             target_status_code=200
         )
         self.assertEqual(0, self.program.excluded_course_runs.all().count())
-        self.assertEqual(4, len(self.program.course_runs.all()))
+        self.assertEqual(4, sum(1 for _ in self.program.course_runs))
         response = self.client.get(reverse('admin_metadata:update_course_runs', args=(self.program.id,)))
         self.assertNotContains(response, '<input checked="checked")')