Merge pull request #417 from edx/bderusha/manually-update-course-run-parent-ECOM-6226

Add 'canonical' field

course_discovery/apps/course_metadata/admin.py

@@ -4,7 +4,7 @@ from django.http import HttpResponseRedirect
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 
-from course_discovery.apps.course_metadata.forms import ProgramAdminForm
+from course_discovery.apps.course_metadata.forms import ProgramAdminForm, CourseAdminForm
 from course_discovery.apps.course_metadata.models import *  # pylint: disable=wildcard-import
 from course_discovery.apps.course_metadata.publishers import ProgramPublisherException
 
@@ -42,6 +42,7 @@ class CorporateEndorsementsInline(admin.TabularInline):
 
 @admin.register(Course)
 class CourseAdmin(admin.ModelAdmin):
+    form = CourseAdminForm
     list_display = ('uuid', 'key', 'title',)
     list_filter = ('partner',)
     ordering = ('key', 'title',)

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -107,8 +107,20 @@ class CoursesApiDataLoader(AbstractDataLoader):
 
             try:
                 body = self.clean_strings(body)
-                course = self.update_course(body)
-                self.update_course_run(course, body)
+                course_run = self.get_course_run(body)
+
+                if course_run:
+                    self.update_course_run(course_run, body)
+                    course = getattr(course_run, 'canonical_for_course', False)
+                    if course:
+                        course = self.update_course(course, body)
+                        logger.info('Processed course with key [%s].', course.key)
+                else:
+                    course, created = self.get_or_create_course(body)
+                    course_run = self.create_course_run(course, body)
+                    if created:
+                        course.canonical_course_run = course_run
+                        course.save()
             except:  # pylint: disable=bare-except
                 msg = 'An error occurred while updating {course_run} from {api_url}'.format(
                     course_run=course_run_id,
@@ -116,14 +128,33 @@ class CoursesApiDataLoader(AbstractDataLoader):
                 )
                 logger.exception(msg)
 
-    def update_course(self, body):
+    def get_course_run(self, body):
+        course_run_key = body['id']
+        try:
+            return CourseRun.objects.get(key__iexact=course_run_key)
+        except CourseRun.DoesNotExist:
+            return None
+
+    def update_course_run(self, course_run, body):
+        validated_data = self.format_course_run_data(body)
+        self._update_instance(course_run, validated_data)
+
+        logger.info('Processed course run with UUID [%s].', course_run.uuid)
+
+    def create_course_run(self, course, body):
+        defaults = self.format_course_run_data(body, course=course)
+
+        return CourseRun.objects.create(**defaults)
+
+    def get_or_create_course(self, body):
         course_run_key = CourseKey.from_string(body['id'])
         course_key = self.get_course_key_from_course_run_key(course_run_key)
+        defaults = self.format_course_data(body)
+        # We need to add the key to the defaults because django ignores kwargs with __
+        # separators when constructing the create request
+        defaults['key'] = course_key
+        defaults['partner'] = self.partner
 
-        defaults = {
-            'key': course_key,
-            'title': body['name'],
-        }
         course, created = Course.objects.get_or_create(key__iexact=course_key, partner=self.partner, defaults=defaults)
 
         if created:
@@ -133,16 +164,27 @@ class CoursesApiDataLoader(AbstractDataLoader):
             defaults = {'key': key}
             organization, __ = Organization.objects.get_or_create(key__iexact=key, partner=self.partner,
                                                                   defaults=defaults)
+
             course.authoring_organizations.add(organization)
 
-        logger.info('Processed course with key [%s].', course_key)
+        return (course, created)
+
+    def update_course(self, course, body):
+        validated_data = self.format_course_data(body)
+        self._update_instance(course, validated_data)
+
+        logger.info('Processed course with key [%s].', course.key)
+
         return course
 
-    def update_course_run(self, course, body):
-        key = body['id']
+    def _update_instance(self, instance, validated_data):
+        for attr, value in validated_data.items():
+            setattr(instance, attr, value)
+        instance.save()
 
+    def format_course_run_data(self, body, course=None):
         defaults = {
-            'key': key,
+            'key': body['id'],
             'end': self.parse_date(body['end']),
             'enrollment_start': self.parse_date(body['enrollment_start']),
             'enrollment_end': self.parse_date(body['enrollment_end']),
@@ -162,10 +204,17 @@ class CoursesApiDataLoader(AbstractDataLoader):
                 'mobile_available': body.get('mobile_available') or False,
             })
 
-        course_run, __ = course.course_runs.update_or_create(key__iexact=key, defaults=defaults)
+        if course:
+            defaults['course'] = course
+
+        return defaults
+
+    def format_course_data(self, body):
+        defaults = {
+            'title': body['name'],
+        }
 
-        logger.info('Processed course run with key [%s].', course_run.key)
-        return course_run
+        return defaults
 
     def get_pacing_type(self, body):
         pacing = body.get('pacing')

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

@@ -377,73 +377,78 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
         return kwargs
 
     def process_node(self, data):
-        course_run_key = CourseKey.from_string(data['field_course_id'])
-        key = self.get_course_key_from_course_run_key(course_run_key)
+        course_run = self.get_course_run(data)
 
-        # Clean the title for the course and course run
-        data['field_course_course_title']['value'] = self.clean_html(data['field_course_course_title']['value'])
+        if course_run:
+            self.update_course_run(course_run, data)
+            try:
+                course = self.update_course(course_run.canonical_for_course, data)
+                self.set_subjects(course, data)
+                self.set_authoring_organizations(course, data)
+                logger.info('Processed course with key [%s].', course.key)
+            except AttributeError:
+                pass
+        else:
+            course, created = self.get_or_create_course(data)
+            course_run = self.create_course_run(course, data)
+            if created:
+                course.canonical_course_run = course_run
+                course.save()
 
-        defaults = {
-            'key': key,
-            'title': self.clean_html(data['field_course_course_title']['value']),
-            'number': data['field_course_code'],
-            'full_description': self.get_description(data),
-            'video': self.get_video(data),
-            'short_description': self.clean_html(data['field_course_sub_title_short']),
-            'level_type': self.get_level_type(data['field_course_level']),
-            'card_image_url': self._get_nested_url(data.get('field_course_image_promoted')),
-        }
-        course, created = Course.objects.get_or_create(key__iexact=key, partner=self.partner, defaults=defaults)
+    def get_course_run(self, data):
+        course_run_key = data['field_course_id']
+        try:
+            return CourseRun.objects.get(key__iexact=course_run_key)
+        except CourseRun.DoesNotExist:
+            return None
 
-        # If the course already exists update the fields only if the course_run we got from drupal is published.
-        # People often put temp data into required drupal fields for unpublished courses. We don't want to  overwrite
-        # the course info with this data, so we only update course info from published sources.
-        published = self.get_course_run_status(data) == CourseRunStatus.Published
-        if not created and published:
-            for attr, value in defaults.items():
-                setattr(course, attr, value)
-            course.save()
+    def update_course_run(self, course_run, data):
+        validated_data = self.format_course_run_data(data, course_run.course)
+        self._update_instance(course_run, validated_data)
+        self.set_course_run_staff(course_run, data)
+        self.set_course_run_transcript_languages(course_run, data)
 
-        self.set_subjects(course, data)
-        self.set_authoring_organizations(course, data)
-        self.create_course_run(course, data)
+        logger.info('Processed course run with UUID [%s].', course_run.uuid)
 
-        logger.info('Processed course with key [%s].', key)
-        return course
+    def create_course_run(self, course, data):
+        defaults = self.format_course_run_data(data, course)
 
-    def get_description(self, data):
-        description = (data.get('field_course_body', {}) or {}).get('value')
-        description = description or (data.get('field_course_description', {}) or {}).get('value')
-        description = description or ''
-        description = self.clean_html(description)
-        return description
+        course_run = CourseRun.objects.create(**defaults)
+        self.set_course_run_staff(course_run, data)
+        self.set_course_run_transcript_languages(course_run, data)
 
-    def get_course_run_status(self, data):
-        return CourseRunStatus.Published if bool(int(data['status'])) else CourseRunStatus.Unpublished
+        return course_run
 
-    def get_level_type(self, name):
-        level_type = None
+    def get_or_create_course(self, data):
+        course_run_key = CourseKey.from_string(data['field_course_id'])
+        key = self.get_course_key_from_course_run_key(course_run_key)
+        defaults = self.format_course_data(data, key=key)
 
-        if name:
-            level_type, __ = LevelType.objects.get_or_create(name=name)
+        course, created = Course.objects.get_or_create(key__iexact=key, partner=self.partner, defaults=defaults)
 
-        return level_type
+        if created:
+            self.set_subjects(course, data)
+            self.set_authoring_organizations(course, data)
 
-    def get_video(self, data):
-        video_url = self._get_nested_url(data.get('field_course_video') or data.get('field_product_video'))
-        image_url = self._get_nested_url(data.get('field_course_image_featured_card'))
-        return self.get_or_create_video(video_url, image_url)
+        return (course, created)
 
-    def get_pacing_type(self, data):
-        self_paced = data.get('field_course_self_paced', False)
-        return CourseRunPacing.Self if self_paced else CourseRunPacing.Instructor
+    def update_course(self, course, data):
+        validated_data = self.format_course_data(data)
+        self._update_instance(course, validated_data)
 
-    def get_hidden(self, data):
-        # 'couse' [sic]. The field is misspelled on Drupal. ಠ_ಠ
-        hidden = data.get('field_couse_is_hidden', False)
-        return hidden is True
+        if self.get_course_run_status(data) != CourseRunStatus.Published:
+            logger.warning(
+                'Updating course [%s] with data from unpublished course_run [%s].', course.uuid, data['field_course_id']
+            )
 
-    def create_course_run(self, course, data):
+        return course
+
+    def _update_instance(self, instance, validated_data):
+        for attr, value in validated_data.items():
+            setattr(instance, attr, value)
+        instance.save()
+
+    def format_course_run_data(self, data, course):
         uuid = data['uuid']
         key = data['field_course_id']
         slug = data['url'].split('/')[-1]
@@ -457,7 +462,6 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
 
         defaults = {
             'key': key,
-            'course': course,
             'uuid': uuid,
             'title_override': self.clean_html(data['field_course_course_title']['value']),
             'language': language,
@@ -470,6 +474,7 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
             'weeks_to_complete': None,
             'mobile_available': data.get('field_course_enrollment_mobile') or False,
             'video': course.video,
+            'course': course,
         }
 
         if weeks_to_complete:
@@ -478,18 +483,57 @@ class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
             weeks_to_complete = rrule.rrule(rrule.WEEKLY, dtstart=start, until=end).count()
             defaults['weeks_to_complete'] = int(weeks_to_complete)
 
-        try:
-            course_run, __ = CourseRun.objects.update_or_create(key__iexact=key, defaults=defaults)
-        except TypeError:
-            # TODO Fix the data in Drupal (ECOM-5304)
-            logger.error('Multiple course runs are identified by the key [%s] or UUID [%s].', key, uuid)
-            return None
+        return defaults
 
-        self.set_course_run_staff(course_run, data)
-        self.set_course_run_transcript_languages(course_run, data)
+    def format_course_data(self, data, key=None):
+        if not key:
+            course_run_key = CourseKey.from_string(data['field_course_id'])
+            key = self.get_course_key_from_course_run_key(course_run_key)
 
-        logger.info('Processed course run with UUID [%s].', uuid)
-        return course_run
+        defaults = {
+            'key': key,
+            'title': self.clean_html(data['field_course_course_title']['value']),
+            'number': data['field_course_code'],
+            'full_description': self.get_description(data),
+            'video': self.get_video(data),
+            'short_description': self.clean_html(data['field_course_sub_title_short']),
+            'level_type': self.get_level_type(data['field_course_level']),
+            'card_image_url': self._get_nested_url(data.get('field_course_image_promoted')),
+        }
+
+        return defaults
+
+    def get_description(self, data):
+        description = (data.get('field_course_body', {}) or {}).get('value')
+        description = description or (data.get('field_course_description', {}) or {}).get('value')
+        description = description or ''
+        description = self.clean_html(description)
+        return description
+
+    def get_course_run_status(self, data):
+        return CourseRunStatus.Published if bool(int(data['status'])) else CourseRunStatus.Unpublished
+
+    def get_level_type(self, name):
+        level_type = None
+
+        if name:
+            level_type, __ = LevelType.objects.get_or_create(name=name)
+
+        return level_type
+
+    def get_video(self, data):
+        video_url = self._get_nested_url(data.get('field_course_video') or data.get('field_product_video'))
+        image_url = self._get_nested_url(data.get('field_course_image_featured_card'))
+        return self.get_or_create_video(video_url, image_url)
+
+    def get_pacing_type(self, data):
+        self_paced = data.get('field_course_self_paced', False)
+        return CourseRunPacing.Self if self_paced else CourseRunPacing.Instructor
+
+    def get_hidden(self, data):
+        # 'couse' [sic]. The field is misspelled on Drupal. ಠ_ಠ
+        hidden = data.get('field_couse_is_hidden', False)
+        return hidden is True
 
     def _get_objects_by_uuid(self, object_type, raw_objects_data):
         uuids = [_object.get('uuid') for _object in raw_objects_data]

course_discovery/apps/course_metadata/data_loaders/tests/mock_data.py

@@ -90,6 +90,75 @@ COURSES_API_BODIES = [
     },
 ]
 
+COURSES_API_BODY_ORIGINAL = {
+    'effort': None,
+    'end': None,
+    'enrollment_start': None,
+    'enrollment_end': None,
+    'id': 'course-v1:KyotoUx+000x+3T2016',
+    'media': {
+        'course_image': {
+            'uri': '/asset-v1:KyotoUx+000x+3T2016+type@asset+block@000x-course_imagec-378x225.jpg'
+        },
+        'course_video': {
+            'uri': None
+        }
+    },
+    'name': 'Evolution of the Human Sociality ORIGINAL',
+    'number': '000x',
+    'org': 'KyotoUx',
+    'short_description': '',
+    'start': None,
+    'mobile_available': None,
+    'hidden': False,
+}
+
+COURSES_API_BODY_SECOND = {
+    'effort': None,
+    'end': None,
+    'enrollment_start': None,
+    'enrollment_end': None,
+    'id': 'course-v1:KyotoUx+000x+1T2020',
+    'media': {
+        'course_image': {
+            'uri': '/asset-v1:KyotoUx+000x+1T2020+type@asset+block@000x-course_imagec-378x225.jpg'
+        },
+        'course_video': {
+            'uri': None
+        }
+    },
+    'name': 'Evolution of the Human Sociality SECOND',
+    'number': '000x',
+    'org': 'KyotoUx',
+    'short_description': '',
+    'start': None,
+    'mobile_available': None,
+    'hidden': False,
+}
+
+COURSES_API_BODY_UPDATED = {
+    'effort': None,
+    'end': None,
+    'enrollment_start': None,
+    'enrollment_end': None,
+    'id': 'course-v1:KyotoUx+000x+3T2016',
+    'media': {
+        'course_image': {
+            'uri': '/asset-v1:KyotoUx+000x+3T2016+type@asset+block@000x-course_imagec-378x225.jpg'
+        },
+        'course_video': {
+            'uri': None
+        }
+    },
+    'name': 'Evolution of the Human Sociality UPDATED',
+    'number': '000x',
+    'org': 'KyotoUx',
+    'short_description': '',
+    'start': None,
+    'mobile_available': None,
+    'hidden': True,
+}
+
 ECOMMERCE_API_BODIES = [
     {
         "id": "audit/course/run",
@@ -1173,9 +1242,7 @@ MARKETING_SITE_API_PERSON_BODIES = [
     }
 ]
 
-MULTI_COURSE_RUN_COURSE_NUMBER = 'CB22x'
-
-MARKETING_SITE_API_COURSE_BODIES = [
+UNIQUE_MARKETING_SITE_API_COURSE_BODIES = [
     {
         'field_course_code': 'CS50x',
         'field_course_course_title': {
@@ -1781,7 +1848,7 @@ MARKETING_SITE_API_COURSE_BODIES = [
         'vuuid': '28da5064-b570-4883-8c53-330d1893ab49'
     },
     {
-        'field_course_code': MULTI_COURSE_RUN_COURSE_NUMBER,
+        'field_course_code': 'CB22x',
         'field_course_course_title': {
             'value': 'The Ancient Greek Hero',
             'format': 'basic_html'
@@ -2051,14 +2118,14 @@ MARKETING_SITE_API_COURSE_BODIES = [
     }
 ]
 
-MARKETING_SITE_API_UNPUBLISHED_COPY_COURSE_BODY = {
-    'field_course_code': MULTI_COURSE_RUN_COURSE_NUMBER,
+ORIGINAL_MARKETING_SITE_API_COURSE_BODY = {
+    'field_course_code': 'CB22x',
     'field_course_course_title': {
-        'value': 'The Ancient Greek Hero UNPUBLISHED',
+        'value': 'The Ancient Greek Hero ORIGINAL',
         'format': 'basic_html'
     },
     'field_course_description': {
-        'value': 'UNPUBLISHED <p><b>NOTE ABOUT OUR START DATE:</b> Although the course was launched on March 13th, '
+        'value': 'ORIGINAL <p><b>NOTE ABOUT OUR START DATE:</b> Although the course was launched on March 13th, '
                  'it\u0027s not too late to start participating! New participants will be joining the course until '
                  '<strong>registration closes on July 11</strong>. We offer everyone a flexible schedule and '
                  'multiple paths for participation. You can work through the course videos and readings at your '
@@ -2189,7 +2256,7 @@ MARKETING_SITE_API_UNPUBLISHED_COPY_COURSE_BODY = {
         'name': 'tombstone_courses.jpg',
         'mime': 'image/jpeg',
         'size': '34861',
-        'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses_UNPUBLISHED.jpg',
+        'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses_ORIGINAL.jpg',
         'timestamp': '1384348699',
         'owner': {
             'uri': 'https://www.edx.org/user/1',
@@ -2284,7 +2351,7 @@ MARKETING_SITE_API_UNPUBLISHED_COPY_COURSE_BODY = {
     'field_course_enrollment_credit': None,
     'field_course_is_disabled': None,
     'field_course_tags': [],
-    'field_course_sub_title_short': 'UNPUBLISHED A survey of ancient Greek literature focusing on classical concepts of'
+    'field_course_sub_title_short': 'ORIGINAL A survey of ancient Greek literature focusing on classical concepts of'
                                     ' the hero and how they can inform our understanding of the human condition.',
     'field_course_length_weeks': '23 weeks',
     'field_course_start_date_style': None,
@@ -2321,14 +2388,284 @@ MARKETING_SITE_API_UNPUBLISHED_COPY_COURSE_BODY = {
     'vuuid': 'e0f8c80a-b377-4546-b247-1c94ab3a218d'
 }
 
-MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY = {
-    'field_course_code': MULTI_COURSE_RUN_COURSE_NUMBER,
+UPDATED_MARKETING_SITE_API_COURSE_BODY = {
+    'field_course_code': 'CB22x',
+    'field_course_course_title': {
+        'value': 'The Ancient Greek Hero UPDATED',
+        'format': 'basic_html'
+    },
+    'field_course_description': {
+        'value': 'UPDATED <p><b>NOTE ABOUT OUR START DATE:</b> Although the course was launched on March 13th, '
+                 'it\u0027s not too late to start participating! New participants will be joining the course until '
+                 '<strong>registration closes on July 11</strong>. We offer everyone a flexible schedule and '
+                 'multiple paths for participation. You can work through the course videos and readings at your '
+                 'own pace to complete the associated exercises <strong>by August 26</strong>, the official course '
+                 'end date. Or, you may choose to \u0022audit\u0022 the course by exploring just the particular '
+                 'videos and readings that seem most suited to your interests. You are free to do as much or as '
+                 'little as you would like!</p>\n<h3>\n\tOverview</h3>\n<p>What is it to be human, and how can '
+                 'ancient concepts of the heroic and anti-heroic inform our understanding of the human condition? '
+                 'That question is at the core of The Ancient Greek Hero, which introduces (or reintroduces) '
+                 'students to the great texts of classical Greek culture by focusing on concepts of the Hero in an '
+                 'engaging, highly comparative way.</p>\n<p>The classical Greeks\u0027 concepts of Heroes and the '
+                 '\u0022heroic\u0022 were very different from the way we understand the term today. In this '
+                 'course, students analyze Greek heroes and anti-heroes in their own historical contexts, in order '
+                 'to gain an understanding of these concepts as they were originally understood while also '
+                 'learning how they can inform our understanding of the human condition in general.</p>\n<p>In '
+                 'Greek tradition, a hero was a human, male or female, of the remote past, who was endowed with '
+                 'superhuman abilities by virtue of being descended from an immortal god. Rather than being '
+                 'paragons of virtue, as heroes are viewed in many modern cultures, ancient Greek heroes had all '
+                 'of the qualities and faults of their fellow humans, but on a much larger scale. Further, despite '
+                 'their mortality, heroes, like the gods, were objects of cult worship \u2013 a dimension which is '
+                 'also explored in depth in the course.</p>\n<p>The original sources studied in this course include'
+                 ' the Homeric Iliad and Odyssey; tragedies of Aeschylus, Sophocles, and Euripides; songs of Sappho'
+                 ' and Pindar; dialogues of Plato; historical texts of Herodotus; and more, including the '
+                 'intriguing but rarely studied dialogue \u0022On Heroes\u0022 by Philostratus. All works are '
+                 'presented in English translation, with attention to the subtleties of the original Greek. These '
+                 'original sources are frequently supplemented both by ancient art and by modern comparanda, '
+                 'including opera and cinema (from Jacques Offenbach\u0027s opera Tales of Hoffman to Ridley '
+                 'Scott\u0027s science fiction classic Blade Runner).</p>',
+        'format': 'standard_html'
+    },
+    'field_course_start_date': '1363147200',
+    'field_course_effort': '4-6 hours / week',
+    'field_course_school_node': [
+        {
+            'uri': 'https://www.edx.org/node/242',
+            'id': '242',
+            'resource': 'node',
+            'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6'
+        }
+    ],
+    'field_course_end_date': '1376971200',
+    'field_course_video': [],
+    'field_course_resources': [],
+    'field_course_sub_title_long': {
+        'value': '<p>A survey of ancient Greek literature focusing on classical concepts of the hero and how they '
+                 'can inform our understanding of the human condition.</p>\n',
+        'format': 'plain_text'
+    },
+    'field_course_subject': [
+        {
+            'uri': 'https://www.edx.org/node/652',
+            'id': '652',
+            'resource': 'node',
+            'uuid': 'c8579e1c-99f2-4a95-988c-3542909f055e'
+        },
+        {
+            'uri': 'https://www.edx.org/node/653',
+            'id': '653',
+            'resource': 'node',
+            'uuid': '00e5d5e0-ce45-4114-84a1-50a5be706da5'
+        },
+        {
+            'uri': 'https://www.edx.org/node/655',
+            'id': '655',
+            'resource': 'node',
+            'uuid': '74b6ed2a-3ba0-49be-adc9-53f7256a12e1'
+        }
+    ],
+    'field_course_statement_title': None,
+    'field_course_statement_body': [],
+    'field_course_status': 'past',
+    'field_course_start_override': None,
+    'field_course_email': None,
+    'field_course_syllabus': [],
+    'field_course_staff': [
+        {
+            'uri': 'https://www.edx.org/node/564',
+            'id': '564',
+            'resource': 'node',
+            'uuid': 'ae56688a-f2b6-4981-9aa7-5c66b68cb13e'
+        },
+        {
+            'uri': 'https://www.edx.org/node/565',
+            'id': '565',
+            'resource': 'node',
+            'uuid': '56d13e72-353f-48fd-9be7-6f20ef467bb7'
+        },
+        {
+            'uri': 'https://www.edx.org/node/566',
+            'id': '566',
+            'resource': 'node',
+            'uuid': '69a415db-3db7-436a-8d02-e571c4c4c75a'
+        },
+        {
+            'uri': 'https://www.edx.org/node/567',
+            'id': '567',
+            'resource': 'node',
+            'uuid': '1639460f-598c-45b7-90c2-bbdbf87cdd54'
+        },
+        {
+            'uri': 'https://www.edx.org/node/568',
+            'id': '568',
+            'resource': 'node',
+            'uuid': '09154d2c-7f31-477c-9d3c-d8cba9af846e'
+        },
+        {
+            'uri': 'https://www.edx.org/node/820',
+            'id': '820',
+            'resource': 'node',
+            'uuid': '05b7ab45-de9a-49d6-8010-04c68fc9fd55'
+        },
+        {
+            'uri': 'https://www.edx.org/node/821',
+            'id': '821',
+            'resource': 'node',
+            'uuid': '8a8d68c4-ab5b-40c5-b897-2d44aed2194d'
+        },
+        {
+            'uri': 'https://www.edx.org/node/822',
+            'id': '822',
+            'resource': 'node',
+            'uuid': 'c3e16519-a23f-4f21-908b-463375b492df'
+        }
+    ],
+    'field_course_staff_override': 'G. Nagy, L. Muellner...',
+    'field_course_image_promoted': {
+        'fid': '32381',
+        'name': 'tombstone_courses.jpg',
+        'mime': 'image/jpeg',
+        'size': '34861',
+        'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses_UPDATED.jpg',
+        'timestamp': '1384348699',
+        'owner': {
+            'uri': 'https://www.edx.org/user/1',
+            'id': '1',
+            'resource': 'user',
+            'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+        },
+        'uuid': '1471888c-a451-4f97-9bb2-ad20c9a43c2d'
+    },
+    'field_course_image_banner': {
+        'fid': '32285',
+        'name': 'cb22x_608x211.jpg',
+        'mime': 'image/jpeg',
+        'size': '25909',
+        'url': 'https://www.edx.org/sites/default/files/course/image/banner/cb22x_608x211.jpg',
+        'timestamp': '1384348498',
+        'owner': {
+            'uri': 'https://www.edx.org/user/1',
+            'id': '1',
+            'resource': 'user',
+            'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+        },
+        'uuid': '15022bf7-e367-4a5c-b115-3755016de286'
+    },
+    'field_course_image_tile': {
+        'fid': '32475',
+        'name': 'cb22x-listing-banner.jpg',
+        'mime': 'image/jpeg',
+        'size': '47678',
+        'url': 'https://www.edx.org/sites/default/files/course/image/tile/cb22x-listing-banner.jpg',
+        'timestamp': '1384348906',
+        'owner': {
+            'uri': 'https://www.edx.org/user/1',
+            'id': '1',
+            'resource': 'user',
+            'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+        },
+        'uuid': '71735cc4-7ac3-4065-ad92-6f18f979eb0e'
+    },
+    'field_course_image_video': {
+        'fid': '32573',
+        'name': 'h_no_video_320x211_1_0.jpg',
+        'mime': 'image/jpeg',
+        'size': '2829',
+        'url': 'https://www.edx.org/sites/default/files/course/image/video/h_no_video_320x211_1_0.jpg',
+        'timestamp': '1384349121',
+        'owner': {
+            'uri': 'https://www.edx.org/user/1',
+            'id': '1',
+            'resource': 'user',
+            'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+        },
+        'uuid': '4d18789f-0909-4289-9d58-2292e5d03aee'
+    },
+    'field_course_id': 'HarvardX/CB22x/2014_Spring',
+    'field_course_image_sample_cert': [],
+    'field_course_image_sample_thumb': [],
+    'field_course_enrollment_audit': True,
+    'field_course_enrollment_honor': False,
+    'field_course_enrollment_verified': False,
+    'field_course_xseries_enable': False,
+    'field_course_statement_image': [],
+    'field_course_image_card': [],
+    'field_course_image_featured_card': [],
+    'field_course_code_override': None,
+    'field_course_video_link_mp4': [],
+    'field_course_video_duration': None,
+    'field_course_self_paced': True,
+    'field_course_new': None,
+    'field_course_registration_dates': {
+        'value': '1384348442',
+        'value2': None,
+        'duration': None
+    },
+    'field_course_enrollment_prof_ed': None,
+    'field_course_enrollment_ap_found': None,
+    'field_cource_price': None,
+    'field_course_additional_keywords': 'Free,',
+    'field_course_enrollment_mobile': None,
+    'field_course_part_of_products': [],
+    'field_course_level': None,
+    'field_course_what_u_will_learn': [],
+    'field_course_video_locale_lang': [],
+    'field_course_languages': [],
+    'field_couse_is_hidden': None,
+    'field_xseries_display_override': [],
+    'field_course_extra_description': [],
+    'field_course_extra_desc_title': None,
+    'field_course_body': [],
+    'field_course_enrollment_no_id': None,
+    'field_course_has_prerequisites': True,
+    'field_course_enrollment_credit': None,
+    'field_course_is_disabled': None,
+    'field_course_tags': [],
+    'field_course_sub_title_short': 'UPDATED A survey of ancient Greek literature focusing on classical concepts of'
+                                    ' the hero and how they can inform our understanding of the human condition.',
+    'field_course_length_weeks': '23 weeks',
+    'field_course_start_date_style': None,
+    'field_course_head_prom_bkg_color': None,
+    'field_course_head_promo_image': [],
+    'field_course_head_promo_text': [],
+    'field_course_outcome': None,
+    'field_course_required_weeks': None,
+    'field_course_required_days': None,
+    'field_course_required_hours': None,
+    'nid': '563',
+    'vid': '8080',
+    'is_new': False,
+    'type': 'course',
+    'title': 'HarvardX: CB22x: The Ancient Greek Hero',
+    'language': 'und',
+    'url': 'https://www.edx.org/course/ancient-greek-hero-harvardx-cb22x',
+    'edit_url': 'https://www.edx.org/node/563/edit',
+    'status': '1',
+    'promote': '0',
+    'sticky': '0',
+    'created': '1384348442',
+    'changed': '1443028625',
+    'author': {
+        'uri': 'https://www.edx.org/user/143',
+        'id': '143',
+        'resource': 'user',
+        'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+    },
+    'log': 'Updated by FeedsNodeProcessor',
+    'revision': None,
+    'body': [],
+    'uuid': '6b8b779f-f567-4e98-aa41-a265d6fa073d',
+    'vuuid': 'e0f8c80a-b377-4546-b247-1c94ab3a218d'
+}
+
+NEW_RUN_MARKETING_SITE_API_COURSE_BODY = {
+    'field_course_code': 'CB22x',
     'field_course_course_title': {
-        'value': 'The Ancient Greek Hero PUBLISHED',
+        'value': 'The Ancient Greek Hero NEW_RUN',
         'format': 'basic_html'
     },
     'field_course_description': {
-        'value': 'PUBLISHED <p><b>NOTE ABOUT OUR START DATE:</b> Although the course was launched on March 13th, '
+        'value': 'NEW_RUN <p><b>NOTE ABOUT OUR START DATE:</b> Although the course was launched on March 13th, '
                  'it\u0027s not too late to start participating! New participants will be joining the course until '
                  '<strong>registration closes on July 11</strong>. We offer everyone a flexible schedule and '
                  'multiple paths for participation. You can work through the course videos and readings at your '
@@ -2459,7 +2796,7 @@ MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY = {
         'name': 'tombstone_courses.jpg',
         'mime': 'image/jpeg',
         'size': '34861',
-        'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses_PUBLISHED.jpg',
+        'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses_NEW_RUN.jpg',
         'timestamp': '1384348699',
         'owner': {
             'uri': 'https://www.edx.org/user/1',
@@ -2514,7 +2851,7 @@ MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY = {
         },
         'uuid': '4d18789f-0909-4289-9d58-2292e5d03aee'
     },
-    'field_course_id': 'HarvardX/CB22x/2015_Spring',
+    'field_course_id': 'HarvardX/CB22x/2016_Spring',
     'field_course_image_sample_cert': [],
     'field_course_image_sample_thumb': [],
     'field_course_enrollment_audit': True,
@@ -2554,7 +2891,7 @@ MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY = {
     'field_course_enrollment_credit': None,
     'field_course_is_disabled': None,
     'field_course_tags': [],
-    'field_course_sub_title_short': 'PUBLISHED A survey of ancient Greek literature focusing on classical concepts of'
+    'field_course_sub_title_short': 'NEW_RUN A survey of ancient Greek literature focusing on classical concepts of'
                                     ' the hero and how they can inform our understanding of the human condition.',
     'field_course_length_weeks': '23 weeks',
     'field_course_start_date_style': None,
@@ -2587,6 +2924,6 @@ MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY = {
     'log': 'Updated by FeedsNodeProcessor',
     'revision': None,
     'body': [],
-    'uuid': '6b8b779f-f567-4e98-aa41-a265d6fa073e',
-    'vuuid': 'e0f8c80a-b377-4546-b247-1c94ab3a218e'
+    'uuid': '6b8b779f-f567-4e98-aa41-a265d6fa073a',
+    'vuuid': 'e0f8c80a-b377-4546-b247-1c94ab3a218a'
 }

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

@@ -133,7 +133,8 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
     def api_url(self):
         return self.partner.courses_api_url
 
-    def mock_api(self):
+    def mock_api(self, bodies=None):
+        if not bodies:
             bodies = mock_data.COURSES_API_BODIES
         url = self.api_url + 'courses/'
         responses.add_callback(
@@ -185,6 +186,8 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
         for field, value in expected_values.items():
             self.assertEqual(getattr(course_run, field), value, 'Field {} is invalid.'.format(field))
 
+        return course_run
+
     @responses.activate
     @ddt.data(True, False)
     def test_ingest(self, partner_has_marketing_site):
@@ -227,6 +230,39 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
                 )
                 mock_logger.exception.assert_called_with(msg)
 
+    @responses.activate
+    def test_ingest_canonical(self):
+        """ Verify the method ingests data from the Courses API. """
+        self.assertEqual(Course.objects.count(), 0)
+        self.assertEqual(CourseRun.objects.count(), 0)
+
+        self.mock_api([
+            mock_data.COURSES_API_BODY_ORIGINAL,
+            mock_data.COURSES_API_BODY_SECOND,
+            mock_data.COURSES_API_BODY_UPDATED,
+        ])
+        self.loader.ingest()
+
+        # Verify the CourseRun was created correctly by no errors raised
+        course_run_orig = CourseRun.objects.get(key=mock_data.COURSES_API_BODY_ORIGINAL['id'])
+
+        # Verify that a course has been created and set as canonical by no errors raised
+        course = course_run_orig.canonical_for_course
+
+        # Verify the CourseRun was created correctly by no errors raised
+        course_run_second = CourseRun.objects.get(key=mock_data.COURSES_API_BODY_SECOND['id'])
+
+        # Verify not set as canonical
+        with self.assertRaises(AttributeError):
+            course_run_second.canonical_for_course  # pylint: disable=pointless-statement
+
+        # Verify second course not used to update course
+        self.assertNotEqual(mock_data.COURSES_API_BODY_SECOND['name'], course.title)
+        # Verify udpated canonical course used to update course
+        self.assertEqual(mock_data.COURSES_API_BODY_UPDATED['name'], course.title)
+        # Verify the updated course run updated the original course run
+        self.assertEqual(mock_data.COURSES_API_BODY_UPDATED['hidden'], course_run_orig.hidden)
+
     def test_get_pacing_type_field_missing(self):
         """ Verify the method returns None if the API response does not include a pacing field. """
         self.assertIsNone(self.loader.get_pacing_type({}))

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -325,9 +325,7 @@ class PersonMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
 @ddt.ddt
 class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
     loader_class = CourseMarketingSiteDataLoader
-    mocked_data = mock_data.MARKETING_SITE_API_COURSE_BODIES
-    mocked_data.append(mock_data.MARKETING_SITE_API_UNPUBLISHED_COPY_COURSE_BODY)
-    mocked_data.append(mock_data.MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY)
+    mocked_data = mock_data.UNIQUE_MARKETING_SITE_API_COURSE_BODIES
 
     def _get_uuids(self, items):
         return [item['uuid'] for item in items]
@@ -513,6 +511,8 @@ class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
         expected_transcript_languages = self.loader.get_language_tags_from_names(language_names)
         self.assertEqual(list(course_run.transcript_languages.all()), list(expected_transcript_languages))
 
+        return course_run
+
     def _get_course(self, data):
         course_run_key = CourseKey.from_string(data['field_course_id'])
         return Course.objects.get(key=self.loader.get_course_key_from_course_run_key(course_run_key),
@@ -522,20 +522,33 @@ class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
     def test_ingest(self):
         self.mock_login_response()
         data = self.mock_api()
-        published_course_run_key = mock_data.MARKETING_SITE_API_PUBLISHED_COPY_COURSE_BODY['field_course_id']
 
         self.loader.ingest()
 
         for datum in data:
             self.assert_course_run_loaded(datum)
-
-            if datum['field_course_code'] == mock_data.MULTI_COURSE_RUN_COURSE_NUMBER:
-                # For the original course and the unpublished course ensure course fields are not present.
-                if datum['field_course_id'] != published_course_run_key:
-                    self.assert_no_override_unpublished_course_fields(datum)
-                # For the latest published course ensure course fields match the latest saved course.
-                else:
             self.assert_course_loaded(datum)
 
-            else:
-                self.assert_course_loaded(datum)
+    @responses.activate
+    def test_canonical(self):
+        self.mocked_data = [
+            mock_data.ORIGINAL_MARKETING_SITE_API_COURSE_BODY,
+            mock_data.NEW_RUN_MARKETING_SITE_API_COURSE_BODY,
+            mock_data.UPDATED_MARKETING_SITE_API_COURSE_BODY,
+        ]
+        self.mock_login_response()
+        self.mock_api()
+
+        self.loader.ingest()
+
+        course_run = self.assert_course_run_loaded(mock_data.UPDATED_MARKETING_SITE_API_COURSE_BODY)
+        self.assert_course_loaded(mock_data.UPDATED_MARKETING_SITE_API_COURSE_BODY)
+        self.assertTrue(course_run.canonical_for_course)
+
+        course_run = self.assert_course_run_loaded(mock_data.NEW_RUN_MARKETING_SITE_API_COURSE_BODY)
+        course = course_run.course
+
+        new_run_title = mock_data.NEW_RUN_MARKETING_SITE_API_COURSE_BODY['field_course_course_title']['value']
+        self.assertNotEqual(course.title, new_run_title)
+        with self.assertRaises(AttributeError):
+            course_run.canonical_for_course  # pylint: disable=pointless-statement

course_discovery/apps/course_metadata/forms.py

@@ -5,7 +5,7 @@ from django.forms.utils import ErrorList
 from django.utils.translation import ugettext_lazy as _
 
 from course_discovery.apps.course_metadata.choices import ProgramStatus
-from course_discovery.apps.course_metadata.models import Program, CourseRun
+from course_discovery.apps.course_metadata.models import Program, CourseRun, Course
 
 
 def filter_choices_to_render_with_order_preserved(self, selected_choices):
@@ -99,3 +99,17 @@ class CourseRunSelectionForm(forms.ModelForm):
         self.fields['excluded_course_runs'].queryset = CourseRun.objects.filter(
             course__id__in=query_set
         )
+
+
+class CourseAdminForm(forms.ModelForm):
+    class Meta:
+        model = Course
+        fields = '__all__'
+        widgets = {
+            'canonical_course_run': autocomplete.ModelSelect2(
+                url='admin_metadata:course-run-autocomplete',
+                attrs={
+                    'data-minimum-input-length': 3,
+                }
+            ),
+        }

course_discovery/apps/course_metadata/lookups.py

 from django.db.models import Q
 
 from dal import autocomplete
-from .models import Course, Organization, Video
+from .models import Course, CourseRun, Organization, Video
 
 
 class CourseAutocomplete(autocomplete.Select2QuerySetView):
@@ -16,6 +16,18 @@ class CourseAutocomplete(autocomplete.Select2QuerySetView):
         return []
 
 
+class CourseRunAutocomplete(autocomplete.Select2QuerySetView):
+    def get_queryset(self):
+        if self.request.user.is_authenticated() and self.request.user.is_staff:
+            qs = CourseRun.objects.all().select_related('course')
+            if self.q:
+                qs = qs.filter(Q(key__icontains=self.q) | Q(course__title__icontains=self.q))
+
+            return qs
+
+        return []
+
+
 class OrganizationAutocomplete(autocomplete.Select2QuerySetView):
     def get_queryset(self):
         if self.request.user.is_authenticated() and self.request.user.is_staff:

course_discovery/apps/course_metadata/migrations/0036_course_canonical_course_run.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('course_metadata', '0035_auto_20161103_2129'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='course',
+            name='canonical_course_run',
+            field=models.OneToOneField(null=True, default=None, blank=True, to='course_metadata.CourseRun', related_name='canonical_for_course'),
+        ),
+    ]

course_discovery/apps/course_metadata/migrations/0037_migrate_courses_with_canonical.py

+from django.db import migrations
+
+from course_discovery.apps.course_metadata.choices import CourseRunStatus
+
+
+def create_canonical(apps, schema_editor):
+    """Create the canonical course run associations."""
+    Course = apps.get_model('course_metadata', 'Course')
+
+    courses = Course.objects.prefetch_related('course_runs').all()
+    for course in courses:
+        course_runs = course.course_runs.all().order_by('-start')
+        published_course_runs = course_runs.filter(status=CourseRunStatus.Published)
+        if published_course_runs:
+            # If there is a published course_run use the latest
+            canonical_course_run = published_course_runs[0]
+        else:
+            # otherwise just use the latest in general
+            canonical_course_run = course_runs.first()
+
+        course.canonical_course_run = canonical_course_run
+        course.save()
+
+
+def delete_canonical(apps, schema_editor):
+    """Delete the canonical course run associations."""
+    Course = apps.get_model('course_metadata', 'Course')
+    Course.objects.all().update(canonical_course_run=None)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('course_metadata', '0036_course_canonical_course_run'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_canonical, reverse_code=delete_canonical),
+    ]

course_discovery/apps/course_metadata/models.py

@@ -232,6 +232,9 @@ class Course(TimeStampedModel):
     """ Course model. """
     partner = models.ForeignKey(Partner)
     uuid = models.UUIDField(default=uuid4, editable=False, verbose_name=_('UUID'))
+    canonical_course_run = models.OneToOneField(
+        'course_metadata.CourseRun', related_name='canonical_for_course', default=None, null=True, blank=True
+    )
     key = models.CharField(max_length=255)
     title = models.CharField(max_length=255, default=None, null=True, blank=True)
     short_description = models.CharField(max_length=255, default=None, null=True, blank=True)

course_discovery/apps/course_metadata/tests/test_lookups.py

@@ -17,6 +17,8 @@ class AutocompleteTests(TestCase):
         self.user = UserFactory(is_staff=True)
         self.client.login(username=self.user.username, password=USER_PASSWORD)
         self.courses = factories.CourseFactory.create_batch(3, title='Some random course title')
+        for course in self.courses:
+            factories.CourseRunFactory(course=course)
         self.organizations = factories.OrganizationFactory.create_batch(3)
 
     @ddt.data('dum', 'ing')
@@ -43,6 +45,34 @@ class AutocompleteTests(TestCase):
         data = json.loads(response.content.decode('utf-8'))
         self.assertEqual(data['results'], [])
 
+    @ddt.data('ing', 'dum')
+    def test_course_run_autocomplete(self, search_key):
+        """ Verify course run autocomplete returns the data. """
+        response = self.client.get(reverse('admin_metadata:course-run-autocomplete'))
+        data = json.loads(response.content.decode('utf-8'))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(data['results']), 3)
+        # update the first course title
+        course = self.courses[0]
+        course.title = 'this is some thing new'
+        course.save()
+        course_run = self.courses[0].course_runs.first()
+        course_run.key = 'edx/dummy/testrun'
+        course_run.save()
+
+        response = self.client.get(
+            reverse('admin_metadata:course-run-autocomplete') + '?q={q}'.format(q=search_key)
+        )
+        data = json.loads(response.content.decode('utf-8'))
+        self.assertEqual(data['results'][0]['text'], str(course_run))
+
+    def test_course_run_autocomplete_un_authorize_user(self):
+        """ Verify course run autocomplete returns empty list for un-authorized users. """
+        self._make_user_non_staff()
+        response = self.client.get(reverse('admin_metadata:course-run-autocomplete'))
+        data = json.loads(response.content.decode('utf-8'))
+        self.assertEqual(data['results'], [])
+
     @ddt.data('irc', 'ing')
     def test_organization_autocomplete(self, search_key):
         """ Verify Organization autocomplete returns the data. """
@@ -77,7 +107,7 @@ class AutocompleteTests(TestCase):
         response = self.client.get(reverse('admin_metadata:video-autocomplete'))
         data = json.loads(response.content.decode('utf-8'))
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(data['results']), 3)
+        self.assertEqual(len(data['results']), 6)
 
         self.courses[0].video.src = 'http://www.youtube.com/dummyurl'
         self.courses[0].video.description = 'testing description'

course_discovery/apps/course_metadata/urls.py

@@ -5,12 +5,13 @@ from django.conf.urls import url
 
 from course_discovery.apps.course_metadata.views import CourseRunSelectionAdmin
 from course_discovery.apps.course_metadata.lookups import (
-    CourseAutocomplete, OrganizationAutocomplete, VideoAutocomplete
+    CourseAutocomplete, CourseRunAutocomplete, OrganizationAutocomplete, VideoAutocomplete
 )
 
 urlpatterns = [
     url(r'^update_course_runs/(?P<pk>\d+)/$', CourseRunSelectionAdmin.as_view(), name='update_course_runs',),
     url(r'^course-autocomplete/$', CourseAutocomplete.as_view(), name='course-autocomplete',),
+    url(r'^course-run-autocomplete/$', CourseRunAutocomplete.as_view(), name='course-run-autocomplete',),
     url(r'^organisation-autocomplete/$', OrganizationAutocomplete.as_view(), name='organisation-autocomplete',),
     url(r'^video-autocomplete/$', VideoAutocomplete.as_view(), name='video-autocomplete',),
 ]