Merge branch 'master' into feature/cdodge/cms-master-merge4

Conflicts:
	common/lib/xmodule/xmodule/course_module.py

common/djangoapps/student/views.py

@@ -27,7 +27,7 @@ from bs4 import BeautifulSoup
 from django.core.cache import cache
 
 from django_future.csrf import ensure_csrf_cookie, csrf_exempt
-from student.models import (Registration, UserProfile, TestCenterUser, TestCenterUserForm, 
+from student.models import (Registration, UserProfile, TestCenterUser, TestCenterUserForm,
                             TestCenterRegistration, TestCenterRegistrationForm,
                             PendingNameChange, PendingEmailChange,
                             CourseEnrollment, unique_id_for_user,
@@ -42,7 +42,7 @@ from xmodule.modulestore.django import modulestore
 #from datetime import date
 from collections import namedtuple
 
-from courseware.courses import get_courses
+from courseware.courses import get_courses, sort_by_announcement
 from courseware.access import has_access
 
 from statsd import statsd
@@ -78,10 +78,7 @@ def index(request, extra_context={}, user=None):
         domain = request.META.get('HTTP_HOST')
 
     courses = get_courses(None, domain=domain)
-
-    # Sort courses by how far are they from they start day
-    key = lambda course: course.days_until_start
-    courses = sorted(courses, key=key, reverse=True)
+    courses = sort_by_announcement(courses)
 
     # Get the 3 most recent news
     top_news = _get_news(top=3)
@@ -211,7 +208,7 @@ def _cert_info(user, course, cert_status):
 def dashboard(request):
     user = request.user
     enrollments = CourseEnrollment.objects.filter(user=user)
-    
+
     # Build our courses list for the user, but ignore any courses that no longer
     # exist (because the course IDs have changed). Still, we don't delete those
     # enrollments, because it could have been a data push snafu.
@@ -473,7 +470,7 @@ def _do_create_account(post_vars):
     except (ValueError, KeyError):
         # If they give us garbage, just ignore it instead
         # of asking them to put an integer.
-        profile.year_of_birth = None  
+        profile.year_of_birth = None
     try:
         profile.save()
     except Exception:
@@ -613,7 +610,7 @@ def exam_registration_info(user, course):
     exam_info = course.current_test_center_exam
     if exam_info is None:
         return None
-    
+
     exam_code = exam_info.exam_series_code
     registrations = get_testcenter_registration(user, course.id, exam_code)
     if registrations:
@@ -621,7 +618,7 @@ def exam_registration_info(user, course):
     else:
         registration = None
     return registration
-    
+
 @login_required
 @ensure_csrf_cookie
 def begin_exam_registration(request, course_id):
@@ -647,7 +644,7 @@ def begin_exam_registration(request, course_id):
 
     # determine if the user is registered for this course:
     registration = exam_registration_info(user, course)
-        
+
     # we want to populate the registration page with the relevant information,
     # if it already exists.  Create an empty object otherwise.
     try:
@@ -655,7 +652,7 @@ def begin_exam_registration(request, course_id):
     except TestCenterUser.DoesNotExist:
         testcenteruser = TestCenterUser()
         testcenteruser.user = user
-        
+
     context = {'course': course,
                'user': user,
                'testcenteruser': testcenteruser,
@@ -672,8 +669,8 @@ def create_exam_registration(request, post_override=None):
     Called by form in test_center_register.html
     '''
     post_vars = post_override if post_override else request.POST
-    
-    # first determine if we need to create a new TestCenterUser, or if we are making any update 
+
+    # first determine if we need to create a new TestCenterUser, or if we are making any update
     # to an existing TestCenterUser.
     username = post_vars['username']
     user = User.objects.get(username=username)
@@ -686,10 +683,10 @@ def create_exam_registration(request, post_override=None):
     for fieldname in TestCenterUser.user_provided_fields():
         if fieldname in post_vars:
             demographic_data[fieldname] = (post_vars[fieldname]).strip()
-        
+
     try:
         testcenter_user = TestCenterUser.objects.get(user=user)
-        needs_updating = testcenter_user.needs_update(demographic_data) 
+        needs_updating = testcenter_user.needs_update(demographic_data)
         log.info("User {0} enrolled in course {1} {2}updating demographic info for exam registration".format(user.username, course_id, "" if needs_updating else "not "))
     except TestCenterUser.DoesNotExist:
         # do additional initialization here:
@@ -699,7 +696,7 @@ def create_exam_registration(request, post_override=None):
 
     # perform validation:
     if needs_updating:
-        # first perform validation on the user information 
+        # first perform validation on the user information
         # using a Django Form.
         form = TestCenterUserForm(instance=testcenter_user, data=demographic_data)
         if form.is_valid():
@@ -710,7 +707,7 @@ def create_exam_registration(request, post_override=None):
             response_data['field_errors'] = form.errors
             response_data['non_field_errors'] = form.non_field_errors()
             return HttpResponse(json.dumps(response_data), mimetype="application/json")
-        
+
     # create and save the registration:
     needs_saving = False
     exam = course.current_test_center_exam
@@ -720,12 +717,12 @@ def create_exam_registration(request, post_override=None):
         registration = registrations[0]
         # NOTE: we do not bother to check here to see if the registration has changed,
         # because at the moment there is no way for a user to change anything about their
-        # registration.  They only provide an optional accommodation request once, and 
+        # registration.  They only provide an optional accommodation request once, and
         # cannot make changes to it thereafter.
         # It is possible that the exam_info content has been changed, such as the
         # scheduled exam dates, but those kinds of changes should not be handled through
-        # this registration screen.   
-        
+        # this registration screen.
+
     else:
         accommodation_request = post_vars.get('accommodation_request','')
         registration = TestCenterRegistration.create(testcenter_user, exam, accommodation_request)
@@ -733,7 +730,7 @@ def create_exam_registration(request, post_override=None):
         log.info("User {0} enrolled in course {1} creating new exam registration".format(user.username, course_id))
 
     if needs_saving:
-        # do validation of registration.  (Mainly whether an accommodation request is too long.)        
+        # do validation of registration.  (Mainly whether an accommodation request is too long.)
         form = TestCenterRegistrationForm(instance=registration, data=post_vars)
         if form.is_valid():
             form.update_and_save()
@@ -743,14 +740,14 @@ def create_exam_registration(request, post_override=None):
             response_data['field_errors'] = form.errors
             response_data['non_field_errors'] = form.non_field_errors()
             return HttpResponse(json.dumps(response_data), mimetype="application/json")
-         
+
 
     # only do the following if there is accommodation text to send,
     # and a destination to which to send it.
     # TODO: still need to create the accommodation email templates
 #    if 'accommodation_request' in post_vars and 'TESTCENTER_ACCOMMODATION_REQUEST_EMAIL' in settings:
 #        d = {'accommodation_request': post_vars['accommodation_request'] }
-#        
+#
 #        # composes accommodation email
 #        subject = render_to_string('emails/accommodation_email_subject.txt', d)
 #        # Email subject *must not* contain newlines

common/lib/xmodule/xmodule/course_module.py

 import logging
 from cStringIO import StringIO
+from math import exp, erf
 from lxml import etree
 from path import path  # NOTE (THK): Only used for detecting presence of syllabus
 import requests
@@ -361,35 +362,66 @@ class CourseDescriptor(SequenceDescriptor):
 
     @property
     def is_new(self):
-        # The course is "new" if either if the metadata flag is_new is
-        # true or if the course has not started yet
+        """
+        Returns if the course has been flagged as new in the metadata. If
+        there is no flag, return a heuristic value considering the
+        announcement and the start dates.
+        """
         flag = self.metadata.get('is_new', None)
         if flag is None:
-            return self.days_until_start > 1
+            # Use a heuristic if the course has not been flagged
+            announcement, start, now = self._sorting_dates()
+            if announcement and (now - announcement).days < 30:
+                # The course has been announced for less that month
+                return True
+            elif (now - start).days < 1:
+                # The course has not started yet
+                return True
+            else:
+                return False
         elif isinstance(flag, basestring):
             return flag.lower() in ['true', 'yes', 'y']
         else:
             return bool(flag)
 
     @property
-    def days_until_start(self):
-        def convert_to_datetime(timestamp):
+    def sorting_score(self):
+        """
+        Returns a number that can be used to sort the courses according
+        the how "new"" they are. The "newness"" score is computed using a
+        heuristic that takes into account the announcement and
+        (advertized) start dates of the course if available.
+
+        The lower the number the "newer" the course.
+        """
+        # Make courses that have an announcement date shave a lower
+        # score than courses than don't, older courses should have a
+        # higher score.
+        announcement, start, now = self._sorting_dates()
+        scale = 300.0  # about a year
+        if announcement:
+            days = (now - announcement).days
+            score = -exp(-days/scale)
+        else:
+            days = (now - start).days
+            score = exp(days/scale)
+        return score
+
+    def _sorting_dates(self):
+        # utility function to get datetime objects for dates used to
+        # compute the is_new flag and the sorting_score
+        def to_datetime(timestamp):
             return datetime.fromtimestamp(time.mktime(timestamp))
 
-        start_date = convert_to_datetime(self.start)
+        def get_date(field):
+            timetuple = self._try_parse_time(field)
+            return to_datetime(timetuple) if timetuple else None
 
-        #  Try to use course advertised date if we can parse it
-        advertised_start = self.metadata.get('advertised_start', None)
-        if advertised_start:
-            try:
-                start_date = datetime.strptime(advertised_start,
-                                               "%Y-%m-%dT%H:%M")
-            except ValueError:
-                pass  # Invalid date, keep using 'start''
+        announcement = get_date('announcement')
+        start = get_date('advertised_start') or to_datetime(self.start)
+        now = to_datetime(time.gmtime())
 
-        now = convert_to_datetime(time.gmtime())
-        days_until_start = (start_date - now).days
-        return days_until_start
+        return announcement, start, now
 
     @lazyproperty
     def grading_context(self):
@@ -565,9 +597,9 @@ class CourseDescriptor(SequenceDescriptor):
             self.first_eligible_appointment_date = self._try_parse_time('First_Eligible_Appointment_Date')
             if self.first_eligible_appointment_date is None:
                 raise ValueError("First appointment date must be specified")
-            # TODO: If defaulting the last appointment date, it should be the 
+            # TODO: If defaulting the last appointment date, it should be the
             # *end* of the same day, not the same time.  It's going to be used as the
-            # end of the exam overall, so we don't want the exam to disappear too soon.  
+            # end of the exam overall, so we don't want the exam to disappear too soon.
             # It's also used optionally as the registration end date, so time matters there too.
             self.last_eligible_appointment_date = self._try_parse_time('Last_Eligible_Appointment_Date') # or self.first_eligible_appointment_date
             if self.last_eligible_appointment_date is None:
@@ -581,7 +613,7 @@ class CourseDescriptor(SequenceDescriptor):
                 raise ValueError("First appointment date must be before last appointment date")
             if self.registration_end_date > self.last_eligible_appointment_date:
                 raise ValueError("Registration end date must be before last appointment date")
-            
+
 
         def _try_parse_time(self, key):
             """
@@ -612,7 +644,7 @@ class CourseDescriptor(SequenceDescriptor):
         def is_registering(self):
             now = time.gmtime()
             return now >= self.registration_start_date and now <= self.registration_end_date
-            
+
         @property
         def first_eligible_appointment_date_text(self):
             return time.strftime("%b %d, %Y", self.first_eligible_appointment_date)
@@ -629,7 +661,7 @@ class CourseDescriptor(SequenceDescriptor):
     def current_test_center_exam(self):
         exams = [exam for exam in self.test_center_exams if exam.has_started_registration() and not exam.has_ended()]
         if len(exams) > 1:
-            # TODO: output some kind of warning.  This should already be 
+            # TODO: output some kind of warning.  This should already be
             # caught if we decide to do validation at load time.
             return exams[0]
         elif len(exams) == 1:

common/lib/xmodule/xmodule/tests/test_course_module.py

 import unittest
-from time import strptime, gmtime
+from time import strptime
 from fs.memoryfs import MemoryFS
 
 from mock import Mock, patch
@@ -39,52 +39,81 @@ class DummySystem(ImportSystem):
 class IsNewCourseTestCase(unittest.TestCase):
     """Make sure the property is_new works on courses"""
     @staticmethod
-    def get_dummy_course(start, is_new=None, load_error_modules=True):
+    def get_dummy_course(start, announcement=None, is_new=None):
         """Get a dummy course"""
 
-        system = DummySystem(load_error_modules)
-        is_new = '' if is_new is None else 'is_new="{0}"'.format(is_new).lower()
+        system = DummySystem(load_error_modules=True)
+
+        def to_attrb(n, v):
+            return '' if v is None else '{0}="{1}"'.format(n, v).lower()
+
+        is_new = to_attrb('is_new', is_new)
+        announcement = to_attrb('announcement', announcement)
 
         start_xml = '''
          <course org="{org}" course="{course}"
                 graceperiod="1 day" url_name="test"
                 start="{start}"
+                {announcement}
                 {is_new}>
             <chapter url="hi" url_name="ch" display_name="CH">
                 <html url_name="h" display_name="H">Two houses, ...</html>
             </chapter>
          </course>
-         '''.format(org=ORG, course=COURSE, start=start, is_new=is_new)
+         '''.format(org=ORG, course=COURSE, start=start, is_new=is_new,
+                    announcement=announcement)
 
         return system.process_xml(start_xml)
 
     @patch('xmodule.course_module.time.gmtime')
-    def test_non_started_yet(self, gmtime_mock):
-        descriptor = self.get_dummy_course(start='2013-01-05T12:00')
-        gmtime_mock.return_value = NOW
-        assert(descriptor.is_new == True)
-        assert(descriptor.days_until_start == 4)
-
-    @patch('xmodule.course_module.time.gmtime')
-    def test_already_started(self, gmtime_mock):
+    def test_sorting_score(self, gmtime_mock):
         gmtime_mock.return_value = NOW
+        dates = [('2012-10-01T12:00', '2012-09-01T12:00'),  # 0
+                 ('2012-12-01T12:00', '2012-11-01T12:00'),  # 1
+                 ('2013-02-01T12:00', '2012-12-01T12:00'),  # 2
+                 ('2013-02-01T12:00', '2012-11-10T12:00'),  # 3
+                 ('2013-02-01T12:00', None),                # 4
+                 ('2013-03-01T12:00', None),                # 5
+                 ('2013-04-01T12:00', None),                # 6
+                 ('2012-11-01T12:00', None),                # 7
+                 ('2012-09-01T12:00', None),                # 8
+                 ('1990-01-01T12:00', None),                # 9
+                 ('2013-01-02T12:00', None),                # 10
+                 ('2013-01-10T12:00', '2012-12-31T12:00'),  # 11
+                 ('2013-01-10T12:00', '2013-01-01T12:00'),  # 12
+        ]
+
+        data = []
+        for i, d in enumerate(dates):
+            descriptor = self.get_dummy_course(start=d[0], announcement=d[1])
+            score = descriptor.sorting_score
+            data.append((score, i))
+
+        result = [d[1] for d in sorted(data)]
+        assert(result == [12, 11, 2, 3, 1, 0, 6, 5, 4, 10, 7, 8, 9])
 
-        descriptor = self.get_dummy_course(start='2012-12-02T12:00')
-        assert(descriptor.is_new == False)
-        assert(descriptor.days_until_start < 0)
 
     @patch('xmodule.course_module.time.gmtime')
-    def test_is_new_set(self, gmtime_mock):
+    def test_is_new(self, gmtime_mock):
         gmtime_mock.return_value = NOW
 
         descriptor = self.get_dummy_course(start='2012-12-02T12:00', is_new=True)
-        assert(descriptor.is_new == True)
-        assert(descriptor.days_until_start < 0)
+        assert(descriptor.is_new is True)
 
         descriptor = self.get_dummy_course(start='2013-02-02T12:00', is_new=False)
-        assert(descriptor.is_new == False)
-        assert(descriptor.days_until_start > 0)
+        assert(descriptor.is_new is False)
 
         descriptor = self.get_dummy_course(start='2013-02-02T12:00', is_new=True)
-        assert(descriptor.is_new == True)
-        assert(descriptor.days_until_start > 0)
+        assert(descriptor.is_new is True)
+
+        descriptor = self.get_dummy_course(start='2013-01-15T12:00')
+        assert(descriptor.is_new is True)
+
+        descriptor = self.get_dummy_course(start='2013-03-00T12:00')
+        assert(descriptor.is_new is True)
+
+        descriptor = self.get_dummy_course(start='2012-10-15T12:00')
+        assert(descriptor.is_new is False)
+
+        descriptor = self.get_dummy_course(start='2012-12-31T12:00')
+        assert(descriptor.is_new is True)

lms/djangoapps/courseware/courses.py

@@ -95,6 +95,7 @@ def course_image_url(course):
         path = StaticContent.get_url_path_from_location(loc)
         return path
 
+
 def find_file(fs, dirs, filename):
     """
     Looks for a filename in a list of dirs on a filesystem, in the specified order.
@@ -111,6 +112,7 @@ def find_file(fs, dirs, filename):
             return filepath
     raise ResourceNotFoundError("Could not find {0}".format(filename))
 
+
 def get_course_about_section(course, section_key):
     """
     This returns the snippet of html to be rendered on the course about page,
@@ -256,4 +258,18 @@ def get_courses(user, domain=None):
     courses = [c for c in courses if has_access(user, c, 'see_exists')]
 
     courses = sorted(courses, key=lambda course:course.number)
+
+    return courses
+
+
+def sort_by_announcement(courses):
+    """
+    Sorts a list of courses by their announcement date. If the date is
+    not available, sort them by their start date.
+    """
+
+    # Sort courses by how far are they from they start day
+    key = lambda course: course.sorting_score
+    courses = sorted(courses, key=key)
+
     return courses

lms/djangoapps/courseware/views.py

@@ -17,7 +17,8 @@ from django.views.decorators.cache import cache_control
 
 from courseware import grades
 from courseware.access import has_access
-from courseware.courses import (get_courses, get_course_with_access, get_courses_by_university)
+from courseware.courses import (get_courses, get_course_with_access,
+                                get_courses_by_university, sort_by_announcement)
 import courseware.tabs as tabs
 from courseware.models import StudentModuleCache
 from module_render import toc_for_course, get_module, get_instance_module, get_module_for_descriptor
@@ -67,11 +68,8 @@ def courses(request):
     '''
     Render "find courses" page.  The course selection work is done in courseware.courses.
     '''
-    courses = get_courses(request.user, domain=request.META.get('HTTP_HOST'))
-
-    # Sort courses by how far are they from they start day
-    key = lambda course: course.days_until_start
-    courses = sorted(courses, key=key, reverse=True)
+    courses = get_courses(request.user, request.META.get('HTTP_HOST'))
+    courses = sort_by_announcement(courses)
 
     return render_to_response("courseware/courses.html", {'courses': courses})
 
@@ -437,10 +435,7 @@ def university_profile(request, org_id):
     # Only grab courses for this org...
     courses = get_courses_by_university(request.user,
                                         domain=request.META.get('HTTP_HOST'))[org_id]
-
-    # Sort courses by how far are they from they start day
-    key = lambda course: course.days_until_start
-    courses = sorted(courses, key=key, reverse=True)
+    courses = sort_by_announcement(courses)
 
     context = dict(courses=courses, org_id=org_id)
     template_file = "university_profile/{0}.html".format(org_id).lower()

lms/djangoapps/instructor/views.py

@@ -111,6 +111,7 @@ def instructor_dashboard(request, course_id):
         except Group.DoesNotExist:
             group = Group(name=grpname)     # create the group
             group.save()
+        return group
 
     def get_beta_group(course):
         """