Merge commit 'c7546271' into edx-west/rc-20130918

Conflicts:
	CHANGELOG.rst
	common/djangoapps/track/views.py
	common/lib/xmodule/xmodule/js/src/peergrading/peer_grading_problem.coffee

AUTHORS

@@ -86,3 +86,5 @@ Yarko Tymciurak <yarkot1@gmail.com>
 Miles Steele <miles@milessteele.com>
 Kevin Luo <kevluo@edx.org>
 Akshay Jagadeesh <akjags@gmail.com>
+Marko Seric <marko.seric@math.uzh.ch>
+

CHANGELOG.rst

@@ -5,10 +5,15 @@ These are notable changes in edx-platform.  This is a rolling list of changes,
 in roughly chronological order, most recent first.  Add your entries at or near
 the top.  Include a label indicating the component affected.
 
-LMS: Added bulk email for course feature, with option to optout of individual
-course emails.
+Studio/LMS: Added ability to set due date formatting through Studio's Advanced Settings.
+The key is due_date_display_format, and the value should be a format supported by Python's
+strftime function.
 
-Blades: Added Learning Tools Interoperability (LTI) blade. Now LTI components 
+Common: Added configurable backends for tracking events. Tracking events using
+the python logging module is the default backend. Support for MongoDB and a
+Django database is also available.
+
+Blades: Added Learning Tools Interoperability (LTI) blade. Now LTI components
 can be included to courses.
 
 LMS: Added alphabetical sorting of forum categories and subcategories.
@@ -289,4 +294,3 @@ Common: Updated CodeJail.
 Common: Allow setting of authentication session cookie name.
 
 LMS: Option to email students when enroll/un-enroll them.
-

cms/djangoapps/contentstore/features/advanced-settings.py

@@ -11,7 +11,6 @@ DISPLAY_NAME_KEY = "display_name"
 DISPLAY_NAME_VALUE = '"Robot Super Course"'
 
 
-############### ACTIONS ####################
 @step('I select the Advanced Settings$')
 def i_select_advanced_settings(step):
     world.click_course_settings()
@@ -45,7 +44,6 @@ def create_value_not_in_quotes(step):
     change_display_name_value(step, 'quote me')
 
 
-############### RESULTS ####################
 @step('I see default advanced settings$')
 def i_see_default_advanced_settings(step):
     # Test only a few of the existing properties (there are around 34 of them)
@@ -88,12 +86,13 @@ def the_policy_key_value_is_changed(step):
     assert_equal(get_display_name_value(), '"foo"')
 
 
-############# HELPERS ###############
 def assert_policy_entries(expected_keys, expected_values):
     for key, value in zip(expected_keys, expected_values):
         index = get_index_of(key)
         assert_false(index == -1, "Could not find key: {key}".format(key=key))
-        assert_equal(value, world.css_find(VALUE_CSS)[index].value, "value is incorrect")
+        found_value = world.css_find(VALUE_CSS)[index].value
+        assert_equal(value, found_value,
+            "Expected {} to have value {} but found {}".format(key, value, found_value))
 
 
 def get_index_of(expected_key):

cms/djangoapps/contentstore/features/common.py

@@ -2,7 +2,7 @@
 # pylint: disable=W0621
 
 from lettuce import world, step
-from nose.tools import assert_true  # pylint: disable=E0611
+from nose.tools import assert_true, assert_in, assert_false  # pylint: disable=E0611
 
 from auth.authz import get_user_by_email, get_course_groupname_for_role
 from django.conf import settings
@@ -19,8 +19,6 @@ from terrain.browser import reset_data
 
 TEST_ROOT = settings.COMMON_TEST_DATA_ROOT
 
-###########  STEP HELPERS ##############
-
 
 @step('I (?:visit|access|open) the Studio homepage$')
 def i_visit_the_studio_homepage(_step):
@@ -66,20 +64,32 @@ def select_new_course(_step, whom):
 
 @step(u'I press the "([^"]*)" notification button$')
 def press_the_notification_button(_step, name):
-    css = 'a.action-%s' % name.lower()
-
-    # The button was clicked if either the notification bar is gone,
-    # or we see an error overlaying it (expected for invalid inputs).
-    def button_clicked():
-        confirmation_dismissed = world.is_css_not_present('.is-shown.wrapper-notification-warning')
-        error_showing = world.is_css_present('.is-shown.wrapper-notification-error')
-        return confirmation_dismissed or error_showing
+    # TODO: fix up this code. Selenium is not dealing well with css transforms,
+    # as it thinks that the notification and the buttons are always visible
+
+    # First wait for the notification to pop up
+    notification_css = 'div#page-notification div.wrapper-notification'
+    world.wait_for_visible(notification_css)
+
+    # You would think that the above would have worked, but it doesn't.
+    # Brute force wait for now.
+    world.wait(.5)
+
+    # Now make sure the button is there
+    btn_css = 'div#page-notification a.action-%s' % name.lower()
+    world.wait_for_visible(btn_css)
+
+    # You would think that the above would have worked, but it doesn't.
+    # Brute force wait for now.
+    world.wait(.5)
+
     if world.is_firefox():
-        # This is done to explicitly make the changes save on firefox.  It will remove focus from the previously focused element
-        world.trigger_event(css, event='focus')
-        world.browser.execute_script("$('{}').click()".format(css))
+        # This is done to explicitly make the changes save on firefox.
+        # It will remove focus from the previously focused element
+        world.trigger_event(btn_css, event='focus')
+        world.browser.execute_script("$('{}').click()".format(btn_css))
     else:
-        world.css_click(css, success_condition=button_clicked), '%s button not clicked after 5 attempts.' % name
+        world.css_click(btn_css)
 
 
 @step('I change the "(.*)" field to "(.*)"$')
@@ -110,7 +120,6 @@ def i_see_a_confirmation(step):
     assert world.is_css_present(confirmation_css)
 
 
-####### HELPER FUNCTIONS ##############
 def open_new_course():
     world.clear_courses()
     create_studio_user()
@@ -156,8 +165,8 @@ def log_into_studio(
     world.log_in(username=uname, password=password, email=email, name=name)
     # Navigate to the studio dashboard
     world.visit('/')
+    assert_in(uname, world.css_text('h2.title', timeout=10))
 
-    assert uname in world.css_text('h2.title', max_attempts=15)
 
 def create_a_course():
     course = world.CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
@@ -232,17 +241,37 @@ def open_new_unit(step):
     world.css_click('a.new-unit-item')
 
 
-@step('the save button is disabled$')
+@step('the save notification button is disabled')
 def save_button_disabled(step):
     button_css = '.action-save'
     disabled = 'is-disabled'
     assert world.css_has_class(button_css, disabled)
 
 
+@step('the "([^"]*)" button is disabled')
+def button_disabled(step, value):
+    button_css = 'input[value="%s"]' % value
+    assert world.css_has_class(button_css, 'is-disabled')
+
+
 @step('I confirm the prompt')
 def confirm_the_prompt(step):
-    prompt_css = 'a.button.action-primary'
-    world.css_click(prompt_css, success_condition=lambda: not world.css_visible(prompt_css))
+
+    def click_button(btn_css):
+        world.css_click(btn_css)
+        return world.css_find(btn_css).visible == False
+
+    prompt_css = 'div.prompt.has-actions'
+    world.wait_for_visible(prompt_css)
+
+    btn_css = 'a.button.action-primary'
+    world.wait_for_visible(btn_css)
+
+    # Sometimes you can do a click before the prompt is up.
+    # Thus we need some retry logic here.
+    world.wait_for(lambda _driver: click_button(btn_css))
+
+    assert_false(world.css_find(btn_css).visible)
 
 
 @step(u'I am shown a (.*)$')
@@ -251,6 +280,7 @@ def i_am_shown_a_notification(step, notification_type):
 
 
 def type_in_codemirror(index, text):
+    world.wait(1)  # For now, slow this down so that it works. TODO: fix it.
     world.css_click("div.CodeMirror-lines", index=index)
     world.browser.execute_script("$('div.CodeMirror.CodeMirror-focused > div').css('overflow', '')")
     g = world.css_find("div.CodeMirror.CodeMirror-focused > div > textarea")

cms/djangoapps/contentstore/features/component_settings_editor_helpers.py

@@ -48,9 +48,7 @@ def click_component_from_menu(category, boilerplate, expected_css):
         elem_css = "a[data-category='{}']:not([data-boilerplate])".format(category)
     elements = world.css_find(elem_css)
     assert_equal(len(elements), 1)
-    world.wait_for(lambda _driver: world.css_visible(elem_css))
-    world.css_click(elem_css, success_condition=lambda: 1 == len(world.css_find(expected_css)))
-
+    world.css_click(elem_css)
 
 @world.absorb
 def edit_component_and_select_settings():

cms/djangoapps/contentstore/features/course-settings.feature

@@ -87,7 +87,7 @@ Feature: Course Settings
     Given I have opened a new course in Studio
     When I select Schedule and Details
     And I change the "Course Start Date" field to ""
-    Then the save button is disabled
+    Then the save notification button is disabled
 
   Scenario: User can upload course image
     Given I have opened a new course in Studio

cms/djangoapps/contentstore/features/course-settings.py

@@ -113,7 +113,7 @@ def test_i_have_entered_a_new_course_start_date(step):
 
 @step('The warning about course start date goes away$')
 def test_the_warning_about_course_start_date_goes_away(step):
-    assert_equal(0, len(world.css_find('.message-error')))
+    assert world.is_css_not_present('.message-error')
     assert_false('error' in world.css_find(COURSE_START_DATE_CSS).first._element.get_attribute('class'))
     assert_false('error' in world.css_find(COURSE_START_TIME_CSS).first._element.get_attribute('class'))
 

cms/djangoapps/contentstore/features/course-team.py

@@ -5,7 +5,7 @@ from lettuce import world, step
 from common import create_studio_user
 from django.contrib.auth.models import Group
 from auth.authz import get_course_groupname_for_role, get_user_by_email
-from nose.tools import assert_true  # pylint: disable=E0611
+from nose.tools import assert_true, assert_in  # pylint: disable=E0611
 
 PASSWORD = 'test'
 EMAIL_EXTENSION = '@edx.org'
@@ -110,36 +110,36 @@ def other_user_login(_step, name):
 
 @step(u'I( do not)? see the course on my page')
 @step(u's?he does( not)? see the course on (his|her) page')
-def see_course(_step, inverted, gender='self'):
+def see_course(_step, do_not_see, gender='self'):
     class_css = 'h3.course-title'
-    all_courses = world.css_find(class_css, wait_time=1)
-    all_names = [item.html for item in all_courses]
-    if inverted:
-        assert not world.scenario_dict['COURSE'].display_name in all_names
+    if do_not_see:
+        assert world.is_css_not_present(class_css)
     else:
-        assert world.scenario_dict['COURSE'].display_name in all_names
+        all_courses = world.css_find(class_css)
+        all_names = [item.html for item in all_courses]
+        assert_in(world.scenario_dict['COURSE'].display_name, all_names)
 
 
 @step(u'"([^"]*)" should( not)? be marked as an admin')
-def marked_as_admin(_step, name, inverted):
+def marked_as_admin(_step, name, not_marked_admin):
     flag_css = '.user-item[data-email="{email}"] .flag-role.flag-role-admin'.format(
         email=name+EMAIL_EXTENSION)
-    if inverted:
+    if not_marked_admin:
         assert world.is_css_not_present(flag_css)
     else:
         assert world.is_css_present(flag_css)
 
 
 @step(u'I should( not)? be marked as an admin')
-def self_marked_as_admin(_step, inverted):
-    return marked_as_admin(_step, "robot+studio", inverted)
+def self_marked_as_admin(_step, not_marked_admin):
+    return marked_as_admin(_step, "robot+studio", not_marked_admin)
 
 
 @step(u'I can(not)? delete users')
 @step(u's?he can(not)? delete users')
-def can_delete_users(_step, inverted):
+def can_delete_users(_step, can_not_delete):
     to_delete_css = 'a.remove-user'
-    if inverted:
+    if can_not_delete:
         assert world.is_css_not_present(to_delete_css)
     else:
         assert world.is_css_present(to_delete_css)
@@ -147,9 +147,9 @@ def can_delete_users(_step, inverted):
 
 @step(u'I can(not)? add users')
 @step(u's?he can(not)? add users')
-def can_add_users(_step, inverted):
+def can_add_users(_step, can_not_add):
     add_css = 'a.create-user-button'
-    if inverted:
+    if can_not_add:
         assert world.is_css_not_present(add_css)
     else:
         assert world.is_css_present(add_css)
@@ -157,13 +157,13 @@ def can_add_users(_step, inverted):
 
 @step(u'I can(not)? make ("([^"]*)"|myself) a course team admin')
 @step(u's?he can(not)? make ("([^"]*)"|me) a course team admin')
-def can_make_course_admin(_step, inverted, outer_capture, name):
+def can_make_course_admin(_step, can_not_make_admin, outer_capture, name):
     if outer_capture == "myself":
         email = world.scenario_dict["USER"].email
     else:
         email = name + EMAIL_EXTENSION
     add_button_css = '.user-item[data-email="{email}"] .add-admin-role'.format(email=email)
-    if inverted:
+    if can_not_make_admin:
         assert world.is_css_not_present(add_button_css)
     else:
         assert world.is_css_present(add_button_css)

cms/djangoapps/contentstore/features/course-updates.py

@@ -4,6 +4,7 @@
 from lettuce import world, step
 from selenium.webdriver.common.keys import Keys
 from common import type_in_codemirror
+from nose.tools import assert_in  # pylint: disable=E0611
 
 
 @step(u'I go to the course updates page')
@@ -21,14 +22,17 @@ def add_update(_step, text):
     change_text(text)
 
 
-@step(u'I should( not)? see the update "([^"]*)"$')
-def check_update(_step, doesnt_see_update, text):
+@step(u'I should see the update "([^"]*)"$')
+def check_update(_step, text):
     update_css = 'div.update-contents'
-    update = world.css_find(update_css, wait_time=1)
-    if doesnt_see_update:
-        assert len(update) == 0 or not text in update.html
-    else:
-        assert text in update.html
+    update_html = world.css_find(update_css).html
+    assert_in(text, update_html)
+
+
+@step(u'I should not see the update "([^"]*)"$')
+def check_no_update(_step, text):
+    update_css = 'div.update-contents'
+    assert world.is_css_not_present(update_css)
 
 
 @step(u'I modify the text to "([^"]*)"$')

cms/djangoapps/contentstore/features/courses.feature

@@ -11,3 +11,19 @@ Feature: Create Course
     And I press the "Create" button
     Then the Courseware page has loaded in Studio
     And I see a link for adding a new section
+
+  Scenario: Error message when org/course/run tuple is too long
+    Given There are no courses
+    And I am logged into Studio
+    When I click the New Course button
+    And I create a course with "course name", "012345678901234567890123456789", "012345678901234567890123456789", and "0123456"
+    Then I see an error about the length of the org/course/run tuple
+    And the "Create" button is disabled
+
+  Scenario: Course name is not included in the "too long" computation
+    Given There are no courses
+    And I am logged into Studio
+    When I click the New Course button
+    And I create a course with "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", "org", "coursenum", and "run"
+    And I press the "Create" button
+    Then the Courseware page has loaded in Studio

cms/djangoapps/contentstore/features/courses.py

@@ -23,6 +23,11 @@ def i_fill_in_a_new_course_information(step):
     fill_in_course_info()
 
 
+@step('I create a course with "([^"]*)", "([^"]*)", "([^"]*)", and "([^"]*)"')
+def i_create_course(step, name, org, number, run):
+    fill_in_course_info(name=name, org=org, num=number, run=run)
+
+
 @step('I create a new course$')
 def i_create_a_course(step):
     create_a_course()
@@ -33,6 +38,11 @@ def i_click_the_course_link_in_my_courses(step):
     course_css = 'a.course-link'
     world.css_click(course_css)
 
+
+@step('I see an error about the length of the org/course/run tuple')
+def i_see_error_about_length(step):
+    assert world.css_has_text('#course_creation_error', 'The combined length of the organization, course number, and course run fields cannot be more than 65 characters.')
+
 ############ ASSERTIONS ###################
 
 

cms/djangoapps/contentstore/features/grading.feature

@@ -86,7 +86,7 @@ Feature: Course Grading
         And I have populated the course
         And I am viewing the grading settings
         When I change assignment type "Homework" to ""
-        Then the save button is disabled
+        Then the save notification button is disabled
 
     # IE and Safari cannot type in grade range name
     @skip_internetexplorer

cms/djangoapps/contentstore/features/grading.py

@@ -5,6 +5,7 @@ from lettuce import world, step
 from common import *
 from terrain.steps import reload_the_page
 from selenium.common.exceptions import InvalidElementStateException
+from nose.tools import assert_in, assert_not_in  # pylint: disable=E0611
 
 
 @step(u'I am viewing the grading settings')
@@ -65,21 +66,25 @@ def change_assignment_name(step, old_name, new_name):
 
 @step(u'I go back to the main course page')
 def main_course_page(step):
-    main_page_link_css = 'a[href="/%s/%s/course/%s"]' % (world.scenario_dict['COURSE'].org,
-                                                        world.scenario_dict['COURSE'].number,
-                                                        world.scenario_dict['COURSE'].display_name.replace(' ', '_'),)
-    world.css_click(main_page_link_css)
+    main_page_link = '/{}/{}/course/{}'.format(world.scenario_dict['COURSE'].org,
+                                               world.scenario_dict['COURSE'].number,
+                                               world.scenario_dict['COURSE'].display_name.replace(' ', '_'),)
+    world.visit(main_page_link)
+    assert_in('Course Outline', world.css_text('h1.page-header'))
 
 
 @step(u'I do( not)? see the assignment name "([^"]*)"$')
 def see_assignment_name(step, do_not, name):
     assignment_menu_css = 'ul.menu > li > a'
+    # First assert that it is there, make take a bit to redraw
+    assert world.css_find(assignment_menu_css)
+
     assignment_menu = world.css_find(assignment_menu_css)
     allnames = [item.html for item in assignment_menu]
     if do_not:
-        assert not name in allnames
+        assert_not_in(name, allnames)
     else:
-        assert name in allnames
+        assert_in(name, allnames)
 
 
 @step(u'I delete the assignment type "([^"]*)"$')

cms/djangoapps/contentstore/features/problem-editor.py

@@ -2,7 +2,7 @@
 #pylint: disable=C0111
 
 from lettuce import world, step
-from nose.tools import assert_equal  # pylint: disable=E0611
+from nose.tools import assert_equal, assert_true  # pylint: disable=E0611
 from common import type_in_codemirror
 
 DISPLAY_NAME = "Display Name"
@@ -197,9 +197,20 @@ def high_level_source_in_editor(step):
 
 
 def verify_high_level_source_links(step, visible):
-    assert_equal(visible, world.is_css_present('.launch-latex-compiler'))
+    if visible:
+        assert_true(world.is_css_present('.launch-latex-compiler'),
+                    msg="Expected to find the latex button but it is not present.")
+    else:
+        assert_true(world.is_css_not_present('.launch-latex-compiler'),
+                    msg="Expected not to find the latex button but it is present.")
+
     world.cancel_component(step)
-    assert_equal(visible, world.is_css_present('.upload-button'))
+    if visible:
+        assert_true(world.is_css_present('.upload-button'),
+                    msg="Expected to find the upload button but it is not present.")
+    else:
+        assert_true(world.is_css_not_present('.upload-button'),
+                    msg="Expected not to find the upload button but it is present.")
 
 
 def verify_modified_weight():

cms/djangoapps/contentstore/features/signup.py

@@ -12,7 +12,7 @@ def i_fill_in_the_registration_form(step):
         register_form.find_by_name('password').fill('test')
         register_form.find_by_name('username').fill('robot-studio')
         register_form.find_by_name('name').fill('Robot Studio')
-        register_form.find_by_name('terms_of_service').check()
+        register_form.find_by_name('terms_of_service').click()
     world.retry_on_exception(fill_in_reg_form)
 
 

cms/djangoapps/contentstore/features/static-pages.py

@@ -2,12 +2,11 @@
 #pylint: disable=W0621
 
 from lettuce import world, step
-from selenium.webdriver.common.keys import Keys
 from nose.tools import assert_true # pylint: disable=E0611
 
 
 @step(u'I go to the static pages page')
-def go_to_static(_step):
+def go_to_static(step):
     menu_css = 'li.nav-course-courseware'
     static_css = 'li.nav-course-courseware-pages a'
     world.css_click(menu_css)
@@ -15,25 +14,37 @@ def go_to_static(_step):
 
 
 @step(u'I add a new page')
-def add_page(_step):
+def add_page(step):
     button_css = 'a.new-button'
     world.css_click(button_css)
 
 
-@step(u'I should( not)? see a "([^"]*)" static page$')
-def see_page(_step, doesnt, page):
+@step(u'I should not see a "([^"]*)" static page$')
+def not_see_page(step, page):
+    # Either there are no pages, or there are pages but
+    # not the one I expect not to exist.
 
-    should_exist = not doesnt
+    # Since our only test for deletion right now deletes
+    # the only static page that existed, our success criteria
+    # will be that there are no static pages.
+    # In the future we can refactor if necessary.
+    tabs_css = 'li.component'
+    assert (world.is_css_not_present(tabs_css, wait_time=30))
+
+
+@step(u'I should see a "([^"]*)" static page$')
+def see_page(step, page):
 
     # Need to retry here because the element
     # will sometimes exist before the HTML content is loaded
-    exists_func = lambda(driver): page_exists(page) == should_exist
+    exists_func = lambda(driver): page_exists(page)
+
     world.wait_for(exists_func)
     assert_true(exists_func(None))
 
 
 @step(u'I "([^"]*)" the "([^"]*)" page$')
-def click_edit_delete(_step, edit_delete, page):
+def click_edit_delete(step, edit_delete, page):
     button_css = 'a.%s-button' % edit_delete
     index = get_index(page)
     assert index is not None
@@ -41,7 +52,7 @@ def click_edit_delete(_step, edit_delete, page):
 
 
 @step(u'I change the name to "([^"]*)"$')
-def change_name(_step, new_name):
+def change_name(step, new_name):
     settings_css = '#settings-mode a'
     world.css_click(settings_css)
     input_css = 'input.setting-input'
@@ -56,9 +67,10 @@ def get_index(name):
     page_name_css = 'section[data-type="HTMLModule"]'
     all_pages = world.css_find(page_name_css)
     for i in range(len(all_pages)):
-        if world.css_html(page_name_css, index=i) == '\n    {name}\n'.format(name=name):
+        if world.retry_on_exception(lambda: all_pages[i].html) == '\n    {name}\n'.format(name=name):
             return i
     return None
 
+
 def page_exists(page):
     return get_index(page) is not None

cms/djangoapps/contentstore/features/upload.py

@@ -7,6 +7,8 @@ import requests
 import string
 import random
 import os
+from nose.tools import assert_equal, assert_not_equal # pylint: disable=E0611
+
 
 TEST_ROOT = settings.COMMON_TEST_DATA_ROOT
 
@@ -32,7 +34,7 @@ def upload_file(_step, file_name):
 
 
 @step(u'I upload the files (".*")$')
-def upload_file(_step, files_string):
+def upload_files(_step, files_string):
     # Turn files_string to a list of file names
     files = files_string.split(",")
     files = map(lambda x: string.strip(x, ' "\''), files)
@@ -48,19 +50,29 @@ def upload_file(_step, files_string):
     world.css_click(close_css)
 
 
-@step(u'I should( not)? see the file "([^"]*)" was uploaded$')
-def check_upload(_step, do_not_see_file, file_name):
+@step(u'I should not see the file "([^"]*)" was uploaded$')
+def check_not_there(_step, file_name):
+    # Either there are no files, or there are files but
+    # not the one I expect not to exist.
+
+    # Since our only test for deletion right now deletes
+    # the only file that was uploaded, our success criteria
+    # will be that there are no files.
+    # In the future we can refactor if necessary.
+    names_css = 'td.name-col > a.filename'
+    assert(world.is_css_not_present(names_css))
+
+
+@step(u'I should see the file "([^"]*)" was uploaded$')
+def check_upload(_step, file_name):
     index = get_index(file_name)
-    if do_not_see_file:
-        assert index == -1
-    else:
-        assert index != -1
+    assert_not_equal(index, -1)
 
 
 @step(u'The url for the file "([^"]*)" is valid$')
 def check_url(_step, file_name):
     r = get_file(file_name)
-    assert r.status_code == 200
+    assert_equal(r.status_code , 200)
 
 
 @step(u'I delete the file "([^"]*)"$')
@@ -71,7 +83,7 @@ def delete_file(_step, file_name):
     world.css_click(delete_css, index=index)
 
     prompt_confirm_css = 'li.nav-item > a.action-primary'
-    world.css_click(prompt_confirm_css, success_condition=lambda: not world.css_visible(prompt_confirm_css))
+    world.css_click(prompt_confirm_css)
 
 
 @step(u'I should see only one "([^"]*)"$')

cms/djangoapps/contentstore/management/commands/delete_course.py

@@ -2,13 +2,8 @@
 ### Script for cloning a course
 ###
 from django.core.management.base import BaseCommand, CommandError
-from xmodule.modulestore.store_utilities import delete_course
-from xmodule.modulestore.django import modulestore
-from xmodule.contentstore.django import contentstore
-from xmodule.course_module import CourseDescriptor
 from .prompt import query_yes_no
-
-from auth.authz import _delete_course_group
+from contentstore.utils import delete_course_and_groups
 
 
 #
@@ -30,20 +25,6 @@ class Command(BaseCommand):
         if commit:
             print 'Actually going to delete the course from DB....'
 
-        ms = modulestore('direct')
-        cs = contentstore()
-
-        org, course_num, run = course_id.split("/")
-        ms.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
-
         if query_yes_no("Deleting course {0}. Confirm?".format(course_id), default="no"):
             if query_yes_no("Are you sure. This action cannot be undone!", default="no"):
-                loc = CourseDescriptor.id_to_location(course_id)
-                if delete_course(ms, cs, loc, commit):
-                    print 'removing User permissions from course....'
-                    # in the django layer, we need to remove all the user permissions groups associated with this course
-                    if commit:
-                        try:
-                            _delete_course_group(loc)
-                        except Exception as err:
-                            print("Error in deleting course groups for {0}: {1}".format(loc, err))
+                delete_course_and_groups(course_id, commit)

cms/djangoapps/contentstore/tests/test_checklists.py

 """ Unit tests for checklist methods in views.py. """
 from contentstore.utils import get_modulestore
+from contentstore.views.checklist import expand_checklist_action_url
 from xmodule.modulestore.inheritance import own_metadata
 from xmodule.modulestore.tests.factories import CourseFactory
 from django.core.urlresolvers import reverse
@@ -22,20 +23,16 @@ class ChecklistTestCase(CourseTestCase):
     def compare_checklists(self, persisted, request):
         """
         Handles url expansion as possible difference and descends into guts
-        :param persisted:
-        :param request:
         """
         self.assertEqual(persisted['short_description'], request['short_description'])
-        compare_urls = (persisted.get('action_urls_expanded') == request.get('action_urls_expanded'))
-        pers, req = None, None
-        for pers, req in zip(persisted['items'], request['items']):
+        expanded_checklist = expand_checklist_action_url(self.course, persisted)
+        for pers, req in zip(expanded_checklist['items'], request['items']):
             self.assertEqual(pers['short_description'], req['short_description'])
-        self.assertEqual(pers['long_description'], req['long_description'])
-        self.assertEqual(pers['is_checked'], req['is_checked'])
-        if compare_urls:
+            self.assertEqual(pers['long_description'], req['long_description'])
+            self.assertEqual(pers['is_checked'], req['is_checked'])
             self.assertEqual(pers['action_url'], req['action_url'])
-        self.assertEqual(pers['action_text'], req['action_text'])
-        self.assertEqual(pers['action_external'], req['action_external'])
+            self.assertEqual(pers['action_text'], req['action_text'])
+            self.assertEqual(pers['action_external'], req['action_external'])
 
     def test_get_checklists(self):
         """ Tests the get checklists method. """
@@ -46,6 +43,11 @@ class ChecklistTestCase(CourseTestCase):
         })
         response = self.client.get(checklists_url)
         self.assertContains(response, "Getting Started With Studio")
+        # Verify expansion of action URL happened.
+        self.assertContains(response, '/mitX/333/team/Checklists_Course')
+        # Verify persisted checklist does NOT have expanded URL.
+        checklist_0 = self.get_persisted_checklists()[0]
+        self.assertEqual('ManageUsers', get_action_url(checklist_0, 0))
         payload = response.content
 
         # Now delete the checklists from the course and verify they get repopulated (for courses
@@ -67,7 +69,11 @@ class ChecklistTestCase(CourseTestCase):
             'name': self.course.location.name})
 
         returned_checklists = json.loads(self.client.get(update_url).content)
-        for pay, resp in zip(self.get_persisted_checklists(), returned_checklists):
+        # Verify that persisted checklists do not have expanded action URLs.
+        # compare_checklists will verify that returned_checklists DO have expanded action URLs.
+        pers = self.get_persisted_checklists()
+        self.assertEqual('CourseOutline', get_first_item(pers[1]).get('action_url'))
+        for pay, resp in zip(pers, returned_checklists):
             self.compare_checklists(pay, resp)
 
     def test_update_checklists_index_ignored_on_get(self):
@@ -103,19 +109,21 @@ class ChecklistTestCase(CourseTestCase):
         update_url = reverse('checklists_updates', kwargs={'org': self.course.location.org,
                                                            'course': self.course.location.course,
                                                            'name': self.course.location.name,
-                                                           'checklist_index': 2})
-
-        def get_first_item(checklist):
-            return checklist['items'][0]
+                                                           'checklist_index': 1})
 
-        payload = self.course.checklists[2]
+        payload = self.course.checklists[1]
         self.assertFalse(get_first_item(payload).get('is_checked'))
+        self.assertEqual('CourseOutline', get_first_item(payload).get('action_url'))
         get_first_item(payload)['is_checked'] = True
 
         returned_checklist = json.loads(self.client.post(update_url, json.dumps(payload), "application/json").content)
         self.assertTrue(get_first_item(returned_checklist).get('is_checked'))
-        pers = self.get_persisted_checklists()
-        self.compare_checklists(pers[2], returned_checklist)
+        persisted_checklist = self.get_persisted_checklists()[1]
+        # Verify that persisted checklist does not have expanded action URLs.
+        # compare_checklists will verify that returned_checklist DOES have expanded action URLs.
+        self.assertEqual('CourseOutline', get_first_item(persisted_checklist).get('action_url'))
+        self.compare_checklists(persisted_checklist, returned_checklist)
+
 
     def test_update_checklists_delete_unsupported(self):
         """ Delete operation is not supported. """
@@ -125,3 +133,36 @@ class ChecklistTestCase(CourseTestCase):
                                                            'checklist_index': 100})
         response = self.client.delete(update_url)
         self.assertEqual(response.status_code, 405)
+
+    def test_expand_checklist_action_url(self):
+        """
+        Tests the method to expand checklist action url.
+        """
+
+        def test_expansion(checklist, index, stored, expanded):
+            """
+            Tests that the expected expanded value is returned for the item at the given index.
+
+            Also verifies that the original checklist is not modified.
+            """
+            self.assertEqual(get_action_url(checklist, index), stored)
+            expanded_checklist = expand_checklist_action_url(self.course, checklist)
+            self.assertEqual(get_action_url(expanded_checklist, index), expanded)
+            # Verify no side effect in the original list.
+            self.assertEqual(get_action_url(checklist, index), stored)
+
+        test_expansion(self.course.checklists[0], 0, 'ManageUsers', '/mitX/333/team/Checklists_Course')
+        test_expansion(self.course.checklists[1], 1, 'CourseOutline', '/mitX/333/course/Checklists_Course')
+        test_expansion(self.course.checklists[2], 0, 'http://help.edge.edx.org/', 'http://help.edge.edx.org/')
+
+
+def get_first_item(checklist):
+    """ Returns the first item from the checklist. """
+    return checklist['items'][0]
+
+
+def get_action_url(checklist, index):
+    """
+    Returns the action_url for the item at the specified index in the given checklist.
+    """
+    return checklist['items'][index]['action_url']

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -55,6 +55,8 @@ from uuid import uuid4
 from pymongo import MongoClient
 from student.models import CourseEnrollment
 
+from contentstore.utils import delete_course_and_groups
+
 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
 TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex
 
@@ -1290,6 +1292,28 @@ class ContentStoreTest(ModuleStoreTestCase):
         test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
         self.assertTrue(are_permissions_roles_seeded(self._get_course_id(test_course_data)))
 
+    def test_forum_unseeding_on_delete(self):
+        """Test new course creation and verify forum unseeding """
+        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
+        self.assertTrue(are_permissions_roles_seeded(self._get_course_id(test_course_data)))
+        course_id = self._get_course_id(test_course_data)
+        delete_course_and_groups(course_id, commit=True)
+        self.assertFalse(are_permissions_roles_seeded(course_id))
+
+    def test_forum_unseeding_with_multiple_courses(self):
+        """Test new course creation and verify forum unseeding when there are multiple courses"""
+        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
+        second_course_data = self.assert_created_course(number_suffix=uuid4().hex)
+
+        # unseed the forums for the first course
+        course_id = self._get_course_id(test_course_data)
+        delete_course_and_groups(course_id, commit=True)
+        self.assertFalse(are_permissions_roles_seeded(course_id))
+
+        second_course_id = self._get_course_id(second_course_data)
+        # permissions should still be there for the other course
+        self.assertTrue(are_permissions_roles_seeded(second_course_id))
+
     def _get_course_id(self, test_course_data):
         """Returns the course ID (org/number/run)."""
         return "{org}/{number}/{run}".format(**test_course_data)

cms/djangoapps/contentstore/utils.py

@@ -5,12 +5,16 @@ from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.contentstore.content import StaticContent
-from django.core.urlresolvers import reverse
+from xmodule.contentstore.django import contentstore
 import copy
 import logging
 import re
 from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
 from django.utils.translation import ugettext as _
+from django_comment_common.utils import unseed_permissions_roles
+from auth.authz import _delete_course_group
+from xmodule.modulestore.store_utilities import delete_course
+from xmodule.course_module import CourseDescriptor
 
 log = logging.getLogger(__name__)
 
@@ -20,6 +24,31 @@ NOTES_PANEL = {"name": _("My Notes"), "type": "notes"}
 EXTRA_TAB_PANELS = dict([(p['type'], p) for p in [OPEN_ENDED_PANEL, NOTES_PANEL]])
 
 
+def delete_course_and_groups(course_id, commit=False):
+    """
+    This deletes the courseware associated with a course_id as well as cleaning update_item
+    the various user table stuff (groups, permissions, etc.)
+    """
+    module_store = modulestore('direct')
+    content_store = contentstore()
+
+    org, course_num, run = course_id.split("/")
+    module_store.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
+
+    loc = CourseDescriptor.id_to_location(course_id)
+    if delete_course(module_store, content_store, loc, commit):
+        print 'removing forums permissions and roles...'
+        unseed_permissions_roles(course_id)
+
+        print 'removing User permissions from course....'
+        # in the django layer, we need to remove all the user permissions groups associated with this course
+        if commit:
+            try:
+                _delete_course_group(loc)
+            except Exception as err:
+                log.error("Error in deleting course groups for {0}: {1}".format(loc, err))
+
+
 def get_modulestore(category_or_location):
     """
     Returns the correct modulestore to use for modifying the specified location

cms/djangoapps/contentstore/views/checklist.py

 import json
+import copy
 
 from util.json_request import JsonResponse
 from django.http import HttpResponseBadRequest
@@ -32,19 +33,16 @@ def get_checklists(request, org, course, name):
 
     # If course was created before checklists were introduced, copy them over
     # from the template.
-    copied = False
     if not course_module.checklists:
         course_module.checklists = CourseDescriptor.checklists.default
-        copied = True
-
-    checklists, modified = expand_checklist_action_urls(course_module)
-    if copied or modified:
         course_module.save()
         modulestore.update_metadata(location, own_metadata(course_module))
+
+    expanded_checklists = expand_all_action_urls(course_module)
     return render_to_response('checklists.html',
                               {
                                   'context_course': course_module,
-                                  'checklists': checklists
+                                  'checklists': expanded_checklists
                               })
 
 
@@ -68,14 +66,20 @@ def update_checklist(request, org, course, name, checklist_index=None):
     if request.method in ("POST", "PUT"):
         if checklist_index is not None and 0 <= int(checklist_index) < len(course_module.checklists):
             index = int(checklist_index)
-            course_module.checklists[index] = json.loads(request.body)
+            persisted_checklist = course_module.checklists[index]
+            modified_checklist = json.loads(request.body)
+            # Only thing the user can modify is the "checked" state.
+            # We don't want to persist what comes back from the client because it will
+            # include the expanded action URLs (which are non-portable).
+            for item_index, item in enumerate(modified_checklist.get('items')):
+                persisted_checklist['items'][item_index]['is_checked'] = item['is_checked']
             # seeming noop which triggers kvs to record that the metadata is
             # not default
             course_module.checklists = course_module.checklists
-            checklists, _ = expand_checklist_action_urls(course_module)
             course_module.save()
             modulestore.update_metadata(location, own_metadata(course_module))
-            return JsonResponse(checklists[index])
+            expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
+            return JsonResponse(expanded_checklist)
         else:
             return HttpResponseBadRequest(
                 ( "Could not save checklist state because the checklist index "
@@ -85,23 +89,30 @@ def update_checklist(request, org, course, name, checklist_index=None):
     elif request.method == 'GET':
         # In the JavaScript view initialize method, we do a fetch to get all
         # the checklists.
-        checklists, modified = expand_checklist_action_urls(course_module)
-        if modified:
-            course_module.save()
-            modulestore.update_metadata(location, own_metadata(course_module))
-        return JsonResponse(checklists)
+        expanded_checklists = expand_all_action_urls(course_module)
+        return JsonResponse(expanded_checklists)
+
+
+def expand_all_action_urls(course_module):
+    """
+    Gets the checklists out of the course module and expands their action urls.
+
+    Returns a copy of the checklists with modified urls, without modifying the persisted version
+    of the checklists.
+    """
+    expanded_checklists = []
+    for checklist in course_module.checklists:
+        expanded_checklists.append(expand_checklist_action_url(course_module, checklist))
+    return expanded_checklists
 
 
-def expand_checklist_action_urls(course_module):
+def expand_checklist_action_url(course_module, checklist):
     """
-    Gets the checklists out of the course module and expands their action urls
-    if they have not yet been expanded.
+    Expands the action URLs for a given checklist and returns the modified version.
 
-    Returns the checklists with modified urls, as well as a boolean
-    indicating whether or not the checklists were modified.
+    The method does a copy of the input checklist and does not modify the input argument.
     """
-    checklists = course_module.checklists
-    modified = False
+    expanded_checklist = copy.deepcopy(checklist)
     urlconf_map = {
         "ManageUsers": "manage_users",
         "SettingsDetails": "settings_details",
@@ -109,19 +120,15 @@ def expand_checklist_action_urls(course_module):
         "CourseOutline": "course_index",
         "Checklists": "checklists",
     }
-    for checklist in checklists:
-        if not checklist.get('action_urls_expanded', False):
-            for item in checklist.get('items'):
-                action_url = item.get('action_url')
-                if action_url not in urlconf_map:
-                    continue
-                urlconf_name = urlconf_map[action_url]
-                item['action_url'] = reverse(urlconf_name, kwargs={
-                    'org': course_module.location.org,
-                    'course': course_module.location.course,
-                    'name': course_module.location.name,
-                })
-            checklist['action_urls_expanded'] = True
-            modified = True
-
-    return checklists, modified
+    for item in expanded_checklist.get('items'):
+        action_url = item.get('action_url')
+        if action_url not in urlconf_map:
+            continue
+        urlconf_name = urlconf_map[action_url]
+        item['action_url'] = reverse(urlconf_name, kwargs={
+            'org': course_module.location.org,
+            'course': course_module.location.course,
+            'name': course_module.location.name,
+        })
+
+    return expanded_checklist

cms/djangoapps/contentstore/views/import_export.py

@@ -60,7 +60,7 @@ def import_course(request, org, course, name):
         `filename` is truncted on creation. Additionally removes dirname on
         exit.
         """
-        open("file", "w").close()
+        open(filename, "w").close()
         try:
             yield filename
         finally:
@@ -90,7 +90,7 @@ def import_course(request, org, course, name):
         try:
             matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
             content_range = matches.groupdict()
-        except KeyError:    # Single chunk 
+        except KeyError:    # Single chunk
             # no Content-Range header, so make one that will work
             content_range = {'start': 0, 'stop': 1, 'end': 2}
 
@@ -154,7 +154,7 @@ def import_course(request, org, course, name):
                     sf.write("Extracting")
 
                 tar_file = tarfile.open(temp_filepath)
-                tar_file.extractall(course_dir + '/')
+                tar_file.extractall((course_dir + '/').encode('utf-8'))
 
                 with open(status_file, 'w+') as sf:
                     sf.write("Verifying")

cms/djangoapps/contentstore/views/item.py

+import logging
 from uuid import uuid4
 
 from django.core.exceptions import PermissionDenied
 from django.contrib.auth.decorators import login_required
+from django.http import HttpResponseBadRequest
 
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
@@ -18,6 +20,7 @@ __all__ = ['save_item', 'create_item', 'delete_item']
 # cdodge: these are categories which should not be parented, they are detached from the hierarchy
 DETACHED_CATEGORIES = ['about', 'static_tab', 'course_info']
 
+log = logging.getLogger(__name__)
 
 @login_required
 @expect_json
@@ -32,7 +35,25 @@ def save_item(request):
     """
     # The nullout is a bit of a temporary copout until we can make module_edit.coffee and the metadata editors a
     # little smarter and able to pass something more akin to {unset: [field, field]}
-    item_location = request.POST['id']
+
+    try:
+        item_location = request.POST['id']
+    except KeyError:
+        import inspect
+
+        log.exception(
+            '''Request missing required attribute 'id'.
+                Request info:
+                %s
+                Caller:
+                Function %s in file %s
+            ''',
+            request.META,
+            inspect.currentframe().f_back.f_code.co_name,
+            inspect.currentframe().f_back.f_code.co_filename
+        )
+        return HttpResponseBadRequest()
+
 
     # check permissions for this user within this course
     if not has_access(request.user, item_location):

cms/djangoapps/models/settings/course_metadata.py

@@ -2,7 +2,6 @@ from xmodule.modulestore import Location
 from contentstore.utils import get_modulestore
 from xmodule.modulestore.inheritance import own_metadata
 from xblock.fields import Scope
-from xmodule.course_module import CourseDescriptor
 from cms.xmodule_namespace import CmsBlockMixin
 
 
@@ -20,7 +19,9 @@ class CourseMetadata(object):
                      'enrollment_end',
                      'tabs',
                      'graceperiod',
-                     'checklists']
+                     'checklists',
+                     'show_timezone'
+    ]
 
     @classmethod
     def fetch(cls, course_location):

cms/envs/aws.py

@@ -127,6 +127,10 @@ LOGGING = get_logger_config(LOG_DIR,
 #theming start:
 PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', 'edX')
 
+# Event Tracking
+if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
+    TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")
+
 
 ################ SECURE AUTH ITEMS ###############################
 # Secret things: passwords, access keys, etc.
@@ -147,7 +151,12 @@ MODULESTORE = AUTH_TOKENS['MODULESTORE']
 CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 
 # Datadog for events!
-DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
+DATADOG = AUTH_TOKENS.get("DATADOG", {})
+DATADOG.update(ENV_TOKENS.get("DATADOG", {}))
+
+# TODO: deprecated (compatibility with previous settings)
+if 'DATADOG_API' in AUTH_TOKENS:
+    DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']
 
 # Celery Broker
 CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
@@ -161,3 +170,6 @@ BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT,
                                             CELERY_BROKER_PASSWORD,
                                             CELERY_BROKER_HOSTNAME,
                                             CELERY_BROKER_VHOST)
+
+# Event tracking
+TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))

cms/envs/common.py

@@ -218,9 +218,6 @@ USE_L10N = True
 # Localization strings (e.g. django.po) are under this directory
 LOCALE_PATHS = (REPO_ROOT + '/conf/locale',)  # mitx/conf/locale/
 
-# Tracking
-TRACK_MAX_EVENT = 10000
-
 # Messages
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
@@ -358,6 +355,9 @@ INSTALLED_APPS = (
     # Tracking
     'track',
 
+    # Monitoring
+    'datadog',
+
     # For asset pipelining
     'mitxmako',
     'pipeline',
@@ -391,3 +391,20 @@ MKTG_URL_LINK_MAP = {
 }
 
 COURSES_WITH_UNSAFE_CODE = []
+
+############################## EVENT TRACKING #################################
+
+TRACK_MAX_EVENT = 10000
+
+TRACKING_BACKENDS = {
+    'logger': {
+        'ENGINE': 'track.backends.logger.LoggerBackend',
+        'OPTIONS': {
+            'name': 'tracking'
+        }
+    }
+}
+
+# We're already logging events, and we don't want to capture user
+# names/passwords.  Heartbeat events are likely not interesting.
+TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat']

cms/static/coffee/src/views/module_edit.coffee

@@ -60,8 +60,8 @@ class CMS.Views.ModuleEdit extends Backbone.View
     payload.parent_location = parent
     $.post(
         "/create_item"
-        payload 
-        (data) => 
+        payload
+        (data) =>
             @model.set(id: data.id)
             @$el.data('id', data.id)
             @render()
@@ -85,7 +85,7 @@ class CMS.Views.ModuleEdit extends Backbone.View
     data.metadata = _.extend(data.metadata || {}, @changedMetadata())
     @hideModal()
     saving = new CMS.Views.Notification.Mini
-      title: gettext('Saving') + '…'
+      title: gettext('Saving…')
     saving.show()
     @model.save(data).done( =>
     #   # showToastMessage("Your changes have been saved.", null, 3)

cms/static/coffee/src/views/tabs.coffee

@@ -21,7 +21,7 @@ class CMS.Views.TabsEdit extends Backbone.View
       forcePlaceholderSize: true
       axis: 'y'
       items: '> .component'
-    )    
+    )
 
   tabMoved: (event, ui) =>
     tabs = []
@@ -34,7 +34,7 @@ class CMS.Views.TabsEdit extends Backbone.View
 
     $.ajax({
       type:'POST',
-      url: '/reorder_static_tabs', 
+      url: '/reorder_static_tabs',
       data: JSON.stringify({
         tabs : tabs
       }),
@@ -78,7 +78,7 @@ class CMS.Views.TabsEdit extends Backbone.View
               course: course_location_analytics
               id: $component.data('id')
             deleting = new CMS.Views.Notification.Mini
-              title: gettext('Deleting') + '…'
+              title: gettext('Deleting…')
             deleting.show()
             $.post('/delete_item', {
               id: $component.data('id')

cms/static/coffee/src/views/unit.coffee

@@ -42,7 +42,7 @@ class CMS.Views.UnitEdit extends Backbone.View
 
         payload = children : @components()
         saving = new CMS.Views.Notification.Mini
-          title: gettext('Saving') + '…'
+          title: gettext('Saving…')
         saving.show()
         options = success : =>
           @model.unset('children')
@@ -130,7 +130,7 @@ class CMS.Views.UnitEdit extends Backbone.View
           click: (view) =>
             view.hide()
             deleting = new CMS.Views.Notification.Mini
-              title: gettext('Deleting') + '…',
+              title: gettext('Deleting…'),
             deleting.show()
             $component = $(event.currentTarget).parents('.component')
             $.post('/delete_item', {

cms/static/js/base.js

@@ -395,7 +395,7 @@ function _deleteItem($el, type) {
                     });
 
                     var deleting = new CMS.Views.Notification.Mini({
-                        title: gettext('Deleting') + '…'
+                        title: gettext('Deleting…')
                     });
                     deleting.show();
 
@@ -626,25 +626,25 @@ function addNewCourse(e) {
             return gettext('Please do not use any spaces or special characters in this field.');
         }
         return '';
-    }
+    };
 
-    // Ensure that all items are less than 80 characters.
+    // Ensure that org/course_num/run < 65 chars.
     var validateTotalCourseItemsLength = function() {
         var totalLength = _.reduce(
-            ['.new-course-name', '.new-course-org', '.new-course-number', '.new-course-run'],
+            ['.new-course-org', '.new-course-number', '.new-course-run'],
             function(sum, ele) {
                 return sum + $(ele).val().length;
         }, 0
         );
-        if(totalLength > 80) {
+        if(totalLength > 65) {
             $('.wrap-error').addClass('is-shown');
-            $('#course_creation_error').html('<p>' + gettext('Course fields must have a combined length of no more than 80 characters.') + '</p>');
+            $('#course_creation_error').html('<p>' + gettext('The combined length of the organization, course number, and course run fields cannot be more than 65 characters.') + '</p>');
             $('.new-course-save').addClass('is-disabled');
         }
         else {
             $('.wrap-error').removeClass('is-shown');
         }
-    }
+    };
 
     // Handle validation asynchronously
     _.each(
@@ -840,7 +840,7 @@ function saveSetSectionScheduleDate(e) {
     });
 
     var saving = new CMS.Views.Notification.Mini({
-        title: gettext("Saving") + "&hellip;"
+        title: gettext("Saving&hellip;")
     });
     saving.show();
     // call into server to commit the new order

cms/static/js/models/section.js

@@ -23,7 +23,7 @@ CMS.Models.Section = Backbone.Model.extend({
     showNotification: function() {
         if(!this.msg) {
             this.msg = new CMS.Views.Notification.Mini({
-                title: gettext("Saving") + "…"
+                title: gettext("Saving…")
             });
         }
         this.msg.show();

cms/static/js/views/course_info_edit.js

@@ -118,7 +118,7 @@ CMS.Views.ClassInfoUpdateView = Backbone.View.extend({
         targetModel.set({ date : this.dateEntry(event).val(), content : this.$codeMirror.getValue() });
         // push change to display, hide the editor, submit the change
         var saving = new CMS.Views.Notification.Mini({
-            title: gettext('Saving') + '…'
+            title: gettext('Saving…')
         });
         saving.show();
         var ele = this.modelDom(event);
@@ -183,7 +183,7 @@ CMS.Views.ClassInfoUpdateView = Backbone.View.extend({
                         });
                         self.modelDom(event).remove();
                         var deleting = new CMS.Views.Notification.Mini({
-                            title: gettext('Deleting') + '…'
+                            title: gettext('Deleting…')
                         });
                         deleting.show();
                         targetModel.destroy({
@@ -327,7 +327,7 @@ CMS.Views.ClassInfoHandoutsView = Backbone.View.extend({
     onSave: function(event) {
         this.model.set('data', this.$codeMirror.getValue());
         var saving = new CMS.Views.Notification.Mini({
-            title: gettext('Saving') + '…'
+            title: gettext('Saving…')
         });
         saving.show();
         this.model.save({}, {

cms/static/js/views/grader-select-view.js

 CMS.Models.AssignmentGrade = Backbone.Model.extend({
-	defaults : {
-		graderType : null, // the type label (string). May be "Not Graded" which implies None. I'd like to use id but that's ephemeral
-		location : null	// A location object
-	},
-	initialize : function(attrs) {
-		if (attrs['assignmentUrl']) {
-			this.set('location', new CMS.Models.Location(attrs['assignmentUrl'], {parse: true}));
-		}
-	},
-	parse : function(attrs) {
-		if (attrs && attrs['location']) {
-			attrs.location = new CMS.Models.Location(attrs['location'], {parse: true});
-		}
-	},
-	urlRoot : function() {
-		if (this.has('location')) {
-			var location = this.get('location');
-			return '/' + location.get('org') + "/" + location.get('course') + '/' + location.get('category') + '/'
-			+ location.get('name') + '/gradeas/';
-		}
-		else return "";
-	}
+    defaults : {
+        graderType : null, // the type label (string). May be "Not Graded" which implies None. I'd like to use id but that's ephemeral
+        location : null // A location object
+    },
+    initialize : function(attrs) {
+        if (attrs['assignmentUrl']) {
+            this.set('location', new CMS.Models.Location(attrs['assignmentUrl'], {parse: true}));
+        }
+    },
+    parse : function(attrs) {
+        if (attrs && attrs['location']) {
+            attrs.location = new CMS.Models.Location(attrs['location'], {parse: true});
+        }
+    },
+    urlRoot : function() {
+        if (this.has('location')) {
+            var location = this.get('location');
+            return '/' + location.get('org') + "/" + location.get('course') + '/' + location.get('category') + '/'
+            + location.get('name') + '/gradeas/';
+        }
+        else return "";
+    }
 });
 
 CMS.Views.OverviewAssignmentGrader = Backbone.View.extend({
-	// instantiate w/ { graders : CourseGraderCollection, el : <the gradable-status div> }
-	events : {
-		"click .menu-toggle" : "showGradeMenu",
-		"click .menu li" : "selectGradeType"
-	},
-	initialize : function() {
-		// call template w/ {assignmentType : formatname, graders : CourseGraderCollection instance }
-		this.template = _.template(
-				// TODO move to a template file
-				'<h4 class="status-label"><%= assignmentType %></h4>' +
-				'<a data-tooltip="Mark/unmark this subsection as graded" class="menu-toggle" href="#">' +
-					'<% if (!hideSymbol) {%><i class="icon-ok"></i><%};%>' +
-				'</a>' +
-				'<ul class="menu">' +
-					'<% graders.each(function(option) { %>' +
-						'<li><a <% if (option.get("type") == assignmentType) {%>class="is-selected" <%}%> href="#"><%= option.get("type") %></a></li>' +
-					'<% }) %>' +
-					'<li><a class="gradable-status-notgraded" href="#">Not Graded</a></li>' +
-				'</ul>');
-		this.assignmentGrade = new CMS.Models.AssignmentGrade({
-			assignmentUrl : this.$el.closest('.id-holder').data('id'),
-			graderType : this.$el.data('initial-status')});
-		// TODO throw exception if graders is null
-		this.graders = this.options['graders'];
-		var cachethis = this;
-		// defining here to get closure around this
-		this.removeMenu = function(e) {
-			e.preventDefault();
-			cachethis.$el.removeClass('is-active');
-			$(document).off('click', cachethis.removeMenu);
-		}
-		this.hideSymbol = this.options['hideSymbol'];
-		this.render();
-	},
-	render : function() {
-		this.$el.html(this.template({ assignmentType : this.assignmentGrade.get('graderType'), graders : this.graders,
-			hideSymbol : this.hideSymbol }));
-		if (this.assignmentGrade.has('graderType') && this.assignmentGrade.get('graderType') != "Not Graded") {
-			this.$el.addClass('is-set');
-		}
-		else {
-			this.$el.removeClass('is-set');
-		}
-	},
-	showGradeMenu : function(e) {
-		e.preventDefault();
-		// I sure hope this doesn't break anything but it's needed to keep the removeMenu from activating
-		e.stopPropagation();
-		// nasty global event trap :-(
-		$(document).on('click', this.removeMenu);
-		this.$el.addClass('is-active');
-	},
-	selectGradeType : function(e) {
-	      e.preventDefault();
+    // instantiate w/ { graders : CourseGraderCollection, el : <the gradable-status div> }
+    events : {
+        "click .menu-toggle" : "showGradeMenu",
+        "click .menu li" : "selectGradeType"
+    },
+    initialize : function() {
+        // call template w/ {assignmentType : formatname, graders : CourseGraderCollection instance }
+        this.template = _.template(
+                // TODO move to a template file
+                '<h4 class="status-label"><%= assignmentType %></h4>' +
+                '<a data-tooltip="Mark/unmark this subsection as graded" class="menu-toggle" href="#">' +
+                    '<% if (!hideSymbol) {%><i class="icon-ok"></i><%};%>' +
+                '</a>' +
+                '<ul class="menu">' +
+                    '<% graders.each(function(option) { %>' +
+                        '<li><a <% if (option.get("type") == assignmentType) {%>class="is-selected" <%}%> href="#"><%= option.get("type") %></a></li>' +
+                    '<% }) %>' +
+                    '<li><a class="gradable-status-notgraded" href="#">Not Graded</a></li>' +
+                '</ul>');
+        this.assignmentGrade = new CMS.Models.AssignmentGrade({
+            assignmentUrl : this.$el.closest('.id-holder').data('id'),
+            graderType : this.$el.data('initial-status')});
+        // TODO throw exception if graders is null
+        this.graders = this.options['graders'];
+        var cachethis = this;
+        // defining here to get closure around this
+        this.removeMenu = function(e) {
+            e.preventDefault();
+            cachethis.$el.removeClass('is-active');
+            $(document).off('click', cachethis.removeMenu);
+        };
+        this.hideSymbol = this.options['hideSymbol'];
+        this.render();
+    },
+    render : function() {
+        this.$el.html(this.template({ assignmentType : this.assignmentGrade.get('graderType'), graders : this.graders,
+            hideSymbol : this.hideSymbol }));
+        if (this.assignmentGrade.has('graderType') && this.assignmentGrade.get('graderType') != "Not Graded") {
+            this.$el.addClass('is-set');
+        }
+        else {
+            this.$el.removeClass('is-set');
+        }
+    },
+    showGradeMenu : function(e) {
+        e.preventDefault();
+        // I sure hope this doesn't break anything but it's needed to keep the removeMenu from activating
+        e.stopPropagation();
+        // nasty global event trap :-(
+        $(document).on('click', this.removeMenu);
+        this.$el.addClass('is-active');
+    },
+    selectGradeType : function(e) {
+          e.preventDefault();
 
-	      this.removeMenu(e);
+          this.removeMenu(e);
 
               var saving = new CMS.Views.Notification.Mini({
-                  title: gettext('Saving') + '&hellip;'
+                  title: gettext('Saving&hellip;')
               });
               saving.show();
 
-	      // TODO I'm not happy with this string fetch via the html for what should be an id. I'd rather use the id attr
-	      // of the CourseGradingPolicy model or null for Not Graded (NOTE, change template's if check for is-selected accordingly)
-	      this.assignmentGrade.save(
+          // TODO I'm not happy with this string fetch via the html for what should be an id. I'd rather use the id attr
+          // of the CourseGradingPolicy model or null for Not Graded (NOTE, change template's if check for is-selected accordingly)
+          this.assignmentGrade.save(
                   'graderType',
                   $(e.target).text(),
                   {success: function () { saving.hide(); }}
               );
 
-	      this.render();
-	}
-})
+          this.render();
+    }
+});

cms/static/js/views/overview.js

@@ -226,7 +226,7 @@ function _handleReorder(event, ui, parentIdField, childrenSelector) {
                 children.push(ui.draggable.data('id'));
         }
         var saving = new CMS.Views.Notification.Mini({
-            title: gettext('Saving') + '…'
+            title: gettext('Saving…')
         });
         saving.show();
         $.ajax({

cms/static/js/views/textbook.js

@@ -35,7 +35,7 @@ CMS.Views.ShowTextbook = Backbone.View.extend({
                     click: function(view) {
                         view.hide();
                         var delmsg = new CMS.Views.Notification.Mini({
-                            title: gettext("Deleting") + "…"
+                            title: gettext("Deleting…")
                         }).show();
                         textbook.destroy({
                             complete: function() {
@@ -122,7 +122,7 @@ CMS.Views.EditTextbook = Backbone.View.extend({
         this.setValues();
         if(!this.model.isValid()) { return; }
         var saving = new CMS.Views.Notification.Mini({
-            title: gettext("Saving") + "…"
+            title: gettext("Saving…")
         }).show();
         var that = this;
         this.model.save({}, {

common/djangoapps/course_modes/views.py

@@ -66,7 +66,7 @@ class ChooseModeView(View):
             return HttpResponseBadRequest(_("Enrollment mode not supported"))
 
         if requested_mode in ("audit", "honor"):
-            CourseEnrollment.enroll(user, course_id)
+            CourseEnrollment.enroll(user, course_id, requested_mode)
             return redirect('dashboard')
 
         mode_info = allowed_modes[requested_mode]

common/djangoapps/datadog/startup.py

 from django.conf import settings
-from dogapi import dog_http_api, dog_stats_api
+
+from dogapi import dog_stats_api, dog_http_api
+
 
 def run():
     """
     Initialize connection to datadog during django startup.
 
-    Expects the datadog api key in the DATADOG_API settings key
+    Can be configured using a dictionary named DATADOG in the django
+    project settings.
+
     """
-    if hasattr(settings, 'DATADOG_API'):
-        dog_http_api.api_key = settings.DATADOG_API
-        dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
+
+    # By default use the statsd agent
+    options = {'statsd': True}
+
+    if hasattr(settings, 'DATADOG'):
+        options.update(settings.DATADOG)
+
+    # Not all arguments are documented.
+    # Look at the source code for details.
+    dog_stats_api.start(**options)
+
+    dog_http_api.api_key = options.get('api_key')

common/djangoapps/django_comment_common/utils.py

@@ -32,8 +32,30 @@ def seed_permissions_roles(course_id):
     administrator_role.inherit_permissions(moderator_role)
 
 
-def are_permissions_roles_seeded(course_id):
+def _remove_permission_role(course_id, name):
+    try:
+        role = Role.objects.get(name=name, course_id=course_id)
+        if role.course_id == course_id:
+            role.delete()
+    except Role.DoesNotExist:
+        pass
+
 
+def unseed_permissions_roles(course_id):
+    """
+    A utility method to clean up all forum related permissions and roles
+    """
+    _remove_permission_role(name="Administrator", course_id=course_id)
+    _remove_permission_role(name="Moderator", course_id=course_id)
+    _remove_permission_role(name="Community TA", course_id=course_id)
+    _remove_permission_role(name="Student", course_id=course_id)
+
+
+def are_permissions_roles_seeded(course_id):
+    """
+    Returns whether the forums permissions for a course have been provisioned in
+    the database
+    """
     try:
         administrator_role = Role.objects.get(name="Administrator", course_id=course_id)
         moderator_role = Role.objects.get(name="Moderator", course_id=course_id)

common/djangoapps/external_auth/tests/test_helper.py

+"""
+Tests for utility functions in external_auth module
+"""
+from django.test import TestCase
+from external_auth.views import _safe_postlogin_redirect
+
+
+class ExternalAuthHelperFnTest(TestCase):
+    """
+    Unit tests for the external_auth.views helper function
+    """
+    def test__safe_postlogin_redirect(self):
+        """
+        Tests the _safe_postlogin_redirect function with different values of next
+        """
+        HOST = 'testserver'                               # pylint: disable=C0103
+        ONSITE1 = '/dashboard'                            # pylint: disable=C0103
+        ONSITE2 = '/courses/org/num/name/courseware'      # pylint: disable=C0103
+        ONSITE3 = 'http://{}/my/custom/url'.format(HOST)  # pylint: disable=C0103
+        OFFSITE1 = 'http://www.attacker.com'              # pylint: disable=C0103
+
+        for redirect_to in [ONSITE1, ONSITE2, ONSITE3]:
+            redir = _safe_postlogin_redirect(redirect_to, HOST)
+            self.assertEqual(redir.status_code, 302)
+            self.assertEqual(redir['location'], redirect_to)
+
+        redir2 = _safe_postlogin_redirect(OFFSITE1, HOST)
+        self.assertEqual(redir2.status_code, 302)
+        self.assertEqual("/", redir2['location'])

common/djangoapps/student/management/commands/get_grades.py

 from courseware import grades, courses
+from certificates.models import GeneratedCertificate
 from django.test.client import RequestFactory
 from django.core.management.base import BaseCommand, CommandError
 import os
@@ -28,6 +29,13 @@ class Command(BaseCommand):
     Generate a list of grades for all students
     that are enrolled in a course.
 
+    CSV will include the following:
+      - username
+      - email
+      - grade in the certificate table if it exists
+      - computed grade
+      - grade breakdown
+
     Outputs grades to a csv file.
 
     Example:
@@ -57,8 +65,7 @@ class Command(BaseCommand):
         course_id = options['course']
         print "Fetching enrolled students for {0}".format(course_id)
         enrolled_students = User.objects.filter(
-            courseenrollment__course_id=course_id).prefetch_related(
-                "groups").order_by('username')
+            courseenrollment__course_id=course_id)
         factory = RequestMock()
         request = factory.get('/')
 
@@ -69,6 +76,11 @@ class Command(BaseCommand):
         start = datetime.datetime.now()
         rows = []
         header = None
+        print "Fetching certificate data"
+        cert_grades = {cert.user.username: cert.grade
+                       for cert in list(GeneratedCertificate.objects.filter(
+                       course_id=course_id).prefetch_related('user'))}
+        print "Grading students"
         for count, student in enumerate(enrolled_students):
             count += 1
             if count % STATUS_INTERVAL == 0:
@@ -86,10 +98,13 @@ class Command(BaseCommand):
             grade = grades.grade(student, request, course)
             if not header:
                 header = [section['label'] for section in grade[u'section_breakdown']]
-                rows.append(["email", "username"] + header)
+                rows.append(["email", "username", "certificate-grade", "grade"] + header)
             percents = {section['label']: section['percent'] for section in grade[u'section_breakdown']}
             row_percents = [percents[label] for label in header]
-            rows.append([student.email, student.username] + row_percents)
+            if student.username in cert_grades:
+                rows.append([student.email, student.username, cert_grades[student.username], grade['percent']] + row_percents)
+            else:
+                rows.append([student.email, student.username, "N/A", grade['percent']] + row_percents)
         with open(options['output'], 'wb') as f:
             writer = csv.writer(f)
             writer.writerows(rows)

common/djangoapps/student/management/commands/pearson_import_conf_zip.py

 import csv
-
-from zipfile import ZipFile, is_zipfile
 from time import strptime, strftime
-
 from datetime import datetime
+from zipfile import ZipFile, is_zipfile
+
 from dogapi import dog_http_api
+from pytz import UTC
 
 from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
 
+import django_startup
+
 from student.models import TestCenterUser, TestCenterRegistration
-from pytz import UTC
+
+
+django_startup.autostartup()
 
 
 class Command(BaseCommand):
 
-    dog_http_api.api_key = settings.DATADOG_API
     args = '<input zip file>'
     help = """
     Import Pearson confirmation files and update TestCenterUser

common/djangoapps/student/management/commands/pearson_transfer.py

-import os
 from optparse import make_option
+import os
 from stat import S_ISDIR
 
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.core.management import call_command
+import boto
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
-import boto
 
-dog_http_api.api_key = settings.DATADOG_API
-dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import BaseCommand, CommandError
+
+import django_startup
+
+
+django_startup.autostartup()
 
 
 class Command(BaseCommand):

common/djangoapps/student/management/commands/tests/test_pearson.py

@@ -303,15 +303,13 @@ class PearsonTransferTestCase(PearsonTestCase):
     '''
 
     def test_transfer_config(self):
-        with self.settings(DATADOG_API='FAKE_KEY'):
-            # TODO: why is this failing with the wrong error message?!
-            stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
-            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
-        with self.settings(DATADOG_API='FAKE_KEY'):
-            stderrmsg = get_command_error_text('pearson_transfer')
-            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
+        self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+
+        stderrmsg = get_command_error_text('pearson_transfer')
+        self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'LOCAL_IMPORT': self.import_dir}):
             stderrmsg = get_command_error_text('pearson_transfer')
             self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
@@ -319,8 +317,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_export_missing_dest_dir(self):
         raise SkipTest()
         create_multiple_registrations('export_missing_dest')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'SFTP_EXPORT': 'this/does/not/exist',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -336,8 +333,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_export(self):
         raise SkipTest()
         create_multiple_registrations("transfer_export")
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'SFTP_EXPORT': 'results/topvue',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -354,8 +350,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_import_missing_source_dir(self):
         raise SkipTest()
         create_multiple_registrations('import_missing_src')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+        with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
                                     'SFTP_IMPORT': 'this/does/not/exist',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -371,8 +366,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_import(self):
         raise SkipTest()
         create_multiple_registrations('import_missing_src')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+        with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
                                     'SFTP_IMPORT': 'results',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,

common/djangoapps/student/tests/test_login.py

@@ -2,14 +2,26 @@
 Tests for student activation and login
 '''
 import json
+import unittest
 from mock import patch
 
 from django.test import TestCase
 from django.test.client import Client
+from django.test.utils import override_settings
+from django.conf import settings
 from django.core.cache import cache
 from django.core.urlresolvers import reverse, NoReverseMatch
 from student.tests.factories import UserFactory, RegistrationFactory, UserProfileFactory
+from student.views import _parse_course_id_from_string, _get_course_enrollment_domain
 
+from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
+from xmodule.modulestore.inheritance import own_metadata
+from xmodule.modulestore.django import editable_modulestore
+
+from external_auth.models import ExternalAuthMap
+
+TEST_DATA_MIXED_MODULESTORE = mixed_store_config(settings.COMMON_TEST_DATA_ROOT, {})
 
 class LoginTest(TestCase):
     '''
@@ -154,13 +166,109 @@ class LoginTest(TestCase):
 
     def _assert_audit_log(self, mock_audit_log, level, log_strings):
         """
-        Check that the audit log has received the expected call.
+        Check that the audit log has received the expected call as its last call.
         """
         method_calls = mock_audit_log.method_calls
-        self.assertEquals(len(method_calls), 1)
-        name, args, _kwargs = method_calls[0]
+        name, args, _kwargs = method_calls[-1]
         self.assertEquals(name, level)
         self.assertEquals(len(args), 1)
         format_string = args[0]
         for log_string in log_strings:
             self.assertIn(log_string, format_string)
+
+
+class UtilFnTest(TestCase):
+    """
+    Tests for utility functions in student.views
+    """
+    def test__parse_course_id_from_string(self):
+        """
+        Tests the _parse_course_id_from_string util function
+        """
+        COURSE_ID = u'org/num/run'                                # pylint: disable=C0103
+        COURSE_URL = u'/courses/{}/otherstuff'.format(COURSE_ID)  # pylint: disable=C0103
+        NON_COURSE_URL = u'/blahblah'                             # pylint: disable=C0103
+        self.assertEqual(_parse_course_id_from_string(COURSE_URL), COURSE_ID)
+        self.assertIsNone(_parse_course_id_from_string(NON_COURSE_URL))
+
+
+@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
+class ExternalAuthShibTest(ModuleStoreTestCase):
+    """
+    Tests how login_user() interacts with ExternalAuth, in particular Shib
+    """
+    def setUp(self):
+        self.store = editable_modulestore()
+        self.course = CourseFactory.create(org='Stanford', number='456', display_name='NO SHIB')
+        self.shib_course = CourseFactory.create(org='Stanford', number='123', display_name='Shib Only')
+        self.shib_course.enrollment_domain = 'shib:https://idp.stanford.edu/'
+        metadata = own_metadata(self.shib_course)
+        metadata['enrollment_domain'] = self.shib_course.enrollment_domain
+        self.store.update_metadata(self.shib_course.location.url(), metadata)
+        self.user_w_map = UserFactory.create(email='withmap@stanford.edu')
+        self.extauth = ExternalAuthMap(external_id='withmap@stanford.edu',
+                                       external_email='withmap@stanford.edu',
+                                       external_domain='shib:https://idp.stanford.edu/',
+                                       external_credentials="",
+                                       user=self.user_w_map)
+        self.user_w_map.save()
+        self.extauth.save()
+        self.user_wo_map = UserFactory.create(email='womap@gmail.com')
+        self.user_wo_map.save()
+
+    @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
+    def test_login_page_redirect(self):
+        """
+        Tests that when a shib user types their email address into the login page, they get redirected
+        to the shib login.
+        """
+        response = self.client.post(reverse('login'), {'email': self.user_w_map.email, 'password': ''})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.content, json.dumps({'success': False, 'redirect': reverse('shib-login')}))
+
+    @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
+    def test__get_course_enrollment_domain(self):
+        """
+        Tests the _get_course_enrollment_domain utility function
+        """
+        self.assertIsNone(_get_course_enrollment_domain("I/DONT/EXIST"))
+        self.assertIsNone(_get_course_enrollment_domain(self.course.id))
+        self.assertEqual(self.shib_course.enrollment_domain, _get_course_enrollment_domain(self.shib_course.id))
+
+    @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
+    def test_login_required_dashboard(self):
+        """
+        Tests redirects to when @login_required to dashboard, which should always be the normal login,
+        since there is no course context
+        """
+        response = self.client.get(reverse('dashboard'))
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response['Location'], 'http://testserver/accounts/login?next=/dashboard')
+
+    @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), "AUTH_USE_SHIB not set")
+    def test_externalauth_login_required_course_context(self):
+        """
+        Tests the redirects when visiting course-specific URL with @login_required.
+        Should vary by course depending on its enrollment_domain
+        """
+        TARGET_URL = reverse('courseware', args=[self.course.id])            # pylint: disable=C0103
+        noshib_response = self.client.get(TARGET_URL, follow=True)
+        self.assertEqual(noshib_response.redirect_chain[-1],
+                         ('http://testserver/accounts/login?next={url}'.format(url=TARGET_URL), 302))
+        self.assertContains(noshib_response, ("<title>Log into your {platform_name} Account</title>"
+                                              .format(platform_name=settings.PLATFORM_NAME)))
+        self.assertEqual(noshib_response.status_code, 200)
+
+        TARGET_URL_SHIB = reverse('courseware', args=[self.shib_course.id])  # pylint: disable=C0103
+        shib_response = self.client.get(**{'path': TARGET_URL_SHIB,
+                                           'follow': True,
+                                           'REMOTE_USER': self.extauth.external_id,
+                                           'Shib-Identity-Provider': 'https://idp.stanford.edu/'})
+        # Test that the shib-login redirect page with ?next= and the desired page are part of the redirect chain
+        # The 'courseware' page actually causes a redirect itself, so it's not the end of the chain and we
+        # won't test its contents
+        self.assertEqual(shib_response.redirect_chain[-3],
+                         ('http://testserver/shib-login/?next={url}'.format(url=TARGET_URL_SHIB), 302))
+        self.assertEqual(shib_response.redirect_chain[-2],
+                         ('http://testserver{url}'.format(url=TARGET_URL_SHIB), 302))
+        self.assertEqual(shib_response.status_code, 200)

common/djangoapps/student/views.py

@@ -23,7 +23,8 @@ from django.core.urlresolvers import reverse
 from django.core.validators import validate_email, validate_slug, ValidationError
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import IntegrityError, transaction
-from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseNotAllowed, Http404
+from django.http import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden,
+                         HttpResponseNotAllowed, Http404)
 from django.shortcuts import redirect
 from django_future.csrf import ensure_csrf_cookie
 from django.utils.http import cookie_date, base36_to_int, urlencode
@@ -54,12 +55,13 @@ from courseware.courses import get_courses, sort_by_announcement
 from courseware.access import has_access
 
 from external_auth.models import ExternalAuthMap
+import external_auth.views
 
 from bulk_email.models import Optout
 from cme_registration.views import cme_register_user, cme_create_account
 import track.views
 
-from statsd import statsd
+from dogapi import dog_stats_api
 from pytz import UTC
 
 log = logging.getLogger("mitx.student")
@@ -105,7 +107,7 @@ def index(request, extra_context={}, user=None):
     # The course selection work is done in courseware.courses.
     domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')  # normally False
     # do explicit check, because domain=None is valid
-    if domain == False:
+    if domain is False:
         domain = request.META.get('HTTP_HOST')
 
     courses = get_courses(user, domain=domain)
@@ -268,6 +270,8 @@ def register_user(request, extra_context=None):
     if extra_context is not None:
         context.update(extra_context)
 
+    if context.get("extauth_domain", '').startswith(external_auth.views.SHIBBOLETH_DOMAIN_PREFIX):
+        return render_to_response('register-shib.html', context)
     return render_to_response('register.html', context)
 
 
@@ -404,10 +408,12 @@ def change_enrollment(request):
             )
 
         org, course_num, run = course_id.split("/")
-        statsd.increment("common.student.enrollment",
-                         tags=["org:{0}".format(org),
-                               "course:{0}".format(course_num),
-                               "run:{0}".format(run)])
+        dog_stats_api.increment(
+            "common.student.enrollment",
+            tags=["org:{0}".format(org),
+                  "course:{0}".format(course_num),
+                  "run:{0}".format(run)]
+        )
 
         CourseEnrollment.enroll(user, course.id)
 
@@ -418,10 +424,12 @@ def change_enrollment(request):
             CourseEnrollment.unenroll(user, course_id)
 
             org, course_num, run = course_id.split("/")
-            statsd.increment("common.student.unenrollment",
-                             tags=["org:{0}".format(org),
-                                   "course:{0}".format(course_num),
-                                   "run:{0}".format(run)])
+            dog_stats_api.increment(
+                "common.student.unenrollment",
+                tags=["org:{0}".format(org),
+                      "course:{0}".format(course_num),
+                      "run:{0}".format(run)]
+            )
 
             return HttpResponse()
         except CourseEnrollment.DoesNotExist:
@@ -429,11 +437,49 @@ def change_enrollment(request):
     else:
         return HttpResponseBadRequest(_("Enrollment action is invalid"))
 
+
+def _parse_course_id_from_string(input_str):
+    """
+    Helper function to determine if input_str (typically the queryparam 'next') contains a course_id.
+    @param input_str:
+    @return: the course_id if found, None if not
+    """
+    m_obj = re.match(r'^/courses/(?P<course_id>[^/]+/[^/]+/[^/]+)', input_str)
+    if m_obj:
+        return m_obj.group('course_id')
+    return None
+
+
+def _get_course_enrollment_domain(course_id):
+    """
+    Helper function to get the enrollment domain set for a course with id course_id
+    @param course_id:
+    @return:
+    """
+    try:
+        course = course_from_id(course_id)
+        return course.enrollment_domain
+    except ItemNotFoundError:
+        return None
+
+
 @ensure_csrf_cookie
-def accounts_login(request, error=""):
+def accounts_login(request):
+    """
+    This view is mainly used as the redirect from the @login_required decorator.  I don't believe that
+    the login path linked from the homepage uses it.
+    """
     if settings.MITX_FEATURES.get('AUTH_USE_CAS'):
         return redirect(reverse('cas-login'))
-    return render_to_response('login.html', {'error': error})
+    # see if the "next" parameter has been set, whether it has a course context, and if so, whether
+    # there is a course-specific place to redirect
+    redirect_to = request.GET.get('next')
+    if redirect_to:
+        course_id = _parse_course_id_from_string(redirect_to)
+        if course_id and _get_course_enrollment_domain(course_id):
+            return external_auth.views.course_specific_login(request, course_id)
+    return render_to_response('login.html')
+
 
 # Need different levels of logging
 @ensure_csrf_cookie
@@ -451,6 +497,18 @@ def login_user(request, error=""):
         AUDIT_LOG.warning(u"Login failed - Unknown user email: {0}".format(email))
         user = None
 
+    # check if the user has a linked shibboleth account, if so, redirect the user to shib-login
+    # This behavior is pretty much like what gmail does for shibboleth.  Try entering some @stanford.edu
+    # address into the Gmail login.
+    if settings.MITX_FEATURES.get('AUTH_USE_SHIB') and user:
+        try:
+            eamap = ExternalAuthMap.objects.get(user=user)
+            if eamap.external_domain.startswith(external_auth.views.SHIBBOLETH_DOMAIN_PREFIX):
+                return HttpResponse(json.dumps({'success': False, 'redirect': reverse('shib-login')}))
+        except ExternalAuthMap.DoesNotExist:
+            # This is actually the common case, logging in user without external linked login
+            AUDIT_LOG.info("User %s w/o external auth attempting login", user)
+
     # if the user doesn't exist, we want to set the username to an invalid
     # username so that authentication is guaranteed to fail and we can take
     # advantage of the ratelimited backend
@@ -487,7 +545,7 @@ def login_user(request, error=""):
 
         redirect_url = try_change_enrollment(request)
 
-        statsd.increment("common.student.successful_login")
+        dog_stats_api.increment("common.student.successful_login")
         response = HttpResponse(json.dumps({'success': True, 'redirect_url': redirect_url}))
 
         # set the login cookie for the edx marketing site
@@ -655,9 +713,10 @@ def create_account(request, post_override=None):
         return HttpResponse(json.dumps(js))
 
     # Can't have terms of service for certain SHIB users, like at Stanford
-    tos_not_required = settings.MITX_FEATURES.get("AUTH_USE_SHIB") \
-                       and settings.MITX_FEATURES.get('SHIB_DISABLE_TOS') \
-                       and DoExternalAuth and ("shib" in eamap.external_domain)
+    tos_not_required = (settings.MITX_FEATURES.get("AUTH_USE_SHIB") and
+                        settings.MITX_FEATURES.get('SHIB_DISABLE_TOS') and
+                        DoExternalAuth and
+                        eamap.external_domain.startswith(external_auth.views.SHIBBOLETH_DOMAIN_PREFIX))
 
     if not tos_not_required:
         if post_vars.get('terms_of_service', 'false') != u'true':
@@ -759,7 +818,7 @@ def create_account(request, post_override=None):
 
     redirect_url = try_change_enrollment(request)
 
-    statsd.increment("common.student.account_created")
+    dog_stats_api.increment("common.student.account_created")
 
     response_params = {'success': True,
                        'redirect_url': redirect_url}

common/djangoapps/terrain/browser.py

@@ -12,7 +12,7 @@ from django.core.management import call_command
 from django.conf import settings
 from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-from requests import put
+import requests
 from base64 import encodestring
 from json import dumps
 
@@ -54,12 +54,12 @@ def set_job_status(jobid, passed=True):
     """
     Sets the job status on sauce labs
     """
-    body_content = dumps({"passed": passed})
     config = get_username_and_key()
+    url = 'http://saucelabs.com/rest/v1/{}/jobs/{}'.format(config['username'], world.jobid)
+    body_content = dumps({"passed": passed})
     base64string = encodestring('{}:{}'.format(config['username'], config['access-key']))[:-1]
-    result = put('http://saucelabs.com/rest/v1/{}/jobs/{}'.format(config['username'], world.jobid),
-        data=body_content,
-        headers={"Authorization": "Basic {}".format(base64string)})
+    headers = {"Authorization": "Basic {}".format(base64string)}
+    result = requests.put(url, data=body_content, headers=headers)
     return result.status_code == 200
 
 
@@ -75,7 +75,8 @@ def make_desired_capabilities():
     desired_capabilities['build'] = settings.SAUCE.get('BUILD')
     desired_capabilities['video-upload-on-pass'] = False
     desired_capabilities['sauce-advisor'] = False
-    desired_capabilities['record-screenshots'] = False
+    desired_capabilities['capture-html'] = True
+    desired_capabilities['record-screenshots'] = True
     desired_capabilities['selenium-version'] = "2.34.0"
     desired_capabilities['max-duration'] = 3600
     desired_capabilities['public'] = 'public restricted'
@@ -164,15 +165,18 @@ def reset_databases(scenario):
     xmodule.modulestore.django.clear_existing_modulestores()
 
 
-# Uncomment below to trigger a screenshot on error
-# @after.each_scenario
+@after.each_scenario
 def screenshot_on_error(scenario):
     """
     Save a screenshot to help with debugging.
     """
     if scenario.failed:
-        world.browser.driver.save_screenshot('/tmp/last_failed_scenario.png')
-
+        try:
+            output_dir = '{}/log'.format(settings.TEST_ROOT)
+            image_name = '{}/{}.png'.format(output_dir, scenario.name.replace(' ', '_'))
+            world.browser.driver.save_screenshot(image_name)
+        except WebDriverException:
+            LOGGER.error('Could not capture a screenshot')
 
 @after.all
 def teardown_browser(total):

common/djangoapps/track/backends/__init__.py

+"""
+Event tracking backend module.
+
+Contains the base class for event trackers, and implementation of some
+backends.
+
+"""
+
+from __future__ import absolute_import
+
+import abc
+
+
+# pylint: disable=unused-argument
+class BaseBackend(object):
+    """
+    Abstract Base Class for event tracking backends.
+
+    """
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def send(self, event):
+        """Send event to tracker."""
+        pass

common/djangoapps/track/backends/django.py

+"""
+Event tracker backend that saves events to a Django database.
+
+"""
+
+# TODO: this module is very specific to the event schema, and is only
+# brought here for legacy support. It should be updated when the
+# schema changes or eventually deprecated.
+
+from __future__ import absolute_import
+
+import logging
+
+from django.db import models
+
+from track.backends import BaseBackend
+
+
+log = logging.getLogger('track.backends.django')
+
+
+LOGFIELDS = [
+    'username',
+    'ip',
+    'event_source',
+    'event_type',
+    'event',
+    'agent',
+    'page',
+    'time',
+    'host'
+]
+
+
+class TrackingLog(models.Model):
+    """Defines the fields that are stored in the tracking log database."""
+
+    dtcreated = models.DateTimeField('creation date', auto_now_add=True)
+    username = models.CharField(max_length=32, blank=True)
+    ip = models.CharField(max_length=32, blank=True)
+    event_source = models.CharField(max_length=32)
+    event_type = models.CharField(max_length=512, blank=True)
+    event = models.TextField(blank=True)
+    agent = models.CharField(max_length=256, blank=True)
+    page = models.CharField(max_length=512, blank=True, null=True)
+    time = models.DateTimeField('event time')
+    host = models.CharField(max_length=64, blank=True)
+
+    class Meta:
+        app_label = 'track'
+        db_table = 'track_trackinglog'
+
+    def __unicode__(self):
+        fmt = (
+            u"[{self.time}] {self.username}@{self.ip}: "
+            u"{self.event_source}| {self.event_type} | "
+            u"{self.page} | {self.event}"
+        )
+        return fmt.format(self=self)
+
+
+class DjangoBackend(BaseBackend):
+    """Event tracker backend that saves to a Django database"""
+    def __init__(self, name='default', **options):
+        """
+        Configure database used by the backend.
+
+        :Parameters:
+
+          - `name` is the name of the database as specified in the project
+            settings.
+
+        """
+        super(DjangoBackend, self).__init__(**options)
+        self.name = name
+
+    def send(self, event):
+        field_values = {x: event.get(x, '') for x in LOGFIELDS}
+        tldat = TrackingLog(**field_values)
+        try:
+            tldat.save(using=self.name)
+        except Exception as e:  # pylint: disable=broad-except
+            log.exception(e)

common/djangoapps/track/backends/logger.py

+"""Event tracker backend that saves events to a python logger."""
+
+from __future__ import absolute_import
+
+import logging
+import json
+
+from django.conf import settings
+
+from track.backends import BaseBackend
+from track.utils import DateTimeJSONEncoder
+
+log = logging.getLogger('track.backends.logger')
+
+
+class LoggerBackend(BaseBackend):
+    """Event tracker backend that uses a python logger.
+
+    Events are logged to the INFO level as JSON strings.
+
+    """
+
+    def __init__(self, name, **kwargs):
+        """Event tracker backend that uses a python logger.
+
+        :Parameters:
+          - `name`: identifier of the logger, which should have
+            been configured using the default python mechanisms.
+
+        """
+        super(LoggerBackend, self).__init__(**kwargs)
+
+        self.event_logger = logging.getLogger(name)
+
+    def send(self, event):
+        event_str = json.dumps(event, cls=DateTimeJSONEncoder)
+
+        # TODO: remove trucation of the serialized event, either at a
+        # higher level during the emittion of the event, or by
+        # providing warnings when the events exceed certain size.
+        event_str = event_str[:settings.TRACK_MAX_EVENT]
+
+        self.event_logger.info(event_str)

common/djangoapps/track/backends/mongodb.py

+"""MongoDB event tracker backend."""
+
+from __future__ import absolute_import
+
+import logging
+
+import pymongo
+from pymongo import MongoClient
+from pymongo.errors import PyMongoError
+
+from track.backends import BaseBackend
+
+
+log = logging.getLogger('track.backends.mongodb')
+
+
+class MongoBackend(BaseBackend):
+    """Class for a MongoDB event tracker Backend"""
+
+    def __init__(self, **kwargs):
+        """
+        Connect to a MongoDB.
+
+        :Parameters:
+
+          - `host`: hostname
+          - `port`: port
+          - `user`: collection username
+          - `password`: collection user password
+          - `database`: name of the database
+          - `collection`: name of the collection
+          - `extra`: parameters to pymongo.MongoClient not listed above
+
+        """
+
+        super(MongoBackend, self).__init__(**kwargs)
+
+        # Extract connection parameters from kwargs
+
+        host = kwargs.get('host', 'localhost')
+        port = kwargs.get('port', 27017)
+
+        user = kwargs.get('user', '')
+        password = kwargs.get('password', '')
+
+        db_name = kwargs.get('database', 'track')
+        collection_name = kwargs.get('collection', 'events')
+
+        # Other mongo connection arguments
+        extra = kwargs.get('extra', {})
+
+        # By default disable write acknowledgments, reducing the time
+        # blocking during an insert
+        extra['w'] = extra.get('w', 0)
+
+        # Make timezone aware by default
+        extra['tz_aware'] = extra.get('tz_aware', True)
+
+        # Connect to database and get collection
+
+        self.connection = MongoClient(
+            host=host,
+            port=port,
+            **extra
+        )
+
+        self.collection = self.connection[db_name][collection_name]
+
+        if user or password:
+            self.collection.database.authenticate(user, password)
+
+        self._create_indexes()
+
+    def _create_indexes(self):
+        # WARNING: The collection will be locked during the index
+        # creation. If the collection has a large number of
+        # documents in it, the operation can take a long time.
+
+        # TODO: The creation of indexes can be moved to a Django
+        # management command or equivalent. There is also an option to
+        # run the indexing on the background, without locking.
+        self.collection.ensure_index([('time', pymongo.DESCENDING)])
+        self.collection.ensure_index('event_type')
+
+    def send(self, event):
+        try:
+            self.collection.insert(event, manipulate=False)
+        except PyMongoError:
+            msg = 'Error inserting to MongoDB event tracker backend'
+            log.exception(msg)

common/djangoapps/track/backends/tests/test_django.py

+from __future__ import absolute_import
+
+from django.test import TestCase
+
+from track.backends.django import DjangoBackend, TrackingLog
+
+
+class TestDjangoBackend(TestCase):
+    def setUp(self):
+        self.backend = DjangoBackend()
+
+    def test_django_backend(self):
+        event = {
+            'username': 'test',
+            'time': '2013-01-01T12:01:00-05:00'
+        }
+        self.backend.send(event)
+
+        results = list(TrackingLog.objects.all())
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].username, 'test')
+
+        # Check if time is stored in UTC
+        self.assertEqual(str(results[0].time), '2013-01-01 17:01:00+00:00')

common/djangoapps/track/backends/tests/test_logger.py

+from __future__ import absolute_import
+
+import json
+import logging
+import datetime
+
+from django.test import TestCase
+
+from track.backends.logger import LoggerBackend
+
+
+class TestLoggerBackend(TestCase):
+    def setUp(self):
+
+        self.handler = MockLoggingHandler()
+        self.handler.setLevel(logging.INFO)
+
+        logger_name = 'track.backends.logger.test'
+        logger = logging.getLogger(logger_name)
+        logger.addHandler(self.handler)
+
+        self.backend = LoggerBackend(name=logger_name)
+
+    def test_logger_backend(self):
+        self.handler.reset()
+
+        # Send a couple of events and check if they were recorded
+        # by the logger. The events are serialized to JSON.
+
+        event = {
+            'test': True,
+            'time': datetime.datetime(2012, 05, 01, 07, 27, 01, 200),
+            'date': datetime.date(2012, 05, 07),
+        }
+
+        self.backend.send(event)
+        self.backend.send(event)
+
+        saved_events = [json.loads(e) for e in self.handler.messages['info']]
+
+        unpacked_event = {
+            'test': True,
+            'time': '2012-05-01T07:27:01.000200+00:00',
+            'date': '2012-05-07'
+        }
+
+        self.assertEqual(saved_events[0], unpacked_event)
+        self.assertEqual(saved_events[1], unpacked_event)
+
+
+class MockLoggingHandler(logging.Handler):
+    """
+    Mock logging handler.
+
+    Stores records in a dictionry of lists by level.
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MockLoggingHandler, self).__init__(*args, **kwargs)
+        self.messages = None
+        self.reset()
+
+    def emit(self, record):
+        level = record.levelname.lower()
+        message = record.getMessage()
+        self.messages[level].append(message)
+
+    def reset(self):
+        self.messages = {
+            'debug': [],
+            'info': [],
+            'warning': [],
+            'error': [],
+            'critical': [],
+        }

common/djangoapps/track/backends/tests/test_mongodb.py

+from __future__ import absolute_import
+
+from uuid import uuid4
+
+from mock import patch
+
+from django.test import TestCase
+
+from track.backends.mongodb import MongoBackend
+
+
+class TestMongoBackend(TestCase):
+    def setUp(self):
+        self.mongo_patcher = patch('track.backends.mongodb.MongoClient')
+        self.addCleanup(self.mongo_patcher.stop)
+        self.mongo_patcher.start()
+
+        self.backend = MongoBackend()
+
+    def test_mongo_backend(self):
+        events = [{'test': 1}, {'test': 2}]
+
+        self.backend.send(events[0])
+        self.backend.send(events[1])
+
+        # Check if we inserted events into the database
+
+        calls = self.backend.collection.insert.mock_calls
+
+        self.assertEqual(len(calls), 2)
+
+        # Unpack the arguments and check if the events were used
+        # as the first argument to collection.insert
+
+        def first_argument(call):
+            _, args, _ = call
+            return args[0]
+
+        self.assertEqual(events[0], first_argument(calls[0]))
+        self.assertEqual(events[1], first_argument(calls[1]))

common/djangoapps/track/middleware.py

 import json
+import re
+
+from django.conf import settings
 
 import views
 
 
-class TrackMiddleware:
+class TrackMiddleware(object):
     def process_request(self, request):
         try:
-            # We're already logging events, and we don't want to capture user
-            # names/passwords.
-            if request.META['PATH_INFO'] in ['/event', '/login']:
+            if not self._should_process_request(request):
                 return
 
             # Removes passwords from the tracking logs
@@ -45,3 +46,14 @@ class TrackMiddleware:
             views.server_track(request, request.META['PATH_INFO'], event)
         except:
             pass
+
+    def _should_process_request(self, request):
+        path = request.META['PATH_INFO']
+
+        ignored_url_patterns = getattr(settings, 'TRACKING_IGNORE_URL_PATTERNS', [])
+        for pattern in ignored_url_patterns:
+            # Note we are explicitly relying on python's internal caching of
+            # compiled regular expressions here.
+            if re.match(pattern, path):
+                return False
+        return True

common/djangoapps/track/models.py

-from django.db import models
-
-
-class TrackingLog(models.Model):
-    """Defines the fields that are stored in the tracking log database"""
-    dtcreated = models.DateTimeField('creation date', auto_now_add=True)
-    username = models.CharField(max_length=32, blank=True)
-    ip = models.CharField(max_length=32, blank=True)
-    event_source = models.CharField(max_length=32)
-    event_type = models.CharField(max_length=512, blank=True)
-    event = models.TextField(blank=True)
-    agent = models.CharField(max_length=256, blank=True)
-    page = models.CharField(max_length=512, blank=True, null=True)
-    time = models.DateTimeField('event time')
-    host = models.CharField(max_length=64, blank=True)
-
-    def __unicode__(self):
-        fmt = (
-            u"[{self.time}] {self.username}@{self.ip}: "
-            u"{self.event_source}| {self.event_type} | "
-            u"{self.page} | {self.event}"
-        )
-        return fmt.format(self=self)
+from track.backends.django import TrackingLog

common/djangoapps/track/tests/test_middleware.py

+import re
+
+from mock import patch
+
+from django.test import TestCase
+from django.test.client import RequestFactory
+from django.test.utils import override_settings
+
+from track.middleware import TrackMiddleware
+
+
+@patch('track.views.server_track')
+class TrackMiddlewareTestCase(TestCase):
+
+    def setUp(self):
+        self.track_middleware = TrackMiddleware()
+        self.request_factory = RequestFactory()
+
+    def test_normal_request(self, mock_server_track):
+        request = self.request_factory.get('/somewhere')
+        self.track_middleware.process_request(request)
+        self.assertTrue(mock_server_track.called)
+
+    def test_default_filters_do_not_render_view(self, mock_server_track):
+        for url in ['/event', '/event/1', '/login', '/heartbeat']:
+            request = self.request_factory.get(url)
+            self.track_middleware.process_request(request)
+            self.assertFalse(mock_server_track.called)
+            mock_server_track.reset_mock()
+
+    @override_settings(TRACKING_IGNORE_URL_PATTERNS=[])
+    def test_reading_filtered_urls_from_settings(self, mock_server_track):
+        request = self.request_factory.get('/event')
+        self.track_middleware.process_request(request)
+        self.assertTrue(mock_server_track.called)
+
+    @override_settings(TRACKING_IGNORE_URL_PATTERNS=[r'^/some/excluded.*'])
+    def test_anchoring_of_patterns_at_beginning(self, mock_server_track):
+        request = self.request_factory.get('/excluded')
+        self.track_middleware.process_request(request)
+        self.assertTrue(mock_server_track.called)
+        mock_server_track.reset_mock()
+
+        request = self.request_factory.get('/some/excluded/url')
+        self.track_middleware.process_request(request)
+        self.assertFalse(mock_server_track.called)

common/djangoapps/track/tests/test_tracker.py

+from django.conf import settings
+from django.test import TestCase
+from django.test.utils import override_settings
+
+import track.tracker as tracker
+from track.backends import BaseBackend
+
+
+SIMPLE_SETTINGS = {
+    'default': {
+        'ENGINE': 'track.tests.test_tracker.DummyBackend',
+        'OPTIONS': {
+            'flag': True
+        }
+    }
+}
+
+MULTI_SETTINGS = {
+    'first': {
+        'ENGINE': 'track.tests.test_tracker.DummyBackend',
+    },
+    'second': {
+        'ENGINE': 'track.tests.test_tracker.DummyBackend',
+    }
+}
+
+
+class TestTrackerInstantiation(TestCase):
+    """Test that a helper function can instantiate backends from their name."""
+    def setUp(self):
+        # pylint: disable=protected-access
+        self.get_backend = tracker._instantiate_backend_from_name
+
+    def test_instatiate_backend(self):
+        name = 'track.tests.test_tracker.DummyBackend'
+        options = {'flag': True}
+        backend = self.get_backend(name, options)
+
+        self.assertIsInstance(backend, DummyBackend)
+        self.assertTrue(backend.flag)
+
+    def test_instatiate_backends_with_invalid_values(self):
+        def get_invalid_backend(name, parameters):
+            return self.get_backend(name, parameters)
+
+        options = {}
+        name = 'track.backends.logger'
+        self.assertRaises(ValueError, get_invalid_backend, name, options)
+
+        name = 'track.backends.logger.Foo'
+        self.assertRaises(ValueError, get_invalid_backend, name, options)
+
+        name = 'this.package.does.not.exists'
+        self.assertRaises(ValueError, get_invalid_backend, name, options)
+
+        name = 'unittest.TestCase'
+        self.assertRaises(ValueError, get_invalid_backend, name, options)
+
+
+class TestTrackerDjangoInstantiation(TestCase):
+    """Test if backends are initialized properly from Django settings."""
+
+    @override_settings(TRACKING_BACKENDS=SIMPLE_SETTINGS)
+    def test_django_simple_settings(self):
+        """Test configuration of a simple backend"""
+
+        backends = self._reload_backends()
+
+        self.assertEqual(len(backends), 1)
+
+        tracker.send({})
+
+        self.assertEqual(backends.values()[0].count, 1)
+
+    @override_settings(TRACKING_BACKENDS=MULTI_SETTINGS)
+    def test_django_multi_settings(self):
+        """Test if multiple backends can be configured properly."""
+
+        backends = self._reload_backends().values()
+
+        self.assertEqual(len(backends), 2)
+
+        event_count = 10
+        for _ in xrange(event_count):
+            tracker.send({})
+
+        self.assertEqual(backends[0].count, event_count)
+        self.assertEqual(backends[1].count, event_count)
+
+    @override_settings(TRACKING_BACKENDS=MULTI_SETTINGS)
+    def test_django_remove_settings(self):
+        """Test if a backend can be remove by setting it to None."""
+
+        settings.TRACKING_BACKENDS.update({'second': None})
+
+        backends = self._reload_backends()
+
+        self.assertEqual(len(backends), 1)
+
+    def _reload_backends(self):
+        # pylint: disable=protected-access
+
+        # Reset backends
+        tracker._initialize_backends_from_django_settings()
+
+        return tracker.backends
+
+
+class DummyBackend(BaseBackend):
+    def __init__(self, **options):
+        super(DummyBackend, self).__init__(**options)
+        self.flag = options.get('flag', False)
+        self.count = 0
+
+    # pylint: disable=unused-argument
+    def send(self, event):
+        self.count += 1

common/djangoapps/track/tests/test_util.py

+from datetime import datetime
+import json
+
+from pytz import UTC
+
+from django.test import TestCase
+
+from track.utils import DateTimeJSONEncoder
+
+
+class TestDateTimeJSONEncoder(TestCase):
+    def test_datetime_encoding(self):
+        a_naive_datetime = datetime(2012, 05, 01, 07, 27, 10, 20000)
+        a_tz_datetime = datetime(2012, 05, 01, 07, 27, 10, 20000, tzinfo=UTC)
+        a_date = a_naive_datetime.date()
+        an_iso_datetime = '2012-05-01T07:27:10.020000+00:00'
+        an_iso_date = '2012-05-01'
+
+        obj = {
+            'number': 100,
+            'string': 'hello',
+            'object': {'a': 1},
+            'a_datetime': a_naive_datetime,
+            'a_tz_datetime': a_tz_datetime,
+            'a_date': a_date,
+        }
+
+        to_json = json.dumps(obj, cls=DateTimeJSONEncoder)
+        from_json = json.loads(to_json)
+
+        self.assertEqual(from_json['number'], 100)
+        self.assertEqual(from_json['string'], 'hello')
+        self.assertEqual(from_json['object'], {'a': 1})
+
+        self.assertEqual(from_json['a_datetime'], an_iso_datetime)
+        self.assertEqual(from_json['a_tz_datetime'], an_iso_datetime)
+        self.assertEqual(from_json['a_date'], an_iso_date)

common/djangoapps/track/tracker.py

+"""
+Module that tracks analytics events by sending them to different
+configurable backends.
+
+The backends can be configured using Django settings as the example
+below::
+
+  TRACKING_BACKENDS = {
+      'tracker_name': {
+          'ENGINE': 'class.name.for.backend',
+          'OPTIONS': {
+              'host': ... ,
+              'port': ... ,
+              ...
+          }
+      }
+  }
+
+"""
+
+import inspect
+from importlib import import_module
+
+from dogapi import dog_stats_api
+
+from django.conf import settings
+
+from track.backends import BaseBackend
+
+
+__all__ = ['send']
+
+
+backends = {}
+
+
+def _initialize_backends_from_django_settings():
+    """
+    Initialize the event tracking backends according to the
+    configuration in django settings
+
+    """
+    backends.clear()
+
+    config = getattr(settings, 'TRACKING_BACKENDS', {})
+
+    for name, values in config.iteritems():
+        # Ignore empty values to turn-off default tracker backends
+        if values:
+            engine = values['ENGINE']
+            options = values.get('OPTIONS', {})
+            backends[name] = _instantiate_backend_from_name(engine, options)
+
+
+def _instantiate_backend_from_name(name, options):
+    """
+    Instantiate an event tracker backend from the full module path to
+    the backend class. Useful when setting backends from configuration
+    files.
+
+    """
+    # Parse backend name
+
+    try:
+        parts = name.split('.')
+        module_name = '.'.join(parts[:-1])
+        class_name = parts[-1]
+    except IndexError:
+        raise ValueError('Invalid event track backend %s' % name)
+
+    # Get and verify the backend class
+
+    try:
+        module = import_module(module_name)
+        cls = getattr(module, class_name)
+        if not inspect.isclass(cls) or not issubclass(cls, BaseBackend):
+            raise TypeError
+    except (ValueError, AttributeError, TypeError, ImportError):
+        raise ValueError('Cannot find event track backend %s' % name)
+
+    backend = cls(**options)
+
+    return backend
+
+
+@dog_stats_api.timed('track.send')
+def send(event):
+    """
+    Send an event object to all the initialized backends.
+
+    """
+    dog_stats_api.increment('track.send.count')
+
+    for name, backend in backends.iteritems():
+        with dog_stats_api.timer('track.send.backend.{0}'.format(name)):
+            backend.send(event)
+
+
+_initialize_backends_from_django_settings()

common/djangoapps/track/utils.py

+"""Utility functions and classes for track backends"""
+
+from datetime import datetime, date
+import json
+
+from pytz import UTC
+
+
+class DateTimeJSONEncoder(json.JSONEncoder):
+    """JSON encoder aware of datetime.datetime and datetime.date objects"""
+
+    def default(self, obj):  # pylint: disable=method-hidden
+        """
+        Serialize datetime and date objects of iso format.
+
+        datatime objects are converted to UTC.
+        """
+
+        if isinstance(obj, datetime):
+            if obj.tzinfo is None:
+                # Localize to UTC naive datetime objects
+                obj = UTC.localize(obj)
+            else:
+                # Convert to UTC datetime objects from other timezones
+                obj = obj.astimezone(UTC)
+            return obj.isoformat()
+        elif isinstance(obj, date):
+            return obj.isoformat()
+
+        return super(DateTimeJSONEncoder, self).default(obj)

common/djangoapps/track/views.py

-import json
-import logging
-import pytz
 import datetime
-import dateutil.parser
+
+import pytz
+from pytz import UTC
 
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.shortcuts import redirect
-from django.conf import settings
-from mitxmako.shortcuts import render_to_response
 
 from django_future.csrf import ensure_csrf_cookie
-from track.models import TrackingLog
-from pytz import UTC
 
-log = logging.getLogger("tracking")
+from mitxmako.shortcuts import render_to_response
 
-LOGFIELDS = ['username', 'ip', 'event_source', 'event_type', 'event', 'agent', 'page', 'time', 'host']
+from track import tracker
+from track.models import TrackingLog
 
 
 def log_event(event):
-    """Write tracking event to log file, and optionally to TrackingLog model."""
-    event_str = json.dumps(event)
-    log.info(event_str[:settings.TRACK_MAX_EVENT])
-    if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
-        event['time'] = dateutil.parser.parse(event['time'])
-        tldat = TrackingLog(**dict((x, event[x]) for x in LOGFIELDS))
-        try:
-            tldat.save()
-        except Exception as err:
-            log.exception(err)
+    """Capture a event by sending it to the register trackers"""
+    tracker.send(event)
 
 
 def user_track(request):
@@ -64,11 +52,12 @@ def user_track(request):
         "event": request.REQUEST.get('event','unknown'),
         "agent": agent,
         "page": request.REQUEST.get('page', 'unknown'),
-        "time": datetime.datetime.now(UTC).isoformat(),
+        "time": datetime.datetime.now(UTC),
         "host": request.META['SERVER_NAME'],
     }
 
     log_event(event)
+
     return HttpResponse('success')
 
 
@@ -92,12 +81,13 @@ def server_track(request, event_type, event, page=None):
         "event": event,
         "agent": agent,
         "page": page,
-        "time": datetime.datetime.now(UTC).isoformat(),
+        "time": datetime.datetime.now(UTC),
         "host": request.META['SERVER_NAME'],
     }
 
-    if event_type.startswith("/event_logs") and request.user.is_staff:  # don't log
-        return
+    if event_type.startswith("/event_logs") and request.user.is_staff:
+        return  # don't log
+
     log_event(event)
 
 
@@ -136,7 +126,7 @@ def task_track(request_info, task_info, event_type, event, page=None):
         "event": full_event,
         "agent": request_info.get('agent', 'unknown'),
         "page": page,
-        "time": datetime.datetime.utcnow().isoformat(),
+        "time": datetime.datetime.now(UTC),
         "host": request_info.get('host', 'unknown')
     }
 

common/lib/calc/calc/preview.py

@@ -113,6 +113,12 @@ def enrich_varname(varname):
              "vartheta iota kappa lambda mu nu xi pi rho sigma tau upsilon "
              "phi varphi chi psi omega").split()
 
+    # add capital greek letters
+    greek += [x.capitalize() for x in greek]
+
+    # add hbar for QM
+    greek.append('hbar')
+
     if varname in greek:
         return ur"\{letter}".format(letter=varname)
     else:

common/lib/capa/capa/safe_exec/safe_exec.py

@@ -4,7 +4,7 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec
 from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
 from codejail.safe_exec import json_safe, SafeExecException
 from . import lazymod
-from statsd import statsd
+from dogapi import dog_stats_api
 
 import hashlib
 
@@ -70,7 +70,7 @@ def update_hash(hasher, obj):
         hasher.update(repr(obj))
 
 
-@statsd.timed('capa.safe_exec.time')
+@dog_stats_api.timed('capa.safe_exec.time')
 def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False):
     """
     Execute python code safely.

common/lib/capa/capa/tests/__init__.py

@@ -36,6 +36,7 @@ def test_system():
         user=Mock(),
         filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")),
         debug=True,
+        hostname="edx.org",
         xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10},
         node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
         anonymous_student_id='student',

common/lib/capa/capa/xqueue_interface.py

@@ -64,7 +64,8 @@ class XQueueInterface(object):
     def __init__(self, url, django_auth, requests_auth=None):
         self.url = url
         self.auth = django_auth
-        self.session = requests.session(auth=requests_auth)
+        self.session = requests.Session()
+        self.session.auth = requests_auth
 
     def send_to_queue(self, header, body, files_to_upload=None):
         """

common/lib/sample-post.py

@@ -39,7 +39,7 @@ username = prompt('username on server', 'victor@edx.org')
 password = prompt('password', 'abc123', safe=True)
 
 print "get csrf cookie"
-session = requests.session()
+session = requests.Session()
 r = session.get(server + '/')
 r.raise_for_status()
 

common/lib/xmodule/xmodule/course_module.py

@@ -337,7 +337,14 @@ class CourseFields(object):
                                       "action_external": False}]}
         ])
     info_sidebar_name = String(scope=Scope.settings, default='Course Handouts')
-    show_timezone = Boolean(help="True if timezones should be shown on dates in the courseware", scope=Scope.settings, default=True)
+    show_timezone = Boolean(
+        help="True if timezones should be shown on dates in the courseware. Deprecated in favor of due_date_display_format.",
+        scope=Scope.settings, default=True
+    )
+    due_date_display_format = String(
+        help="Format supported by strftime for displaying due dates. Takes precedence over show_timezone.",
+        scope=Scope.settings, default=None
+    )
     enrollment_domain = String(help="External login method associated with user accounts allowed to register in course",
                                scope=Scope.settings)
     course_image = String(
@@ -391,7 +398,13 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
             elif isinstance(self.location, CourseLocator):
                 self.wiki_slug = self.location.course_id or self.display_name
 
-        msg = None
+        if self.due_date_display_format is None and self.show_timezone is False:
+            # For existing courses with show_timezone set to False (and no due_date_display_format specified),
+            # set the due_date_display_format to what would have been shown previously (with no timezone).
+            # Then remove show_timezone so that if the user clears out the due_date_display_format,
+            # they get the default date display.
+            self.due_date_display_format = u"%b %d, %Y at %H:%M"
+            delattr(self, 'show_timezone')
 
         # NOTE: relies on the modulestore to call set_grading_policy() right after
         # init.  (Modulestore is in charge of figuring out where to load the policy from)

common/lib/xmodule/xmodule/css/tabs/tabs.scss

@@ -32,7 +32,6 @@
 
     //Component Name
     .component-name {
-      @extend .t-copy-sub1;
       position: relative;
       top: 0;
       left: 0;

common/lib/xmodule/xmodule/css/video/display.scss

@@ -225,6 +225,19 @@ div.video {
             @include transition(none);
             -webkit-font-smoothing: antialiased;
             width: 116px;
+
+            @media (max-width: 1024px) {
+              width: 86px;
+            }
+
+            h3 {
+              display: block;
+
+              @media (max-width: 1024px) {
+                display: none;
+              }
+            }
+
             outline: 0;
 
             &:focus {
@@ -247,6 +260,11 @@ div.video {
               font-weight: bold;
               margin-bottom: 0;
               padding: 0 lh(.5) 0 0;
+
+              @media (max-width: 1024px) {
+                padding: 0 lh(.5) 0 lh(.5);
+              }
+
               line-height: 46px;
               color: #fff;
             }
@@ -268,6 +286,11 @@ div.video {
             opacity: 0.0;
             position: absolute;
             width: 131px;
+
+            @media (max-width: 1024px) {
+              width: 101px;
+            }
+
             z-index: 10;
 
             li {

common/lib/xmodule/xmodule/js/fixtures/lti.html

 <div id="lti_id" class="lti">
 
     <form
-        action=""
+        action="http://www.example.com"
         name="ltiLaunchForm"
         class="ltiLaunchForm"
         method="post"
         target="ltiLaunchFrame"
-        encType="application/x-www-form-urlencoded"
+        enctype="application/x-www-form-urlencoded"
     >
-
-        <input type="hidden" name="launch_presentation_return_url" value="">
-        <input type="hidden" name="lis_outcome_service_url" value="">
-        <input type="hidden" name="lis_result_sourcedid" value="">
-        <input type="hidden" name="lti_message_type" value="basic-lti-launch-request">
-        <input type="hidden" name="lti_version" value="LTI-1p0">
-        <input type="hidden" name="oauth_callback" value="about:blank">
-        <input type="hidden" name="oauth_consumer_key" value=""/>
-        <input type="hidden" name="oauth_nonce" value=""/>
-        <input type="hidden" name="oauth_signature_method" value="HMAC-SHA1"/>
-        <input type="hidden" name="oauth_timestamp" value=""/>
-        <input type="hidden" name="oauth_version" value="1.0"/>
-        <input type="hidden" name="user_id" value="default_user_id">
-        <input type="hidden" name="oauth_signature" value=""/>
+        <input name="launch_presentation_return_url" value="" />
+        <input name="lti_version" value="LTI-1p0" />
+        <input name="user_id" value="student" />
+        <input name="oauth_nonce" value="28347958723982798572" />
+        <input name="oauth_timestamp" value="2389479832" />
+        <input name="oauth_consumer_key" value="" />
+        <input name="lis_result_sourcedid" value="" />
+        <input name="oauth_signature_method" value="HMAC-SHA1" />
+        <input name="oauth_version" value="1.0" />
+        <input name="role" value="student" />
+        <input name="lis_outcome_service_url" value="" />
+        <input name="oauth_signature" value="89ru3289r3ry283y3r82ryr38yr" />
+        <input name="lti_message_type" value="basic-lti-launch-request" />
+        <input name="oauth_callback" value="about:blank" />
 
         <input type="submit" value="Press to Launch" />
     </form>
@@ -31,10 +31,6 @@
         required fields.
     </h3>
 
-    <iframe
-        name="ltiLaunchFrame"
-        class="ltiLaunchFrame"
-        src=""
-    ></iframe>
+    <iframe name="ltiLaunchFrame" class="ltiLaunchFrame" src=""></iframe>
 
 </div>

common/lib/xmodule/xmodule/js/spec/lti/constructor.js

@@ -46,7 +46,7 @@
                 });
 
                 it(
-                    'when URL setting is filled form is not submited',
+                    'when URL setting is not filled form is not submited',
                     function () {
 
                     expect('submit').not.toHaveBeenTriggeredOn(form);
@@ -70,7 +70,7 @@
 
                     // The user "fills in" the necessary settings, and the
                     // form will get an action URL.
-                    form.attr('action', 'http://www.example.com/');
+                    form.attr('action', 'http://www.example.com/test_submit');
 
                     LTI(element);
                 });

common/lib/xmodule/xmodule/js/src/lti/lti.js

@@ -16,9 +16,9 @@ window.LTI = (function () {
 
         // If the Form's action attribute is set (i.e. we can perform a normal
         // submit), then we submit the form and make the frame shown.
-        if (form.attr('action')) {
+        if (form.attr('action') && form.attr('action') !== 'http://www.example.com') {
             form.submit();
-            element.find('.lti').addClass('rendered')
+            element.find('.lti').addClass('rendered');
         }
     }
 

common/lib/xmodule/xmodule/lti_module.py

 """
 Module that allows to insert LTI tools to page.
 
-Module uses current edx-platform 0.14.2 version of requests (oauth part).
-Please update code when upgrading requests.
-
 Protocol is oauth1, LTI version is 1.1.1:
 http://www.imsglobal.org/LTI/v1p1p1/ltiIMGv1p1p1.html
 """
 
 import logging
-import requests
+import oauthlib.oauth1
 import urllib
 
 from xmodule.editing_module import MetadataOnlyEditingDescriptor
@@ -41,9 +38,12 @@ class LTIFields(object):
         vbid=put_book_id_here
         book_location=page/put_page_number_here
 
+    Default non-empty url for `launch_url` is needed due to oauthlib demand (url scheme should be presented)::
+
+    https://github.com/idan/oauthlib/blob/master/oauthlib/oauth1/rfc5849/signature.py#L136
     """
     lti_id = String(help="Id of the tool", default='', scope=Scope.settings)
-    launch_url = String(help="URL of the tool", default='', scope=Scope.settings)
+    launch_url = String(help="URL of the tool", default='http://www.example.com', scope=Scope.settings)
     custom_parameters = List(help="Custom parameters (vbid, book_location, etc..)", scope=Scope.settings)
 
 
@@ -192,7 +192,7 @@ class LTIModule(LTIFields, XModule):
         Also *anonymous student id* is passed to template and therefore to LTI provider.
         """
 
-        client = requests.auth.Client(
+        client = oauthlib.oauth1.Client(
             client_key=unicode(client_key),
             client_secret=unicode(client_secret)
         )
@@ -215,14 +215,26 @@ class LTIModule(LTIFields, XModule):
         # appending custom parameter for signing
         body.update(custom_parameters)
 
-        # This is needed for body encoding:
-        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+        headers = {
+            # This is needed for body encoding:
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+
+        try:
+            __, headers, __ = client.sign(
+                unicode(self.launch_url),
+                http_method=u'POST',
+                body=body,
+                headers=headers)
+        except ValueError:  # scheme not in url
+            #https://github.com/idan/oauthlib/blob/master/oauthlib/oauth1/rfc5849/signature.py#L136
+            #Stubbing headers for now:
+            headers = {
+                u'Content-Type': u'application/x-www-form-urlencoded',
+                u'Authorization': u'OAuth oauth_nonce="80966668944732164491378916897", \
+oauth_timestamp="1378916897", oauth_version="1.0", oauth_signature_method="HMAC-SHA1", \
+oauth_consumer_key="", oauth_signature="frVp4JuvT1mVXlxktiAUjQ7%2F1cw%3D"'}
 
-        __, headers, __ = client.sign(
-            unicode(self.launch_url),
-            http_method=u'POST',
-            body=body,
-            headers=headers)
         params = headers['Authorization']
         # parse headers to pass to template as part of context:
         params = dict([param.strip().replace('"', '').split('=') for param in params.split(',')])
@@ -230,8 +242,8 @@ class LTIModule(LTIFields, XModule):
         params[u'oauth_nonce'] = params[u'OAuth oauth_nonce']
         del params[u'OAuth oauth_nonce']
 
-        # 0.14.2 (current) version of requests oauth library encodes signature,
-        # with 'Content-Type': 'application/x-www-form-urlencoded'
+        # oauthlib encodes signature with
+        # 'Content-Type': 'application/x-www-form-urlencoded'
         # so '='' becomes '%3D'.
         # We send form via browser, so browser will encode it again,
         # So we need to decode signature back:

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -10,7 +10,6 @@ from collections import namedtuple
 
 from .exceptions import InvalidLocationError, InsufficientSpecificationError
 from xmodule.errortracker import make_error_tracker
-from bson.son import SON
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
@@ -449,13 +448,3 @@ class ModuleStoreBase(ModuleStore):
             if c.id == course_id:
                 return c
         return None
-
-
-def namedtuple_to_son(namedtuple, prefix=''):
-    """
-    Converts a namedtuple into a SON object with the same key order
-    """
-    son = SON()
-    for idx, field_name in enumerate(namedtuple._fields):
-        son[prefix + field_name] = namedtuple[idx]
-    return son

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -17,6 +17,7 @@ import sys
 import logging
 import copy
 
+from bson.son import SON
 from fs.osfs import OSFS
 from itertools import repeat
 from path import path
@@ -31,7 +32,7 @@ from xblock.runtime import DbModel
 from xblock.exceptions import InvalidScopeError
 from xblock.fields import Scope, ScopeIds
 
-from xmodule.modulestore import ModuleStoreBase, Location, namedtuple_to_son, MONGO_MODULESTORE_TYPE
+from xmodule.modulestore import ModuleStoreBase, Location, MONGO_MODULESTORE_TYPE
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.inheritance import own_metadata, InheritanceMixin, inherit_metadata, InheritanceKeyValueStore
 
@@ -215,6 +216,16 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 )
 
 
+def namedtuple_to_son(namedtuple, prefix=''):
+    """
+    Converts a namedtuple into a SON object with the same key order
+    """
+    son = SON()
+    for idx, field_name in enumerate(namedtuple._fields):
+        son[prefix + field_name] = namedtuple[idx]
+    return son
+
+
 def location_to_query(location, wildcard=True):
     """
     Takes a Location and returns a SON object that will query for that location.
@@ -605,8 +616,8 @@ class MongoModuleStore(ModuleStoreBase):
             )
         xblock_class = XModuleDescriptor.load_class(location.category, self.default_class)
         if definition_data is None:
-            if hasattr(xblock_class, 'data') and getattr(xblock_class, 'data').default is not None:
-                definition_data = getattr(xblock_class, 'data').default
+            if hasattr(xblock_class, 'data') and xblock_class.data.default is not None:
+                definition_data = xblock_class.data.default
             else:
                 definition_data = {}
         dbmodel = self._create_new_field_data(location.category, location, definition_data, metadata)

common/lib/xmodule/xmodule/modulestore/mongo/draft.py

@@ -9,10 +9,10 @@ and otherwise returns i4x://org/course/cat/name).
 from datetime import datetime
 
 from xmodule.exceptions import InvalidVersionError
-from xmodule.modulestore import Location, namedtuple_to_son
+from xmodule.modulestore import Location
 from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
 from xmodule.modulestore.inheritance import own_metadata
-from xmodule.modulestore.mongo.base import location_to_query, get_course_id_no_run, MongoModuleStore
+from xmodule.modulestore.mongo.base import location_to_query, namedtuple_to_son, get_course_id_no_run, MongoModuleStore
 import pymongo
 from pytz import UTC
 

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -2,8 +2,10 @@ from pprint import pprint
 # pylint: disable=E0611
 from nose.tools import assert_equals, assert_raises, \
     assert_not_equals, assert_false
+from itertools import ifilter
 # pylint: enable=E0611
 import pymongo
+import logging
 from uuid import uuid4
 
 from xblock.fields import Scope
@@ -19,6 +21,7 @@ from xmodule.contentstore.mongo import MongoContentStore
 
 from xmodule.modulestore.tests.test_modulestore import check_path_to_location
 
+log = logging.getLogger(__name__)
 
 HOST = 'localhost'
 PORT = 27017
@@ -59,7 +62,7 @@ class TestMongoModuleStore(object):
         #
         draft_store = DraftModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
         # Explicitly list the courses to load (don't want the big one)
-        courses = ['toy', 'simple', 'simple_with_draft']
+        courses = ['toy', 'simple', 'simple_with_draft', 'test_unicode']
         import_from_xml(store, DATA_DIR, courses, draft_store=draft_store, static_content_store=content_store)
 
         # also test a course with no importing of static content
@@ -86,6 +89,19 @@ class TestMongoModuleStore(object):
     def tearDown(self):
         pass
 
+    def get_course_by_id(self, name):
+        """
+        Returns the first course with `id` of `name`, or `None` if there are none.
+        """
+        courses = self.store.get_courses()
+        return next(ifilter(lambda x: x.id == name, courses), None)
+
+    def course_with_id_exists(self, name):
+        """
+        Returns true iff there exists some course with `id` of `name`.
+        """
+        return (self.get_course_by_id(name) is not None)
+
     def test_init(self):
         '''Make sure the db loads, and print all the locations in the db.
         Call this directly from failing tests to see what is loaded'''
@@ -100,12 +116,12 @@ class TestMongoModuleStore(object):
     def test_get_courses(self):
         '''Make sure the course objects loaded properly'''
         courses = self.store.get_courses()
-        assert_equals(len(courses), 4)
-        courses.sort(key=lambda c: c.id)
-        assert_equals(courses[0].id, 'edX/simple/2012_Fall')
-        assert_equals(courses[1].id, 'edX/simple_with_draft/2012_Fall')
-        assert_equals(courses[2].id, 'edX/test_import_course/2012_Fall')
-        assert_equals(courses[3].id, 'edX/toy/2012_Fall')
+        assert_equals(len(courses), 5)
+        assert self.course_with_id_exists('edX/simple/2012_Fall')
+        assert self.course_with_id_exists('edX/simple_with_draft/2012_Fall')
+        assert self.course_with_id_exists('edX/test_import_course/2012_Fall')
+        assert self.course_with_id_exists('edX/test_unicode/2012_Fall')
+        assert self.course_with_id_exists('edX/toy/2012_Fall')
 
     def test_loads(self):
         assert_not_equals(
@@ -120,6 +136,22 @@ class TestMongoModuleStore(object):
             self.store.get_item("i4x://edX/toy/video/Welcome"),
             None)
 
+    def test_unicode_loads(self):
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/course/2012_Fall"),
+            None)
+        # All items with ascii-only filenames should load properly.
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
+            None)
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
+            None)
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/chapter/Overview"),
+            None)
+
+
     def test_find_one(self):
         assert_not_equals(
             self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
@@ -153,15 +185,15 @@ class TestMongoModuleStore(object):
             )
 
     def test_static_tab_names(self):
-        courses = self.store.get_courses()
 
         def get_tab_name(index):
             """
             Helper function for pulling out the name of a given static tab.
 
-            Assumes the information is desired for courses[1] ('toy' course).
+            Assumes the information is desired for courses[4] ('toy' course).
             """
-            return courses[2].tabs[index]['name']
+            course = self.get_course_by_id('edX/toy/2012_Fall')
+            return course.tabs[index]['name']
 
         # There was a bug where model.save was not getting called after the static tab name
         # was set set for tabs that have a URL slug. 'Syllabus' and 'Resources' fall into that

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -29,7 +29,7 @@ from .exceptions import ItemNotFoundError
 from .inheritance import compute_inherited_metadata
 
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=False, remove_blank_text=True)
 
 etree.set_default_parser(edx_xml_parser)
 
@@ -173,7 +173,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 # Didn't load properly.  Fall back on loading as an error
                 # descriptor.  This should never error due to formatting.
 
-                msg = "Error loading from xml. " + str(err)[:200]
+                msg = "Error loading from xml. " + unicode(err)[:200]
                 log.warning(msg)
                 # Normally, we don't want lots of exception traces in our logs from common
                 # content problems.  But if you're debugging the xml loading code itself,
@@ -190,7 +190,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     err_msg
                 )
 
-            setattr(descriptor, 'data_dir', course_dir)
+            descriptor.data_dir = course_dir
 
             xmlstore.modules[course_id][descriptor.location] = descriptor
 
@@ -317,7 +317,8 @@ class XMLModuleStore(ModuleStoreBase):
         try:
             course_descriptor = self.load_course(course_dir, errorlog.tracker)
         except Exception as e:
-            msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir, str(e))
+            msg = "ERROR: Failed to load course '{0}': {1}".format(course_dir.encode("utf-8"),
+                    unicode(e))
             log.exception(msg)
             errorlog.tracker(msg)
 
@@ -493,8 +494,9 @@ class XMLModuleStore(ModuleStoreBase):
                     module.save()
                     self.modules[course_descriptor.id][module.location] = module
                 except Exception, e:
-                    logging.exception("Failed to load {0}. Skipping... Exception: {1}".format(filepath, str(e)))
-                    system.error_tracker("ERROR: " + str(e))
+                    logging.exception("Failed to load %s. Skipping... \
+                            Exception: %s", filepath, unicode(e))
+                    system.error_tracker("ERROR: " + unicode(e))
 
     def get_instance(self, course_id, location, depth=0):
         """

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -31,7 +31,7 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
             try:
                 content_path = os.path.join(dirname, filename)
                 if verbose:
-                    log.debug('importing static content {0}...'.format(content_path))
+                    log.debug('importing static content %s...', content_path)
 
                 fullname_with_subpath = content_path.replace(static_dir, '')  # strip away leading path from the name
                 if fullname_with_subpath.startswith('/'):

common/lib/xmodule/xmodule/open_ended_grading_classes/grading_service_module.py

@@ -25,7 +25,7 @@ class GradingService(object):
     def __init__(self, config):
         self.username = config['username']
         self.password = config['password']
-        self.session = requests.session()
+        self.session = requests.Session()
         self.system = config['system']
 
     def _login(self):
@@ -42,7 +42,7 @@ class GradingService(object):
 
         response.raise_for_status()
 
-        return response.json
+        return response.json()
 
     def post(self, url, data, allow_redirects=False):
         """
@@ -88,9 +88,10 @@ class GradingService(object):
         Returns the result of operation().  Does not catch exceptions.
         """
         response = operation()
-        if (response.json
-            and response.json.get('success') is False
-            and response.json.get('error') == 'login_required'):
+        resp_json = response.json()
+        if (resp_json
+                and resp_json.get('success') is False
+                and resp_json.get('error') == 'login_required'):
             # apparrently we aren't logged in.  Try to fix that.
             r = self._login()
             if r and not r.get('success'):

common/lib/xmodule/xmodule/seq_module.py

@@ -133,7 +133,7 @@ class SequenceDescriptor(SequenceFields, MakoModuleDescriptor, XmlDescriptor):
             except Exception as e:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 if system.error_tracker is not None:
-                    system.error_tracker("ERROR: " + str(e))
+                    system.error_tracker("ERROR: " + unicode(e))
                 continue
         return {}, children
 

common/lib/xmodule/xmodule/tests/__init__.py

@@ -62,6 +62,7 @@ def get_test_system(course_id=''):
         user=Mock(is_staff=False),
         filestore=Mock(),
         debug=True,
+        hostname="edx.org",
         xqueue={'interface': None, 'callback_url': '/', 'default_queuename': 'testqueue', 'waittime': 10, 'construct_callback' : Mock(side_effect="/")},
         node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
         xblock_field_data=lambda descriptor: descriptor._field_data,

common/lib/xmodule/xmodule/tests/test_conditional_logic.py

@@ -13,7 +13,7 @@ class ConditionalModuleTest(LogicTest):
         "Make shure that ajax request works correctly"
         # Mock is_condition_satisfied
         self.xmodule.is_condition_satisfied = lambda: True
-        setattr(self.xmodule.descriptor, 'get_children', lambda: [])
+        self.xmodule.descriptor.get_children = lambda: []
 
         response = self.ajax_request('No', {})
         html = response['html']

common/lib/xmodule/xmodule/tests/test_date_utils.py

 """Tests for xmodule.util.date_utils"""
 
 from nose.tools import assert_equals, assert_false  # pylint: disable=E0611
-from xmodule.util.date_utils import get_default_time_display, almost_same_datetime
+from xmodule.util.date_utils import get_default_time_display, get_time_display, almost_same_datetime
 from datetime import datetime, timedelta, tzinfo
 from pytz import UTC
 
@@ -12,25 +12,34 @@ def test_get_default_time_display():
     assert_equals(
         "Mar 12, 1992 at 15:03 UTC",
         get_default_time_display(test_time))
-    assert_equals(
-        "Mar 12, 1992 at 15:03 UTC",
-        get_default_time_display(test_time, True))
-    assert_equals(
-        "Mar 12, 1992 at 15:03",
-        get_default_time_display(test_time, False))
 
 
-def test_get_default_time_display_notz():
+def test_get_dflt_time_disp_notz():
     test_time = datetime(1992, 3, 12, 15, 3, 30)
     assert_equals(
         "Mar 12, 1992 at 15:03 UTC",
         get_default_time_display(test_time))
-    assert_equals(
-        "Mar 12, 1992 at 15:03 UTC",
-        get_default_time_display(test_time, True))
-    assert_equals(
-        "Mar 12, 1992 at 15:03",
-        get_default_time_display(test_time, False))
+
+
+def test_get_time_disp_ret_empty():
+    assert_equals("", get_time_display(None))
+    test_time = datetime(1992, 3, 12, 15, 3, 30, tzinfo=UTC)
+    assert_equals("", get_time_display(test_time, ""))
+
+
+def test_get_time_display():
+    test_time = datetime(1992, 3, 12, 15, 3, 30, tzinfo=UTC)
+    assert_equals("dummy text", get_time_display(test_time, 'dummy text'))
+    assert_equals("Mar 12 1992", get_time_display(test_time, '%b %d %Y'))
+    assert_equals("Mar 12 1992 UTC", get_time_display(test_time, '%b %d %Y %Z'))
+    assert_equals("Mar 12 15:03", get_time_display(test_time, '%b %d %H:%M'))
+
+
+def test_get_time_pass_through():
+    test_time = datetime(1992, 3, 12, 15, 3, 30, tzinfo=UTC)
+    assert_equals("Mar 12, 1992 at 15:03 UTC", get_time_display(test_time))
+    assert_equals("Mar 12, 1992 at 15:03 UTC", get_time_display(test_time, None))
+    assert_equals("Mar 12, 1992 at 15:03 UTC", get_time_display(test_time, "%"))
 
 
 # pylint: disable=W0232
@@ -50,12 +59,6 @@ def test_get_default_time_display_no_tzname():
     assert_equals(
         "Mar 12, 1992 at 15:03-0300",
         get_default_time_display(test_time))
-    assert_equals(
-        "Mar 12, 1992 at 15:03-0300",
-        get_default_time_display(test_time, True))
-    assert_equals(
-        "Mar 12, 1992 at 15:03",
-        get_default_time_display(test_time, False))
 
 
 def test_almost_same_datetime():

common/lib/xmodule/xmodule/tests/test_import.py

@@ -91,6 +91,7 @@ class ImportTestCase(BaseCourseTestCase):
 
         self.assertNotEqual(descriptor1.location, descriptor2.location)
 
+    @unittest.skip('Temporarily disabled')
     def test_reimport(self):
         '''Make sure an already-exported error xml tag loads properly'''
 
@@ -368,6 +369,32 @@ class ImportTestCase(BaseCourseTestCase):
         html = modulestore.get_instance(course_id, loc)
         self.assertEquals(html.display_name, "Toy lab")
 
+    def test_unicode(self):
+        """Check that courses with unicode characters in filenames and in
+        org/course/name import properly. Currently, this means: (a) Having
+        files with unicode names does not prevent import; (b) if files are not
+        loaded because of unicode filenames, there are appropriate
+        exceptions/errors to that effect."""
+
+        print("Starting import")
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['test_unicode'])
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+
+        print("course errors:")
+
+        # Expect to find an error/exception about characters in "®esources"
+        expect = "Invalid characters in '®esources'"
+        errors = [(msg.encode("utf-8"), err.encode("utf-8"))
+                    for msg, err in
+                    modulestore.get_item_errors(course.location)]
+
+        self.assertTrue(any(expect in msg or expect in err
+            for msg, err in errors))
+        chapters = course.get_children()
+        self.assertEqual(len(chapters), 3)
+
     def test_url_name_mangling(self):
         """
         Make sure that url_names are only mangled once.

common/lib/xmodule/xmodule/tests/test_video.py

@@ -48,6 +48,12 @@ class VideoModuleTest(LogicTest):
         output = VideoDescriptor._parse_time('00:04:07')
         self.assertEqual(output, expected)
 
+    def test_parse_time_with_float(self):
+        """Ensure that times are parsed correctly into seconds."""
+        expected = 247
+        output = VideoDescriptor._parse_time('247.0')
+        self.assertEqual(output, expected)
+
     def test_parse_youtube(self):
         """Test parsing old-style Youtube ID strings into a dict."""
         youtube_str = '0.75:jNCf2gIqpeE,1.00:ZwkTiUPN0mg,1.25:rsq9auxASqI,1.50:kMyNdzVHHgg'
@@ -412,6 +418,35 @@ class VideoDescriptorImportTestCase(unittest.TestCase):
             'data': ''
         })
 
+    def test_import_with_float_times(self):
+        """
+        Ensure that Video is able to read VideoModule's model data.
+        """
+        module_system = DummySystem(load_error_modules=True)
+        xml_data = """
+            <video display_name="Test Video"
+                   youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
+                   show_captions="false"
+                   from="1.0"
+                   to="60.0">
+              <source src="http://www.example.com/source.mp4"/>
+              <track src="http://www.example.com/track"/>
+            </video>
+        """
+        video = VideoDescriptor.from_xml(xml_data, module_system)
+        self.assert_attributes_equal(video, {
+            'youtube_id_0_75': 'izygArpw-Qo',
+            'youtube_id_1_0': 'p2Q6BrNhdh8',
+            'youtube_id_1_25': '1EeWXzPdhSA',
+            'youtube_id_1_5': 'rABDYkeK0x8',
+            'show_captions': False,
+            'start_time': 1.0,
+            'end_time': 60.0,
+            'track': 'http://www.example.com/track',
+            'html5_sources': ['http://www.example.com/source.mp4'],
+            'data': ''
+        })
+
 
 class VideoExportTestCase(unittest.TestCase):
     """

common/lib/xmodule/xmodule/util/date_utils.py

@@ -2,32 +2,46 @@
 Convenience methods for working with datetime objects
 """
 from datetime import timedelta
-from django.utils.translation import ugettext as _
 
 
-def get_default_time_display(dt, show_timezone=True):
+def get_default_time_display(dtime):
     """
     Converts a datetime to a string representation. This is the default
     representation used in Studio and LMS.
-    It is of the form "Apr 09, 2013 at 16:00" or "Apr 09, 2013 at 16:00 UTC",
-    depending on the value of show_timezone.
+    It is of the form "Apr 09, 2013 at 16:00 UTC".
 
     If None is passed in for dt, an empty string will be returned.
-    The default value of show_timezone is True.
     """
-    if dt is None:
+    if dtime is None:
         return u""
-    timezone = u""
-    if show_timezone:
-        if dt.tzinfo is not None:
-            try:
-                timezone = u" " + dt.tzinfo.tzname(dt)
-            except NotImplementedError:
-                timezone = dt.strftime('%z')
-        else:
-            timezone = u" UTC"
-    return unicode(dt.strftime(u"%b %d, %Y {at} %H:%M{tz}")).format(
-        at=_(u"at"), tz=timezone).strip()
+    if dtime.tzinfo is not None:
+        try:
+            timezone = u" " + dtime.tzinfo.tzname(dtime)
+        except NotImplementedError:
+            timezone = dtime.strftime('%z')
+    else:
+        timezone = u" UTC"
+    return unicode(dtime.strftime(u"%b %d, %Y at %H:%M{tz}")).format(
+        tz=timezone).strip()
+
+
+def get_time_display(dtime, format_string=None):
+    """
+    Converts a datetime to a string representation.
+
+    If None is passed in for dt, an empty string will be returned.
+
+    If the format_string is None, or if format_string is improperly
+    formatted, this method will return the value from `get_default_time_display`.
+
+    format_string should be a unicode string that is a valid argument for datetime's strftime method.
+    """
+    if dtime is None or format_string is None:
+        return get_default_time_display(dtime)
+    try:
+        return unicode(dtime.strftime(format_string))
+    except ValueError:
+        return get_default_time_display(dtime)
 
 
 def almost_same_datetime(dt1, dt2, allowed_delta=timedelta(minutes=1)):

common/lib/xmodule/xmodule/video_module.py

@@ -385,12 +385,16 @@ class VideoDescriptor(VideoFields, TabsEditingDescriptor, EmptyDataRawDescriptor
         if not str_time:
             return ''
         else:
-            obj_time = time.strptime(str_time, '%H:%M:%S')
-            return datetime.timedelta(
-                hours=obj_time.tm_hour,
-                minutes=obj_time.tm_min,
-                seconds=obj_time.tm_sec
-            ).total_seconds()
+            try:
+                obj_time = time.strptime(str_time, '%H:%M:%S')
+                return datetime.timedelta(
+                    hours=obj_time.tm_hour,
+                    minutes=obj_time.tm_min,
+                    seconds=obj_time.tm_sec
+                ).total_seconds()
+            except ValueError:
+                # We've seen serialized versions of float in this field
+                return float(str_time)
 
 
 def _create_youtube_string(module):

common/lib/xmodule/xmodule/x_module.py

@@ -349,7 +349,7 @@ class ResourceTemplates(object):
     @classmethod
     def get_template_dir(cls):
         if getattr(cls, 'template_dir_name', None):
-            dirname = os.path.join('templates', getattr(cls, 'template_dir_name'))
+            dirname = os.path.join('templates', cls.template_dir_name)
             if not resource_isdir(__name__, dirname):
                 log.warning("No resource directory {dir} found when loading {cls_name} templates".format(
                     dir=dirname,
@@ -619,14 +619,6 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         raise NotImplementedError(
             'Modules must implement export_to_xml to enable xml export')
 
-    # =============================== Testing ==================================
-    def get_sample_state(self):
-        """
-        Return a list of tuples of instance_state, shared_state. Each tuple
-        defines a sample case for this module
-        """
-        return [('{}', '{}')]
-
     @property
     def xblock_kvs(self):
         """
@@ -847,7 +839,7 @@ class ModuleSystem(Runtime):
     def __init__(
             self, ajax_url, track_function, get_module, render_template,
             replace_urls, xblock_field_data, user=None, filestore=None,
-            debug=False, xqueue=None, publish=None, node_path="",
+            debug=False, hostname="", xqueue=None, publish=None, node_path="",
             anonymous_student_id='', course_id=None,
             open_ended_grading_interface=None, s3_interface=None,
             cache=None, can_execute_unsafe_code=None, replace_course_urls=None,
@@ -911,6 +903,7 @@ class ModuleSystem(Runtime):
         self.get_module = get_module
         self.render_template = render_template
         self.DEBUG = self.debug = debug
+        self.HOSTNAME = self.hostname = hostname
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path

common/lib/xmodule/xmodule/xml_module.py

@@ -164,7 +164,7 @@ class XmlDescriptor(XModuleDescriptor):
                          # Used for storing xml attributes between import and export, for roundtrips
                          'xml_attributes')
 
-    metadata_to_export_to_policy = ('discussion_topics')
+    metadata_to_export_to_policy = ('discussion_topics', 'checklists')
 
     @classmethod
     def get_map_for_field(cls, attr):

common/static/coffee/src/discussion/views/discussion_thread_list_view.coffee

@@ -64,10 +64,8 @@ if Backbone?
 
       sidebar = $(".sidebar")
       if scrollTop > discussionsBodyTop - @sidebar_padding
-        sidebar.addClass('fixed');
-        sidebar.css('top', @sidebar_padding);
+        sidebar.css('top', scrollTop - discussionsBodyTop + @sidebar_padding);
       else
-        sidebar.removeClass('fixed');
         sidebar.css('top', '0');
 
       sidebarWidth = .31 * $(".discussion-body").width();

common/templates/course_modes/choose.html

@@ -82,7 +82,7 @@ $(document).ready(function() {
                   <dl class="list-faq">
                     <dt class="faq-question">${_("Why do I have to pay?")}</dt>
                     <dd class="faq-answer">
-                      <p>${_("As a not-for-profit, edX uses your contribution to support our mission to provide quality education to everyone around the world. While we have established a minimum fee, we ask that you contribute as much as you can.")}</p>
+                      <p>${_("As a not-for-profit, edX uses your contribution to support our mission to provide quality education to everyone around the world, and to improve learning through research. While we have established a minimum fee, we ask that you contribute as much as you can.")}</p>
                     </dd>
 
                     <dt class="faq-question">${_("I'd like to pay more than the minimum.  Is my contribution tax deductible?")}</dt>
@@ -93,7 +93,7 @@ $(document).ready(function() {
                     % if "honor" in modes:
                     <dt class="faq-question">${_("What if I can't afford it or don't have the necessary equipment?")}</dt>
                     <dd class="faq-answer">
-                      <p>${_("If you can't afford the minimum fee or don't meet the requirements, you can audit the course for free. You may also elect to pursue an Honor Code certificate, but you will need to tell us why you would like the fee waived below. Then click the 'Select Certificate' button to complete your registration.")}</p>
+                      <p>${_("If you can't afford the minimum fee or don't meet the requirements, you can audit the course or elect to pursue an honor code certificate at no cost. If you would like to pursue the honor code certificate, please check the honor code certificate box, tell us why you can't pursue the verified certificate below, and then click the 'Select Certificate' button to complete your registration.")}</p>
 
                       <ul class="list-fields">
                         <li class="field field-honor-code checkbox">
@@ -102,7 +102,7 @@ $(document).ready(function() {
                         </li>
 
                         <li class="field field-explain">
-                          <label for="explain"><span class="sr">${_("Explain your situation: ")}</span>${_("Please write a few sentences about why you would like the fee waived for this course")}</label>
+                          <label for="explain"><span class="sr">${_("Explain your situation: ")}</span>${_("Please write a few sentences about why you'd like to opt out of the paid verified certificate to pursue the honor code certificate:")}</label>
                           <textarea name="explain"></textarea>
                         </li>
                       </ul>

common/test/data/due_date/about/overview.html

+<section class="about">
+  <h2>About This Course</h2>
+  <p>Include your long course description here. The long course description should contain 150-400 words.</p>
+
+  <p>This is paragraph 2 of the long course description. Add more paragraphs as needed. Make sure to enclose them in paragraph tags.</p>
+</section>
+
+<section class="prerequisites">
+  <h2>Prerequisites</h2>
+  <p>Add information about course prerequisites here.</p>
+</section>
+
+<section class="course-staff">
+  <h2>Course Staff</h2>
+  <article class="teacher">
+    <div class="teacher-image">
+      <img src="/static/images/pl-faculty.png" align="left" style="margin:0 20 px 0">
+    </div>
+
+    <h3>Staff Member #1</h3>
+    <p>Biography of instructor/staff member #1</p>
+  </article>
+
+  <article class="teacher">
+    <div class="teacher-image">
+      <img src="/static/images/pl-faculty.png" align="left" style="margin:0 20 px 0">
+    </div>
+
+    <h3>Staff Member #2</h3>
+    <p>Biography of instructor/staff member #2</p>
+  </article>
+</section>
+
+<section class="faq">
+  <section class="responses">
+    <h2>Frequently Asked Questions</h2>
+    <article class="response">
+      <h3>Do I need to buy a textbook?</h3>
+      <p>No, a free online version of Chemistry: Principles, Patterns, and Applications, First Edition by Bruce Averill and Patricia Eldredge will be available, though you can purchase a printed version (published by FlatWorld Knowledge) if you’d like.</p>
+    </article>
+
+    <article class="response">
+      <h3>Question #2</h3>
+      <p>Your answer would be displayed here.</p>
+    </article>
+  </section>
+</section>

common/test/data/due_date/chapter/c8ee0db7e5a84c85bac80b7013cf6512.xml

+<chapter display_name="Section">
+  <sequential url_name="c804fa32227142a1bd9d5bc183d4a20d"/>
+</chapter>