Merge branch 'master' into pbaratta/calc-add-trig

.gitignore

@@ -10,6 +10,8 @@
 .AppleDouble
 database.sqlite
 requirements/private.txt
+lms/envs/private.py
+cms/envs/private.py
 courseware/static/js/mathjax/*
 flushdb.sh
 build
@@ -27,6 +29,7 @@ conf/locale/en/LC_MESSAGES/*.po
 !messages.po
 lms/static/sass/*.css
 lms/static/sass/application.scss
+lms/static/sass/course.scss
 cms/static/sass/*.css
 lms/lib/comment_client/python
 nosetests.xml

AUTHORS

@@ -72,4 +72,7 @@ Giulio Gratta <giulio@giuliogratta.com>
 David Baumgold <david@davidbaumgold.com>
 Jason Bau <jbau@stanford.edu>
 Frances Botsford <frances@edx.org>
+Jonah Stanley <Jonah_Stanley@brown.edu>
 Slater Victoroff <slater.r.victoroff@gmail.com>
+Peter Fogg <peter.p.fogg@gmail.com>
+Renzo Lucioni <renzolucioni@gmail.com>
\ No newline at end of file

CHANGELOG.rst

+Change Log
+----------
+
+These are notable changes in edx-platform.  This is a rolling list of changes,
+in roughly chronological order, most recent first.  Add your entries at or near
+the top.  Include a label indicating the component affected.
+
+
+LMS: Some errors handling Non-ASCII data in XML courses have been fixed.
+
+LMS: Add page-load tracking using segment-io (if SEGMENT_IO_LMS_KEY and
+SEGMENT_IO_LMS feature flag is on)
+
+LMS: Background colors on login, register, and courseware have been corrected
+back to white.
+
+LMS: Accessibility improvements have been made to several courseware and
+navigation elements.
+
+LMS: Small design/presentation changes to login and register views.
+
+LMS: Functionality added to instructor enrollment tab in LMS such that invited
+student can be auto-enrolled in course or when activating if not current
+student.
+
+Blades: Staff debug info is now accessible for Graphical Slider Tool problems.
+
+Blades: For Video Alpha the events ready, play, pause, seek, and speed change
+are logged on the server (in the logs). 
+
+Common: Developers can now have private Django settings files.
+
+Common: Safety code added to prevent anything above the vertical level in the
+course tree from being marked as version='draft'. It will raise an exception if
+the code tries to so mark a node. We need the backtraces to figure out where
+this very infrequent intermittent marking was occurring. It was making courses
+look different in Studio than in LMS.
+
+Deploy: MKTG_URLS is now read from env.json.
+
+Common: Theming makes it possible to change the look of the site, from
+Stanford.
+
+Common: Accessibility UI fixes.
+
+Common: The "duplicate email" error message is more informative.
+
+Studio: Component metadata settings editor.
+
+Studio: Autoplay is disabled (only in Studio).
+
+Studio: Single-click creation for video and discussion components.
+
+Studio: fixed a bad link in the activation page.
+
+LMS: Changed the help button text.
+
+LMS: Fixed failing numeric response (decimal but no trailing digits).
+
+LMS: XML Error module no longer shows students a stack trace.
+
+Blades: Videoalpha.
+
+XModules: Added partial credit for foldit module.
+
+XModules: Added "randomize" XModule to list of XModule types.
+
+XModules: Show errors with full descriptors.
+
+XQueue: Fixed (hopefully) worker crash when the connection to RabbitMQ is
+dropped suddenly.
+
+XQueue: Upload file submissions to a specially named bucket in S3.
+
+Common: Removed request debugger.
+
+Common: Updated Django to version 1.4.5.
+
+Common: Updated CodeJail.
+
+Common: Allow setting of authentication session cookie name.
+

README.md

@@ -111,11 +111,11 @@ CMS templates. Fortunately, `rake` will do all of this for you! Just run:
 
     $ rake django-admin[syncdb]
     $ rake django-admin[migrate]
-    $ rake django-admin[update_templates]
+    $ rake cms:update_templates
 
 If you are running these commands using the [`zsh`](http://www.zsh.org/) shell,
 zsh will assume that you are doing
-[shell globbing](https://en.wikipedia.org/wiki/Glob_(programming)), search for
+[shell globbing](https://en.wikipedia.org/wiki/Glob_%28programming%29), search for
 a file in your directory named `django-adminsyncdb` or `django-adminmigrate`,
 and fail. To fix this, just surround the argument with quotation marks, so that
 you're running `rake "django-admin[syncdb]"`.

cms/djangoapps/contentstore/features/advanced-settings.feature

@@ -11,8 +11,6 @@ Feature: Advanced (manual) course policy
     Given I am on the Advanced Course Settings page in Studio
     Then the settings are alphabetized
 
-  # Skipped because Ubuntu ChromeDriver cannot click notification "Cancel"
-  @skip
   Scenario: Test cancel editing key value
     Given I am on the Advanced Course Settings page in Studio
     When I edit the value of a policy key
@@ -21,8 +19,6 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then the policy key value is unchanged
 
-  # Skipped because Ubuntu ChromeDriver cannot click notification "Save"
-  @skip
   Scenario: Test editing key value
     Given I am on the Advanced Course Settings page in Studio
     When I edit the value of a policy key and save
@@ -30,8 +26,6 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then the policy key value is changed
 
-  # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input
-  @skip
   Scenario: Test how multi-line input appears
     Given I am on the Advanced Course Settings page in Studio
     When I create a JSON object as a value
@@ -39,8 +33,6 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then it is displayed as formatted
 
-  # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input
-  @skip
   Scenario: Test automatic quoting of non-JSON values
     Given I am on the Advanced Course Settings page in Studio
     When I create a non-JSON value not in quotes

cms/djangoapps/contentstore/features/advanced-settings.py

@@ -42,8 +42,9 @@ def edit_the_value_of_a_policy_key(step):
     It is hard to figure out how to get into the CodeMirror
     area, so cheat and do it from the policy key field :)
     """
-    e = world.css_find(KEY_CSS)[get_index_of(DISPLAY_NAME_KEY)]
-    e._element.send_keys(Keys.TAB, Keys.END, Keys.ARROW_LEFT, ' ', 'X')
+    world.css_find(".CodeMirror")[get_index_of(DISPLAY_NAME_KEY)].click()
+    g = world.css_find("div.CodeMirror.CodeMirror-focused > div > textarea")
+    g._element.send_keys(Keys.ARROW_LEFT, ' ', 'X')
 
 
 @step(u'I edit the value of a policy key and save$')
@@ -123,10 +124,12 @@ def get_display_name_value():
 
 
 def change_display_name_value(step, new_value):
-    e = world.css_find(KEY_CSS)[get_index_of(DISPLAY_NAME_KEY)]
+
+    world.css_find(".CodeMirror")[get_index_of(DISPLAY_NAME_KEY)].click()
+    g = world.css_find("div.CodeMirror.CodeMirror-focused > div > textarea")
     display_name = get_display_name_value()
     for count in range(len(display_name)):
-        e._element.send_keys(Keys.TAB, Keys.END, Keys.BACK_SPACE)
+        g._element.send_keys(Keys.END, Keys.BACK_SPACE)
         # Must delete "" before typing the JSON value
-    e._element.send_keys(Keys.TAB, Keys.END, Keys.BACK_SPACE, Keys.BACK_SPACE, new_value)
+    g._element.send_keys(Keys.END, Keys.BACK_SPACE, Keys.BACK_SPACE, new_value)
     press_the_notification_button(step, "Save")

cms/djangoapps/contentstore/features/common.py

@@ -161,3 +161,11 @@ def i_created_a_video_component(step):
         'i4x://edx/templates/video/default',
         '.xmodule_VideoModule'
     )
+
+
+@step('I have clicked the new unit button')
+def open_new_unit(step):
+    step.given('I have opened a new course section in Studio')
+    step.given('I have added a new subsection')
+    step.given('I expand the first section')
+    world.css_click('a.new-unit-item')

cms/djangoapps/contentstore/features/component_settings_editor_helpers.py

@@ -14,20 +14,27 @@ def create_component_instance(step, component_button_css, instance_id, expected_
 
 @world.absorb
 def click_new_component_button(step, component_button_css):
-    step.given('I have opened a new course section in Studio')
-    step.given('I have added a new subsection')
-    step.given('I expand the first section')
-    world.css_click('a.new-unit-item')
+    step.given('I have clicked the new unit button')
     world.css_click(component_button_css)
 
 
 @world.absorb
 def click_component_from_menu(instance_id, expected_css):
+    """
+    Creates a component from `instance_id`. For components with more
+    than one template, clicks on `elem_css` to create the new
+    component. Components with only one template are created as soon
+    as the user clicks the appropriate button, so we assert that the
+    expected component is present.
+    """
     elem_css = "a[data-location='%s']" % instance_id
-    assert_equal(1, len(world.css_find(elem_css)))
-    world.css_click(elem_css)
+    elements = world.css_find(elem_css)
+    assert(len(elements) == 1)
+    if elements[0]['id'] == instance_id:  # If this is a component with multiple templates
+        world.css_click(elem_css)
     assert_equal(1, len(world.css_find(expected_css)))
 
+
 @world.absorb
 def edit_component_and_select_settings():
     world.css_click('a.edit-button')

cms/djangoapps/contentstore/features/courses.py

@@ -47,12 +47,6 @@ def i_see_the_course_in_my_courses(step):
     assert world.css_has_text(course_css, 'Robot Super Course')
 
 
-@step('the course is loaded$')
-def course_is_loaded(step):
-    class_css = 'a.class-name'
-    assert world.css_has_text(course_css, 'Robot Super Cousre')
-
-
 @step('I am on the "([^"]*)" tab$')
 def i_am_on_tab(step, tab_name):
     header_css = 'div.inner-wrapper h1'

cms/djangoapps/contentstore/features/discussion-editor.feature

@@ -11,3 +11,7 @@ Feature: Discussion Component Editor
     And I edit and select Settings
     Then I can modify the display name
     And my display name change is persisted on save
+
+  Scenario: Creating a discussion takes a single click
+    Given I have clicked the new unit button
+    Then creating a discussion takes a single click

cms/djangoapps/contentstore/features/discussion-editor.py

@@ -21,3 +21,10 @@ def i_see_only_the_settings_and_values(step):
             ['Display Name', "Discussion Tag", True],
             ['Subcategory', "Topic-Level Student-Visible Label", True]
         ])
+
+
+@step('creating a discussion takes a single click')
+def discussion_takes_a_single_click(step):
+    assert(not world.is_css_present('.xmodule_DiscussionModule'))
+    world.css_click("a[data-location='i4x://edx/templates/discussion/Discussion_Tag']")
+    assert(world.is_css_present('.xmodule_DiscussionModule'))

cms/djangoapps/contentstore/features/problem-editor.feature

@@ -52,7 +52,7 @@ Feature: Problem Editor
   Scenario: User cannot type out of range values in an integer number field
     Given I have created a Blank Common Problem
     And I edit and select Settings
-    Then if I set the max attempts to "-3", it displays initially as "-3", and is persisted as "1"
+    Then if I set the max attempts to "-3", it displays initially as "-3", and is persisted as "0"
 
   Scenario: Settings changes are not saved on Cancel
     Given I have created a Blank Common Problem

cms/djangoapps/contentstore/features/section.feature

@@ -26,11 +26,9 @@ Feature: Create Section
     And I save a new section release date
     Then the section release date is updated
 
-  # Skipped because Ubuntu ChromeDriver hangs on alert
-  @skip
   Scenario: Delete section
     Given I have opened a new course in Studio
     And I have added a new section
-    When I press the "section" delete icon
-    And I confirm the alert
+    When I will confirm all alerts
+    And I press the "section" delete icon
     Then the section does not exist

cms/djangoapps/contentstore/features/section.py

@@ -9,34 +9,34 @@ from nose.tools import assert_equal
 
 
 @step('I click the new section link$')
-def i_click_new_section_link(step):
+def i_click_new_section_link(_step):
     link_css = 'a.new-courseware-section-button'
     world.css_click(link_css)
 
 
 @step('I enter the section name and click save$')
-def i_save_section_name(step):
+def i_save_section_name(_step):
     save_section_name('My Section')
 
 
 @step('I enter a section name with a quote and click save$')
-def i_save_section_name_with_quote(step):
+def i_save_section_name_with_quote(_step):
     save_section_name('Section with "Quote"')
 
 
 @step('I have added a new section$')
-def i_have_added_new_section(step):
+def i_have_added_new_section(_step):
     add_section()
 
 
 @step('I click the Edit link for the release date$')
-def i_click_the_edit_link_for_the_release_date(step):
+def i_click_the_edit_link_for_the_release_date(_step):
     button_css = 'div.section-published-date a.edit-button'
     world.css_click(button_css)
 
 
 @step('I save a new section release date$')
-def i_save_a_new_section_release_date(step):
+def i_save_a_new_section_release_date(_step):
     set_date_and_time('input.start-date.date.hasDatepicker', '12/25/2013',
         'input.start-time.time.ui-timepicker-input', '00:00')
     world.browser.click_link_by_text('Save')
@@ -46,35 +46,35 @@ def i_save_a_new_section_release_date(step):
 
 
 @step('I see my section on the Courseware page$')
-def i_see_my_section_on_the_courseware_page(step):
+def i_see_my_section_on_the_courseware_page(_step):
     see_my_section_on_the_courseware_page('My Section')
 
 
 @step('I see my section name with a quote on the Courseware page$')
-def i_see_my_section_name_with_quote_on_the_courseware_page(step):
+def i_see_my_section_name_with_quote_on_the_courseware_page(_step):
     see_my_section_on_the_courseware_page('Section with "Quote"')
 
 
 @step('I click to edit the section name$')
-def i_click_to_edit_section_name(step):
+def i_click_to_edit_section_name(_step):
     world.css_click('span.section-name-span')
 
 
 @step('I see the complete section name with a quote in the editor$')
-def i_see_complete_section_name_with_quote_in_editor(step):
+def i_see_complete_section_name_with_quote_in_editor(_step):
     css = '.section-name-edit input[type=text]'
     assert world.is_css_present(css)
     assert_equal(world.browser.find_by_css(css).value, 'Section with "Quote"')
 
 
 @step('the section does not exist$')
-def section_does_not_exist(step):
-    css = 'span.section-name-span'
-    assert world.browser.is_element_not_present_by_css(css)
+def section_does_not_exist(_step):
+    css = 'h3[data-name="My Section"]'
+    assert world.is_css_not_present(css)
 
 
 @step('I see a release date for my section$')
-def i_see_a_release_date_for_my_section(step):
+def i_see_a_release_date_for_my_section(_step):
     import re
 
     css = 'span.published-status'
@@ -83,26 +83,32 @@ def i_see_a_release_date_for_my_section(step):
 
     # e.g. 11/06/2012 at 16:25
     msg = 'Will Release:'
-    date_regex = '[01][0-9]\/[0-3][0-9]\/[12][0-9][0-9][0-9]'
-    time_regex = '[0-2][0-9]:[0-5][0-9]'
-    match_string = '%s %s at %s' % (msg, date_regex, time_regex)
+    date_regex = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d\d?, \d{4}'
+    if not re.search(date_regex, status_text):
+        print status_text, date_regex
+    time_regex = r'[0-2]\d:[0-5]\d( \w{3})?'
+    if not re.search(time_regex, status_text):
+        print status_text, time_regex
+    match_string = r'%s\s+%s at %s' % (msg, date_regex, time_regex)
+    if not re.match(match_string, status_text):
+        print status_text, match_string
     assert re.match(match_string, status_text)
 
 
 @step('I see a link to create a new subsection$')
-def i_see_a_link_to_create_a_new_subsection(step):
+def i_see_a_link_to_create_a_new_subsection(_step):
     css = 'a.new-subsection-item'
     assert world.is_css_present(css)
 
 
 @step('the section release date picker is not visible$')
-def the_section_release_date_picker_not_visible(step):
+def the_section_release_date_picker_not_visible(_step):
     css = 'div.edit-subsection-publish-settings'
     assert not world.css_visible(css)
 
 
 @step('the section release date is updated$')
-def the_section_release_date_is_updated(step):
+def the_section_release_date_is_updated(_step):
     css = 'span.published-status'
     status_text = world.css_text(css)
     assert_equal(status_text, 'Will Release: 12/25/2013 at 00:00 UTC')

cms/djangoapps/contentstore/features/studio-overview-togglesection.feature

 Feature: Overview Toggle Section
-	In order to quickly view the details of a course's section or to scan the inventory of sections
+    In order to quickly view the details of a course's section or to scan the inventory of sections
     As a course author
     I want to toggle the visibility of each section's  subsection details in the overview listing
 
-	Scenario: The default layout for the overview page is to show sections in expanded view
-		Given I have a course with multiple sections
-    When I navigate to the course overview page
-    Then I see the "Collapse All Sections" link
-    And all sections are expanded
-
-	Scenario: Expand /collapse for a course with no sections
-		Given I have a course with no sections
-    When I navigate to the course overview page
-    Then I do not see the "Collapse All Sections" link
-
-	Scenario: Collapse link appears after creating first section of a course
-		Given I have a course with no sections
-    When I navigate to the course overview page
-    And I add a section
-    Then I see the "Collapse All Sections" link
-    And all sections are expanded
-
-    # Skipped because Ubuntu ChromeDriver hangs on alert
-    @skip
-	Scenario: Collapse link is not removed after last section of a course is deleted
-		Given I have a course with 1 section
-    And I navigate to the course overview page
-    When I press the "section" delete icon
-    And I confirm the alert
-		Then I see the "Collapse All Sections" link
-
-	Scenario: Collapsing all sections when all sections are expanded
-		Given I navigate to the courseware page of a course with multiple sections
-		And all sections are expanded
-		When I click the "Collapse All Sections" link
-		Then I see the "Expand All Sections" link
-		And all sections are collapsed
-
-	Scenario: Collapsing all sections when 1 or more sections are already collapsed
-		Given I navigate to the courseware page of a course with multiple sections
-		And all sections are expanded
-		When I collapse the first section
-		And I click the "Collapse All Sections" link
-		Then I see the "Expand All Sections" link
-		And all sections are collapsed
-
-	Scenario: Expanding all sections when all sections are collapsed
-		Given I navigate to the courseware page of a course with multiple sections
-		And I click the "Collapse All Sections" link
-		When I click the "Expand All Sections" link
-		Then I see the "Collapse All Sections" link
-		And all sections are expanded
-
-	Scenario: Expanding all sections when 1 or more sections are already expanded
-		Given I navigate to the courseware page of a course with multiple sections
-		And I click the "Collapse All Sections" link
-		When I expand the first section
-		And I click the "Expand All Sections" link
-		Then I see the "Collapse All Sections" link
-		And all sections are expanded
+    Scenario: The default layout for the overview page is to show sections in expanded view
+        Given I have a course with multiple sections
+        When I navigate to the course overview page
+        Then I see the "Collapse All Sections" link
+        And all sections are expanded
+
+    Scenario: Expand /collapse for a course with no sections
+        Given I have a course with no sections
+        When I navigate to the course overview page
+        Then I do not see the "Collapse All Sections" link
+
+    Scenario: Collapse link appears after creating first section of a course
+        Given I have a course with no sections
+        When I navigate to the course overview page
+        And I add a section
+        Then I see the "Collapse All Sections" link
+        And all sections are expanded
+
+    Scenario: Collapse link is not removed after last section of a course is deleted
+        Given I have a course with 1 section
+        And I navigate to the course overview page
+        When I will confirm all alerts
+        And I press the "section" delete icon
+        Then I see the "Collapse All Sections" link
+
+    Scenario: Collapsing all sections when all sections are expanded
+        Given I navigate to the courseware page of a course with multiple sections
+        And all sections are expanded
+        When I click the "Collapse All Sections" link
+        Then I see the "Expand All Sections" link
+        And all sections are collapsed
+
+    Scenario: Collapsing all sections when 1 or more sections are already collapsed
+        Given I navigate to the courseware page of a course with multiple sections
+        And all sections are expanded
+        When I collapse the first section
+        And I click the "Collapse All Sections" link
+        Then I see the "Expand All Sections" link
+        And all sections are collapsed
+
+    Scenario: Expanding all sections when all sections are collapsed
+        Given I navigate to the courseware page of a course with multiple sections
+        And I click the "Collapse All Sections" link
+        When I click the "Expand All Sections" link
+        Then I see the "Collapse All Sections" link
+        And all sections are expanded
+
+    Scenario: Expanding all sections when 1 or more sections are already expanded
+        Given I navigate to the courseware page of a course with multiple sections
+        And I click the "Collapse All Sections" link
+        When I expand the first section
+        And I click the "Expand All Sections" link
+        Then I see the "Collapse All Sections" link
+        And all sections are expanded

cms/djangoapps/contentstore/features/studio-overview-togglesection.py

@@ -112,7 +112,7 @@ def all_sections_are_expanded(step):
 
 
 @step(u'all sections are collapsed$')
-def all_sections_are_expanded(step):
+def all_sections_are_collapsed(step):
     subsection_locator = 'div.subsection-list'
     subsections = world.css_find(subsection_locator)
     for s in subsections:

cms/djangoapps/contentstore/features/subsection.feature

@@ -32,12 +32,10 @@ Feature: Create Subsection
     And I reload the page
     Then I see the correct dates
 
-  # Skipped because Ubuntu ChromeDriver hangs on alert
-  @skip
   Scenario: Delete a subsection
     Given I have opened a new course section in Studio
     And I have added a new subsection
     And I see my subsection on the Courseware page
-    When I press the "subsection" delete icon
-    And I confirm the alert
+    When I will confirm all alerts
+    And I press the "subsection" delete icon
     Then the subsection does not exist

cms/djangoapps/contentstore/features/video.feature

@@ -4,3 +4,12 @@ Feature: Video Component
   Scenario: Autoplay is disabled in Studio
     Given I have created a Video component
     Then when I view the video it does not have autoplay enabled
+
+  Scenario: Creating a video takes a single click
+    Given I have clicked the new unit button
+    Then creating a video takes a single click
+
+  Scenario: Captions are shown correctly
+    Given I have created a Video component
+    And I have hidden captions
+    Then when I view the video it does not show the captions

cms/djangoapps/contentstore/features/video.py

@@ -9,3 +9,20 @@ from lettuce import world, step
 def does_not_autoplay(step):
     assert world.css_find('.video')[0]['data-autoplay'] == 'False'
     assert world.css_find('.video_control')[0].has_class('play')
+
+
+@step('creating a video takes a single click')
+def video_takes_a_single_click(step):
+    assert(not world.is_css_present('.xmodule_VideoModule'))
+    world.css_click("a[data-location='i4x://edx/templates/video/default']")
+    assert(world.is_css_present('.xmodule_VideoModule'))
+
+
+@step('I have hidden captions')
+def set_show_captions_false(step):
+    world.css_click('a.hide-subtitles')
+
+
+@step('when I view the video it does not show the captions')
+def does_not_show_captions(step):
+    assert world.css_find('.video')[0].has_class('closed')

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -37,6 +37,9 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from contentstore.views.component import ADVANCED_COMPONENT_TYPES
 
 from django_comment_common.utils import are_permissions_roles_seeded
+from xmodule.exceptions import InvalidVersionError
+import datetime
+from pytz import UTC
 
 TEST_DATA_MODULESTORE = copy.deepcopy(settings.MODULESTORE)
 TEST_DATA_MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
@@ -77,14 +80,25 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         self.client = Client()
         self.client.login(username=uname, password=password)
 
-    def test_advanced_components_in_edit_unit(self):
+    def check_components_on_page(self, component_types, expected_types):
+        """
+        Ensure that the right types end up on the page.
+
+        component_types is the list of advanced components.
+
+        expected_types is the list of elements that should appear on the page.
+
+        expected_types and component_types should be similar, but not
+        exactly the same -- for example, 'videoalpha' in
+        component_types should cause 'Video Alpha' to be present.
+        """
         store = modulestore('direct')
         import_from_xml(store, 'common/test/data/', ['simple'])
 
         course = store.get_item(Location(['i4x', 'edX', 'simple',
                                           'course', '2012_Fall', None]), depth=None)
 
-        course.advanced_modules = ADVANCED_COMPONENT_TYPES
+        course.advanced_modules = component_types
 
         store.update_metadata(course.location, own_metadata(course))
 
@@ -94,13 +108,31 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
         self.assertEqual(resp.status_code, 200)
 
+        for expected in expected_types:
+            self.assertIn(expected, resp.content)
+
+    def test_advanced_components_in_edit_unit(self):
         # This could be made better, but for now let's just assert that we see the advanced modules mentioned in the page
         # response HTML
-        self.assertIn('Video Alpha', resp.content)
-        self.assertIn('Word cloud', resp.content)
-        self.assertIn('Annotation', resp.content)
-        self.assertIn('Open Ended Response', resp.content)
-        self.assertIn('Peer Grading Interface', resp.content)
+        self.check_components_on_page(ADVANCED_COMPONENT_TYPES, ['Video Alpha',
+                                                                 'Word cloud',
+                                                                 'Annotation',
+                                                                 'Open Ended Response',
+                                                                 'Peer Grading Interface'])
+
+    def test_advanced_components_require_two_clicks(self):
+        self.check_components_on_page(['videoalpha'], ['Video Alpha'])
+
+    def test_malformed_edit_unit_request(self):
+        store = modulestore('direct')
+        import_from_xml(store, 'common/test/data/', ['simple'])
+
+        # just pick one vertical
+        descriptor = store.get_items(Location('i4x', 'edX', 'simple', 'vertical', None, None))[0]
+        location = descriptor.location._replace(name='.' + descriptor.location.name)
+
+        resp = self.client.get(reverse('edit_unit', kwargs={'location': location.url()}))
+        self.assertEqual(resp.status_code, 400)
 
     def check_edit_unit(self, test_course_name):
         import_from_xml(modulestore('direct'), 'common/test/data/', [test_course_name])
@@ -239,7 +271,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         )
         self.assertTrue(getattr(draft_problem, 'is_draft', False))
 
-        #now requery with depth
+        # now requery with depth
         course = modulestore('draft').get_item(
             Location(['i4x', 'edX', 'simple', 'course', '2012_Fall', None]),
             depth=None
@@ -386,6 +418,32 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
             resp = self.client.get(reverse('edit_unit', kwargs={'location': new_loc.url()}))
             self.assertEqual(resp.status_code, 200)
 
+    def test_illegal_draft_crud_ops(self):
+        draft_store = modulestore('draft')
+        direct_store = modulestore('direct')
+
+        CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
+
+        location = Location('i4x://MITx/999/chapter/neuvo')
+        self.assertRaises(InvalidVersionError, draft_store.clone_item, 'i4x://edx/templates/chapter/Empty',
+            location)
+        direct_store.clone_item('i4x://edx/templates/chapter/Empty', location)
+        self.assertRaises(InvalidVersionError, draft_store.clone_item, location,
+            location)
+
+        self.assertRaises(InvalidVersionError, draft_store.update_item, location,
+            'chapter data')
+
+        # taking advantage of update_children and other functions never checking that the ids are valid
+        self.assertRaises(InvalidVersionError, draft_store.update_children, location,
+            ['i4x://MITx/999/problem/doesntexist'])
+
+        self.assertRaises(InvalidVersionError, draft_store.update_metadata, location,
+            {'due': datetime.datetime.now(UTC)})
+
+        self.assertRaises(InvalidVersionError, draft_store.unpublish, location)
+
+
     def test_bad_contentstore_request(self):
         resp = self.client.get('http://localhost:8001/c4x/CDX/123123/asset/&images_circuits_Lab7Solution2.png')
         self.assertEqual(resp.status_code, 400)
@@ -468,6 +526,9 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         # check for custom_tags
         self.verify_content_existence(module_store, root_dir, location, 'custom_tags', 'custom_tag_template')
 
+        # check for about content
+        self.verify_content_existence(module_store, root_dir, location, 'about', 'about', '.html')
+
         # check for graiding_policy.json
         filesystem = OSFS(root_dir / 'test_export/policies/6.002_Spring_2012')
         self.assertTrue(filesystem.exists('grading_policy.json'))
@@ -478,7 +539,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
             on_disk = loads(grading_policy.read())
             self.assertEqual(on_disk, course.grading_policy)
 
-        #check for policy.json
+        # check for policy.json
         self.assertTrue(filesystem.exists('policy.json'))
 
         # compare what's on disk to what we have in the course module

cms/djangoapps/contentstore/tests/test_course_settings.py

@@ -54,6 +54,7 @@ class CourseDetailsTestCase(CourseTestCase):
     def test_virgin_fetch(self):
         details = CourseDetails.fetch(self.course_location)
         self.assertEqual(details.course_location, self.course_location, "Location not copied into")
+        self.assertIsNotNone(details.start_date.tzinfo)
         self.assertIsNone(details.end_date, "end date somehow initialized " + str(details.end_date))
         self.assertIsNone(details.enrollment_start, "enrollment_start date somehow initialized " + str(details.enrollment_start))
         self.assertIsNone(details.enrollment_end, "enrollment_end date somehow initialized " + str(details.enrollment_end))
@@ -67,7 +68,6 @@ class CourseDetailsTestCase(CourseTestCase):
         jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
         jsondetails = json.loads(jsondetails)
         self.assertTupleEqual(Location(jsondetails['course_location']), self.course_location, "Location !=")
-        # Note, start_date is being initialized someplace. I'm not sure why b/c the default will make no sense.
         self.assertIsNone(jsondetails['end_date'], "end date somehow initialized ")
         self.assertIsNone(jsondetails['enrollment_start'], "enrollment_start date somehow initialized ")
         self.assertIsNone(jsondetails['enrollment_end'], "enrollment_end date somehow initialized ")
@@ -76,6 +76,23 @@ class CourseDetailsTestCase(CourseTestCase):
         self.assertIsNone(jsondetails['intro_video'], "intro_video somehow initialized")
         self.assertIsNone(jsondetails['effort'], "effort somehow initialized")
 
+    def test_ooc_encoder(self):
+        """
+        Test the encoder out of its original constrained purpose to see if it functions for general use
+        """
+        details = {'location': Location(['tag', 'org', 'course', 'category', 'name']),
+            'number': 1,
+            'string': 'string',
+            'datetime': datetime.datetime.now(UTC())}
+        jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
+        jsondetails = json.loads(jsondetails)
+
+        self.assertIn('location', jsondetails)
+        self.assertIn('org', jsondetails['location'])
+        self.assertEquals('org', jsondetails['location'][1])
+        self.assertEquals(1, jsondetails['number'])
+        self.assertEqual(jsondetails['string'], 'string')
+
     def test_update_and_fetch(self):
         # # NOTE: I couldn't figure out how to validly test time setting w/ all the conversions
         jsondetails = CourseDetails.fetch(self.course_location)
@@ -116,11 +133,8 @@ class CourseDetailsViewTest(CourseTestCase):
         self.compare_details_with_encoding(json.loads(resp.content), details.__dict__, field + str(val))
 
     @staticmethod
-    def convert_datetime_to_iso(datetime):
-        if datetime is not None:
-            return datetime.isoformat("T")
-        else:
-            return None
+    def convert_datetime_to_iso(dt):
+        return Date().to_json(dt)
 
     def test_update_and_fetch(self):
         details = CourseDetails.fetch(self.course_location)
@@ -151,22 +165,12 @@ class CourseDetailsViewTest(CourseTestCase):
         self.assertEqual(details['intro_video'], encoded.get('intro_video', None), context + " intro_video not ==")
         self.assertEqual(details['effort'], encoded['effort'], context + " efforts not ==")
 
-    @staticmethod
-    def struct_to_datetime(struct_time):
-        return datetime.datetime(*struct_time[:6], tzinfo=UTC())
-
     def compare_date_fields(self, details, encoded, context, field):
         if details[field] is not None:
             date = Date()
             if field in encoded and encoded[field] is not None:
-                encoded_encoded = date.from_json(encoded[field])
-                dt1 = CourseDetailsViewTest.struct_to_datetime(encoded_encoded)
-
-                if isinstance(details[field], datetime.datetime):
-                    dt2 = details[field]
-                else:
-                    details_encoded = date.from_json(details[field])
-                    dt2 = CourseDetailsViewTest.struct_to_datetime(details_encoded)
+                dt1 = date.from_json(encoded[field])
+                dt2 = details[field]
 
                 expected_delta = datetime.timedelta(0)
                 self.assertEqual(dt1 - dt2, expected_delta, str(dt1) + "!=" + str(dt2) + " at " + context)

cms/djangoapps/contentstore/tests/test_utils.py

@@ -4,6 +4,7 @@ import mock
 import collections
 import copy
 from django.test import TestCase
+from django.test.utils import override_settings
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 
@@ -11,11 +12,52 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 class LMSLinksTestCase(TestCase):
     """ Tests for LMS links. """
     def about_page_test(self):
-        """ Get URL for about page. """
+        """ Get URL for about page, no marketing site """
+        # default for ENABLE_MKTG_SITE is False.
+        self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")
+
+    @override_settings(MKTG_URLS={'ROOT': 'dummy-root'})
+    def about_page_marketing_site_test(self):
+        """ Get URL for about page, marketing root present. """
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': True}):
+            self.assertEquals(self.get_about_page_link(), "//dummy-root/courses/mitX/101/test/about")
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': False}):
+            self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")
+
+    @override_settings(MKTG_URLS={'ROOT': 'http://www.dummy'})
+    def about_page_marketing_site_remove_http_test(self):
+        """ Get URL for about page, marketing root present, remove http://. """
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': True}):
+            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")
+
+    @override_settings(MKTG_URLS={'ROOT': 'https://www.dummy'})
+    def about_page_marketing_site_remove_https_test(self):
+        """ Get URL for about page, marketing root present, remove https://. """
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': True}):
+            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")
+
+    @override_settings(MKTG_URLS={'ROOT': 'www.dummyhttps://x'})
+    def about_page_marketing_site_https__edge_test(self):
+        """ Get URL for about page, only remove https:// at the beginning of the string. """
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': True}):
+            self.assertEquals(self.get_about_page_link(), "//www.dummyhttps://x/courses/mitX/101/test/about")
+
+    @override_settings(MKTG_URLS={})
+    def about_page_marketing_urls_not_set_test(self):
+        """ Error case. ENABLE_MKTG_SITE is True, but there is either no MKTG_URLS, or no MKTG_URLS Root property. """
+        with mock.patch.dict('django.conf.settings.MITX_FEATURES', {'ENABLE_MKTG_SITE': True}):
+            self.assertEquals(self.get_about_page_link(), None)
+
+    @override_settings(LMS_BASE=None)
+    def about_page_no_lms_base_test(self):
+        """ No LMS_BASE, nor is ENABLE_MKTG_SITE True """
+        self.assertEquals(self.get_about_page_link(), None)
+
+    def get_about_page_link(self):
+        """ create mock course and return the about page link """
         location = 'i4x', 'mitX', '101', 'course', 'test'
         utils.get_course_id = mock.Mock(return_value="mitX/101/test")
-        link = utils.get_lms_link_for_about_page(location)
-        self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/about")
+        return utils.get_lms_link_for_about_page(location)
 
     def lms_link_test(self):
         """ Tests get_lms_link_for_item. """

cms/djangoapps/contentstore/utils.py

@@ -4,8 +4,11 @@ from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from django.core.urlresolvers import reverse
 import copy
+import logging
+import re
+from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
 
-DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_tab', 'course_info']
+log = logging.getLogger(__name__)
 
 #In order to instantiate an open ended tab automatically, need to have this data
 OPEN_ENDED_PANEL = {"name": "Open Ended Panel", "type": "open_ended"}
@@ -107,9 +110,29 @@ def get_lms_link_for_about_page(location):
     """
     Returns the url to the course about page from the location tuple.
     """
-    if settings.LMS_BASE is not None:
-        lms_link = "//{lms_base}/courses/{course_id}/about".format(
-            lms_base=settings.LMS_BASE,
+    if settings.MITX_FEATURES.get('ENABLE_MKTG_SITE', False):
+        if not hasattr(settings, 'MKTG_URLS'):
+            log.exception("ENABLE_MKTG_SITE is True, but MKTG_URLS is not defined.")
+            about_base = None
+        else:
+            marketing_urls = settings.MKTG_URLS
+            if marketing_urls.get('ROOT', None) is None:
+                log.exception('There is no ROOT defined in MKTG_URLS')
+                about_base = None
+            else:
+                # Root will be "https://www.edx.org". The complete URL will still not be exactly correct,
+                # but redirects exist from www.edx.org to get to the Drupal course about page URL.
+                about_base = marketing_urls.get('ROOT')
+                # Strip off https:// (or http://) to be consistent with the formatting of LMS_BASE.
+                about_base = re.sub(r"^https?://", "", about_base)
+    elif settings.LMS_BASE is not None:
+        about_base = settings.LMS_BASE
+    else:
+        about_base = None
+
+    if about_base is not None:
+        lms_link = "//{about_base_url}/courses/{course_id}/about".format(
+            about_base_url=about_base,
             course_id=get_course_id(location)
         )
     else:
@@ -205,7 +228,7 @@ def add_extra_panel_tab(tab_type, course):
     course_tabs = copy.copy(course.tabs)
     changed = False
     #Check to see if open ended panel is defined in the course
-    
+
     tab_panel = EXTRA_TAB_PANELS.get(tab_type)
     if tab_panel not in course_tabs:
         #Add panel to the tabs if it is not defined

cms/djangoapps/contentstore/views/assets.py

@@ -62,7 +62,7 @@ def asset_index(request, org, course, name):
         asset_id = asset['_id']
         display_info = {}
         display_info['displayname'] = asset['displayname']
-        display_info['uploadDate'] = get_default_time_display(asset['uploadDate'].timetuple())
+        display_info['uploadDate'] = get_default_time_display(asset['uploadDate'])
 
         asset_location = StaticContent.compute_location(asset_id['org'], asset_id['course'], asset_id['name'])
         display_info['url'] = StaticContent.get_url_path_from_location(asset_location)
@@ -103,6 +103,9 @@ def upload_asset(request, org, course, coursename):
         logging.error('Could not find course' + location)
         return HttpResponseBadRequest()
 
+    if 'file' not in request.FILES:
+        return HttpResponseBadRequest()
+
     # compute a 'filename' which is similar to the location formatting, we're using the 'filename'
     # nomenclature since we're using a FileSystem paradigm here. We're just imposing
     # the Location string formatting expectations to keep things a bit more consistent
@@ -131,7 +134,7 @@ def upload_asset(request, org, course, coursename):
     readback = contentstore().find(content.location)
 
     response_payload = {'displayname': content.name,
-                        'uploadDate': get_default_time_display(readback.last_modified_at.timetuple()),
+                        'uploadDate': get_default_time_display(readback.last_modified_at),
                         'url': StaticContent.get_url_path_from_location(content.location),
                         'thumb_url': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_content is not None else None,
                         'msg': 'Upload completed'
@@ -227,11 +230,9 @@ def generate_export_course(request, org, course, name):
     root_dir = path(mkdtemp())
 
     # export out to a tempdir
-
     logging.debug('root = {0}'.format(root_dir))
 
     export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
-    #filename = root_dir / name + '.tar.gz'
 
     logging.debug('tar file being generated at {0}'.format(export_file.name))
     tar_file = tarfile.open(name=export_file.name, mode='w:gz')

cms/djangoapps/contentstore/views/component.py

@@ -7,7 +7,7 @@ from django.contrib.auth.decorators import login_required
 from django.core.exceptions import PermissionDenied
 from django_future.csrf import ensure_csrf_cookie
 from django.conf import settings
-
+from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
 from mitxmako.shortcuts import render_to_response
 
 from xmodule.modulestore import Location
@@ -50,11 +50,18 @@ ADVANCED_COMPONENT_POLICY_KEY = 'advanced_modules'
 @login_required
 def edit_subsection(request, location):
     # check that we have permissions to edit this item
-    course = get_course_for_item(location)
+    try:
+        course = get_course_for_item(location)
+    except InvalidLocationError:
+        return HttpResponseBadRequest()
+
     if not has_access(request.user, course.location):
         raise PermissionDenied()
 
-    item = modulestore().get_item(location, depth=1)
+    try:
+        item = modulestore().get_item(location, depth=1)
+    except ItemNotFoundError:
+        return HttpResponseBadRequest()
 
     lms_link = get_lms_link_for_item(location, course_id=course.location.course_id)
     preview_link = get_lms_link_for_item(location, course_id=course.location.course_id, preview=True)
@@ -113,11 +120,18 @@ def edit_unit(request, location):
 
     id: A Location URL
     """
-    course = get_course_for_item(location)
+    try:
+        course = get_course_for_item(location)
+    except InvalidLocationError:
+        return HttpResponseBadRequest()
+
     if not has_access(request.user, course.location):
         raise PermissionDenied()
 
-    item = modulestore().get_item(location, depth=1)
+    try:
+        item = modulestore().get_item(location, depth=1)
+    except ItemNotFoundError:
+        return HttpResponseBadRequest()
 
     lms_link = get_lms_link_for_item(item.location, course_id=course.location.course_id)
 

cms/djangoapps/contentstore/views/course.py

@@ -2,7 +2,6 @@
 Views related to operations on course objects
 """
 import json
-import time
 
 from django.contrib.auth.decorators import login_required
 from django_future.csrf import ensure_csrf_cookie
@@ -32,6 +31,8 @@ from .component import OPEN_ENDED_COMPONENT_TYPES, \
      NOTE_COMPONENT_TYPES, ADVANCED_COMPONENT_POLICY_KEY
 
 from django_comment_common.utils import seed_permissions_roles
+import datetime
+from django.utils.timezone import UTC
 
 # TODO: should explicitly enumerate exports with __all__
 
@@ -130,7 +131,7 @@ def create_new_course(request):
         new_course.display_name = display_name
 
     # set a default start date to now
-    new_course.start = time.gmtime()
+    new_course.start = datetime.datetime.now(UTC())
 
     initialize_course_tabs(new_course)
 
@@ -357,49 +358,49 @@ def course_advanced_updates(request, org, course, name):
         # Whether or not to filter the tabs key out of the settings metadata
         filter_tabs = True
 
-        #Check to see if the user instantiated any advanced components. This is a hack
-        #that does the following : 
-        #   1) adds/removes the open ended panel tab to a course automatically if the user 
+        # Check to see if the user instantiated any advanced components. This is a hack
+        # that does the following :
+        #   1) adds/removes the open ended panel tab to a course automatically if the user
         #   has indicated that they want to edit the combinedopendended or peergrading module
         #   2) adds/removes the notes panel tab to a course automatically if the user has
         #   indicated that they want the notes module enabled in their course
         # TODO refactor the above into distinct advanced policy settings
         if ADVANCED_COMPONENT_POLICY_KEY in request_body:
-            #Get the course so that we can scrape current tabs
+            # Get the course so that we can scrape current tabs
             course_module = modulestore().get_item(location)
 
-            #Maps tab types to components 
+            # Maps tab types to components
             tab_component_map = {
-                'open_ended': OPEN_ENDED_COMPONENT_TYPES, 
+                'open_ended': OPEN_ENDED_COMPONENT_TYPES,
                 'notes': NOTE_COMPONENT_TYPES,
             }
 
-            #Check to see if the user instantiated any notes or open ended components
+            # Check to see if the user instantiated any notes or open ended components
             for tab_type in tab_component_map.keys():
                 component_types = tab_component_map.get(tab_type)
                 found_ac_type = False
                 for ac_type in component_types:
                     if ac_type in request_body[ADVANCED_COMPONENT_POLICY_KEY]:
-                        #Add tab to the course if needed
+                        # Add tab to the course if needed
                         changed, new_tabs = add_extra_panel_tab(tab_type, course_module)
-                        #If a tab has been added to the course, then send the metadata along to CourseMetadata.update_from_json
+                        # If a tab has been added to the course, then send the metadata along to CourseMetadata.update_from_json
                         if changed:
                             course_module.tabs = new_tabs
                             request_body.update({'tabs': new_tabs})
-                            #Indicate that tabs should not be filtered out of the metadata
+                            # Indicate that tabs should not be filtered out of the metadata
                             filter_tabs = False
-                        #Set this flag to avoid the tab removal code below.
+                        # Set this flag to avoid the tab removal code below.
                         found_ac_type = True
                         break
-                #If we did not find a module type in the advanced settings,
+                # If we did not find a module type in the advanced settings,
                 # we may need to remove the tab from the course.
                 if not found_ac_type:
-                    #Remove tab from the course if needed
+                    # Remove tab from the course if needed
                     changed, new_tabs = remove_extra_panel_tab(tab_type, course_module)
                     if changed:
                         course_module.tabs = new_tabs
                         request_body.update({'tabs': new_tabs})
-                        #Indicate that tabs should *not* be filtered out of the metadata
+                        # Indicate that tabs should *not* be filtered out of the metadata
                         filter_tabs = False
 
         response_json = json.dumps(CourseMetadata.update_from_json(location,

cms/djangoapps/models/settings/course_details.py

@@ -3,26 +3,26 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.inheritance import own_metadata
 import json
 from json.encoder import JSONEncoder
-import time
 from contentstore.utils import get_modulestore
 from models.settings import course_grading
 from contentstore.utils import update_item
 from xmodule.fields import Date
 import re
 import logging
+import datetime
 
 
 class CourseDetails(object):
     def __init__(self, location):
-        self.course_location = location    # a Location obj
+        self.course_location = location  # a Location obj
         self.start_date = None  # 'start'
-        self.end_date = None    # 'end'
+        self.end_date = None  # 'end'
         self.enrollment_start = None
         self.enrollment_end = None
-        self.syllabus = None    # a pdf file asset
-        self.overview = ""      # html to render as the overview
-        self.intro_video = None    # a video pointer
-        self.effort = None      # int hours/week
+        self.syllabus = None  # a pdf file asset
+        self.overview = ""  # html to render as the overview
+        self.intro_video = None  # a video pointer
+        self.effort = None  # int hours/week
 
     @classmethod
     def fetch(cls, course_location):
@@ -73,9 +73,9 @@ class CourseDetails(object):
         """
         Decode the json into CourseDetails and save any changed attrs to the db
         """
-        ## TODO make it an error for this to be undefined & for it to not be retrievable from modulestore
+        # TODO make it an error for this to be undefined & for it to not be retrievable from modulestore
         course_location = jsondict['course_location']
-        ## Will probably want to cache the inflight courses because every blur generates an update
+        # Will probably want to cache the inflight courses because every blur generates an update
         descriptor = get_modulestore(course_location).get_item(course_location)
 
         dirty = False
@@ -181,7 +181,7 @@ class CourseSettingsEncoder(json.JSONEncoder):
             return obj.__dict__
         elif isinstance(obj, Location):
             return obj.dict()
-        elif isinstance(obj, time.struct_time):
+        elif isinstance(obj, datetime.datetime):
             return Date().to_json(obj)
         else:
             return JSONEncoder.default(self, obj)

cms/envs/acceptance.py

@@ -23,7 +23,7 @@ MODULESTORE_OPTIONS = {
     'db': 'test_xmodule',
     'collection': 'acceptance_modulestore',
     'fs_root': TEST_ROOT / "data",
-    'render_template': 'mitxmako.shortcuts.render_to_string',
+    'render_template': 'mitxmako.shortcuts.render_to_string'
 }
 
 MODULESTORE = {

cms/envs/aws.py

@@ -91,11 +91,19 @@ CACHES = ENV_TOKENS['CACHES']
 
 SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
 
+# allow for environments to specify what cookie name our login subsystem should use
+# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can
+# happen with some browsers (e.g. Firefox)
+if ENV_TOKENS.get('SESSION_COOKIE_NAME', None):
+    # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str()
+    SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME'))
+
 #Email overrides
 DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL)
 DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL)
 ADMINS = ENV_TOKENS.get('ADMINS', ADMINS)
 SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL)
+MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
 
 #Timezone overrides
 TIME_ZONE = ENV_TOKENS.get('TIME_ZONE', TIME_ZONE)

cms/envs/common.py

@@ -25,6 +25,7 @@ Longer TODO:
 
 import sys
 import lms.envs.common
+from lms.envs.common import USE_TZ
 from path import path
 
 ############################ FEATURE CONFIGURATION #############################
@@ -34,8 +35,8 @@ MITX_FEATURES = {
     'GITHUB_PUSH': False,
     'ENABLE_DISCUSSION_SERVICE': False,
     'AUTH_USE_MIT_CERTIFICATES': False,
-    'STUB_VIDEO_FOR_TESTING': False,   # do not display video when running automated acceptance tests
-    'STAFF_EMAIL': '',			# email address for staff (eg to request course creation)
+    'STUB_VIDEO_FOR_TESTING': False,  # do not display video when running automated acceptance tests
+    'STAFF_EMAIL': '',  # email address for staff (eg to request course creation)
     'STUDIO_NPS_SURVEY': True,
     'SEGMENT_IO': True,
 
@@ -183,7 +184,7 @@ STATICFILES_DIRS = [
 
 # Locale/Internationalization
 TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
-LANGUAGE_CODE = 'en'            # http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html
 
 USE_I18N = True
 USE_L10N = True
@@ -323,6 +324,7 @@ INSTALLED_APPS = (
     'track',
 
     # For asset pipelining
+    'mitxmako',
     'pipeline',
     'staticfiles',
     'static_replace',
@@ -334,3 +336,14 @@ INSTALLED_APPS = (
 ################# EDX MARKETING SITE ##################################
 
 EDXMKTG_COOKIE_NAME = 'edxloggedin'
+MKTG_URLS = {}
+MKTG_URL_LINK_MAP = {
+    'ABOUT': 'about_edx',
+    'CONTACT': 'contact',
+    'FAQ': 'help_edx',
+    'COURSES': 'courses',
+    'ROOT': 'root',
+    'TOS': 'tos',
+    'HONOR': 'honor',
+    'PRIVACY': 'privacy_edx',
+}

cms/envs/dev.py

@@ -22,7 +22,7 @@ modulestore_options = {
     'db': 'xmodule',
     'collection': 'modulestore',
     'fs_root': GITHUB_REPO_ROOT,
-    'render_template': 'mitxmako.shortcuts.render_to_string',
+    'render_template': 'mitxmako.shortcuts.render_to_string'
 }
 
 MODULESTORE = {
@@ -64,7 +64,7 @@ REPOS = {
     },
     'content-mit-6002x': {
         'branch': 'master',
-        #'origin': 'git@github.com:MITx/6002x-fall-2012.git',
+        # 'origin': 'git@github.com:MITx/6002x-fall-2012.git',
         'origin': 'git@github.com:MITx/content-mit-6002x.git',
     },
     '6.00x': {
@@ -165,3 +165,11 @@ MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True
 
 # segment-io key for dev
 SEGMENT_IO_KEY = 'mty8edrrsg'
+
+
+#####################################################################
+# Lastly, see if the developer has any local overrides.
+try:
+    from .private import *
+except ImportError:
+    pass

cms/envs/test.py

@@ -48,7 +48,7 @@ MODULESTORE_OPTIONS = {
     'db': 'test_xmodule',
     'collection': 'test_modulestore',
     'fs_root': TEST_ROOT / "data",
-    'render_template': 'mitxmako.shortcuts.render_to_string',
+    'render_template': 'mitxmako.shortcuts.render_to_string'
 }
 
 MODULESTORE = {
@@ -121,7 +121,7 @@ CELERY_RESULT_BACKEND = 'cache'
 BROKER_TRANSPORT = 'memory'
 
 ################### Make tests faster
-#http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
+# http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
 PASSWORD_HASHERS = (
     'django.contrib.auth.hashers.SHA1PasswordHasher',
     'django.contrib.auth.hashers.MD5PasswordHasher',

cms/pydev_manage.py

+'''
+Used for pydev eclipse. Should be innocuous for everyone else.
+Created on May 8, 2013
+
+@author: dmitchell
+'''
+#!/home/<username>/mitx_all/python/bin/python
+from django.core import management
+
+if __name__ == '__main__':
+    management.execute_from_command_line()

cms/static/coffee/src/views/unit.coffee

 class CMS.Views.UnitEdit extends Backbone.View
   events:
-    'click .new-component .new-component-type a': 'showComponentTemplates'
+    'click .new-component .new-component-type a.multiple-templates': 'showComponentTemplates'
+    'click .new-component .new-component-type a.single-template': 'saveNewComponent'
     'click .new-component .cancel-button': 'closeNewComponent'
     'click .new-component-templates .new-component-template a': 'saveNewComponent'
     'click .new-component-templates .cancel-button': 'closeNewComponent'

cms/static/js/base.js

@@ -411,8 +411,12 @@ function showFileSelectionMenu(e) {
 }
 
 function startUpload(e) {
+    var files = $('.file-input').get(0).files;
+    if (files.length === 0)
+        return;
+
     $('.upload-modal h1').html(gettext('Uploading…'));
-    $('.upload-modal .file-name').html($('.file-input').val().replace('C:\\fakepath\\', ''));
+    $('.upload-modal .file-name').html(files[0].name);
     $('.upload-modal .file-chooser').ajaxSubmit({
         beforeSend: resetUploadBar,
         uploadProgress: showUploadFeedback,

cms/static/sass/_base.scss

@@ -14,7 +14,7 @@ body {
   color: $gray-d2;
 }
 
-body, input {
+body, input, button {
   font-family: 'Open Sans', sans-serif;
 }
 

cms/templates/edit_subsection.html

 <%inherit file="base.html" />
 <%!
    import logging
-   from xmodule.util.date_utils import get_time_struct_display
+   from xmodule.util.date_utils import get_default_time_display
 %>
 
 <%! from django.core.urlresolvers import reverse %>
@@ -36,11 +36,15 @@
         <div class="datepair" data-language="javascript">
           <div class="field field-start-date">
             <label for="start_date">Release Day</label>
-            <input type="text" id="start_date" name="start_date" value="${get_time_struct_display(subsection.lms.start, '%m/%d/%Y')}" placeholder="MM/DD/YYYY" class="date" size='15' autocomplete="off"/>
+            <input type="text" id="start_date" name="start_date" 
+                value="${subsection.lms.start.strftime('%m/%d/%Y') if subsection.lms.start else ''}" 
+                placeholder="MM/DD/YYYY" class="date" size='15' autocomplete="off"/>
           </div>
           <div class="field field-start-time">
             <label for="start_time">Release Time (<abbr title="Coordinated Universal Time">UTC</abbr>)</label>
-            <input type="text" id="start_time" name="start_time" value="${get_time_struct_display(subsection.lms.start, '%H:%M')}" placeholder="HH:MM" class="time" size='10' autocomplete="off"/>
+            <input type="text" id="start_time" name="start_time" 
+                value="${subsection.lms.start.strftime('%H:%M') if subsection.lms.start else ''}" 
+                placeholder="HH:MM" class="time" size='10' autocomplete="off"/>
           </div>
         </div>
         % if subsection.lms.start != parent_item.lms.start and subsection.lms.start:
@@ -48,7 +52,7 @@
             <p class="notice">The date above differs from the release date of ${parent_item.display_name_with_default}, which is unset.
             % else:
             <p class="notice">The date above differs from the release date of ${parent_item.display_name_with_default} –
-                ${get_time_struct_display(parent_item.lms.start, '%m/%d/%Y at %H:%M UTC')}.
+                ${get_default_time_display(parent_item.lms.start)}.
             % endif
             <a href="#" class="sync-date no-spinner">Sync to ${parent_item.display_name_with_default}.</a></p>
         % endif
@@ -65,11 +69,15 @@
       <div class="datepair date-setter">
         <div class="field field-start-date">
           <label for="due_date">Due Day</label>
-          <input type="text" id="due_date" name="due_date" value="${get_time_struct_display(subsection.lms.due, '%m/%d/%Y')}" placeholder="MM/DD/YYYY" class="date" size='15' autocomplete="off"/>
+          <input type="text" id="due_date" name="due_date" 
+          value="${subsection.lms.due.strftime('%m/%d/%Y') if subsection.lms.due else ''}" 
+          placeholder="MM/DD/YYYY" class="date" size='15' autocomplete="off"/>
         </div>
         <div class="field field-start-time">
           <label for="due_time">Due Time (<abbr title="Coordinated Universal Time">UTC</abbr>)</label>
-          <input type="text" id="due_time" name="due_time" value="${get_time_struct_display(subsection.lms.due, '%H:%M')}" placeholder="HH:MM" class="time" size='10' autocomplete="off"/>
+          <input type="text" id="due_time" name="due_time" 
+            value="${subsection.lms.due.strftime('%H:%M') if subsection.lms.due else ''}" 
+            placeholder="HH:MM" class="time" size='10' autocomplete="off"/>
         </div>
         <a href="#" class="remove-date">Remove due date</a>
       </div>

cms/templates/overview.html

 <%inherit file="base.html" />
 <%!
    import logging
-   from xmodule.util.date_utils import get_time_struct_display
+   from xmodule.util import date_utils
 %>
 <%! from django.core.urlresolvers import reverse %>
 <%block name="title">Course Outline</%block>
@@ -154,14 +154,19 @@
               <h3 class="section-name" data-name="${section.display_name_with_default | h}"></h3>
               <div class="section-published-date">
                 <%
-                  start_date_str = get_time_struct_display(section.lms.start, '%m/%d/%Y')
-                  start_time_str = get_time_struct_display(section.lms.start, '%H:%M')
+                  if section.lms.start is not None:
+                      start_date_str = section.lms.start.strftime('%m/%d/%Y')
+                      start_time_str = section.lms.start.strftime('%H:%M')
+                  else:
+                      start_date_str = ''
+                      start_time_str = ''
                 %>
                 %if section.lms.start is None:
                   <span class="published-status">This section has not been released.</span>
                   <a href="#" class="schedule-button" data-date="" data-time="" data-id="${section.location}">Schedule</a>
                 %else:
-                  <span class="published-status"><strong>Will Release:</strong> ${get_time_struct_display(section.lms.start, '%m/%d/%Y at %H:%M UTC')}</span>
+                  <span class="published-status"><strong>Will Release:</strong> 
+                  ${date_utils.get_default_time_display(section.lms.start)}</span>
                   <a href="#" class="edit-button" data-date="${start_date_str}" data-time="${start_time_str}" data-id="${section.location}">Edit</a>
                 %endif
               </div>

cms/templates/unit.html

@@ -53,9 +53,15 @@
               <div class="new-component">
                 <h5>Add New Component</h5>
                 <ul class="new-component-type">
-                  % for type in sorted(component_templates.keys()):
+                  % for type, templates in sorted(component_templates.items()):
                   <li>
-                    <a href="#" data-type="${type}">
+                  % if type == 'advanced' or len(templates) > 1:
+                    <a href="#" class="multiple-templates" data-type="${type}">
+                  % else:
+                    % for _, location, _ in templates:
+                    <a href="#" class="single-template" data-type="${type}" data-location="${location}">
+                    % endfor
+                  % endif
                       <span class="large-template-icon large-${type}-icon"></span>
                       <span class="name">${type}</span>
                     </a>
@@ -64,50 +70,52 @@
                 </ul>
               </div>
                 % for type, templates in sorted(component_templates.items()):
+                  % if len(templates) > 1 or type == 'advanced':
                 <div class="new-component-templates new-component-${type}">
                     % if type == "problem":
-                    <div class="tab-group tabs">
-                            <ul class="problem-type-tabs nav-tabs">
-                              <li class="current">
-                                <a class="link-tab" href="#tab1">Common Problem Types</a>
-                              </li>
-                              <li>
-                                <a class="link-tab" href="#tab2">Advanced</a>
-                              </li>
-                            </ul>
-                    % endif
-                        <div class="tab current" id="tab1">
-                          <ul class="new-component-template">
-                              % for name, location, has_markdown in templates:
-                                % if has_markdown or type != "problem":
-                                  <li class="editor-md">
-                                    <a href="#" id="${location}" data-location="${location}">
-                                      <span class="name"> ${name}</span>
-                                    </a>
-                                  </li>
-                                % endif
-
-                              %endfor
-                          </ul>
-                        </div>
-                    % if type == "problem":
-                            <div class="tab" id="tab2">
-                              <ul class="new-component-template">
-                                  % for name, location, has_markdown in templates:
-                                    % if not has_markdown:
-                                      <li class="editor-manual">
-                                        <a href="#" id="${location}" data-location="${location}">
-                                          <span class="name"> ${name}</span>
-                                        </a>
-                                      </li>
-                                    % endif
-                                  % endfor
+                      <div class="tab-group tabs">
+                              <ul class="problem-type-tabs nav-tabs">
+                                <li class="current">
+                                  <a class="link-tab" href="#tab1">Common Problem Types</a>
+                                </li>
+                                <li>
+                                  <a class="link-tab" href="#tab2">Advanced</a>
+                                </li>
                               </ul>
-                            </div>
-                    </div>
-                    % endif
-                    <a href="#" class="cancel-button">Cancel</a>
-                </div>
+                      % endif
+                          <div class="tab current" id="tab1">
+                            <ul class="new-component-template">
+                                % for name, location, has_markdown in templates:
+                                  % if has_markdown or type != "problem":
+                                    <li class="editor-md">
+                                      <a href="#" id="${location}" data-location="${location}">
+                                        <span class="name"> ${name}</span>
+                                      </a>
+                                    </li>
+                                  % endif
+  
+                                %endfor
+                            </ul>
+                          </div>
+                      % if type == "problem":
+                              <div class="tab" id="tab2">
+                                <ul class="new-component-template">
+                                    % for name, location, has_markdown in templates:
+                                      % if not has_markdown:
+                                        <li class="editor-manual">
+                                          <a href="#" id="${location}" data-location="${location}">
+                                            <span class="name"> ${name}</span>
+                                          </a>
+                                        </li>
+                                      % endif
+                                    % endfor
+                                </ul>
+                              </div>
+                      </div>
+                      % endif
+                      <a href="#" class="cancel-button">Cancel</a>
+                  </div>
+                  % endif
                 % endfor
             </li>
           </ol>

common/djangoapps/contentserver/middleware.py

-import logging
-import time
-
-from django.http import HttpResponse, Http404, HttpResponseNotModified
+from django.http import HttpResponse, HttpResponseNotModified
 
 from xmodule.contentstore.django import contentstore
 from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
@@ -20,7 +17,7 @@ class StaticContentServer(object):
                 # return a 'Bad Request' to browser as we have a malformed Location
                 response = HttpResponse()
                 response.status_code = 400
-                return response     
+                return response
 
             # first look in our cache so we don't have to round-trip to the DB
             content = get_cached_content(loc)

common/djangoapps/mitxmako/makoloader.py

@@ -44,7 +44,11 @@ class MakoLoader(object):
 
         if source.startswith("## mako\n"):
             # This is a mako template
-            template = Template(filename=file_path, module_directory=self.module_directory, uri=template_name)
+            template = Template(filename=file_path,
+                                module_directory=self.module_directory,
+                                input_encoding='utf-8',
+                                output_encoding='utf-8',
+                                uri=template_name)
             return template, None
         else:
             # This is a regular template

common/djangoapps/mitxmako/management/commands/preprocess_assets.py

+"""
+Preprocess templatized asset files, enabling asset authors to use
+Python/Django inside of Sass and CoffeeScript. This preprocessing
+will happen before the invocation of the asset compiler (currently
+handled by the asset Rakefile).
+
+For this to work, assets need to be named with the appropriate
+template extension (e.g., .mako for Mako templates). Currently Mako
+is the only template engine supported.
+"""
+import os
+
+from django.core.management.base import NoArgsCommand
+from django.conf import settings
+
+from mako.template import Template
+import textwrap
+
+class Command(NoArgsCommand):
+    """
+    Basic management command to preprocess asset template files.
+    """
+
+    help = "Preprocess asset template files to ready them for compilation."
+
+    def handle_noargs(self, **options):
+        """
+        Walk over all of the static files directories specified in the
+        settings file, looking for asset template files (indicated by
+        a file extension like .mako).
+        """
+        for staticfiles_dir in getattr(settings, "STATICFILES_DIRS", []):
+            # Cribbed from the django-staticfiles app at:
+            # https://github.com/jezdez/django-staticfiles/blob/develop/staticfiles/finders.py#L52
+            if isinstance(staticfiles_dir, (list, tuple)):
+                prefix, staticfiles_dir = staticfiles_dir
+
+            # Walk over the current static files directory tree,
+            # preprocessing files that have a template extension.
+            for root, dirs, files in os.walk(staticfiles_dir):
+                for filename in files:
+                    outfile, extension = os.path.splitext(filename)
+                    # We currently only handle Mako templates
+                    if extension == ".mako":
+                        self.__preprocess(os.path.join(root, filename),
+                                          os.path.join(root, outfile))
+
+
+    def __context(self):
+        """
+        Return a dict that contains all of the available context
+        variables to the asset template.
+        """
+        # TODO: do we need to include anything else?
+        # TODO: do this with the django-settings-context-processor
+        return { "THEME_NAME" : getattr(settings, "THEME_NAME", None) }
+
+
+    def __preprocess(self, infile, outfile):
+        """
+        Run `infile` through the Mako template engine, storing the
+        result in `outfile`.
+        """
+        with open(outfile, "w") as _outfile:
+            _outfile.write(textwrap.dedent("""\
+            /*
+             * This file is dynamically generated and ignored by Git.
+             * DO NOT MAKE CHANGES HERE. Instead, go edit its template:
+             * %s
+             */
+            """ % infile))
+            _outfile.write(Template(filename=str(infile)).render(env=self.__context()))
+

common/djangoapps/mitxmako/shortcuts.py

@@ -41,7 +41,9 @@ def marketing_link(name):
         return settings.MKTG_URLS.get('ROOT') + settings.MKTG_URLS.get(name)
     # only link to the old pages when the marketing site isn't on
     elif not settings.MITX_FEATURES.get('ENABLE_MKTG_SITE') and name in link_map:
-        return reverse(link_map[name])
+        # don't try to reverse disabled marketing links
+        if link_map[name] is not None:
+            return reverse(link_map[name])
     else:
         log.warning("Cannot find corresponding link for name: {name}".format(name=name))
         return '#'

common/djangoapps/mitxmako/tests.py

 from django.test import TestCase
 from django.test.utils import override_settings
 from django.core.urlresolvers import reverse
-from django.conf import settings
 from mitxmako.shortcuts import marketing_link
 from mock import patch
+from util.testing import UrlResetMixin
 
 
-class ShortcutsTests(TestCase):
+class ShortcutsTests(UrlResetMixin, TestCase):
     """
     Test the mitxmako shortcuts file
     """
-
     @override_settings(MKTG_URLS={'ROOT': 'dummy-root', 'ABOUT': '/about-us'})
     @override_settings(MKTG_URL_LINK_MAP={'ABOUT': 'login'})
     def test_marketing_link(self):

common/djangoapps/student/management/commands/assigngroups.py

@@ -14,6 +14,7 @@ import sys
 import datetime
 
 import json
+from pytz import UTC
 
 middleware.MakoMiddleware()
 
@@ -32,7 +33,7 @@ def group_from_value(groups, v):
 
 
 class Command(BaseCommand):
-    help =  \
+    help = \
 ''' Assign users to test groups. Takes a list
 of groups:
 a:0.3,b:0.4,c:0.3 file.txt "Testing something"
@@ -75,7 +76,7 @@ Will log what happened to file.txt.
             utg = UserTestGroup()
             utg.name = group
             utg.description = json.dumps({"description": args[2]},
-                                         {"time": datetime.datetime.utcnow().isoformat()})
+                                         {"time": datetime.datetime.now(UTC).isoformat()})
             group_objects[group] = utg
             group_objects[group].save()
 

common/djangoapps/student/management/commands/pearson_export_cdd.py

@@ -8,6 +8,7 @@ from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterUser
+from pytz import UTC
 
 
 class Command(BaseCommand):
@@ -58,7 +59,7 @@ class Command(BaseCommand):
     def handle(self, **options):
         # update time should use UTC in order to be comparable to the user_updated_at
         # field
-        uploaded_at = datetime.utcnow()
+        uploaded_at = datetime.now(UTC)
 
         # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
@@ -100,7 +101,7 @@ class Command(BaseCommand):
                                     extrasaction='ignore')
             writer.writeheader()
             for tcu in TestCenterUser.objects.order_by('id'):
-                if tcu.needs_uploading:   # or dump_all
+                if tcu.needs_uploading:  # or dump_all
                     record = dict((csv_field, ensure_encoding(getattr(tcu, model_field)))
                                   for csv_field, model_field
                                   in Command.CSV_TO_MODEL_FIELDS.items())

common/djangoapps/student/management/commands/pearson_export_ead.py

@@ -8,6 +8,7 @@ from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
 from student.models import TestCenterRegistration, ACCOMMODATION_REJECTED_CODE
+from pytz import UTC
 
 
 class Command(BaseCommand):
@@ -51,7 +52,7 @@ class Command(BaseCommand):
     def handle(self, **options):
         # update time should use UTC in order to be comparable to the user_updated_at
         # field
-        uploaded_at = datetime.utcnow()
+        uploaded_at = datetime.now(UTC)
 
         # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,

common/djangoapps/student/management/commands/pearson_import_conf_zip.py

@@ -13,6 +13,7 @@ from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
 
 from student.models import TestCenterUser, TestCenterRegistration
+from pytz import UTC
 
 
 class Command(BaseCommand):
@@ -68,7 +69,7 @@ class Command(BaseCommand):
                     Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile.name)
                     # now update the record:
                     registration.upload_status = row['Status']
-                    registration.upload_error_message =  row['Message']
+                    registration.upload_error_message = row['Message']
                     try:
                         registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
                     except ValueError as ve:
@@ -80,7 +81,7 @@ class Command(BaseCommand):
                         except ValueError as ve:
                             Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
 
-                    registration.confirmed_at = datetime.utcnow()
+                    registration.confirmed_at = datetime.now(UTC)
                     registration.save()
                 except TestCenterRegistration.DoesNotExist:
                     Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)

common/djangoapps/student/management/commands/pearson_make_tc_registration.py

 from optparse import make_option
-from time import strftime
 
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
@@ -128,8 +127,8 @@ class Command(BaseCommand):
             exam = CourseDescriptor.TestCenterExam(course_id, exam_name, exam_info)
             # update option values for date_first and date_last to use YYYY-MM-DD format
             # instead of YYYY-MM-DDTHH:MM
-            our_options['eligibility_appointment_date_first'] = strftime("%Y-%m-%d", exam.first_eligible_appointment_date)
-            our_options['eligibility_appointment_date_last'] = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
+            our_options['eligibility_appointment_date_first'] = exam.first_eligible_appointment_date.strftime("%Y-%m-%d")
+            our_options['eligibility_appointment_date_last'] = exam.last_eligible_appointment_date.strftime("%Y-%m-%d")
 
         if exam is None:
             raise CommandError("Exam for course_id {} does not exist".format(course_id))

common/djangoapps/student/models.py

@@ -16,7 +16,6 @@ import json
 import logging
 import uuid
 from random import randint
-from time import strftime
 
 
 from django.conf import settings
@@ -27,6 +26,7 @@ from django.dispatch import receiver
 from django.forms import ModelForm, forms
 
 import comment_client as cc
+from pytz import UTC
 
 
 log = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ class UserProfile(models.Model):
     class Meta:
         db_table = "auth_userprofile"
 
-    ## CRITICAL TODO/SECURITY
+    # CRITICAL TODO/SECURITY
     # Sanitize all fields.
     # This is not visible to other users, but could introduce holes later
     user = models.OneToOneField(User, unique=True, db_index=True, related_name='profile')
@@ -254,7 +254,7 @@ class TestCenterUserForm(ModelForm):
     def update_and_save(self):
         new_user = self.save(commit=False)
         # create additional values here:
-        new_user.user_updated_at = datetime.utcnow()
+        new_user.user_updated_at = datetime.now(UTC)
         new_user.upload_status = ''
         new_user.save()
         log.info("Updated demographic information for user's test center exam registration: username \"{}\" ".format(new_user.user.username))
@@ -429,8 +429,8 @@ class TestCenterRegistration(models.Model):
         registration.course_id = exam.course_id
         registration.accommodation_request = accommodation_request.strip()
         registration.exam_series_code = exam.exam_series_code
-        registration.eligibility_appointment_date_first = strftime("%Y-%m-%d", exam.first_eligible_appointment_date)
-        registration.eligibility_appointment_date_last = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
+        registration.eligibility_appointment_date_first = exam.first_eligible_appointment_date.strftime("%Y-%m-%d")
+        registration.eligibility_appointment_date_last = exam.last_eligible_appointment_date.strftime("%Y-%m-%d")
         registration.client_authorization_id = cls._create_client_authorization_id()
         # accommodation_code remains blank for now, along with Pearson confirmation information
         return registration
@@ -556,7 +556,7 @@ class TestCenterRegistrationForm(ModelForm):
     def update_and_save(self):
         registration = self.save(commit=False)
         # create additional values here:
-        registration.user_updated_at = datetime.utcnow()
+        registration.user_updated_at = datetime.now(UTC)
         registration.upload_status = ''
         registration.save()
         log.info("Updated registration information for user's test center exam registration: username \"{}\" course \"{}\", examcode \"{}\"".format(registration.testcenter_user.user.username, registration.course_id, registration.exam_series_code))
@@ -598,7 +598,7 @@ def unique_id_for_user(user):
     return h.hexdigest()
 
 
-## TODO: Should be renamed to generic UserGroup, and possibly
+# TODO: Should be renamed to generic UserGroup, and possibly
 # Given an optional field for type of group
 class UserTestGroup(models.Model):
     users = models.ManyToManyField(User, db_index=True)
@@ -626,7 +626,6 @@ class Registration(models.Model):
     def activate(self):
         self.user.is_active = True
         self.user.save()
-        #self.delete()
 
 
 class PendingNameChange(models.Model):
@@ -648,7 +647,7 @@ class CourseEnrollment(models.Model):
     created = models.DateTimeField(auto_now_add=True, null=True, db_index=True)
 
     class Meta:
-        unique_together = (('user', 'course_id'), )
+        unique_together = (('user', 'course_id'),)
 
     def __unicode__(self):
         return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
@@ -662,16 +661,17 @@ class CourseEnrollmentAllowed(models.Model):
     """
     email = models.CharField(max_length=255, db_index=True)
     course_id = models.CharField(max_length=255, db_index=True)
+    auto_enroll = models.BooleanField(default=0)
 
     created = models.DateTimeField(auto_now_add=True, null=True, db_index=True)
 
     class Meta:
-        unique_together = (('email', 'course_id'), )
+        unique_together = (('email', 'course_id'),)
 
     def __unicode__(self):
         return "[CourseEnrollmentAllowed] %s: %s (%s)" % (self.email, self.course_id, self.created)
 
-#cache_relation(User.profile)
+# cache_relation(User.profile)
 
 #### Helper methods for use from python manage.py shell and other classes.
 

common/djangoapps/student/tests/factories.py

@@ -5,6 +5,7 @@ from django.contrib.auth.models import Group
 from datetime import datetime
 from factory import DjangoModelFactory, SubFactory, PostGenerationMethodCall, post_generation, Sequence
 from uuid import uuid4
+from pytz import UTC
 
 # Factories don't have __init__ methods, and are self documenting
 # pylint: disable=W0232
@@ -46,8 +47,8 @@ class UserFactory(DjangoModelFactory):
     is_staff = False
     is_active = True
     is_superuser = False
-    last_login = datetime(2012, 1, 1)
-    date_joined = datetime(2011, 1, 1)
+    last_login = datetime(2012, 1, 1, tzinfo=UTC)
+    date_joined = datetime(2011, 1, 1, tzinfo=UTC)
 
     @post_generation
     def profile(obj, create, extracted, **kwargs):

common/djangoapps/student/views.py

@@ -32,7 +32,7 @@ from student.models import (Registration, UserProfile, TestCenterUser, TestCente
                             TestCenterRegistration, TestCenterRegistrationForm,
                             PendingNameChange, PendingEmailChange,
                             CourseEnrollment, unique_id_for_user,
-                            get_testcenter_registration)
+                            get_testcenter_registration, CourseEnrollmentAllowed)
 
 from certificates.models import CertificateStatuses, certificate_status_for_student
 
@@ -49,6 +49,7 @@ from courseware.views import get_module_for_descriptor, jump_to
 from courseware.model_data import ModelDataCache
 
 from statsd import statsd
+from pytz import UTC
 
 log = logging.getLogger("mitx.student")
 Article = namedtuple('Article', 'title url author image deck publication publish_date')
@@ -77,7 +78,7 @@ def index(request, extra_context={}, user=None):
     '''
 
     # The course selection work is done in courseware.courses.
-    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')     # normally False
+    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')  # normally False
     # do explicit check, because domain=None is valid
     if domain == False:
         domain = request.META.get('HTTP_HOST')
@@ -264,7 +265,6 @@ def dashboard(request):
     if not user.is_active:
         message = render_to_string('registration/activate_account_notice.html', {'email': user.email})
 
-
     # Global staff can see what courses errored on their dashboard
     staff_access = False
     errored_courses = {}
@@ -355,7 +355,7 @@ def change_enrollment(request):
             course = course_from_id(course_id)
         except ItemNotFoundError:
             log.warning("User {0} tried to enroll in non-existent course {1}"
-                      .format(user.username, course_id))
+                        .format(user.username, course_id))
             return HttpResponseBadRequest("Course id is invalid")
 
         if not has_access(user, course, 'enroll'):
@@ -363,9 +363,9 @@ def change_enrollment(request):
 
         org, course_num, run = course_id.split("/")
         statsd.increment("common.student.enrollment",
-                        tags=["org:{0}".format(org),
-                              "course:{0}".format(course_num),
-                              "run:{0}".format(run)])
+                         tags=["org:{0}".format(org),
+                               "course:{0}".format(course_num),
+                               "run:{0}".format(run)])
 
         try:
             enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
@@ -382,9 +382,9 @@ def change_enrollment(request):
 
             org, course_num, run = course_id.split("/")
             statsd.increment("common.student.unenrollment",
-                            tags=["org:{0}".format(org),
-                                  "course:{0}".format(course_num),
-                                  "run:{0}".format(run)])
+                             tags=["org:{0}".format(org),
+                                   "course:{0}".format(course_num),
+                                   "run:{0}".format(run)])
 
             return HttpResponse()
         except CourseEnrollment.DoesNotExist:
@@ -454,7 +454,6 @@ def login_user(request, error=""):
             expires_time = time.time() + max_age
             expires = cookie_date(expires_time)
 
-
         response.set_cookie(settings.EDXMKTG_COOKIE_NAME,
                             'true', max_age=max_age,
                             expires=expires, domain=settings.SESSION_COOKIE_DOMAIN,
@@ -515,8 +514,8 @@ def _do_create_account(post_vars):
     Note: this function is also used for creating test users.
     """
     user = User(username=post_vars['username'],
-             email=post_vars['email'],
-             is_active=False)
+                email=post_vars['email'],
+                is_active=False)
     user.set_password(post_vars['password'])
     registration = Registration()
     # TODO: Rearrange so that if part of the process fails, the whole process fails.
@@ -632,7 +631,7 @@ def create_account(request, post_override=None):
 
     # Ok, looks like everything is legit.  Create the account.
     ret = _do_create_account(post_vars)
-    if isinstance(ret, HttpResponse):		# if there was an error then return that
+    if isinstance(ret, HttpResponse):  # if there was an error then return that
         return ret
     (user, profile, registration) = ret
 
@@ -670,7 +669,7 @@ def create_account(request, post_override=None):
 
     if DoExternalAuth:
         eamap.user = login_user
-        eamap.dtsignup = datetime.datetime.now()
+        eamap.dtsignup = datetime.datetime.now(UTC)
         eamap.save()
         log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'], eamap))
 
@@ -698,7 +697,6 @@ def create_account(request, post_override=None):
         expires_time = time.time() + max_age
         expires = cookie_date(expires_time)
 
-
     response.set_cookie(settings.EDXMKTG_COOKIE_NAME,
                         'true', max_age=max_age,
                         expires=expires, domain=settings.SESSION_COOKIE_DOMAIN,
@@ -708,7 +706,6 @@ def create_account(request, post_override=None):
     return response
 
 
-
 def exam_registration_info(user, course):
     """ Returns a Registration object if the user is currently registered for a current
     exam of the course.  Returns None if the user is not registered, or if there is no
@@ -849,7 +846,6 @@ def create_exam_registration(request, post_override=None):
             response_data['non_field_errors'] = form.non_field_errors()
             return HttpResponse(json.dumps(response_data), mimetype="application/json")
 
-
     # only do the following if there is accommodation text to send,
     # and a destination to which to send it.
     # TODO: still need to create the accommodation email templates
@@ -872,7 +868,6 @@ def create_exam_registration(request, post_override=None):
 #            response_data['non_field_errors'] =  [ 'Could not send accommodation e-mail.', ]
 #            return HttpResponse(json.dumps(response_data), mimetype="application/json")
 
-
     js = {'success': True}
     return HttpResponse(json.dumps(js), mimetype="application/json")
 
@@ -916,6 +911,16 @@ def activate_account(request, key):
         if not r[0].user.is_active:
             r[0].activate()
             already_active = False
+
+        #Enroll student in any pending courses he/she may have if auto_enroll flag is set
+        student = User.objects.filter(id=r[0].user_id)
+        if student:
+            ceas = CourseEnrollmentAllowed.objects.filter(email=student[0].email)
+            for cea in ceas:
+                if cea.auto_enroll:
+                    course_id = cea.course_id
+                    enrollment, created = CourseEnrollment.objects.get_or_create(user_id=student[0].id, course_id=course_id)
+
         resp = render_to_response("registration/activation_complete.html", {'user_logged_in': user_logged_in, 'already_active': already_active})
         return resp
     if len(r) == 0:
@@ -1194,6 +1199,10 @@ def accept_name_change(request):
 def _get_news(top=None):
     "Return the n top news items on settings.RSS_URL"
 
+    # Don't return anything if we're in a themed site
+    if settings.MITX_FEATURES["USE_CUSTOM_THEME"]:
+        return None
+
     feed_data = cache.get("students_index_rss_feed_data")
     if feed_data is None:
         if hasattr(settings, 'RSS_URL'):

common/djangoapps/terrain/browser.py

@@ -87,8 +87,8 @@ def reset_data(scenario):
     LOGGER.debug("Flushing the test database...")
     call_command('flush', interactive=False)
 
-
-@after.each_scenario
+# Uncomment below to trigger a screenshot on error
+# @after.each_scenario
 def screenshot_on_error(scenario):
     """
     Save a screenshot to help with debugging.

common/djangoapps/terrain/steps.py

@@ -129,9 +129,12 @@ def should_have_link_with_id_and_text(step, link_id, text):
     assert_equals(link.text, text)
 
 
-@step(r'should see "(.*)" (?:somewhere|anywhere) in (?:the|this) page')
-def should_see_in_the_page(step, text):
-    assert_in(text, world.css_text('body'))
+@step(r'should( not)? see "(.*)" (?:somewhere|anywhere) (?:in|on) (?:the|this) page')
+def should_see_in_the_page(step, doesnt_appear, text):
+    if doesnt_appear:
+        assert world.browser.is_text_not_present(text, wait_time=5)
+    else:
+        assert world.browser.is_text_present(text, wait_time=5)
 
 
 @step('I am logged in$')
@@ -156,3 +159,33 @@ def registered_edx_user(step, uname):
 @step(u'All dialogs should be closed$')
 def dialogs_are_closed(step):
     assert world.dialogs_closed()
+
+
+@step('I will confirm all alerts')
+def i_confirm_all_alerts(step):
+    """
+    Please note: This method must be called RIGHT BEFORE an expected alert
+    Window variables are page local and thus all changes are removed upon navigating to a new page
+    In addition, this method changes the functionality of ONLY future alerts
+    """
+    world.browser.execute_script('window.confirm = function(){return true;} ; window.alert = function(){return;}')
+
+
+@step('I will cancel all alerts')
+def i_cancel_all_alerts(step):
+    """
+    Please note: This method must be called RIGHT BEFORE an expected alert
+    Window variables are page local and thus all changes are removed upon navigating to a new page
+    In addition, this method changes the functionality of ONLY future alerts
+    """
+    world.browser.execute_script('window.confirm = function(){return false;} ; window.alert = function(){return;}')
+
+
+@step('I will answer all prompts with "([^"]*)"')
+def i_answer_prompts_with(step, prompt):
+    """
+    Please note: This method must be called RIGHT BEFORE an expected alert
+    Window variables are page local and thus all changes are removed upon navigating to a new page
+    In addition, this method changes the functionality of ONLY future alerts
+    """
+    world.browser.execute_script('window.prompt = function(){return %s;}') % prompt

common/djangoapps/terrain/ui_helpers.py

@@ -32,8 +32,13 @@ def url_equals(url):
 
 
 @world.absorb
-def is_css_present(css_selector):
-    return world.browser.is_element_present_by_css(css_selector, wait_time=4)
+def is_css_present(css_selector, wait_time=5):
+    return world.browser.is_element_present_by_css(css_selector, wait_time=wait_time)
+
+
+@world.absorb
+def is_css_not_present(css_selector, wait_time=5):
+    return world.browser.is_element_not_present_by_css(css_selector, wait_time=wait_time)
 
 
 @world.absorb
@@ -42,11 +47,11 @@ def css_has_text(css_selector, text):
 
 
 @world.absorb
-def css_find(css):
+def css_find(css, wait_time=5):
     def is_visible(driver):
         return EC.visibility_of_element_located((By.CSS_SELECTOR, css,))
 
-    world.browser.is_element_present_by_css(css, 5)
+    world.browser.is_element_present_by_css(css, wait_time=wait_time)
     wait_for(is_visible)
     return world.browser.find_by_css(css)
 
@@ -56,6 +61,7 @@ def css_click(css_selector):
     """
     Perform a click on a CSS selector, retrying if it initially fails
     """
+    assert is_css_present(css_selector)
     try:
         world.browser.find_by_css(css_selector).click()
 
@@ -89,6 +95,7 @@ def id_click(elem_id):
 
 @world.absorb
 def css_fill(css_selector, text):
+    assert is_css_present(css_selector)
     world.browser.find_by_css(css_selector).first.fill(text)
 
 
@@ -114,6 +121,7 @@ def css_text(css_selector):
 
 @world.absorb
 def css_visible(css_selector):
+    assert is_css_present(css_selector)
     return world.browser.find_by_css(css_selector).visible
 
 

common/djangoapps/tests.py

+'''
+Created on Jun 6, 2013
+
+@author: dmitchell
+'''
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from student.tests.factories import AdminFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+import xmodule_modifiers
+import datetime
+from pytz import UTC
+from xmodule.modulestore.tests import factories
+
+class TestXmoduleModfiers(ModuleStoreTestCase):
+
+    # FIXME disabled b/c start date inheritance is not occuring and render_... in get_html is failing due
+    # to middleware.lookup['main'] not being defined
+    def _test_add_histogram(self):
+        instructor = AdminFactory.create()
+        self.client.login(username=instructor.username, password='test')
+
+        course = CourseFactory.create(org='test',
+            number='313', display_name='histogram test')
+        section = ItemFactory.create(
+            parent_location=course.location, display_name='chapter hist',
+            template='i4x://edx/templates/chapter/Empty')
+        problem = ItemFactory.create(
+            parent_location=section.location, display_name='problem hist 1',
+            template='i4x://edx/templates/problem/Blank_Common_Problem')
+        problem.has_score = False  # don't trip trying to retrieve db data
+
+        late_problem = ItemFactory.create(
+            parent_location=section.location, display_name='problem hist 2',
+            template='i4x://edx/templates/problem/Blank_Common_Problem')
+        late_problem.lms.start = datetime.datetime.now(UTC) + datetime.timedelta(days=32)
+        late_problem.has_score = False
+
+
+        problem_module = factories.get_test_xmodule_for_descriptor(problem)
+        problem_module.get_html = xmodule_modifiers.add_histogram(lambda:'', problem_module, instructor)
+
+        self.assertRegexpMatches(
+            problem_module.get_html(), r'.*<font color=\'green\'>Not yet</font>.*')
+
+        problem_module = factories.get_test_xmodule_for_descriptor(late_problem)
+        problem_module.get_html = xmodule_modifiers.add_histogram(lambda: '', problem_module, instructor)
+
+        self.assertRegexpMatches(
+            problem_module.get_html(), r'.*<font color=\'red\'>Yes!</font>.*')

common/djangoapps/track/views.py

@@ -14,6 +14,7 @@ from mitxmako.shortcuts import render_to_response
 
 from django_future.csrf import ensure_csrf_cookie
 from track.models import TrackingLog
+from pytz import UTC
 
 log = logging.getLogger("tracking")
 
@@ -59,7 +60,7 @@ def user_track(request):
         "event": request.GET['event'],
         "agent": agent,
         "page": request.GET['page'],
-        "time": datetime.datetime.utcnow().isoformat(),
+        "time": datetime.datetime.now(UTC).isoformat(),
         "host": request.META['SERVER_NAME'],
         }
     log_event(event)
@@ -85,11 +86,11 @@ def server_track(request, event_type, event, page=None):
         "event": event,
         "agent": agent,
         "page": page,
-        "time": datetime.datetime.utcnow().isoformat(),
+        "time": datetime.datetime.now(UTC).isoformat(),
         "host": request.META['SERVER_NAME'],
         }
 
-    if event_type.startswith("/event_logs") and request.user.is_staff:  	# don't log
+    if event_type.startswith("/event_logs") and request.user.is_staff:  # don't log
         return
     log_event(event)
 

common/djangoapps/util/testing.py

+import sys
+
+from django.conf import settings
+from django.core.urlresolvers import clear_url_caches
+
+
+class UrlResetMixin(object):
+    """Mixin to reset urls.py before and after a test
+
+    Django memoizes the function that reads the urls module (whatever module
+    urlconf names). The module itself is also stored by python in sys.modules.
+    To fully reload it, we need to reload the python module, and also clear django's
+    cache of the parsed urls.
+
+    However, the order in which we do this doesn't matter, because neither one will
+    get reloaded until the next request
+
+    Doing this is expensive, so it should only be added to tests that modify settings
+    that affect the contents of urls.py
+    """
+
+    def _reset_urls(self, urlconf=None):
+        if urlconf is None:
+            urlconf = settings.ROOT_URLCONF
+
+        if urlconf in sys.modules:
+            reload(sys.modules[urlconf])
+        clear_url_caches()
+
+    def setUp(self):
+        """Reset django default urlconf before tests and after tests"""
+        super(UrlResetMixin, self).setUp()
+        self._reset_urls()
+        self.addCleanup(self._reset_urls)

common/djangoapps/util/views.py

@@ -12,6 +12,7 @@ from django.core.validators import ValidationError, validate_email
 from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed, HttpResponseServerError
 from django.shortcuts import redirect
 from django_future.csrf import ensure_csrf_cookie
+from dogapi import dog_stats_api
 from mitxmako.shortcuts import render_to_response, render_to_string
 from urllib import urlencode
 import zendesk
@@ -73,11 +74,64 @@ class _ZendeskApi(object):
         self._zendesk_instance.update_ticket(ticket_id=ticket_id, data=update)
 
 
-def submit_feedback_via_zendesk(request):
+def _record_feedback_in_zendesk(realname, email, subject, details, tags, additional_info):
     """
     Create a new user-requested Zendesk ticket.
 
-    If Zendesk submission is not enabled, any request will raise `Http404`.
+    Once created, the ticket will be updated with a private comment containing
+    additional information from the browser and server, such as HTTP headers
+    and user state. Returns a boolean value indicating whether ticket creation
+    was successful, regardless of whether the private comment update succeeded.
+    """
+    zendesk_api = _ZendeskApi()
+
+    additional_info_string = (
+        "Additional information:\n\n" +
+        "\n".join("%s: %s" % (key, value) for (key, value) in additional_info.items() if value is not None)
+    )
+
+    # Tag all issues with LMS to distinguish channel in Zendesk; requested by student support team
+    zendesk_tags = list(tags.values()) + ["LMS"]
+    new_ticket = {
+        "ticket": {
+            "requester": {"name": realname, "email": email},
+            "subject": subject,
+            "comment": {"body": details},
+            "tags": zendesk_tags
+        }
+    }
+    try:
+        ticket_id = zendesk_api.create_ticket(new_ticket)
+    except zendesk.ZendeskError as err:
+        log.error("Error creating Zendesk ticket: %s", str(err))
+        return False
+
+    # Additional information is provided as a private update so the information
+    # is not visible to the user.
+    ticket_update = {"ticket": {"comment": {"public": False, "body": additional_info_string}}}
+    try:
+        zendesk_api.update_ticket(ticket_id, ticket_update)
+    except zendesk.ZendeskError as err:
+        log.error("Error updating Zendesk ticket: %s", str(err))
+        # The update is not strictly necessary, so do not indicate failure to the user
+        pass
+
+    return True
+
+
+DATADOG_FEEDBACK_METRIC = "lms_feedback_submissions"
+
+
+def _record_feedback_in_datadog(tags):
+    datadog_tags = ["{k}:{v}".format(k=k, v=v) for k, v in tags.items()]
+    dog_stats_api.increment(DATADOG_FEEDBACK_METRIC, tags=datadog_tags)
+
+
+def submit_feedback(request):
+    """
+    Create a new user-requested ticket, currently implemented with Zendesk.
+
+    If feedback submission is not enabled, any request will raise `Http404`.
     If any configuration parameter (`ZENDESK_URL`, `ZENDESK_USER`, or
     `ZENDESK_API_KEY`) is missing, any request will raise an `Exception`.
     The request must be a POST request specifying `subject` and `details`.
@@ -85,12 +139,9 @@ def submit_feedback_via_zendesk(request):
     `email`. If the user is authenticated, the `name` and `email` will be
     populated from the user's information. If any required parameter is
     missing, a 400 error will be returned indicating which field is missing and
-    providing an error message. If Zendesk returns any error on ticket
-    creation, a 500 error will be returned with no body. Once created, the
-    ticket will be updated with a private comment containing additional
-    information from the browser and server, such as HTTP headers and user
-    state. Whether or not the update succeeds, if the user's ticket is
-    successfully created, an empty successful response (200) will be returned.
+    providing an error message. If Zendesk ticket creation fails, 500 error
+    will be returned with no body; if ticket creation succeeds, an empty
+    successful response (200) will be returned.
     """
     if not settings.MITX_FEATURES.get('ENABLE_FEEDBACK_SUBMISSION', False):
         raise Http404()
@@ -124,9 +175,9 @@ def submit_feedback_via_zendesk(request):
 
     subject = request.POST["subject"]
     details = request.POST["details"]
-    tags = []
-    if "tag" in request.POST:
-        tags = [request.POST["tag"]]
+    tags = dict(
+        [(tag, request.POST[tag]) for tag in ["issue_type", "course_id"] if tag in request.POST]
+    )
 
     if request.user.is_authenticated():
         realname = request.user.profile.name
@@ -140,41 +191,18 @@ def submit_feedback_via_zendesk(request):
         except ValidationError:
             return build_error_response(400, "email", required_field_errs["email"])
 
-    for header in ["HTTP_REFERER", "HTTP_USER_AGENT"]:
-        additional_info[header] = request.META.get(header)
+    for header, pretty in [
+        ("HTTP_REFERER", "Page"),
+        ("HTTP_USER_AGENT", "Browser"),
+        ("REMOTE_ADDR", "Client IP"),
+        ("SERVER_NAME", "Host")
+    ]:
+        additional_info[pretty] = request.META.get(header)
 
-    zendesk_api = _ZendeskApi()
-
-    additional_info_string = (
-        "Additional information:\n\n" +
-        "\n".join("%s: %s" % (key, value) for (key, value) in additional_info.items() if value is not None)
-    )
-
-    new_ticket = {
-        "ticket": {
-            "requester": {"name": realname, "email": email},
-            "subject": subject,
-            "comment": {"body": details},
-            "tags": tags
-        }
-    }
-    try:
-        ticket_id = zendesk_api.create_ticket(new_ticket)
-    except zendesk.ZendeskError as err:
-        log.error("Error creating Zendesk ticket: %s", str(err))
-        return HttpResponse(status=500)
-
-    # Additional information is provided as a private update so the information
-    # is not visible to the user.
-    ticket_update = {"ticket": {"comment": {"public": False, "body": additional_info_string}}}
-    try:
-        zendesk_api.update_ticket(ticket_id, ticket_update)
-    except zendesk.ZendeskError as err:
-        log.error("Error updating Zendesk ticket: %s", str(err))
-        # The update is not strictly necessary, so do not indicate failure to the user
-        pass
+    success = _record_feedback_in_zendesk(realname, email, subject, details, tags, additional_info)
+    _record_feedback_in_datadog(tags)
 
-    return HttpResponse()
+    return HttpResponse(status=(200 if success else 500))
 
 
 def info(request):

common/djangoapps/xmodule_modifiers.py

 import re
 import json
 import logging
-import time
 import static_replace
 
 from django.conf import settings
@@ -9,6 +8,8 @@ from functools import wraps
 from mitxmako.shortcuts import render_to_string
 from xmodule.seq_module import SequenceModule
 from xmodule.vertical_module import VerticalModule
+import datetime
+from django.utils.timezone import UTC
 
 log = logging.getLogger("mitx.xmodule_modifiers")
 
@@ -83,7 +84,7 @@ def grade_histogram(module_id):
     cursor.execute(q, [module_id])
 
     grades = list(cursor.fetchall())
-    grades.sort(key=lambda x: x[0])          # Add ORDER BY to sql query?
+    grades.sort(key=lambda x: x[0])  # Add ORDER BY to sql query?
     if len(grades) >= 1 and grades[0][0] is None:
         return []
     return grades
@@ -101,7 +102,7 @@ def add_histogram(get_html, module, user):
     @wraps(get_html)
     def _get_html():
 
-        if type(module) in [SequenceModule, VerticalModule]:  	# TODO: make this more general, eg use an XModule attribute instead
+        if type(module) in [SequenceModule, VerticalModule]:  # TODO: make this more general, eg use an XModule attribute instead
             return get_html()
 
         module_id = module.id
@@ -132,7 +133,7 @@ def add_histogram(get_html, module, user):
 
         # useful to indicate to staff if problem has been released or not
         # TODO (ichuang): use _has_access_descriptor.can_load in lms.courseware.access, instead of now>mstart comparison here
-        now = time.gmtime()
+        now = datetime.datetime.now(UTC())
         is_released = "unknown"
         mstart = module.descriptor.lms.start
 

common/lib/capa/capa/capa_problem.py

@@ -470,6 +470,7 @@ class LoncapaProblem(object):
                     python_path=python_path,
                     cache=self.system.cache,
                     slug=self.problem_id,
+                    unsafely=self.system.can_execute_unsafe_code(),
                 )
             except Exception as err:
                 log.exception("Error while execing script code: " + all_code)

common/lib/capa/capa/inputtypes.py

@@ -144,11 +144,11 @@ class InputTypeBase(object):
         self.tag = xml.tag
         self.system = system
 
-        ## NOTE: ID should only come from one place.  If it comes from multiple,
-        ## we use state first, XML second (in case the xml changed, but we have
-        ## existing state with an old id). Since we don't make this guarantee,
-        ## we can swap this around in the future if there's a more logical
-        ## order.
+        # NOTE: ID should only come from one place.  If it comes from multiple,
+        # we use state first, XML second (in case the xml changed, but we have
+        # existing state with an old id). Since we don't make this guarantee,
+        # we can swap this around in the future if there's a more logical
+        # order.
 
         self.input_id = state.get('id', xml.get('id'))
         if self.input_id is None:
@@ -769,7 +769,7 @@ class MatlabInput(CodeInput):
 
         # construct xqueue headers
         qinterface = self.system.xqueue['interface']
-        qtime = datetime.strftime(datetime.utcnow(), xqueue_interface.dateformat)
+        qtime = datetime.utcnow().strftime(xqueue_interface.dateformat)
         callback_url = self.system.xqueue['construct_callback']('ungraded_response')
         anonymous_student_id = self.system.anonymous_student_id
         queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +

common/lib/capa/capa/responsetypes.py

@@ -288,7 +288,14 @@ class LoncapaResponse(object):
             }
 
             try:
-                safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path'], slug=self.id)
+                safe_exec.safe_exec(
+                    code,
+                    globals_dict,
+                    python_path=self.context['python_path'],
+                    slug=self.id,
+                    random_seed=self.context['seed'],
+                    unsafely=self.system.can_execute_unsafe_code(),
+                )
             except Exception as err:
                 msg = 'Error %s in evaluating hint function %s' % (err, hintfn)
                 msg += "\nSee XML source line %s" % getattr(
@@ -973,7 +980,14 @@ class CustomResponse(LoncapaResponse):
                             'ans': ans,
                         }
                         globals_dict.update(kwargs)
-                        safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path'], slug=self.id)
+                        safe_exec.safe_exec(
+                            code,
+                            globals_dict,
+                            python_path=self.context['python_path'],
+                            slug=self.id,
+                            random_seed=self.context['seed'],
+                            unsafely=self.system.can_execute_unsafe_code(),
+                        )
                         return globals_dict['cfn_return']
                     return check_function
 
@@ -1090,7 +1104,14 @@ class CustomResponse(LoncapaResponse):
         # exec the check function
         if isinstance(self.code, basestring):
             try:
-                safe_exec.safe_exec(self.code, self.context, cache=self.system.cache, slug=self.id)
+                safe_exec.safe_exec(
+                    self.code,
+                    self.context,
+                    cache=self.system.cache,
+                    slug=self.id,
+                    random_seed=self.context['seed'],
+                    unsafely=self.system.can_execute_unsafe_code(),
+                )
             except Exception as err:
                 self._handle_exec_exception(err)
 
@@ -1815,7 +1836,14 @@ class SchematicResponse(LoncapaResponse):
         ]
         self.context.update({'submission': submission})
         try:
-            safe_exec.safe_exec(self.code, self.context, cache=self.system.cache, slug=self.id)
+            safe_exec.safe_exec(
+                self.code,
+                self.context,
+                cache=self.system.cache,
+                slug=self.id,
+                random_seed=self.context['seed'],
+                unsafely=self.system.can_execute_unsafe_code(),
+            )
         except Exception as err:
             msg = 'Error %s in evaluating SchematicResponse' % err
             raise ResponseError(msg)

common/lib/capa/capa/safe_exec/safe_exec.py

 """Capa's specialized use of codejail.safe_exec."""
 
 from codejail.safe_exec import safe_exec as codejail_safe_exec
+from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
 from codejail.safe_exec import json_safe, SafeExecException
 from . import lazymod
 from statsd import statsd
@@ -71,7 +72,7 @@ def update_hash(hasher, obj):
 
 
 @statsd.timed('capa.safe_exec.time')
-def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None):
+def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False):
     """
     Execute python code safely.
 
@@ -90,6 +91,8 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None
     `slug` is an arbitrary string, a description that's meaningful to the
     caller, that will be used in log messages.
 
+    If `unsafely` is true, then the code will actually be executed without sandboxing.
+
     """
     # Check the cache for a previous result.
     if cache:
@@ -111,9 +114,15 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None
     # Create the complete code we'll run.
     code_prolog = CODE_PROLOG % random_seed
 
+    # Decide which code executor to use.
+    if unsafely:
+        exec_fn = codejail_not_safe_exec
+    else:
+        exec_fn = codejail_safe_exec
+
     # Run the code!  Results are side effects in globals_dict.
     try:
-        codejail_safe_exec(
+        exec_fn(
             code_prolog + LAZY_IMPORTS + code, globals_dict,
             python_path=python_path, slug=slug,
         )

common/lib/capa/capa/safe_exec/tests/test_safe_exec.py

 """Test safe_exec.py"""
 
 import hashlib
+import os
 import os.path
 import random
 import textwrap
 import unittest
 
+from nose.plugins.skip import SkipTest
+
 from capa.safe_exec import safe_exec, update_hash
 from codejail.safe_exec import SafeExecException
+from codejail.jail_code import is_configured
 
 
 class TestSafeExec(unittest.TestCase):
@@ -68,6 +72,24 @@ class TestSafeExec(unittest.TestCase):
         self.assertIn("ZeroDivisionError", cm.exception.message)
 
 
+class TestSafeOrNot(unittest.TestCase):
+    def test_cant_do_something_forbidden(self):
+        # Can't test for forbiddenness if CodeJail isn't configured for python.
+        if not is_configured("python"):
+            raise SkipTest
+
+        g = {}
+        with self.assertRaises(SafeExecException) as cm:
+            safe_exec("import os; files = os.listdir('/')", g)
+        self.assertIn("OSError", cm.exception.message)
+        self.assertIn("Permission denied", cm.exception.message)
+
+    def test_can_do_something_forbidden_if_run_unsafely(self):
+        g = {}
+        safe_exec("import os; files = os.listdir('/')", g, unsafely=True)
+        self.assertEqual(g['files'], os.listdir('/'))
+
+
 class DictCache(object):
     """A cache implementation over a simple dict, for testing."""
 

common/lib/capa/capa/templates/annotationinput.html

@@ -14,7 +14,7 @@
     <div class="block block-comment">${comment}</div>
 
     <div class="block">${comment_prompt}</div>
-    <textarea class="comment" id="input_${id}_comment" name="input_${id}_comment">${comment_value|h}</textarea>
+    <textarea class="comment" id="input_${id}_comment" name="input_${id}_comment" aria-describedby="answer_${id}">${comment_value|h}</textarea>
 
     <div class="block">${tag_prompt}</div>
     <ul class="tags">
@@ -22,11 +22,11 @@
         <li>
             % if has_options_value:
                 % if all([c == 'correct' for c in option['choice'], status]):
-                <span class="tag-status correct" id="status_${id}"></span>
+                <span class="tag-status correct" id="status_${id}" aria-describedby="input_${id}_comment"><span class="sr">Status: Correct</span></span>
                 % elif all([c == 'partially-correct' for c in option['choice'], status]):
-                <span class="tag-status partially-correct" id="status_${id}"></span>
+                <span class="tag-status partially-correct" id="status_${id}" aria-describedby="input_${id}_comment"><span class="sr">Status: Partially Correct</span></span>
                 % elif all([c == 'incorrect' for c in option['choice'], status]):
-                <span class="tag-status incorrect" id="status_${id}"></span>
+                <span class="tag-status incorrect" id="status_${id}" aria-describedby="input_${id}_comment"><span class="sr">Status: Incorrect</span></span>
                 % endif
             % endif
 
@@ -53,11 +53,11 @@
     % endif
 
     % if status == 'unsubmitted':
-    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+    <span class="unanswered" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: Unanswered</span></span>
     % elif status == 'incomplete':
-    <span class="incorrect" id="status_${id}"></span>
+    <span class="incorrect" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: Incorrect</span></span>
     % elif status == 'incorrect' and not has_options_value:
-    <span class="incorrect" id="status_${id}"></span>
+    <span class="incorrect" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: Incorrect</span></span>
     % endif
 
     <p id="answer_${id}" class="answer answer-annotation"></p>

common/lib/capa/capa/templates/chemicalequationinput.html

@@ -11,13 +11,13 @@
     <div class="incorrect" id="status_${id}">
     % endif
 
-    <input type="text" name="input_${id}" id="input_${id}" data-input-id="${id}" value="${value|h}"
+    <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" data-input-id="${id}" value="${value|h}"
    % if size:
    size="${size}"
    % endif
    />
 
-      <p class="status">
+      <p class="status" aria-describedby="input_${id}">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':

common/lib/capa/capa/templates/choicegroup.html

@@ -3,12 +3,12 @@
     % if input_type == 'checkbox' or not value:
         % if status == 'unsubmitted' or show_correctness == 'never':
         <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-        % elif status == 'correct': 
-        <span class="correct" id="status_${id}"></span>
+        % elif status == 'correct':
+        <span class="correct" id="status_${id}"><span class="sr">Status: correct</span></span>
         % elif status == 'incorrect':
-        <span class="incorrect" id="status_${id}"></span>
+        <span class="incorrect" id="status_${id}"><span class="sr">Status: incorrect</span></span>
         % elif status == 'incomplete':
-        <span class="incorrect" id="status_${id}"></span>
+        <span class="incorrect" id="status_${id}"><span class="sr">Status: incomplete</span></span>
         % endif
     % endif
     </div>
@@ -18,7 +18,7 @@
         % for choice_id, choice_description in choices:
         <label for="input_${id}_${choice_id}"
             % if input_type == 'radio' and ( (isinstance(value, basestring) and (choice_id == value))  or  (not isinstance(value, basestring) and choice_id in value) ):
-            <% 
+            <%
                 if status == 'correct':
                     correctness = 'correct'
                 elif status == 'incorrect':
@@ -31,14 +31,29 @@
             % endif
             % endif
             >
-            <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" value="${choice_id}"
+            <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" aria-describedby="answer_${id}" value="${choice_id}"
             % if input_type == 'radio' and ( (isinstance(value, basestring) and (choice_id == value))  or  (not isinstance(value, basestring) and choice_id in value) ):
-            checked="true" 
+            checked="true"
             % elif input_type != 'radio' and choice_id in value:
             checked="true"
             % endif
 
-            /> ${choice_description} </label>
+            /> ${choice_description}
+
+            % if input_type == 'radio' and ( (isinstance(value, basestring) and (choice_id == value))  or  (not isinstance(value, basestring) and choice_id in value) ):
+            <%
+                if status == 'correct':
+                    correctness = 'correct'
+                elif status == 'incorrect':
+                    correctness = 'incorrect'
+                else:
+                    correctness = None
+            %>
+            % if correctness and not show_correctness=='never':
+            <span class="sr" aria-describedby="input_${id}_${choice_id}">Status: ${correctness}</span>
+            % endif
+            % endif
+        </label>
         % endfor
         <span id="answer_${id}"></span>
     </fieldset>

common/lib/capa/capa/templates/codeinput.html

 <section id="textbox_${id}" class="textbox">
-  <textarea rows="${rows}" cols="${cols}" name="input_${id}" id="input_${id}"
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}"
        % if hidden:
       	    style="display:none;"
        % endif
@@ -7,13 +7,13 @@
 
   <div class="grader-status">
     % if status == 'unsubmitted':
-      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+      <span class="unanswered" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Unanswered</span>
     % elif status == 'correct':
-      <span class="correct" id="status_${id}">Correct</span>
+      <span class="correct" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Correct</span>
     % elif status == 'incorrect':
-      <span class="incorrect" id="status_${id}">Incorrect</span>
+      <span class="incorrect" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Incorrect</span>
     % elif status == 'queued':
-      <span class="processing" id="status_${id}">Queued</span>
+      <span class="processing" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Queued</span>
       <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
 

common/lib/capa/capa/templates/crystallography.html

@@ -20,9 +20,9 @@
     % endif
 
 
-    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}" style="display:none;"/>
+    <input type="text" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}" value="${value|h}" style="display:none;"/>
 
-    <p class="status">
+    <p class="status" aria-describedby="input_${id}">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':

common/lib/capa/capa/templates/designprotein2dinput.html

@@ -11,12 +11,12 @@
     % elif status == 'incomplete':
     <div class="incomplete" id="status_${id}">
     % endif
-    
+
     <div id="protex_container"></div>
     <input type="hidden" name="target_shape" id="target_shape" value ="${target_shape}"></input>
-    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+    <input type="hidden" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"/>
 
-    <p class="status">
+    <p class="status" aria-describedby="input_${id}">
       % if status == 'unsubmitted':
       unanswered
       % elif status == 'correct':

common/lib/capa/capa/templates/drag_and_drop_input.html

@@ -19,10 +19,10 @@
     % endif
 
 
-    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+    <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"
     style="display:none;"/>
 
-    <p class="status">
+    <p class="status" aria-describedby="input_${id}">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':

common/lib/capa/capa/templates/editageneinput.html

@@ -11,13 +11,13 @@
     % elif status == 'incomplete':
     <div class="incomplete" id="status_${id}">
     % endif
-    
+
     <div id="genex_container"></div>
     <input type="hidden" name="genex_dna_sequence" id="genex_dna_sequence" value ="${genex_dna_sequence}"></input>
     <input type="hidden" name="genex_problem_number" id="genex_problem_number" value ="${genex_problem_number}"></input>
-    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+    <input type="hidden" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}" value="${value|h}"/>
 
-    <p class="status">
+    <p class="status" aria-describedby="input_${id}">
       % if status == 'unsubmitted':
       unanswered
       % elif status == 'correct':

common/lib/capa/capa/templates/editamolecule.html

@@ -16,13 +16,13 @@
 
     <br/>
 
-    <input type="hidden" name="input_${id}" id="input_${id}" value="${value|h}"/>
+    <input type="hidden" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"/>
 
     <button id="reset_${id}" class="reset">Reset</button>
 
     <p id="answer_${id}" class="answer"></p>
 
-    <p class="status">
+    <p class="status" aria-describedby="input_${id}">
       % if status == 'unsubmitted':
       unanswered
       % elif status == 'correct':

common/lib/capa/capa/templates/imageinput.html

@@ -5,12 +5,20 @@
   </div>
 
   % if status == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-  % elif status == 'correct': 
-  <span class="correct" id="status_${id}"></span>
+  <span class="unanswered" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: unanswered</span>
+  </span>
+  % elif status == 'correct':
+  <span class="correct" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: correct</span>
+  </span>
   % elif status == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % elif status == 'incomplete':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % endif
 </span>

common/lib/capa/capa/templates/jstextline.html

@@ -19,13 +19,21 @@
   % endif
 
   % if status == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+  <span class="unanswered" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: unanswered</span>
+  </span>
   % elif status == 'correct':
-  <span class="correct" id="status_${id}"></span>
+  <span class="correct" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: correct</span>
+  </span>
   % elif status == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % elif status == 'incomplete':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % endif
   % if msg:
     <br/>

common/lib/capa/capa/templates/matlabinput.html

 <section id="textbox_${id}" class="textbox">
-  <textarea rows="${rows}" cols="${cols}" name="input_${id}" id="input_${id}"
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}"
        % if hidden:
       	    style="display:none;"
        % endif
@@ -7,13 +7,13 @@
 
   <div class="grader-status">
     % if status == 'unsubmitted':
-      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+      <span class="unanswered" style="display:inline-block;" id="status_${id}"><span class="sr">Status: </span>Unanswered</span>
     % elif status == 'correct':
-      <span class="correct" id="status_${id}">Correct</span>
+      <span class="correct" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Correct</span>
     % elif status == 'incorrect':
-      <span class="incorrect" id="status_${id}">Incorrect</span>
+      <span class="incorrect" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Incorrect</span>
     % elif status == 'queued':
-      <span class="processing" id="status_${id}">Queued</span>
+      <span class="processing" id="status_${id}" aria-describedby="input_${id}"><span class="sr">Status: </span>Queued</span>
       <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
 
@@ -71,7 +71,7 @@
           $(parent_elt).find('.action').after(alert_elem);
           $(parent_elt).find('.capa_alert').css({opacity: 0}).animate({opacity: 1}, 700);
       }
-      
+
 
       // hook up the plot button
       var plot = function(event) {
@@ -97,10 +97,10 @@
               }
           }
 
-          var save_callback = function(response) { 
+          var save_callback = function(response) {
                   if(response.success) {
                       // send information to the problem's plot functionality
-                      Problem.inputAjax(url, input_id, 'plot', 
+                      Problem.inputAjax(url, input_id, 'plot',
                         {'submission': submission}, plot_callback);
                   }
                   else {

common/lib/capa/capa/templates/optioninput.html

 <form class="option-input">
-   <select name="input_${id}" id="input_${id}" >
+   <select name="input_${id}" id="input_${id}" aria-describedby="answer_${id}">
       <option value="option_${id}_dummy_default">  </option>
       % for option_id, option_description in options:
           <option value="${option_id}"
@@ -13,12 +13,20 @@
   <span id="answer_${id}"></span>
 
   % if status == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-  % elif status == 'correct': 
-  <span class="correct" id="status_${id}"></span>
+  <span class="unanswered" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: unsubmitted</span>
+  </span>
+  % elif status == 'correct':
+  <span class="correct" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: correct</span>
+  </span>
   % elif status == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % elif status == 'incomplete':
-  <span class="incorrect" id="status_${id}"></span>
+  <span class="incorrect" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incomplete</span>
+  </span>
   % endif
 </form>

common/lib/capa/capa/templates/schematicinput.html

 <span>
-  <input type="hidden" class="schematic" height="${height}" width="${width}" parts="${parts}" analyses="${analyses}" name="input_${id}" id="input_${id}" value="" initial_value=""/>
+  <input type="hidden" class="schematic" height="${height}" width="${width}" parts="${parts}" analyses="${analyses}" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="" initial_value=""/>
 
   <div id="value_${id}" style="display:none">${value}</div>
   <div id="initial_value_${id}" style="display:none">${initial_value}</div>
@@ -13,13 +13,21 @@
 
   <span id="answer_${id}"></span>
   % if status == 'unsubmitted':
-  <span class="ui-icon ui-icon-bullet" style="display:inline-block;" id="status_${id}"></span>
-  % elif status == 'correct': 
-  <span class="ui-icon ui-icon-check" style="display:inline-block;" id="status_${id}"></span>
+  <span class="ui-icon ui-icon-bullet" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: unsubmitted</span>
+  </span>
+  % elif status == 'correct':
+  <span class="ui-icon ui-icon-check" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: correct</span>
+  </span>
   % elif status == 'incorrect':
-  <span class="ui-icon ui-icon-close" style="display:inline-block;" id="status_${id}"></span>
+  <span class="ui-icon ui-icon-close" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incorrect</span>
+  </span>
   % elif status == 'incomplete':
-  <span class="ui-icon ui-icon-close" style="display:inline-block;" id="status_${id}"></span>
+  <span class="ui-icon ui-icon-close" style="display:inline-block;" id="status_${id}" aria-describedby="input_${id}">
+    <span class="sr">Status: incomplete</span>
+  </span>
   % endif
 </span>
 

common/lib/capa/capa/templates/textline.html

@@ -20,7 +20,7 @@
       <div style="display:none;" name="${hidden}" inputid="input_${id}" />
     % endif
 
-  <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+  <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"
         % if do_math:
             class="math"
         % endif
@@ -33,7 +33,7 @@
    />
    ${trailing_text | h}
 
-      <p class="status">
+      <p class="status" aria-describedby="input_${id}">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':

common/lib/capa/capa/templates/vsepr_input.html

@@ -21,11 +21,11 @@
     <div class="incorrect" id="status_${id}">
     % endif
 
-  <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
+  <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"
       style="display:none;"
    />
 
-      <p class="status">
+      <p class="status" aria-describedby="input_${id}">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':

common/lib/capa/capa/tests/test_responsetypes.py

@@ -640,6 +640,23 @@ class StringResponseTest(ResponseTest):
         correct_map = problem.grade_answers(input_dict)
         self.assertEquals(correct_map.get_hint('1_2_1'), "Hello??")
 
+    def test_hint_function_randomization(self):
+        # The hint function should get the seed from the problem.
+        problem = self.build_problem(
+            answer="1",
+            hintfn="gimme_a_random_hint",
+            script=textwrap.dedent("""
+                def gimme_a_random_hint(answer_ids, student_answers, new_cmap, old_cmap):
+                    answer = str(random.randint(0, 1e9))
+                    new_cmap.set_hint_and_mode(answer_ids[0], answer, "always")
+
+            """)
+        )
+        correct_map = problem.grade_answers({'1_2_1': '2'})
+        hint = correct_map.get_hint('1_2_1')
+        r = random.Random(problem.seed)
+        self.assertEqual(hint, str(r.randint(0, 1e9)))
+
 
 class CodeResponseTest(ResponseTest):
     from response_xml_factory import CodeResponseXMLFactory
@@ -948,7 +965,6 @@ class CustomResponseTest(ResponseTest):
     xml_factory_class = CustomResponseXMLFactory
 
     def test_inline_code(self):
-
         # For inline code, we directly modify global context variables
         # 'answers' is a list of answers provided to us
         # 'correct' is a list we fill in with True/False
@@ -961,15 +977,14 @@ class CustomResponseTest(ResponseTest):
         self.assert_grade(problem, '0', 'incorrect')
 
     def test_inline_message(self):
-
         # Inline code can update the global messages list
         # to pass messages to the CorrectMap for a particular input
         # The code can also set the global overall_message (str)
         # to pass a message that applies to the whole response
         inline_script = textwrap.dedent("""
-        messages[0] = "Test Message"
-        overall_message = "Overall message"
-        """)
+            messages[0] = "Test Message"
+            overall_message = "Overall message"
+            """)
         problem = self.build_problem(answer=inline_script)
 
         input_dict = {'1_2_1': '0'}
@@ -983,8 +998,19 @@ class CustomResponseTest(ResponseTest):
         overall_msg = correctmap.get_overall_message()
         self.assertEqual(overall_msg, "Overall message")
 
-    def test_function_code_single_input(self):
+    def test_inline_randomization(self):
+        # Make sure the seed from the problem gets fed into the script execution.
+        inline_script = """messages[0] = str(random.randint(0, 1e9))"""
+        problem = self.build_problem(answer=inline_script)
+
+        input_dict = {'1_2_1': '0'}
+        correctmap = problem.grade_answers(input_dict)
 
+        input_msg = correctmap.get_msg('1_2_1')
+        r = random.Random(problem.seed)
+        self.assertEqual(input_msg, str(r.randint(0, 1e9)))
+
+    def test_function_code_single_input(self):
         # For function code, we pass in these arguments:
         #
         #   'expect' is the expect attribute of the <customresponse>
@@ -1212,6 +1238,29 @@ class CustomResponseTest(ResponseTest):
         with self.assertRaises(ResponseError):
             problem.grade_answers({'1_2_1': '42'})
 
+    def test_setup_randomization(self):
+        # Ensure that the problem setup script gets the random seed from the problem.
+        script = textwrap.dedent("""
+            num = random.randint(0, 1e9)
+            """)
+        problem = self.build_problem(script=script)
+        r = random.Random(problem.seed)
+        self.assertEqual(r.randint(0, 1e9), problem.context['num'])
+
+    def test_check_function_randomization(self):
+        # The check function should get random-seeded from the problem.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {'ok': True, 'msg': str(random.randint(0, 1e9))}
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+        msg = correct_map.get_msg('1_2_1')
+        r = random.Random(problem.seed)
+        self.assertEqual(msg, str(r.randint(0, 1e9)))
+
     def test_module_imports_inline(self):
         '''
         Check that the correct modules are available to custom
@@ -1275,7 +1324,6 @@ class SchematicResponseTest(ResponseTest):
     xml_factory_class = SchematicResponseXMLFactory
 
     def test_grade(self):
-
         # Most of the schematic-specific work is handled elsewhere
         # (in client-side JavaScript)
         # The <schematicresponse> is responsible only for executing the
@@ -1290,7 +1338,7 @@ class SchematicResponseTest(ResponseTest):
 
         # The actual dictionary would contain schematic information
         # sent from the JavaScript simulation
-        submission_dict = {'test': 'test'}
+        submission_dict = {'test': 'the_answer'}
         input_dict = {'1_2_1': json.dumps(submission_dict)}
         correct_map = problem.grade_answers(input_dict)
 
@@ -1299,8 +1347,19 @@ class SchematicResponseTest(ResponseTest):
         # is what we expect)
         self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct')
 
-    def test_script_exception(self):
+    def test_check_function_randomization(self):
+        # The check function should get a random seed from the problem.
+        script = "correct = ['correct' if (submission[0]['num'] == random.randint(0, 1e9)) else 'incorrect']"
+        problem = self.build_problem(answer=script)
 
+        r = random.Random(problem.seed)
+        submission_dict = {'num': r.randint(0, 1e9)}
+        input_dict = {'1_2_1': json.dumps(submission_dict)}
+        correct_map = problem.grade_answers(input_dict)
+
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct')
+
+    def test_script_exception(self):
         # Construct a script that will raise an exception
         script = "raise Exception('test')"
         problem = self.build_problem(answer=script)

common/lib/capa/symmath/README.md

-(Originally written by Ike.)
-
-At a high level, the main challenges of checking symbolic math expressions are (1) making sure the expression is mathematically legal, and (2) simplifying the expression for comparison with what is expected.
-
-(1) Generation (and testing) of legal input is done by using MathJax to provide input math in an XML format known as Presentation MathML (PMathML).  Such expressions typeset correctly, but may not be mathematically legal, like "5 / (1 = 2)".  The PMathML is converted into "Content MathML" (CMathML), which is by definition mathematically legal, using an XSLT 2.0 stylesheet, via a module in SnuggleTeX. CMathML is then converted into a sympy expression.  This work is all done in `lms/lib/symmath/formula.py`
-
-(2) Simplifying the expression and checking against what is expected is done by using sympy, and a set of heuristics based on options flags provided by the problem author.  For example, the problem author may specify that the expected expression is a matrix, in which case the dimensionality of the input expression is checked.  Other options include specifying that the comparison be checked numerically in addition to symbolically.  The checking is done in stages, first with no simplification, then with increasing levels of testing; if a match is found at any stage, then an "ok" is returned.  Helpful messages are also returned, eg if the input expression is of a different type than the expected.  This work is all done in `lms/lib/symmath/symmath_check.py`
-
-Links:
-
-SnuggleTex: http://www2.ph.ed.ac.uk/snuggletex/documentation/overview-and-features.html
-MathML: http://www.w3.org/TR/MathML2/overview.html
-SymPy: http://sympy.org/en/index.html

common/lib/sandbox-packages/setup.py

@@ -4,6 +4,7 @@ setup(
     name="sandbox-packages",
     version="0.1.1",
     packages=[
+        "loncapa",
         "verifiers",
     ],
     py_modules=[

common/lib/symmath/setup.py

+from setuptools import setup
+
+setup(
+    name="symmath",
+    version="0.1",
+    packages=["symmath"],
+    install_requires=[
+        "sympy",
+    ],
+)

common/lib/symmath/symmath/README.md

+(Originally written by Ike.)
+
+At a high level, the main challenges of checking symbolic math expressions are
+(1) making sure the expression is mathematically legal, and (2) simplifying the
+expression for comparison with what is expected.
+
+(1) Generation (and testing) of legal input is done by using MathJax to provide
+input math in an XML format known as Presentation MathML (PMathML).  Such
+expressions typeset correctly, but may not be mathematically legal, like "5 /
+(1 = 2)".  The PMathML is converted into "Content MathML" (CMathML), which is
+by definition mathematically legal, using an XSLT 2.0 stylesheet, via a module
+in SnuggleTeX. CMathML is then converted into a sympy expression.  This work is
+all done in `symmath/formula.py`.
+
+(2) Simplifying the expression and checking against what is expected is done by
+using sympy, and a set of heuristics based on options flags provided by the
+problem author.  For example, the problem author may specify that the expected
+expression is a matrix, in which case the dimensionality of the input
+expression is checked.  Other options include specifying that the comparison be
+checked numerically in addition to symbolically.  The checking is done in
+stages, first with no simplification, then with increasing levels of testing;
+if a match is found at any stage, then an "ok" is returned.  Helpful messages
+are also returned, eg if the input expression is of a different type than the
+expected.  This work is all done in `symmath/symmath_check.py`.
+
+Links:
+
+SnuggleTex: http://www2.ph.ed.ac.uk/snuggletex/documentation/overview-and-features.html
+MathML: http://www.w3.org/TR/MathML2/overview.html
+SymPy: http://sympy.org/en/index.html

common/lib/xmodule/xmodule/annotatable_module.py

@@ -125,6 +125,5 @@ class AnnotatableModule(AnnotatableFields, XModule):
 
 class AnnotatableDescriptor(AnnotatableFields, RawDescriptor):
     module_class = AnnotatableModule
-    stores_state = True
     template_dir_name = "annotatable"
     mako_template = "widgets/raw-edit.html"

common/lib/xmodule/xmodule/capa_module.py

@@ -11,7 +11,7 @@ import sys
 from pkg_resources import resource_string
 
 from capa.capa_problem import LoncapaProblem
-from capa.responsetypes import StudentInputError,\
+from capa.responsetypes import StudentInputError, \
     ResponseError, LoncapaProblemError
 from capa.util import convert_files_to_filenames
 from .progress import Progress
@@ -20,7 +20,7 @@ from xmodule.raw_module import RawDescriptor
 from xmodule.exceptions import NotFoundError, ProcessingError
 from xblock.core import Scope, String, Boolean, Object
 from .fields import Timedelta, Date, StringyInteger, StringyFloat
-from xmodule.util.date_utils import time_to_datetime
+from django.utils.timezone import UTC
 
 log = logging.getLogger("mitx.courseware")
 
@@ -69,7 +69,7 @@ class CapaFields(object):
     max_attempts = StringyInteger(
         display_name="Maximum Attempts",
         help="Defines the number of times a student can try to answer this problem. If the value is not set, infinite attempts are allowed.",
-        values={"min": 1}, scope=Scope.settings
+        values={"min": 0}, scope=Scope.settings
     )
     due = Date(help="Date that this problem is due by", scope=Scope.settings)
     graceperiod = Timedelta(help="Amount of time after the due date that submissions will be accepted", scope=Scope.settings)
@@ -134,7 +134,7 @@ class CapaModule(CapaFields, XModule):
     def __init__(self, system, location, descriptor, model_data):
         XModule.__init__(self, system, location, descriptor, model_data)
 
-        due_date = time_to_datetime(self.due)
+        due_date = self.due
 
         if self.graceperiod is not None and due_date:
             self.close_date = due_date + self.graceperiod
@@ -502,7 +502,7 @@ class CapaModule(CapaFields, XModule):
         Is it now past this problem's due date, including grace period?
         """
         return (self.close_date is not None and
-                datetime.datetime.utcnow() > self.close_date)
+                datetime.datetime.now(UTC()) > self.close_date)
 
     def closed(self):
         ''' Is the student still allowed to submit answers? '''
@@ -747,7 +747,7 @@ class CapaModule(CapaFields, XModule):
 
         # Problem queued. Students must wait a specified waittime before they are allowed to submit
         if self.lcp.is_queued():
-            current_time = datetime.datetime.now()
+            current_time = datetime.datetime.now(UTC())
             prev_submit_time = self.lcp.get_recentmost_queuetime()
             waittime_between_requests = self.system.xqueue['waittime']
             if (current_time - prev_submit_time).total_seconds() < waittime_between_requests:
@@ -902,7 +902,6 @@ class CapaDescriptor(CapaFields, RawDescriptor):
 
     module_class = CapaModule
 
-    stores_state = True
     has_score = True
     template_dir_name = 'problem'
     mako_template = "widgets/problem-edit.html"