Merge pull request #1217 from MITx/feature/zoldak/lms-acceptance-testing

Merge the lettuce acceptance test framework to master

doc/testing.md

 # Testing
 
-Testing is good.  Here is some useful info about how we set up tests--
+Testing is good.  Here is some useful info about how we set up tests.
+More info is [on the wiki](https://edx-wiki.atlassian.net/wiki/display/ENG/Test+Engineering)
 
-### Backend code:
+## Backend code
 
-- TODO
+- The python unit tests can be run via rake tasks. 
+See development.md for more info on how to do this.
 
-### Frontend code:
+## Frontend code
 
-We're using Jasmine to unit-testing the JavaScript files. All the specs are
-written in CoffeeScript for the consistency. To access the test cases, start the
-server in debug mode, navigate to `http://127.0.0.1:[port number]/_jasmine` to
-see the test result.
+### Jasmine
+
+We're using Jasmine to unit/integration test the JavaScript files. 
+More info [on the wiki](https://edx-wiki.atlassian.net/wiki/display/ENG/Jasmine)
+
+All the specs are written in CoffeeScript to be consistent with the code.
+To access the test cases, start the server using the settings file **jasmine.py** using this command:
+    `rake django-admin[runserver,lms,jasmine,12345]`
+
+Then navigate to `http://localhost:12345/_jasmine/` to see the test results.
 
 All the JavaScript codes must have test coverage. Both CMS and LMS
 has its own test directory in `{cms,lms}/static/coffee/spec`  If you haven't
@@ -30,3 +38,31 @@ If you're finishing a feature that contains JavaScript code snippets and do not
 sure how to test, please feel free to open up a pull request and asking people
 for help. (However, the best way to do it would be writing your test first, then
 implement your feature - Test Driven Development.)
+
+### BDD style acceptance tests with Lettuce
+
+We're using Lettuce for end user acceptance testing of features.
+More info [on the wiki](https://edx-wiki.atlassian.net/wiki/display/ENG/Lettuce+Acceptance+Testing)
+
+Lettuce is a port of Cucumber. We're using it to drive Splinter, which is a python wrapper to Selenium.
+To execute the automated test scripts, you'll need to start up the django server separately, then launch the tests.
+Do both use the settings file named **acceptance.py**.
+
+What this will do is to use a sqllite database named mitx_all/db/test_mitx.db. 
+That way it can be flushed etc. without messing up your dev db.
+Note that this also means that you need to syncdb and migrate the db first before starting the server to initialize it if it does not yet exist.
+
+1. Set up the test database (only needs to be done once):
+    rm ../db/test_mitx.db
+    rake django-admin[syncdb,lms,acceptance,--noinput]
+    rake django-admin[migrate,lms,acceptance,--noinput]
+
+2. Start up the django server separately in a shell
+    rake lms[acceptance]
+
+3. Then in another shell, run the tests in different ways as below. Lettuce comes with a new django-admin command called _harvest_. See the [lettuce django docs](http://lettuce.it/recipes/django-lxml.html) for more details.
+* All tests in a specified feature folder: `django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/portal/features/`
+* Only the specified feature's scenarios: `django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/courseware/features/high-level-tabs.feature`
+
+4. Troubleshooting
+* If you get an error msg that says something about harvest not being a command, you probably are missing a requirement. Pip install (test-requirements.txt) and/or brew install as needed.
\ No newline at end of file

lms/djangoapps/courseware/features/courseware.feature

+Feature: View the Courseware Tab
+  As a student in an edX course
+  In order to work on the course
+  I want to view the info on the courseware tab
+
+    Scenario: I can get to the courseware tab when logged in
+    Given I am registered for a course
+    And I log in
+    And I click on View Courseware
+    When I click on the "Courseware" tab
+    Then the "Courseware" tab is active
+
+    # TODO: fix this one? Not sure whether you should get a 404.
+    # Scenario: I cannot get to the courseware tab when not logged in
+    # Given I am not logged in
+    # And I visit the homepage
+    # When I visit the courseware URL    
+    # Then the login dialog is visible

lms/djangoapps/courseware/features/courseware.py

+from lettuce import world, step
+from lettuce.django import django_url
+
+@step('I visit the courseware URL$')
+def i_visit_the_course_info_url(step):
+    url = django_url('/courses/MITx/6.002x/2012_Fall/courseware')
+    world.browser.visit(url)
\ No newline at end of file

lms/djangoapps/courseware/features/courseware_common.py

+from lettuce import world, step
+from lettuce.django import django_url
+
+@step('I click on View Courseware')
+def i_click_on_view_courseware(step):
+    css = 'p.enter-course'
+    world.browser.find_by_css(css).first.click()
+
+@step('I click on the "([^"]*)" tab$')
+def i_click_on_the_tab(step, tab):
+    world.browser.find_link_by_text(tab).first.click()
+    world.save_the_html()
+
+@step('I visit the courseware URL$')
+def i_visit_the_course_info_url(step):
+    url = django_url('/courses/MITx/6.002x/2012_Fall/courseware')
+    world.browser.visit(url)
+
+@step(u'I do not see "([^"]*)" anywhere on the page')
+def i_do_not_see_text_anywhere_on_the_page(step, text):
+    assert world.browser.is_text_not_present(text)    
+
+@step(u'I am on the dashboard page$')
+def i_am_on_the_dashboard_page(step):
+    assert world.browser.is_element_present_by_css('section.courses')
+    assert world.browser.url == django_url('/dashboard')
+
+@step('the "([^"]*)" tab is active$')
+def the_tab_is_active(step, tab):
+    css = '.course-tabs a.active'
+    active_tab = world.browser.find_by_css(css)
+    assert (active_tab.text == tab)
+
+@step('the login dialog is visible$')
+def login_dialog_visible(step):
+    css = 'form#login_form.login_form'
+    assert world.browser.find_by_css(css).visible

lms/djangoapps/courseware/features/high-level-tabs.feature

+Feature: All the high level tabs should work
+  In order to preview the courseware
+  As a student
+  I want to navigate through the high level tabs
+
+# Note this didn't work as a scenario outline because
+# before each scenario was not flushing the database
+# TODO: break this apart so that if one fails the others
+# will still run
+  Scenario: A student can see all tabs of the course
+    Given I am registered for a course
+    And I log in
+    And I click on View Courseware
+    When I click on the "Courseware" tab
+    Then the page title should be "6.002x Courseware"
+    When I click on the "Course Info" tab
+    Then the page title should be "6.002x Course Info"
+    When I click on the "Textbook" tab
+    Then the page title should be "6.002x Textbook"
+    When I click on the "Wiki" tab
+    Then the page title should be "6.002x | edX Wiki"
+    When I click on the "Progress" tab
+    Then the page title should be "6.002x Progress"

lms/djangoapps/courseware/features/openended.feature

+Feature: Open ended grading
+  As a student in an edX course
+  In order to complete the courseware questions
+  I want the machine learning grading to be functional
+
+    Scenario: An answer that is too short is rejected
+    Given I navigate to an openended question
+    And I enter the answer "z"
+    When I press the "Check" button
+    And I wait for "8" seconds
+    And I see the grader status "Submitted for grading"
+    And I press the "Recheck for Feedback" button
+    Then I see the red X
+    And I see the grader score "0"
+
+    Scenario: An answer with too many spelling errors is rejected
+    Given I navigate to an openended question
+    And I enter the answer "az"
+    When I press the "Check" button
+    And I wait for "8" seconds
+    And I see the grader status "Submitted for grading"
+    And I press the "Recheck for Feedback" button
+    Then I see the red X
+    And I see the grader score "0"
+    When I click the link for full output
+    Then I see the spelling grading message "More spelling errors than average."
+
+    Scenario: An answer makes its way to the instructor dashboard
+    Given I navigate to an openended question as staff
+    When I submit the answer "I love Chemistry."
+    And I wait for "8" seconds
+    And I visit the staff grading page
+    Then my answer is queued for instructor grading
\ No newline at end of file

lms/djangoapps/courseware/features/openended.py

+from lettuce import world, step
+from lettuce.django import django_url
+from nose.tools import assert_equals, assert_in
+from logging import getLogger
+logger = getLogger(__name__)
+
+@step('I navigate to an openended question$')
+def navigate_to_an_openended_question(step):
+    world.register_by_course_id('MITx/3.091x/2012_Fall')
+    world.log_in('robot@edx.org','test')
+    problem = '/courses/MITx/3.091x/2012_Fall/courseware/Week_10/Polymer_Synthesis/'
+    world.browser.visit(django_url(problem))
+    tab_css = 'ol#sequence-list > li > a[data-element="5"]'
+    world.browser.find_by_css(tab_css).click()
+
+@step('I navigate to an openended question as staff$')
+def navigate_to_an_openended_question_as_staff(step):
+    world.register_by_course_id('MITx/3.091x/2012_Fall', True)
+    world.log_in('robot@edx.org','test')
+    problem = '/courses/MITx/3.091x/2012_Fall/courseware/Week_10/Polymer_Synthesis/'
+    world.browser.visit(django_url(problem))
+    tab_css = 'ol#sequence-list > li > a[data-element="5"]'
+    world.browser.find_by_css(tab_css).click()
+
+@step(u'I enter the answer "([^"]*)"$')
+def enter_the_answer_text(step, text):
+    textarea_css = 'textarea'
+    world.browser.find_by_css(textarea_css).first.fill(text)
+
+@step(u'I submit the answer "([^"]*)"$')
+def i_submit_the_answer_text(step, text):
+    textarea_css = 'textarea'
+    world.browser.find_by_css(textarea_css).first.fill(text)
+    check_css = 'input.check'
+    world.browser.find_by_css(check_css).click()
+
+@step('I click the link for full output$')
+def click_full_output_link(step):
+    link_css = 'a.full'
+    world.browser.find_by_css(link_css).first.click()
+
+@step(u'I visit the staff grading page$')
+def i_visit_the_staff_grading_page(step):
+    # course_u = '/courses/MITx/3.091x/2012_Fall'
+    # sg_url = '%s/staff_grading' % course_u
+    world.browser.click_link_by_text('Instructor')
+    world.browser.click_link_by_text('Staff grading')    
+    # world.browser.visit(django_url(sg_url))
+
+@step(u'I see the grader message "([^"]*)"$')
+def see_grader_message(step, msg):
+    message_css = 'div.external-grader-message'
+    grader_msg = world.browser.find_by_css(message_css).text
+    assert_in(msg, grader_msg)
+
+@step(u'I see the grader status "([^"]*)"$')
+def see_the_grader_status(step, status):
+    status_css = 'div.grader-status'
+    grader_status = world.browser.find_by_css(status_css).text
+    assert_equals(status, grader_status)
+
+@step('I see the red X$')
+def see_the_red_x(step):
+    x_css = 'div.grader-status > span.incorrect'
+    assert world.browser.find_by_css(x_css)
+
+@step(u'I see the grader score "([^"]*)"$')
+def see_the_grader_score(step, score):
+    score_css = 'div.result-output > p'
+    score_text = world.browser.find_by_css(score_css).text
+    assert_equals(score_text, 'Score: %s' % score)
+
+@step('I see the link for full output$')
+def see_full_output_link(step):
+    link_css = 'a.full'
+    assert world.browser.find_by_css(link_css)
+
+@step('I see the spelling grading message "([^"]*)"$')
+def see_spelling_msg(step, msg):
+    spelling_css = 'div.spelling'
+    spelling_msg = world.browser.find_by_css(spelling_css).text    
+    assert_equals('Spelling: %s' % msg, spelling_msg)
+
+@step(u'my answer is queued for instructor grading$')
+def answer_is_queued_for_instructor_grading(step):
+    list_css = 'ul.problem-list > li > a'
+    actual_msg = world.browser.find_by_css(list_css).text
+    expected_msg = "(0 graded, 1 pending)"
+    assert_in(expected_msg, actual_msg)

lms/djangoapps/courseware/features/smart-accordion.feature

+# Here are all the courses for Fall 2012
+#  MITx/3.091x/2012_Fall
+#  MITx/6.002x/2012_Fall
+#  MITx/6.00x/2012_Fall
+#  HarvardX/CS50x/2012 (we will not be testing this, as it is anomolistic)
+#  HarvardX/PH207x/2012_Fall
+#  BerkeleyX/CS169.1x/2012_Fall
+#  BerkeleyX/CS169.2x/2012_Fall
+#  BerkeleyX/CS184.1x/2012_Fall
+
+#You can load the courses into your data directory with these cmds:
+#  git clone https://github.com/MITx/3.091x.git
+#  git clone https://github.com/MITx/6.00x.git
+#  git clone https://github.com/MITx/content-mit-6002x.git
+#  git clone https://github.com/MITx/content-mit-6002x.git
+#  git clone https://github.com/MITx/content-harvard-id270x.git
+#  git clone https://github.com/MITx/content-berkeley-cs169x.git
+#  git clone https://github.com/MITx/content-berkeley-cs169.2x.git
+#  git clone https://github.com/MITx/content-berkeley-cs184x.git
+
+Feature: There are courses on the homepage
+  In order to compared rendered content to the database
+  As an acceptance test
+  I want to count all the chapters, sections, and tabs for each course
+
+  Scenario: Navigate through course MITx/3.091x/2012_Fall
+    Given I am registered for course "MITx/3.091x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course MITx/6.002x/2012_Fall
+    Given I am registered for course "MITx/6.002x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course MITx/6.00x/2012_Fall
+    Given I am registered for course "MITx/6.00x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course HarvardX/PH207x/2012_Fall
+    Given I am registered for course "HarvardX/PH207x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course BerkeleyX/CS169.1x/2012_Fall
+    Given I am registered for course "BerkeleyX/CS169.1x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course BerkeleyX/CS169.2x/2012_Fall
+    Given I am registered for course "BerkeleyX/CS169.2x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
+
+  Scenario: Navigate through course BerkeleyX/CS184.1x/2012_Fall
+    Given I am registered for course "BerkeleyX/CS184.1x/2012_Fall"
+    And I log in
+    Then I verify all the content of each course
\ No newline at end of file

lms/djangoapps/courseware/features/smart-accordion.py

+from lettuce import world, step
+from re import sub
+from nose.tools import assert_equals
+from xmodule.modulestore.django import modulestore
+from courses import *
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+def check_for_errors():
+    e = world.browser.find_by_css('.outside-app')
+    if len(e) > 0:
+        assert False, 'there was a server error at %s' % (world.browser.url)
+    else:
+        assert True
+
+@step(u'I verify all the content of each course')
+def i_verify_all_the_content_of_each_course(step):
+    all_possible_courses = get_courses()
+    logger.debug('Courses found:')
+    for c in all_possible_courses:
+        logger.debug(c.id)
+    ids = [c.id for c in all_possible_courses]
+
+    # Get a list of all the registered courses
+    registered_courses = world.browser.find_by_css('article.my-course')
+    if len(all_possible_courses) < len(registered_courses):
+        assert False, "user is registered for more courses than are uniquely posssible"
+    else:
+        pass
+
+    for test_course in registered_courses:
+        test_course.find_by_css('a').click()
+        check_for_errors()
+
+        # Get the course. E.g. 'MITx/6.002x/2012_Fall'
+        current_course = sub('/info','', sub('.*/courses/', '', world.browser.url))
+        validate_course(current_course,ids)
+
+        world.browser.find_link_by_text('Courseware').click()
+        assert world.browser.is_element_present_by_id('accordion',wait_time=2)
+        check_for_errors()
+        browse_course(current_course)
+
+        # clicking the user link gets you back to the user's home page
+        world.browser.find_by_css('.user-link').click()
+        check_for_errors()
+
+def browse_course(course_id):
+
+    ## count chapters from xml and page and compare
+    chapters = get_courseware_with_tabs(course_id)
+    num_chapters = len(chapters)
+
+    rendered_chapters = world.browser.find_by_css('#accordion > nav > div')
+    num_rendered_chapters = len(rendered_chapters)
+
+    msg = '%d chapters expected, %d chapters found on page for %s' % (num_chapters, num_rendered_chapters, course_id)
+    #logger.debug(msg)
+    assert num_chapters == num_rendered_chapters, msg
+
+    chapter_it = 0
+
+    ## Iterate the chapters
+    while chapter_it < num_chapters:
+
+        ## click into a chapter
+        world.browser.find_by_css('#accordion > nav > div')[chapter_it].find_by_tag('h3').click()
+
+        ## look for the "there was a server error" div
+        check_for_errors()
+
+        ## count sections from xml and page and compare
+        sections = chapters[chapter_it]['sections']
+        num_sections = len(sections)
+
+        rendered_sections = world.browser.find_by_css('#accordion > nav > div')[chapter_it].find_by_tag('li')
+        num_rendered_sections = len(rendered_sections)
+
+        msg = ('%d sections expected, %d sections found on page, %s - %d - %s' %
+                (num_sections, num_rendered_sections, course_id, chapter_it, chapters[chapter_it]['chapter_name']))
+        #logger.debug(msg)
+        assert num_sections == num_rendered_sections, msg
+
+        section_it = 0
+
+        ## Iterate the sections
+        while section_it < num_sections:
+
+            ## click on a section
+            world.browser.find_by_css('#accordion > nav > div')[chapter_it].find_by_tag('li')[section_it].find_by_tag('a').click()
+
+            ## sometimes the course-content takes a long time to load
+            assert world.browser.is_element_present_by_css('.course-content',wait_time=5)
+
+            ## look for server error div
+            check_for_errors()
+
+            ## count tabs from xml and page and compare
+
+            ## count the number of tabs. If number of tabs is 0, there won't be anything rendered
+            ## so we explicitly set rendered_tabs because otherwise find_elements returns a None object with no length
+            num_tabs = sections[section_it]['clickable_tab_count']
+            if num_tabs != 0:
+                rendered_tabs = world.browser.find_by_css('ol#sequence-list > li')
+                num_rendered_tabs = len(rendered_tabs)
+            else:
+                rendered_tabs = 0
+                num_rendered_tabs = 0
+
+            msg = ('%d tabs expected, %d tabs found, %s - %d - %s' % 
+                        (num_tabs, num_rendered_tabs, course_id, section_it, sections[section_it]['section_name']))
+            #logger.debug(msg)
+
+            # Save the HTML to a file for later comparison
+            world.save_the_course_content('/tmp/%s' % course_id)
+
+            assert num_tabs == num_rendered_tabs, msg
+
+            tabs = sections[section_it]['tabs']
+            tab_it = 0
+
+            ## Iterate the tabs
+            while tab_it < num_tabs:
+
+                rendered_tabs[tab_it].find_by_tag('a').click()
+
+                ## do something with the tab sections[section_it]
+                # e = world.browser.find_by_css('section.course-content section')
+                # process_section(e)
+                tab_children = tabs[tab_it]['children_count']
+                tab_class = tabs[tab_it]['class']
+                if tab_children != 0:
+                    rendered_items = world.browser.find_by_css('div#seq_content > section > ol > li > section')
+                    num_rendered_items = len(rendered_items)              
+                    msg = ('%d items expected, %d items found, %s - %d - %s - tab %d' %
+                        (tab_children, num_rendered_items, course_id, section_it, sections[section_it]['section_name'], tab_it))
+                    #logger.debug(msg)
+                    assert tab_children == num_rendered_items, msg
+
+                tab_it += 1
+
+            section_it += 1
+
+        chapter_it += 1
+
+
+def validate_course(current_course, ids):
+    try:
+        ids.index(current_course)
+    except:
+        assert False, "invalid course id %s" % current_course

lms/djangoapps/portal/README.md

+## acceptance_testing
+
+This fake django app is here to support acceptance testing using <a href="http://lettuce.it/">lettuce</a> + 
+<a href="http://splinter.cobrateam.info/">splinter</a> (which wraps <a href="http://selenium.googlecode.com/svn/trunk/docs/api/py/index.html">selenium</a>).
+
+First you need to make sure that you've installed the requirements. 
+This includes lettuce, selenium, splinter, etc.
+Do this with:
+```pip install -r test-requirements.txt```
+
+The settings.py environment file used is named acceptance.py.
+It uses a test SQLite database defined as ../db/test-mitx.db.
+You need to first start up the server separately, then run the lettuce scenarios.
+
+Full documentation can be found on the wiki at <a href="https://edx-wiki.atlassian.net/wiki/display/ENG/Lettuce+Acceptance+Testing">this link</a>.

lms/djangoapps/portal/features/common.py

+from lettuce import world, step#, before, after
+from factories import *
+from django.core.management import call_command
+from nose.tools import assert_equals, assert_in
+from lettuce.django import django_url
+from django.conf import settings
+from django.contrib.auth.models import User
+from student.models import CourseEnrollment
+import time
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+@step(u'I wait (?:for )?"(\d+)" seconds?$')
+def wait(step, seconds):
+    time.sleep(float(seconds))
+
+@step('I (?:visit|access|open) the homepage$')
+def i_visit_the_homepage(step):
+    world.browser.visit(django_url('/'))
+    assert world.browser.is_element_present_by_css('header.global', 10)
+
+@step(u'I (?:visit|access|open) the dashboard$')
+def i_visit_the_dashboard(step):
+    world.browser.visit(django_url('/dashboard'))
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+
+@step(r'click (?:the|a) link (?:called|with the text) "([^"]*)"$')
+def click_the_link_called(step, text):
+    world.browser.find_link_by_text(text).click()
+
+@step('I should be on the dashboard page$')
+def i_should_be_on_the_dashboard(step):
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+    assert world.browser.title == 'Dashboard'
+
+@step(u'I (?:visit|access|open) the courses page$')
+def i_am_on_the_courses_page(step):
+    world.browser.visit(django_url('/courses'))
+    assert world.browser.is_element_present_by_css('section.courses')
+
+@step('I should see that the path is "([^"]*)"$')
+def i_should_see_that_the_path_is(step, path):
+    assert world.browser.url == django_url(path)
+
+@step(u'the page title should be "([^"]*)"$')
+def the_page_title_should_be(step, title):
+    assert world.browser.title == title
+
+@step(r'should see that the url is "([^"]*)"$')
+def should_have_the_url(step, url):
+    assert_equals(world.browser.url, url)
+
+@step(r'should see (?:the|a) link (?:called|with the text) "([^"]*)"$')
+def should_see_a_link_called(step, text):
+    assert len(world.browser.find_link_by_text(text)) > 0
+
+@step(r'should see "(.*)" (?:somewhere|anywhere) in (?:the|this) page')
+def should_see_in_the_page(step, text):
+    assert_in(text, world.browser.html)
+
+@step('I am logged in$')
+def i_am_logged_in(step):
+    world.create_user('robot')
+    world.log_in('robot@edx.org', 'test')
+
+@step('I am not logged in$')
+def i_am_not_logged_in(step):
+    world.browser.cookies.delete()
+
+@step(u'I am registered for a course$')
+def i_am_registered_for_a_course(step):
+    world.create_user('robot')
+    u = User.objects.get(username='robot')
+    CourseEnrollment.objects.create(user=u, course_id='MITx/6.002x/2012_Fall')
+    world.log_in('robot@edx.org', 'test')
+    
+@step(u'I am an edX user$')
+def i_am_an_edx_user(step):
+    world.create_user('robot')
+
+@step(u'User "([^"]*)" is an edX user$')
+def registered_edx_user(step, uname):
+    world.create_user(uname)

lms/djangoapps/portal/features/factories.py

+import factory
+from student.models import User, UserProfile, Registration
+from datetime import datetime
+import uuid
+
+class UserProfileFactory(factory.Factory):
+    FACTORY_FOR = UserProfile
+
+    user = None
+    name = 'Jack Foo'
+    level_of_education = None
+    gender = 'm'
+    mailing_address = None
+    goals = 'World domination'
+
+class RegistrationFactory(factory.Factory):
+    FACTORY_FOR = Registration
+
+    user = None
+    activation_key = uuid.uuid4().hex
+
+class UserFactory(factory.Factory):
+    FACTORY_FOR = User
+
+    username = 'robot'
+    email = 'robot+test@edx.org'
+    password = 'test'
+    first_name = 'Robot'
+    last_name = 'Test'
+    is_staff = False
+    is_active = True
+    is_superuser = False
+    last_login = datetime(2012, 1, 1)
+    date_joined = datetime(2011, 1, 1)

lms/djangoapps/portal/features/homepage.feature

+Feature: Homepage for web users
+  In order to get an idea what edX is about
+  As a an anonymous web user
+  I want to check the information on the home page
+
+  Scenario: User can see the "Login" button
+    Given I visit the homepage
+    Then I should see a link called "Log In"
+
+  Scenario: User can see the "Sign up" button
+    Given I visit the homepage
+    Then I should see a link called "Sign Up"
+
+  Scenario Outline: User can see main parts of the page
+    Given I visit the homepage
+    Then I should see a link called "<Link>"
+    When I click the link with the text "<Link>"
+    Then I should see that the path is "<Path>"
+
+    Examples:
+    | Link         | Path     |
+    | Find Courses | /courses |
+    | About        | /about   |
+    | Jobs         | /jobs    |
+    | Contact      | /contact |
+
+  Scenario: User can visit the blog
+    Given I visit the homepage
+    When I click the link with the text "Blog"
+    Then I should see that the url is "http://blog.edx.org/"
+
+  # TODO: test according to domain or policy
+  Scenario: User can see the partner institutions
+    Given I visit the homepage
+    Then I should see "<Partner>" in the Partners section
+
+    Examples:
+    | Partner     |
+    | MITx        |
+    | HarvardX    |
+    | BerkeleyX   |
+    | UTx         |    
+    | WellesleyX  |
+    | GeorgetownX |  
+
+  # # TODO: Add scenario that tests the courses available
+  # # using a policy or a configuration file

lms/djangoapps/portal/features/homepage.py

+from lettuce import world, step
+from nose.tools import assert_in
+
+@step('I should see "([^"]*)" in the Partners section$')
+def i_should_see_partner(step, partner):
+    partners = world.browser.find_by_css(".partner .name span")
+    names = set(span.text for span in partners)
+    assert_in(partner, names)

lms/djangoapps/portal/features/login.feature

+Feature: Login in as a registered user
+  As a registered user
+  In order to access my content
+  I want to be able to login in to edX
+
+    Scenario: Login to an unactivated account
+    Given I am an edX user
+    And I am an unactivated user
+    And I visit the homepage
+    When I click the link with the text "Log In"
+    And I submit my credentials on the login form
+    Then I should see the login error message "This account has not been activated"
+
+    Scenario: Login to an activated account
+    Given I am an edX user
+    And I am an activated user
+    And I visit the homepage
+    When I click the link with the text "Log In"
+    And I submit my credentials on the login form
+    Then I should be on the dashboard page
+
+    Scenario: Logout of a signed in account
+    Given I am logged in
+    When I click the dropdown arrow
+    And I click the link with the text "Log Out"
+    Then I should see a link with the text "Log In"
+    And I should see that the path is "/"

lms/djangoapps/portal/features/login.py

+from lettuce import step, world
+from django.contrib.auth.models import User
+
+@step('I am an unactivated user$')
+def i_am_an_unactivated_user(step):
+    user_is_an_unactivated_user('robot')
+
+@step('I am an activated user$')
+def i_am_an_activated_user(step):
+    user_is_an_activated_user('robot')
+
+@step('I submit my credentials on the login form')
+def i_submit_my_credentials_on_the_login_form(step):
+    fill_in_the_login_form('email', 'robot@edx.org')
+    fill_in_the_login_form('password', 'test')
+    login_form = world.browser.find_by_css('form#login_form')
+    login_form.find_by_value('Access My Courses').click()
+    
+@step(u'I should see the login error message "([^"]*)"$')
+def i_should_see_the_login_error_message(step, msg):
+    login_error_div = world.browser.find_by_css('form#login_form #login_error')
+    assert (msg in login_error_div.text)
+
+@step(u'click the dropdown arrow$')
+def click_the_dropdown(step):
+    css = ".dropdown"
+    e = world.browser.find_by_css(css)
+    e.click()
+
+#### helper functions
+
+def user_is_an_unactivated_user(uname):
+    u = User.objects.get(username=uname)
+    u.is_active = False
+    u.save()
+
+def user_is_an_activated_user(uname):
+    u = User.objects.get(username=uname)
+    u.is_active = True
+    u.save()
+
+def fill_in_the_login_form(field, value):
+    login_form = world.browser.find_by_css('form#login_form')
+    form_field = login_form.find_by_name(field)
+    form_field.fill(value)

lms/djangoapps/portal/features/registration.feature

+Feature: Register for a course
+  As a registered user
+  In order to access my class content
+  I want to register for a class on the edX website
+
+  	Scenario: I can register for a course
+  	Given I am logged in
+    And I visit the courses page
+		When I register for the course numbered "6.002x"
+  	Then I should see the course numbered "6.002x" in my dashboard
+
+    Scenario: I can unregister for a course
+    Given I am registered for a course
+    And I visit the dashboard
+    When I click the link with the text "Unregister"
+    And I press the "Unregister" button in the Unenroll dialog
+    Then I should see "Looks like you haven't registered for any courses yet." somewhere in the page
\ No newline at end of file

lms/djangoapps/portal/features/registration.py

+from lettuce import world, step
+
+@step('I register for the course numbered "([^"]*)"$')
+def i_register_for_the_course(step, course):
+    courses_section = world.browser.find_by_css('section.courses')
+    course_link_css = 'article[id*="%s"] a' % course
+    course_link = courses_section.find_by_css(course_link_css).first
+    course_link.click()
+
+    intro_section = world.browser.find_by_css('section.intro')
+    register_link = intro_section.find_by_css('a.register')
+    register_link.click()
+
+    assert world.browser.is_element_present_by_css('section.container.dashboard')
+
+@step(u'I should see the course numbered "([^"]*)" in my dashboard$')
+def i_should_see_that_course_in_my_dashboard(step, course):
+    course_link_css = 'section.my-courses a[href*="%s"]' % course
+    assert world.browser.is_element_present_by_css(course_link_css)
+
+@step(u'I press the "([^"]*)" button in the Unenroll dialog')
+def i_press_the_button_in_the_unenroll_dialog(step, value):
+    button_css = 'section#unenroll-modal input[value="%s"]' % value
+    world.browser.find_by_css(button_css).click()

lms/djangoapps/portal/features/signup.feature

+Feature: Sign in
+  In order to use the edX content
+  As a new user
+  I want to signup for a student account
+
+  Scenario: Sign up from the homepage
+    Given I visit the homepage
+    When I click the link with the text "Sign Up"
+    And I fill in "email" on the registration form with "robot2@edx.org"
+    And I fill in "password" on the registration form with "test"
+    And I fill in "username" on the registration form with "robot2"
+    And I fill in "name" on the registration form with "Robot Two"
+    And I check the checkbox named "terms_of_service"
+    And I check the checkbox named "honor_code"
+    And I press the "Create My Account" button on the registration form
+    Then I should see "THANKS FOR REGISTERING!" in the dashboard banner

lms/djangoapps/portal/features/signup.py

+from lettuce import world, step
+
+@step('I fill in "([^"]*)" on the registration form with "([^"]*)"$')
+def when_i_fill_in_field_on_the_registration_form_with_value(step, field, value):
+    register_form = world.browser.find_by_css('form#register_form')
+    form_field = register_form.find_by_name(field)
+    form_field.fill(value)
+
+@step('I press the "([^"]*)" button on the registration form$')
+def i_press_the_button_on_the_registration_form(step, button):
+    register_form = world.browser.find_by_css('form#register_form')
+    register_form.find_by_value(button).click()
+
+@step('I check the checkbox named "([^"]*)"$')
+def i_check_checkbox(step, checkbox):
+    world.browser.find_by_name(checkbox).check()
+
+@step('I should see "([^"]*)" in the dashboard banner$')
+def i_should_see_text_in_the_dashboard_banner_section(step, text):
+    css_selector = "section.dashboard-banner h2"
+    assert (text in world.browser.find_by_css(css_selector).text)
+    
\ No newline at end of file

lms/djangoapps/terrain/__init__.py

+# Use this as your lettuce terrain file so that the common steps
+# across all lms apps can be put in terrain/common
+# See https://groups.google.com/forum/?fromgroups=#!msg/lettuce-users/5VyU9B4HcX8/USgbGIJdS5QJ
+from terrain.browser import *
+from terrain.steps import *
+from terrain.factories import *
\ No newline at end of file

lms/djangoapps/terrain/browser.py

+from lettuce import before, after, world
+from splinter.browser import Browser
+from logging import getLogger
+import time
+
+logger = getLogger(__name__)
+logger.info("Loading the lettuce acceptance testing terrain file...")
+
+from django.core.management import call_command
+
+@before.harvest
+def initial_setup(server):
+    # Launch firefox
+    world.browser = Browser('firefox')
+
+@before.each_scenario
+def reset_data(scenario):
+    # Clean out the django test database defined in the 
+    # envs/acceptance.py file: mitx_all/db/test_mitx.db
+    logger.debug("Flushing the test database...")
+    call_command('flush', interactive=False)
+
+@after.all
+def teardown_browser(total):
+    # Quit firefox
+    world.browser.quit()

lms/djangoapps/terrain/factories.py

+import factory
+from student.models import User, UserProfile, Registration
+from datetime import datetime
+import uuid
+
+class UserProfileFactory(factory.Factory):
+    FACTORY_FOR = UserProfile
+
+    user = None
+    name = 'Robot Test'
+    level_of_education = None
+    gender = 'm'
+    mailing_address = None
+    goals = 'World domination'
+
+class RegistrationFactory(factory.Factory):
+    FACTORY_FOR = Registration
+
+    user = None
+    activation_key = uuid.uuid4().hex
+
+class UserFactory(factory.Factory):
+    FACTORY_FOR = User
+
+    username = 'robot'
+    email = 'robot+test@edx.org'
+    password = 'test'
+    first_name = 'Robot'
+    last_name = 'Test'
+    is_staff = False
+    is_active = True
+    is_superuser = False
+    last_login = datetime(2012, 1, 1)
+    date_joined = datetime(2011, 1, 1)

lms/djangoapps/terrain/steps.py

+from lettuce import world, step
+from factories import *
+from django.core.management import call_command
+from lettuce.django import django_url
+from django.conf import settings
+from django.contrib.auth.models import User
+from student.models import CourseEnrollment
+from urllib import quote_plus
+from nose.tools import assert_equals
+from bs4 import BeautifulSoup
+import time
+import re
+import os.path
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+@step(u'I wait (?:for )?"(\d+)" seconds?$')
+def wait(step, seconds):
+    time.sleep(float(seconds))
+
+@step('I (?:visit|access|open) the homepage$')
+def i_visit_the_homepage(step):
+    world.browser.visit(django_url('/'))
+    assert world.browser.is_element_present_by_css('header.global', 10)
+
+@step(u'I (?:visit|access|open) the dashboard$')
+def i_visit_the_dashboard(step):
+    world.browser.visit(django_url('/dashboard'))
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+
+@step('I should be on the dashboard page$')
+def i_should_be_on_the_dashboard(step):
+    assert world.browser.is_element_present_by_css('section.container.dashboard', 5)
+    assert world.browser.title == 'Dashboard'
+
+@step(u'I (?:visit|access|open) the courses page$')
+def i_am_on_the_courses_page(step):
+    world.browser.visit(django_url('/courses'))
+    assert world.browser.is_element_present_by_css('section.courses')
+
+@step(u'I press the "([^"]*)" button$')
+def and_i_press_the_button(step, value):
+    button_css = 'input[value="%s"]' % value
+    world.browser.find_by_css(button_css).first.click()
+
+@step('I should see that the path is "([^"]*)"$')
+def i_should_see_that_the_path_is(step, path):
+    assert world.browser.url == django_url(path)
+
+@step(u'the page title should be "([^"]*)"$')
+def the_page_title_should_be(step, title):
+    assert_equals(world.browser.title, title)
+
+@step('I am a logged in user$')
+def i_am_logged_in_user(step):
+    create_user('robot')
+    log_in('robot@edx.org','test')
+
+@step('I am not logged in$')
+def i_am_not_logged_in(step):
+    world.browser.cookies.delete()
+
+@step('I am registered for a course$')
+def i_am_registered_for_a_course(step):
+    create_user('robot')
+    u = User.objects.get(username='robot')
+    CourseEnrollment.objects.get_or_create(user=u, course_id='MITx/6.002x/2012_Fall')
+
+@step('I am registered for course "([^"]*)"$')
+def i_am_registered_for_course_by_id(step, course_id):
+    register_by_course_id(course_id)
+
+@step('I am staff for course "([^"]*)"$')
+def i_am_staff_for_course_by_id(step, course_id):
+    register_by_course_id(course_id, True)
+
+@step('I log in$')
+def i_log_in(step):
+    log_in('robot@edx.org','test')    
+
+@step(u'I am an edX user$')
+def i_am_an_edx_user(step):
+    create_user('robot')
+
+#### helper functions
+@world.absorb
+def create_user(uname):
+    portal_user = UserFactory.build(username=uname, email=uname + '@edx.org')
+    portal_user.set_password('test')
+    portal_user.save()
+
+    registration = RegistrationFactory(user=portal_user)
+    registration.register(portal_user)
+    registration.activate()
+
+    user_profile = UserProfileFactory(user=portal_user)
+
+@world.absorb
+def log_in(email, password):   
+    world.browser.cookies.delete()
+    world.browser.visit(django_url('/'))
+    world.browser.is_element_present_by_css('header.global', 10)       
+    world.browser.click_link_by_href('#login-modal')
+    login_form = world.browser.find_by_css('form#login_form')
+    login_form.find_by_name('email').fill(email)
+    login_form.find_by_name('password').fill(password)
+    login_form.find_by_name('submit').click()
+
+    # wait for the page to redraw
+    assert world.browser.is_element_present_by_css('.content-wrapper', 10)
+
+@world.absorb
+def register_by_course_id(course_id, is_staff=False):
+    create_user('robot')
+    u = User.objects.get(username='robot')
+    if is_staff:
+        u.is_staff=True
+        u.save()        
+    CourseEnrollment.objects.get_or_create(user=u, course_id=course_id)
+
+@world.absorb
+def save_the_html(path='/tmp'):
+    u = world.browser.url
+    html = world.browser.html.encode('ascii', 'ignore')
+    filename = '%s.html' % quote_plus(u)
+    f = open('%s/%s' % (path, filename), 'w')
+    f.write(html)
+    f.close
+
+@world.absorb
+def save_the_course_content(path='/tmp'):
+    html = world.browser.html.encode('ascii', 'ignore')
+    soup = BeautifulSoup(html)
+
+    # get rid of the header, we only want to compare the body
+    soup.head.decompose()
+
+    # for now, remove the data-id attributes, because they are 
+    # causing mismatches between cms-master and master
+    for item in soup.find_all(attrs={'data-id': re.compile('.*')}):
+        del item['data-id']
+
+    # we also need to remove them from unrendered problems, 
+    # where they are contained in the text of divs instead of
+    # in attributes of tags
+    # Be careful of whether or not it was the last attribute
+    # and needs a trailing space
+    for item in soup.find_all(text=re.compile(' data-id=".*?" ')):
+        s = unicode(item.string)
+        item.string.replace_with(re.sub(' data-id=".*?" ', ' ', s))
+
+    for item in soup.find_all(text=re.compile(' data-id=".*?"')):
+        s = unicode(item.string)
+        item.string.replace_with(re.sub(' data-id=".*?"', ' ', s))
+
+    # prettify the html so it will compare better, with
+    # each HTML tag on its own line
+    output = soup.prettify()
+
+    # use string slicing to grab everything after 'courseware/' in the URL
+    u = world.browser.url
+    section_url = u[u.find('courseware/')+11:] 
+
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    filename = '%s.html' % (quote_plus(section_url))
+    f = open('%s/%s' % (path, filename), 'w')
+    f.write(output)
+    f.close

lms/envs/acceptance.py

+"""
+This config file extends the test environment configuration 
+so that we can run the lettuce acceptance tests.
+"""
+from .test import *
+
+# You need to start the server in debug mode,
+# otherwise the browser will not render the pages correctly
+DEBUG = True
+
+# Show the courses that are in the data directory
+COURSES_ROOT = ENV_ROOT / "data"
+DATA_DIR = COURSES_ROOT
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
+        'OPTIONS': {
+            'data_dir': DATA_DIR,
+            'default_class': 'xmodule.hidden_module.HiddenDescriptor',
+        }
+    }
+}
+
+# Set this up so that rake lms[acceptance] and running the 
+# harvest command both use the same (test) database
+# which they can flush without messing up your dev db
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "test_mitx.db",
+        'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db",     
+    }
+}
+
+# Do not display the YouTube videos in the browser while running the
+# acceptance tests. This makes them faster and more reliable
+MITX_FEATURES['STUB_VIDEO_FOR_TESTING'] = True
+
+# Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
+INSTALLED_APPS += ('lettuce.django',)
+LETTUCE_APPS = ('portal',)  # dummy app covers the home page, login, registration, and course enrollment

lms/envs/cms/acceptance.py

+"""
+This config file is a copy of dev environment without the Debug
+Toolbar. I it suitable to run against acceptance tests.
+
+"""
+from .dev import *
+
+#  REMOVE DEBUG TOOLBAR
+
+INSTALLED_APPS = tuple(e for e in INSTALLED_APPS if e != 'debug_toolbar')
+INSTALLED_APPS = tuple(e for e in INSTALLED_APPS if e != 'debug_toolbar_mongo')
+
+MIDDLEWARE_CLASSES = tuple(e for e in MIDDLEWARE_CLASSES \
+                           if e != 'debug_toolbar.middleware.DebugToolbarMiddleware')
+
+
+########################### LETTUCE TESTING ##########################
+MITX_FEATURES['DISPLAY_TOY_COURSES'] = True
+
+INSTALLED_APPS += ('lettuce.django',)
+# INSTALLED_APPS += ('portal',)
+
+LETTUCE_APPS = ('portal',)  # dummy app covers the home page, login, registration, and course enrollment

lms/envs/common.py

@@ -76,6 +76,8 @@ MITX_FEATURES = {
 
     'DISABLE_LOGIN_BUTTON': False,  # used in systems where login is automatic, eg MIT SSL
 
+    'STUB_VIDEO_FOR_TESTING': False, # do not display video when running automated acceptance tests
+
     # extrernal access methods
     'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
     'AUTH_USE_OPENID': False,

lms/templates/video.html

@@ -2,19 +2,21 @@
   <h2> ${display_name} </h2>
 % endif
 
-<div id="video_${id}" class="video" data-streams="${streams}"
-data-caption-data-dir="${data_dir}" data-show-captions="${show_captions}"
-data-start="${start}" data-end="${end}">
-  <div class="tc-wrapper">
-    <article class="video-wrapper">
-      <section class="video-player">
-        <div id="${id}"></div>
-      </section>
-
-      <section class="video-controls"></section>
-    </article>
+%if settings.MITX_FEATURES['STUB_VIDEO_FOR_TESTING']:
+  <div id="stub_out_video_for_testing"></div>
+%else:
+  <div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}" data-show-captions="${show_captions}">
+    <div class="tc-wrapper">
+      <article class="video-wrapper">
+        <section class="video-player">
+          <div id="${id}"></div>
+        </section>
+        <section class="video-controls"></section>
+      </article>
+    </div>
   </div>
-</div>
+%endif
+
 % if source:
 <div class="video-sources">
   <p>Download video <a href="${source}">here</a>.</p>

test-requirements.txt

@@ -3,3 +3,6 @@ coverage
 nosexcover
 pylint
 pep8
+lettuce
+selenium
+factory_boy