Merge pull request #2023 from edx/feature/will/ci-lettuce-tests

Feature/will/ci lettuce tests

cms/djangoapps/contentstore/features/advanced-settings.feature

@@ -11,7 +11,8 @@ Feature: Advanced (manual) course policy
     Given I am on the Advanced Course Settings page in Studio
     Then the settings are alphabetized
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver cannot click notification "Cancel"
+  @skip
   Scenario: Test cancel editing key value
     Given I am on the Advanced Course Settings page in Studio
     When I edit the value of a policy key
@@ -20,7 +21,8 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then the policy key value is unchanged
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver cannot click notification "Save"
+  @skip
   Scenario: Test editing key value
     Given I am on the Advanced Course Settings page in Studio
     When I edit the value of a policy key and save
@@ -28,7 +30,8 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then the policy key value is changed
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input
+  @skip
   Scenario: Test how multi-line input appears
     Given I am on the Advanced Course Settings page in Studio
     When I create a JSON object as a value
@@ -36,7 +39,8 @@ Feature: Advanced (manual) course policy
     And I reload the page
     Then it is displayed as formatted
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input
+  @skip
   Scenario: Test automatic quoting of non-JSON values
     Given I am on the Advanced Course Settings page in Studio
     When I create a non-JSON value not in quotes

cms/djangoapps/contentstore/features/checklists.feature

@@ -10,8 +10,6 @@ Feature: Course checklists
     Then I can check and uncheck tasks in a checklist
     And They are correctly selected after I reload the page
 
-  @skip-phantom
-  @skip-firefox
   Scenario: A task can link to a location within Studio
     Given I have opened Checklists
     When I select a link to the course outline
@@ -19,8 +17,6 @@ Feature: Course checklists
     And I press the browser back button
     Then I am brought back to the course outline in the correct state
 
-  @skip-phantom
-  @skip-firefox
   Scenario: A task can link to a location outside Studio
     Given I have opened Checklists
     When I select a link to help page

cms/djangoapps/contentstore/features/course-settings.feature

 Feature: Course Settings
   As a course author, I want to be able to configure my course settings.
 
-  @skip-phantom
   Scenario: User can set course dates
     Given I have opened a new course in Studio
     When I select Schedule and Details
     And I set course dates
     Then I see the set dates on refresh
 
-  @skip-phantom
   Scenario: User can clear previously set course dates (except start date)
     Given I have set course dates
     And I clear all the dates except start
     Then I see cleared dates on refresh
 
-  @skip-phantom
   Scenario: User cannot clear the course start date
     Given I have set course dates
     And I clear the course start date

cms/djangoapps/contentstore/features/section.feature

@@ -3,7 +3,6 @@ Feature: Create Section
   As a course author
   I want to create and edit sections
 
-  @skip-phantom
   Scenario: Add a new section to a course
     Given I have opened a new course in Studio
     When I click the New Section link
@@ -27,7 +26,8 @@ Feature: Create Section
     And I save a new section release date
     Then the section release date is updated
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver hangs on alert
+  @skip
   Scenario: Delete section
     Given I have opened a new course in Studio
     And I have added a new section

cms/djangoapps/contentstore/features/signup.py

@@ -18,10 +18,7 @@ def i_fill_in_the_registration_form(step):
 @step('I press the Create My Account button on the registration form$')
 def i_press_the_button_on_the_registration_form(step):
     submit_css = 'form#register_form button#submit'
-    # Workaround for click not working on ubuntu
-    # for some unknown reason.
-    e = world.css_find(submit_css)
-    e.type(' ')
+    world.css_click(submit_css)
 
 
 @step('I should see be on the studio home page$')

cms/djangoapps/contentstore/features/studio-overview-togglesection.feature

@@ -14,7 +14,6 @@ Feature: Overview Toggle Section
     When I navigate to the course overview page
     Then I do not see the "Collapse All Sections" link
 
-  @skip-phantom
 	Scenario: Collapse link appears after creating first section of a course
 		Given I have a course with no sections
     When I navigate to the course overview page
@@ -22,7 +21,8 @@ Feature: Overview Toggle Section
     Then I see the "Collapse All Sections" link
     And all sections are expanded
 
-	@skip-phantom
+    # Skipped because Ubuntu ChromeDriver hangs on alert
+    @skip
 	Scenario: Collapse link is not removed after last section of a course is deleted
 		Given I have a course with 1 section
     And I navigate to the course overview page

cms/djangoapps/contentstore/features/subsection.feature

@@ -3,14 +3,12 @@ Feature: Create Subsection
   As a course author
   I want to create and edit subsections
 
-  @skip-phantom
   Scenario: Add a new subsection to a section
     Given I have opened a new course section in Studio
     When I click the New Subsection link
     And I enter the subsection name and click save
     Then I see my subsection on the Courseware page
 
-  @skip-phantom
   Scenario: Add a new subsection (with a name containing a quote) to a section (bug #216)
     Given I have opened a new course section in Studio
     When I click the New Subsection link
@@ -27,7 +25,6 @@ Feature: Create Subsection
     And I reload the page
     Then I see it marked as Homework
 
-  @skip-phantom
   Scenario: Set a due date in a different year (bug #256)
     Given I have opened a new subsection in Studio
     And I have set a release date and due date in different years
@@ -35,7 +32,8 @@ Feature: Create Subsection
     And I reload the page
     Then I see the correct dates
 
-  @skip-phantom
+  # Skipped because Ubuntu ChromeDriver hangs on alert
+  @skip
   Scenario: Delete a subsection
     Given I have opened a new course section in Studio
     And I have added a new subsection

cms/djangoapps/contentstore/features/subsection.py

@@ -63,14 +63,6 @@ def test_have_set_dates_in_different_years(step):
     set_date_and_time('input#due_date', '01/02/2012', 'input#due_time', '04:00')
 
 
-@step('I see the correct dates$')
-def i_see_the_correct_dates(step):
-    assert_equal('12/25/2011', world.css_find('input#start_date').first.value)
-    assert_equal('03:00', world.css_find('input#start_time').first.value)
-    assert_equal('01/02/2012', world.css_find('input#due_date').first.value)
-    assert_equal('04:00', world.css_find('input#due_time').first.value)
-
-
 @step('I mark it as Homework$')
 def i_mark_it_as_homework(step):
     world.css_click('a.menu-toggle')
@@ -101,8 +93,20 @@ def the_subsection_does_not_exist(step):
     assert world.browser.is_element_not_present_by_css(css)
 
 
+@step('I see the correct dates$')
+def i_see_the_correct_dates(step):
+    assert_equal('12/25/2011', get_date('input#start_date'))
+    assert_equal('03:00', get_date('input#start_time'))
+    assert_equal('01/02/2012', get_date('input#due_date'))
+    assert_equal('04:00', get_date('input#due_time'))
+
+
 ############ HELPER METHODS ###################
 
+def get_date(css):
+    return world.css_find(css).first.value.strip()
+
+
 def save_subsection_name(name):
     name_css = 'input.new-subsection-name-input'
     save_css = 'input.new-subsection-name-save'

cms/envs/acceptance.py

@@ -8,27 +8,41 @@ from .test import *
 # otherwise the browser will not render the pages correctly
 DEBUG = True
 
-# Show the courses that are in the data directory
-COURSES_ROOT = ENV_ROOT / "data"
-DATA_DIR = COURSES_ROOT
-# MODULESTORE = {
-#     'default': {
-#         'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
-#         'OPTIONS': {
-#             'data_dir': DATA_DIR,
-#             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-#         }
-#     }
-# }
+# Disable warnings for acceptance tests, to make the logs readable
+import logging
+logging.disable(logging.ERROR)
 
+MODULESTORE_OPTIONS = {
+    'default_class': 'xmodule.raw_module.RawDescriptor',
+    'host': 'localhost',
+    'db': 'test_xmodule',
+    'collection': 'acceptance_modulestore',
+    'fs_root': TEST_ROOT / "data",
+    'render_template': 'mitxmako.shortcuts.render_to_string',
+}
+
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    },
+    'direct': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    },
+    'draft': {
+        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    }
+}
 # Set this up so that rake lms[acceptance] and running the
 # harvest command both use the same (test) database
 # which they can flush without messing up your dev db
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ENV_ROOT / "db" / "test_mitx.db",
-        'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db",
+        'NAME': TEST_ROOT / "db" / "test_mitx.db",
+        'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db",
     }
 }
 

cms/envs/test.py

@@ -41,8 +41,8 @@ MODULESTORE_OPTIONS = {
     'default_class': 'xmodule.raw_module.RawDescriptor',
     'host': 'localhost',
     'db': 'test_xmodule',
-    'collection': 'modulestore',
-    'fs_root': GITHUB_REPO_ROOT,
+    'collection': 'test_modulestore',
+    'fs_root': TEST_ROOT / "data",
     'render_template': 'mitxmako.shortcuts.render_to_string',
 }
 

common/djangoapps/terrain/browser.py

+"""
+Browser set up for acceptance tests.
+"""
+
+#pylint: disable=E1101
+#pylint: disable=W0613
+#pylint: disable=W0611
+
 from lettuce import before, after, world
 from splinter.browser import Browser
 from logging import getLogger
 from django.core.management import call_command
 from django.conf import settings
+from selenium.common.exceptions import WebDriverException
 
 # Let the LMS and CMS do their one-time setup
 # For example, setting up mongo caches
 from lms import one_time_startup
 from cms import one_time_startup
 
-logger = getLogger(__name__)
-logger.info("Loading the lettuce acceptance testing terrain file...")
+# There is an import issue when using django-staticfiles with lettuce
+# Lettuce assumes that we are using django.contrib.staticfiles,
+# but the rest of the app assumes we are using django-staticfiles
+# (in particular, django-pipeline and our mako implementation)
+# To resolve this, we check whether staticfiles is installed,
+# then redirect imports for django.contrib.staticfiles
+# to use staticfiles.
+try:
+    import staticfiles
+except ImportError:
+    pass
+else:
+    import sys
+    sys.modules['django.contrib.staticfiles'] = staticfiles
+
+LOGGER = getLogger(__name__)
+LOGGER.info("Loading the lettuce acceptance testing terrain file...")
+
+MAX_VALID_BROWSER_ATTEMPTS = 20
 
 
 @before.harvest
 def initial_setup(server):
-    '''
-    Launch the browser once before executing the tests
-    '''
+    """
+    Launch the browser once before executing the tests.
+    """
     browser_driver = getattr(settings, 'LETTUCE_BROWSER', 'chrome')
-    world.browser = Browser(browser_driver)
+
+    # There is an issue with ChromeDriver2 r195627 on Ubuntu
+    # in which we sometimes get an invalid browser session.
+    # This is a work-around to ensure that we get a valid session.
+    success = False
+    num_attempts = 0
+    while (not success) and num_attempts < MAX_VALID_BROWSER_ATTEMPTS:
+
+        # Get a browser session
+        world.browser = Browser(browser_driver)
+
+        # Try to visit the main page
+        # If the browser session is invalid, this will
+        # raise a WebDriverException
+        try:
+            world.visit('/')
+
+        except WebDriverException:
+            world.browser.quit()
+            num_attempts += 1
+
+        else:
+            success = True
+
+    # If we were unable to get a valid session within the limit of attempts,
+    # then we cannot run the tests.
+    if not success:
+        raise IOError("Could not acquire valid ChromeDriver browser session.")
 
 
 @before.each_scenario
 def reset_data(scenario):
-    '''
+    """
     Clean out the django test database defined in the
     envs/acceptance.py file: mitx_all/db/test_mitx.db
-    '''
-    logger.debug("Flushing the test database...")
+    """
+    LOGGER.debug("Flushing the test database...")
     call_command('flush', interactive=False)
 
 
 @after.each_scenario
 def screenshot_on_error(scenario):
-    '''
-    Save a screenshot to help with debugging
-    '''
+    """
+    Save a screenshot to help with debugging.
+    """
     if scenario.failed:
         world.browser.driver.save_screenshot('/tmp/last_failed_scenario.png')
 
 
 @after.all
 def teardown_browser(total):
-    '''
-    Quit the browser after executing the tests
-    '''
+    """
+    Quit the browser after executing the tests.
+    """
     world.browser.quit()
-    pass

common/djangoapps/terrain/course_helpers.py

@@ -38,9 +38,11 @@ def create_user(uname):
 
 @world.absorb
 def log_in(username, password):
-    '''
-    Log the user in programatically
-    '''
+    """
+    Log the user in programatically.
+    This will delete any existing cookies to ensure that the user
+    logs in to the correct session.
+    """
 
     # Authenticate the user
     user = authenticate(username=username, password=password)
@@ -60,15 +62,8 @@ def log_in(username, password):
 
     # Retrieve the sessionid and add it to the browser's cookies
     cookie_dict = {settings.SESSION_COOKIE_NAME: request.session.session_key}
-    try:
-        world.browser.cookies.add(cookie_dict)
-
-    # WebDriver has an issue where we cannot set cookies
-    # before we make a GET request, so if we get an error,
-    # we load the '/' page and try again
-    except:
-        world.browser.visit(django_url('/'))
-        world.browser.cookies.add(cookie_dict)
+    world.browser.cookies.delete()
+    world.browser.cookies.add(cookie_dict)
 
 
 @world.absorb

common/djangoapps/terrain/ui_helpers.py

@@ -53,12 +53,9 @@ def css_find(css):
 
 @world.absorb
 def css_click(css_selector):
-    '''
-    First try to use the regular click method,
-    but if clicking in the middle of an element
-    doesn't work it might be that it thinks some other
-    element is on top of it there so click in the upper left
-    '''
+    """
+    Perform a click on a CSS selector, retrying if it initially fails
+    """
     try:
         world.browser.find_by_css(css_selector).click()
 

doc/testing.md

@@ -161,36 +161,36 @@ try running `bundle install` to install the required ruby gems.
 We use [Lettuce](http://lettuce.it/) for acceptance testing.
 Most of our tests use [Splinter](http://splinter.cobrateam.info/)
 to simulate UI browser interactions.  Splinter, in turn,
-uses [Selenium](http://docs.seleniumhq.org/) to control the browser.
+uses [Selenium](http://docs.seleniumhq.org/) to control the Chrome browser.
 
 **Prerequisite**: You must have [ChromeDriver](https://code.google.com/p/selenium/wiki/ChromeDriver) 
-installed to run the tests in Chrome.  
+installed to run the tests in Chrome.  The tests are confirmed to run
+with Chrome (not Chromium) version 26.0.0.1410.63 with ChromeDriver
+version r195636.
 
-Before running the tests, you need to set up the test database:
+To run all the acceptance tests:
 
-    rm ../db/test_mitx.db
-    rake django-admin[syncdb,lms,acceptance,--noinput]
-    rake django-admin[migrate,lms,acceptance,--noinput]
+    rake test_acceptance_lms
+    rake test_acceptance_cms
 
-To run the acceptance tests:
-
-1. Start the Django server locally using the settings in **acceptance.py**:
+To test only a specific feature:
 
-        rake lms[acceptance]
+    rake test_acceptance_lms[lms/djangoapps/courseware/features/problems.feature]
 
-2. In another shell, run the tests:
+To start the debugger on failure, add the `--pdb` option:
 
-        django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/portal/features/
+    rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"]
 
-To test only a specific feature:
+To run tests faster by not collecting static files, you can use
+`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`.
 
-    django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/courseware/features/high-level-tabs.feature
 
 **Troubleshooting**: If you get an error message that says something about harvest not being a command, you probably are missing a requirement.
 Try running:
 
     pip install -r requirements.txt
 
+**Note**: The acceptance tests can *not* currently run in parallel.  
 
 ## Viewing Test Coverage
 

jenkins/test.sh

@@ -78,7 +78,7 @@ rake test_lms[false] || TESTS_FAILED=1
 rake test_common/lib/capa || TESTS_FAILED=1
 rake test_common/lib/xmodule || TESTS_FAILED=1
 
-# Run the jaavascript unit tests
+# Run the javascript unit tests
 rake phantomjs_jasmine_lms || TESTS_FAILED=1
 rake phantomjs_jasmine_cms || TESTS_FAILED=1
 rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1

jenkins/test_acceptance.sh

+#! /bin/bash
+
+set -e
+set -x
+
+git remote prune origin
+
+# Reset the submodule, in case it changed
+git submodule foreach 'git reset --hard HEAD'
+
+# Set the IO encoding to UTF-8 so that askbot will start
+export PYTHONIOENCODING=UTF-8
+
+if [ ! -d /mnt/virtualenvs/"$JOB_NAME" ]; then
+    mkdir -p /mnt/virtualenvs/"$JOB_NAME"
+    virtualenv /mnt/virtualenvs/"$JOB_NAME"
+fi
+
+export PIP_DOWNLOAD_CACHE=/mnt/pip-cache
+
+source /mnt/virtualenvs/"$JOB_NAME"/bin/activate
+rake install_prereqs
+rake clobber
+
+TESTS_FAILED=0
+
+# Assumes that Xvfb has been started by upstart
+# and is capturing display :1
+# The command for this is:
+# /usr/bin/Xvfb :1 -screen 0 1024x268x24
+# This allows us to run Chrome without a display
+export DISPLAY=:1
+
+# Run the lms and cms acceptance tests
+# (the -v flag turns off color in the output)
+rake test_acceptance_lms["-v 3"] || TESTS_FAILED=1
+rake test_acceptance_cms["-v 3"] || TESTS_FAILED=1
+
+[ $TESTS_FAILED == '0' ]

lms/envs/acceptance.py

@@ -8,13 +8,17 @@ from .test import *
 # otherwise the browser will not render the pages correctly
 DEBUG = True
 
+# Disable warnings for acceptance tests, to make the logs readable
+import logging
+logging.disable(logging.ERROR)
+
 # Use the mongo store for acceptance tests
 modulestore_options = {
     'default_class': 'xmodule.raw_module.RawDescriptor',
     'host': 'localhost',
     'db': 'test_xmodule',
-    'collection': 'modulestore',
-    'fs_root': GITHUB_REPO_ROOT,
+    'collection': 'acceptance_modulestore',
+    'fs_root': TEST_ROOT / "data",
     'render_template': 'mitxmako.shortcuts.render_to_string',
 }
 
@@ -33,7 +37,7 @@ CONTENTSTORE = {
     'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
     'OPTIONS': {
         'host': 'localhost',
-        'db': 'test_xcontent',
+        'db': 'test_xmodule',
     }
 }
 
@@ -43,8 +47,8 @@ CONTENTSTORE = {
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ENV_ROOT / "db" / "test_mitx.db",
-        'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db",
+        'NAME': TEST_ROOT / "db" / "test_mitx.db",
+        'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db",
     }
 }
 

rakefiles/tests.rake

@@ -24,6 +24,13 @@ def run_tests(system, report_dir, stop_on_failure=true)
     end
 end
 
+def run_acceptance_tests(system, report_dir, harvest_args)
+    sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
+    sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
+    sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
+end
+
+
 directory REPORT_DIR
 
 task :clean_test_files do
@@ -46,6 +53,17 @@ TEST_TASK_DIRS = []
         run_tests(system, report_dir, args.stop_on_failure)
     end
 
+    # Run acceptance tests
+    desc "Run acceptance tests"
+    task "test_acceptance_#{system}", [:harvest_args] => ["#{system}:gather_assets:acceptance", "fasttest_acceptance_#{system}"]
+
+    desc "Run acceptance tests without collectstatic"
+    task "fasttest_acceptance_#{system}", [:harvest_args] => ["clean_test_files", :predjango, report_dir] do |t, args|
+        args.with_defaults(:harvest_args => '')
+        run_acceptance_tests(system, report_dir, args.harvest_args)
+    end
+
+
     task :fasttest => "fasttest_#{system}"
 
     TEST_TASK_DIRS << system
@@ -116,4 +134,4 @@ namespace :coverage do
             sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc")
         end
     end
-end
\ No newline at end of file
+end