Write xunit reports for acceptance tests

Refactor rake tasks for acceptance tests
Address PR comments

docs/internal/testing.md

@@ -205,24 +205,27 @@ with Chrome (not Chromium) version 28.0.1500.71 with ChromeDriver
 version 2.1.210398.
 
 To run all the acceptance tests:
+    rake test:acceptance
 
-    rake test_acceptance_lms
-    rake test_acceptance_cms
+To run only for lms or cms:
+
+    rake test:acceptance:lms
+    rake test:acceptance:cms
 
 To test only a specific feature:
 
-    rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature"]
+    rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature"]
 
 To test only a specific scenario
 
-    rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature -s 3"]
+    rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature -s 3"]
 
 To start the debugger on failure, add the `--pdb` option:
 
-    rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"]
+    rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature --pdb"]
 
 To run tests faster by not collecting static files, you can use
-`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`.
+`rake test:acceptance:lms:fast` and `rake test:acceptance:cms:fast`.
 
 Acceptance tests will run on a randomized port and can be run in the background of rake cms and lms or unit tests.
 To specify the port, change the LETTUCE_SERVER_PORT constant in cms/envs/acceptance.py and lms/envs/acceptance.py

jenkins/test_acceptance.sh

@@ -44,8 +44,6 @@ if [ "$LETTUCE_SELENIUM_CLIENT" == saucelabs ]; then
 fi
 
 # Run the lms and cms acceptance tests
-# (the -v flag turns off color in the output)
-rake test_acceptance_lms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1
-rake test_acceptance_cms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1
+rake test:acceptance["$SKIP_TESTS"] || TESTS_FAILED=1
 
 [ $TESTS_FAILED == '0' ]

rakelib/acceptance_test.rake

+ACCEPTANCE_DB = 'test_root/db/test_edx.db'
+ACCEPTANCE_REPORT_DIR = report_dir_path('acceptance')
+directory ACCEPTANCE_REPORT_DIR
+
+def run_acceptance_tests(system, harvest_args)
+    # Create the acceptance report directory
+    # because if it doesn't exist then lettuce will give an IOError.
+    report_dir = report_dir_path('acceptance')
+
+    report_file = File.join(ACCEPTANCE_REPORT_DIR, "#{system}.xml")
+    report_args = "--with-xunit --xunit-file #{report_file}"
+    test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', report_args, harvest_args))
+end
+
+task :setup_acceptance_db do
+    # HACK: Since the CMS depends on the existence of some database tables
+    # that are now in common but used to be in LMS (Role/Permissions for Forums)
+    # we need to create/migrate the database tables defined in the LMS.
+    # We might be able to address this by moving out the migrations from
+    # lms/django_comment_client, but then we'd have to repair all the existing
+    # migrations from the upgrade tables in the DB.
+    # But for now for either system (lms or cms), use the lms
+    # definitions to sync and migrate.
+    if File.exists?(ACCEPTANCE_DB)
+        File.delete(ACCEPTANCE_DB)
+    end
+
+    sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
+    sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
+end
+
+task :prep_for_acceptance_tests => [
+    :clean_reports_dir, :clean_test_files, ACCEPTANCE_REPORT_DIR,
+    :install_prereqs, :setup_acceptance_db
+]
+
+namespace :test do
+    namespace :acceptance do
+        task :all, [:harvest_args] => [
+            :prep_for_acceptance_tests,
+            "^^lms:gather_assets:acceptance",
+            "^^cms:gather_assets:acceptance"
+        ] do |t, args|
+            run_acceptance_tests('lms', args.harvest_args)
+            run_acceptance_tests('cms', args.harvest_args)
+        end
+
+        ['lms', 'cms'].each do |system|
+            desc "Run the acceptance tests for the #{system}"
+            task system, [:harvest_args] => [
+                :prep_for_acceptance_tests,
+                "^^#{system}:gather_assets:acceptance"
+            ] do |t, args|
+                args.with_defaults(:harvest_args => '')
+                run_acceptance_tests(system, args.harvest_args)
+            end
+
+            desc "Run acceptance tests for the #{system} without collectstatic or db migrations"
+            task "#{system}:fast", [:harvest_args] => [
+                :clean_reports_dir, ACCEPTANCE_REPORT_DIR,
+            ] do |t, args|
+                args.with_defaults(:harvest_args => '')
+                run_acceptance_tests(system, args.harvest_args)
+            end
+        end
+    end
+    desc "Run the lettuce acceptance tests for lms and cms"
+    task :acceptance, [:harvest_args] do |t, args|
+      Rake::Task["test:acceptance:all"].invoke(args.harvest_args)
+    end
+end

rakelib/deprecated.rake

@@ -24,6 +24,8 @@ end
     deprecated("jasmine:#{system}:phantomjs", "test:js:run", system)
     deprecated("#{system}:check_settings:jasmine", "")
     deprecated("#{system}:gather_assets:jasmine", "")
+    deprecated("test_acceptance_#{system}", "test:acceptance:#{system}")
+    deprecated("fasttest_acceptance_#{system}", "test:acceptance:#{system}:fast")
 end
 
 Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
@@ -49,3 +51,4 @@ deprecated("jasmine:common/static/coffee:phantomjs", "test:js:run", "common")
 deprecated("jasmine", "test:js")
 deprecated("jasmine:phantomjs", "test:js:run")
 deprecated("jasmine:browser", "test:js:dev")
+deprecated("test_acceptance", "test:acceptance")

rakelib/tests.rake

@@ -4,8 +4,6 @@ CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
 # Create the directory to hold coverage reports, if it doesn't already exist.
 directory REPORT_DIR
 
-ACCEPTANCE_DB = 'test_root/db/test_edx.db'
-
 def test_id_dir(path)
     return File.join(".testids", path.to_s)
 end
@@ -32,38 +30,6 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
     test_sh(run_under_coverage(cmd, system))
 end
 
-def create_acceptance_db(system)
-    # HACK: Since now the CMS depends on the existence of some database tables
-    # that used to be in LMS (Role/Permissions for Forums) we need to make
-    # sure the acceptance tests create/migrate the database tables
-    # that are represented in the LMS. We might be able to address this by moving
-    # out the migrations from lms/django_comment_client, but then we'd have to
-    # repair all the existing migrations from the upgrade tables in the DB.
-    if system == :cms
-        sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
-        sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
-    end
-    sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
-    sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
-end
-
-def setup_acceptance_db(system, fasttest=false)
-    # If running under fasttest mode and the database already
-    # exists, skip the migrations.
-    if File.exists?(ACCEPTANCE_DB)
-        if not fasttest
-            File.delete(ACCEPTANCE_DB)
-            create_acceptance_db(system)
-        end
-    else
-        create_acceptance_db(system)
-    end
-end
-
-def run_acceptance_tests(system, report_dir, harvest_args)
-    test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', harvest_args))
-end
-
 # Run documentation tests
 desc "Run documentation tests"
 task :test_docs do
@@ -110,26 +76,6 @@ TEST_TASK_DIRS = []
         run_tests(system, report_dir, args.test_id)
     end
 
-    # Run acceptance tests
-    desc "Run acceptance tests"
-    task "test_acceptance_#{system}", [:harvest_args] => [
-        :clean_test_files, :install_prereqs,
-        "#{system}:gather_assets:acceptance"
-    ] do |t, args|
-        setup_acceptance_db(system)
-        Rake::Task["fasttest_acceptance_#{system}"].invoke(args.harvest_args)
-    end
-
-    desc "Run acceptance tests without collectstatic or database migrations"
-    task "fasttest_acceptance_#{system}", [:harvest_args] => [
-        report_dir, :clean_reports_dir
-    ] do |t, args|
-        args.with_defaults(:harvest_args => '')
-        setup_acceptance_db(system, fasttest=true)
-        run_acceptance_tests(system, report_dir, args.harvest_args)
-    end
-
-
     task :fasttest => "fasttest_#{system}"
 
     TEST_TASK_DIRS << system