Run all tests on jenkins

We used to specify specific rake test tasks so that we could run all of
them even if early ones failed. However, that meant that as new tasks
were added, they weren't being run on jenkins.

Now, there is a facility in the rake scripts so that tests can run using
the test_sh function, which will delay failure until the end of the rake
run, unless the TESTS_FAIL_FAST environment variable is set.

Furthermore, this reorganizes the jasmine test tasks so that we can run
those as part of `rake test` as well.

doc/testing.md

@@ -141,21 +141,36 @@ Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you i
 
 ### Running Javascript Unit Tests
 
-These commands start a development server with jasmine testing enabled, and launch your default browser
-pointing to those tests
+To run all of the javascript unit tests, use
 
-    rake browse_jasmine_{lms,cms}
+    rake jasmine
 
-To run the tests headless, you must install [phantomjs](http://phantomjs.org/download.html), then run:
+If the `phantomjs` binary is on the path, or the `PHANTOMJS_PATH` environment variable is
+set to point to it, then the tests will be run headless. Otherwise, they will be run in
+your default browser
 
-    rake phantomjs_jasmine_{lms,cms}
+    export PATH=/path/to/phantomjs:$PATH
+    rake jasmine  # Runs headless
 
-If the `phantomjs` binary is not on the path, set the `PHANTOMJS_PATH` environment variable to point to it
+or
+
+    PHANTOMJS_PATH=/path/to/phantomjs rake jasmine  # Runs headless
+
+or
+
+    rake jasmine  # Runs in browser
+
+You can also force a run using phantomjs or the browser using the commands
+
+    rake jasmine:browser  # Runs in browser
+    rake jasmine:phantomjs  # Runs headless
+
+You can run tests for a specific subsystems as well
 
-    PHANTOMJS_PATH=/path/to/phantomjs rake phantomjs_jasmine_{lms,cms}
+    rake jasmine:lms  # Runs all lms javascript unit tests using the default method
+    rake jasmine:cms:browser  # Runs all cms javascript unit tests in the browser
 
-Once you have run the `rake` command, your browser should open to
-to `http://localhost/_jasmine/`, which displays the test results.
+Use `rake -T` to get a list of all available subsystems
 
 **Troubleshooting**: If you get an error message while running the `rake` task,
 try running `bundle install` to install the required ruby gems.

jenkins/test.sh

@@ -70,23 +70,11 @@ rake clobber
 rake pep8 > pep8.log || cat pep8.log
 rake pylint > pylint.log || cat pylint.log
 
-TESTS_FAILED=0
-
-# Run the python unit tests
-rake test_cms || TESTS_FAILED=1
-rake test_lms || TESTS_FAILED=1
-rake test_common/lib/capa || TESTS_FAILED=1
-rake test_common/lib/xmodule || TESTS_FAILED=1
-
-# Run the javascript unit tests
-rake phantomjs_jasmine_lms || TESTS_FAILED=1
-rake phantomjs_jasmine_cms || TESTS_FAILED=1
-rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1
-rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1
+# Run the unit tests (use phantomjs for javascript unit tests)
+rake test
 
 rake coverage:xml coverage:html
 
-[ $TESTS_FAILED == '0' ]
 rake autodeploy_properties
 
 github_status state:success "passed"

rakefiles/deprecated.rake

+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by)
+    task deprecated do
+        puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red)
+        sleep(5)
+        Rake::Task[deprecated_by].invoke
+    end
+end
+
+[:lms, :cms].each do |system|
+    deprecated("browse_jasmine_#{system}", "jasmine:#{system}:browser")
+    deprecated("phantomjs_jasmine_#{system}", "jasmine:#{system}:phantomjs")
+end
+
+Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
+    deprecated("browse_jasmine_#{lib}", "jasmine:#{lib}:browser")
+    deprecated("phantomjs_jasmine_#{lib}", "jasmine:#{lib}:phantomjs")
+end
+
+deprecated("browse_jasmine_discussion", "jasmine:common/static/coffee:browser")
+deprecated("phantomjs_jasmine_discussion", "jasmine:common/static/coffee:phantomjs")
\ No newline at end of file

rakefiles/helpers.rb

 require 'digest/md5'
 
+def find_executable(exec)
+    path = %x(which #{exec}).strip
+    $?.exitstatus == 0 ? path : nil
+end
 
 def select_executable(*cmds)
-    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
+    cmds.find_all{ |cmd| !find_executable(cmd).nil? }[0] || fail("No executables found from #{cmds.join(', ')}")
 end
 
 def django_admin(system, env, command, *args)
@@ -85,3 +89,31 @@ def environments(system)
         env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')
     end
 end
+
+$failed_tests = 0
+
+# Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure.
+# Otherwise, a final task will be added that will fail if any tests have failed
+def test_sh(*args)
+    sh(*args) do |ok, res|
+        if ok
+            return
+        end
+
+        if ENV['TESTS_FAIL_FAST']
+            fail("Test failed!")
+        else
+            $failed_tests += 1
+        end
+    end
+end
+
+# Add a task after all other tasks that fails if any tests have failed
+if !ENV['TESTS_FAIL_FAST']
+    task :fail_tests do
+        fail("#{$failed_tests} tests failed!") if $failed_tests > 0
+    end
+
+    Rake.application.top_level_tasks << :fail_tests
+end
+

rakefiles/jasmine.rake

@@ -3,6 +3,11 @@ require 'erb'
 require 'launchy'
 require 'net/http'
 
+PHANTOMJS_PATH = find_executable(ENV['PHANTOMJS_PATH'] || 'phantomjs')
+PREFERRED_METHOD = PHANTOMJS_PATH.nil? ? 'browser' : 'phantomjs'
+if PHANTOMJS_PATH.nil?
+    puts("phantomjs not found on path. Set $PHANTOMJS_PATH. Using browser for jasmine tests".blue)
+end
 
 def django_for_jasmine(system, django_reload)
     if !django_reload
@@ -35,18 +40,6 @@ def django_for_jasmine(system, django_reload)
 end
 
 def template_jasmine_runner(lib)
-    case lib
-    when /common\/lib\/.+/
-        coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"]
-    when /common\/static\/coffee/
-        coffee_files = Dir["#{lib}/**/*.coffee"]
-    else
-        puts('I do not know how to run jasmine tests for #{lib}')
-        exit
-    end
-    if !coffee_files.empty?
-        sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}")
-    end
     phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine")
     jasmine_reporters_path = File.expand_path("node_modules/jasmine-reporters")
     common_js_root = File.expand_path("common/static/js")
@@ -54,8 +47,8 @@ def template_jasmine_runner(lib)
 
     # Get arrays of spec and source files, ordered by how deep they are nested below the library
     # (and then alphabetically) and expanded from a relative to an absolute path
-    spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js")
-    src_glob = File.join("#{lib}", "**", "src", "**", "*.js")
+    spec_glob = File.join(lib, "**", "spec", "**", "*.js")
+    src_glob = File.join(lib, "**", "src", "**", "*.js")
     js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
     js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
 
@@ -68,74 +61,90 @@ def template_jasmine_runner(lib)
     yield File.expand_path(template_output)
 end
 
-def run_phantom_js(url)
-    phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
-    sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
-end
-
-# Open jasmine tests for :system in the default browser. The :env
-# should (always?) be 'jasmine', but it's passed as an arg so that
-# the :assets dependency gets it.
-#
-# This task should be invoked via the wrapper below, so we don't
-# include a description to keep it from showing up in rake -T.
-task :browse_jasmine, [:system, :env] => :assets do |t, args|
-    django_for_jasmine(args.system, true) do |jasmine_url|
-        Launchy.open(jasmine_url)
-        puts "Press ENTER to terminate".red
-        $stdin.gets
-    end
+def jasmine_browser(url, wait=10)
+    # Jitter starting the browser so that the tests don't all try and
+    # start the browser simultaneously
+    sleep(rand(3))
+    sh("python -m webbrowser -t '#{url}'")
+    sleep(wait)
 end
 
-# Use phantomjs to run jasmine tests from the console. The :env
-# should (always?) be 'jasmine', but it's passed as an arg so that
-# the :assets dependency gets it.
-#
-# This task should be invoked via the wrapper below, so we don't
-# include a description to keep it from showing up in rake -T.
-task :phantomjs_jasmine, [:system, :env] => :assets do |t, args|
-    django_for_jasmine(args.system, false) do |jasmine_url|
-        run_phantom_js(jasmine_url)
-    end
+def jasmine_phantomjs(url)
+    fail("phantomjs not found. Add it to your path, or set $PHANTOMJS_PATH") if PHANTOMJS_PATH.nil?
+    test_sh("#{PHANTOMJS_PATH} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
 end
 
 # Wrapper tasks for the real browse_jasmine and phantomjs_jasmine
 # tasks above. These have a nicer UI since there's no arg passing.
 [:lms, :cms].each do |system|
-    desc "Open jasmine tests for #{system} in your default browser"
-    task "browse_jasmine_#{system}" do
-        Rake::Task[:browse_jasmine].invoke(system, 'jasmine')
-    end
+    namespace :jasmine do
+        namespace system do
+            desc "Open jasmine tests for #{system} in your default browser"
+            task :browser do
+                Rake::Task[:assets].invoke(system, 'jasmine')
+                django_for_jasmine(system, true) do |jasmine_url|
+                    jasmine_browser(jasmine_url)
+                end
+            end
+
+            desc "Use phantomjs to run jasmine tests for #{system} from the console"
+            task :phantomjs do
+                Rake::Task[:assets].invoke(system, 'jasmine')
+                phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
+                django_for_jasmine(system, false) do |jasmine_url|
+                    jasmine_phantomjs(jasmine_url)
+                end
+            end
+        end
+
+        desc "Run jasmine tests for #{system} using #{PREFERRED_METHOD}"
+        task system => "jasmine:#{system}:#{PREFERRED_METHOD}"
 
-    desc "Use phantomjs to run jasmine tests for #{system} from the console"
-    task "phantomjs_jasmine_#{system}" do
-        Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine')
+        task :phantomjs => "jasmine:#{system}:phantomjs"
+        multitask :browser => "jasmine:#{system}:browser"
     end
 end
 
-STATIC_JASMINE_TESTS = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
-STATIC_JASMINE_TESTS << 'common/static/coffee'
-
-STATIC_JASMINE_TESTS.each do |lib|
-    desc "Open jasmine tests for #{lib} in your default browser"
-    task "browse_jasmine_#{lib}" do
-        template_jasmine_runner(lib) do |f|
-            sh("python -m webbrowser -t 'file://#{f}'")
-            puts "Press ENTER to terminate".red
-            $stdin.gets
+static_js_dirs = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
+static_js_dirs << 'common/static/coffee'
+static_js_dirs.select!{|lib| !Dir["#{lib}/**/spec"].empty?}
+
+static_js_dirs.each do |dir|
+    namespace :jasmine do
+        namespace dir do
+            desc "Open jasmine tests for #{dir} in your default browser"
+            task :browser do
+                # We need to use either CMS or LMS to preprocess files. Use LMS by default
+                Rake::Task['assets:coffee'].invoke('lms', 'jasmine')
+                template_jasmine_runner(dir) do |f|
+                    jasmine_browser("file://#{f}")
+                end
+            end
+
+            desc "Use phantomjs to run jasmine tests for #{dir} from the console"
+            task :phantomjs do
+                # We need to use either CMS or LMS to preprocess files. Use LMS by default
+                Rake::Task[:assets].invoke('lms', 'jasmine')
+                template_jasmine_runner(dir) do |f|
+                    jasmine_phantomjs(f)
+                end
+            end
         end
-    end
 
-    desc "Use phantomjs to run jasmine tests for #{lib} from the console"
-    task "phantomjs_jasmine_#{lib}" do
-        template_jasmine_runner(lib) do |f|
-            run_phantom_js(f)
-        end
+        desc "Run jasmine tests for #{dir} using #{PREFERRED_METHOD}"
+        task dir => "jasmine:#{dir}:#{PREFERRED_METHOD}"
+
+        task :phantomjs => "jasmine:#{dir}:phantomjs"
+        multitask :browser => "jasmine:#{dir}:browser"
     end
 end
 
-desc "Open jasmine tests for discussion in your default browser"
-task "browse_jasmine_discussion" => "browse_jasmine_common/static/coffee"
+desc "Run all jasmine tests using #{PREFERRED_METHOD}"
+task :jasmine => "jasmine:#{PREFERRED_METHOD}"
+
+['phantomjs', 'browser'].each do |method|
+    desc "Run all jasmine tests using #{method}"
+    task "jasmine:#{method}"
+end
 
-desc "Use phantomjs to run jasmine tests for discussion from the console"
-task "phantomjs_jasmine_discussion" => "phantomjs_jasmine_common/static/coffee"
+task :test => :jasmine

rakefiles/tests.rake

-
 # Set up the clean and clobber tasks
 CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
 
-$failed_tests = 0
-
 def run_under_coverage(cmd, root)
     cmd0, cmd_rest = cmd.split(" ", 2)
     # We use "python -m coverage" so that the proper python will run the importable coverage
@@ -17,12 +14,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
     dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
     test_id = dirs.join(' ') if test_id.nil? or test_id == ''
     cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id)
-    sh(run_under_coverage(cmd, system)) do |ok, res|
-        if !ok and stop_on_failure
-            abort "Test failed!"
-        end
-        $failed_tests += 1 unless ok
-    end
+    test_sh(run_under_coverage(cmd, system))
 end
 
 def run_acceptance_tests(system, report_dir, harvest_args)
@@ -38,7 +30,7 @@ def run_acceptance_tests(system, report_dir, harvest_args)
     end
     sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
     sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
-    sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
+    test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
 end
 
 
@@ -55,13 +47,13 @@ TEST_TASK_DIRS = []
 
     # Per System tasks
     desc "Run all django tests on our djangoapps for the #{system}"
-    task "test_#{system}", [:test_id, :stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
+    task "test_#{system}", [:test_id] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
 
     # Have a way to run the tests without running collectstatic -- useful when debugging without
     # messing with static files.
-    task "fasttest_#{system}", [:test_id, :stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args|
-        args.with_defaults(:stop_on_failure => 'true', :test_id => nil)
-        run_tests(system, report_dir, args.test_id, args.stop_on_failure)
+    task "fasttest_#{system}", [:test_id] => [report_dir, :install_prereqs, :predjango] do |t, args|
+        args.with_defaults(:test_id => nil)
+        run_tests(system, report_dir, args.test_id)
     end
 
     # Run acceptance tests
@@ -89,9 +81,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
     task task_name => report_dir do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
         cmd = "nosetests #{lib}"
-        sh(run_under_coverage(cmd, lib)) do |ok, res|
-            $failed_tests += 1 unless ok
-        end
+        test_sh(run_under_coverage(cmd, lib))
     end
     TEST_TASK_DIRS << lib
 
@@ -107,17 +97,11 @@ TEST_TASK_DIRS.each do |dir|
     report_dir = report_dir_path(dir)
     directory report_dir
     task :report_dirs => [REPORT_DIR, report_dir]
+    task :test => "test_#{dir}"
 end
 
-task :test do
-    TEST_TASK_DIRS.each do |dir|
-        Rake::Task["test_#{dir}"].invoke(nil, false)
-    end
-
-    if $failed_tests > 0
-        abort "Tests failed!"
-    end
-end
+desc "Run all tests"
+task :test
 
 namespace :coverage do
     desc "Build the html coverage reports"