Merge pull request #2428 from edx/will/i18n-test-results

Better test console reports; i18n XUnit results

i18n/config.py

@@ -5,7 +5,7 @@ from path import path
 
 # BASE_DIR is the working directory to execute django-admin commands from.
 # Typically this should be the 'edx-platform' directory.
-BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath()
+BASE_DIR = path(__file__).abspath().dirname().dirname()
 
 # LOCALE_DIR contains the locale files.
 # Typically this should be 'edx-platform/conf/locale'

rakelib/acceptance_test.rake

@@ -10,7 +10,11 @@ def run_acceptance_tests(system, harvest_args)
 
     report_file = File.join(ACCEPTANCE_REPORT_DIR, "#{system}.xml")
     report_args = "--with-xunit --xunit-file #{report_file}"
-    test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', report_args, harvest_args))
+    cmd = django_admin(
+        system, 'acceptance', 'harvest', '--debug-mode',
+        '--verbosity 2', report_args, harvest_args
+    )
+    test_sh("#{system} acceptance tests", cmd)
 end
 
 task :setup_acceptance_db do

rakelib/docs.rake

@@ -42,4 +42,18 @@ desc "Build docs and show them in browser"
 task :doc, [:type, :quiet] =>  :builddocs do |t, args|
     Rake::Task["showdocs"].invoke(args.type, args.quiet)
 end
-# --- Develop and public documentation ---
+
+
+# Run documentation tests
+desc "Run documentation tests"
+task :test_docs do
+    # Be sure that sphinx can build docs w/o exceptions.
+    test_message = "If a docs test fails, you should run '%s' and look at whole output and fix exceptions.
+(You shouldn't fix rst warnings and errors for this to pass, just get rid of exceptions.)"
+    puts (test_message  % ["rake doc[docs,verbose]"]).colorize( :light_green )
+    test_sh('docs', 'rake builddocs')
+end
+
+
+# Add documentation tests to the main test command
+task :test => :'test_docs'

rakelib/helpers.rb

@@ -119,11 +119,16 @@ def environments(system)
     end
 end
 
-$failed_tests = 0
+
+$failed_tests = []
 
 # Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure.
 # Otherwise, a final task will be added that will fail if any tests have failed
-def test_sh(*args)
+def test_sh(name, *args)
+    puts("\n=======================================".green)
+    puts("Running #{name} tests".green)
+    puts("=======================================".green)
+
     sh(*args) do |ok, res|
         if ok
             return
@@ -132,15 +137,24 @@ def test_sh(*args)
         if ENV['TESTS_FAIL_FAST']
             fail("Test failed!")
         else
-            $failed_tests += 1
+            $failed_tests << name
         end
     end
+
+    puts("\n=======================================\n".green)
 end
 
 # Add a task after all other tasks that fails if any tests have failed
 if !ENV['TESTS_FAIL_FAST']
     task :fail_tests do
-        fail("#{$failed_tests} tests failed!") if $failed_tests > 0
+        if $failed_tests.length > 0
+            puts("=======================================".red)
+            puts("Tests failed in these test suites:".red)
+            $failed_tests.each do |test|
+                puts("* #{test}".red)
+            end
+            exit 1
+        end
     end
 
     Rake.application.top_level_tasks << :fail_tests

rakelib/i18n.rake

 # --- Internationalization tasks
 
+I18N_REPORT_DIR = report_dir_path('i18n')
+I18N_XUNIT_REPORT = File.join(I18N_REPORT_DIR, 'nosetests.xml')
+
+directory I18N_REPORT_DIR
+
+
 namespace :i18n do
 
   desc "Extract localizable strings from sources"
@@ -57,10 +63,9 @@ namespace :i18n do
   end
 
   desc "Run tests for the internationalization library"
-  task :test do
-    test = File.join(REPO_ROOT, "i18n", "tests")
+  task :test => [I18N_REPORT_DIR, :clean_reports_dir] do
     pythonpath_prefix = "PYTHONPATH=#{REPO_ROOT}/i18n:$PYTHONPATH"
-    sh("#{pythonpath_prefix} nosetests #{test}")
+    test_sh("i18n", "#{pythonpath_prefix} nosetests #{REPO_ROOT}/i18n/tests --with-xunit --xunit-file=#{I18N_XUNIT_REPORT}")
   end
 
   # Commands for automating the process of including translations in edx-platform.
@@ -79,3 +84,7 @@ namespace :i18n do
   end
 
 end
+
+
+# Add i18n tests to the main test command
+task :test => :'i18n:test'

rakelib/js_test.rake

@@ -40,7 +40,7 @@ def js_test_tool(env, command, do_coverage)
         cmd += " --coverage-xml #{report_dir}"
     end
 
-    test_sh(cmd)
+    test_sh("javascript", cmd)
 end
 
 # Print a list of js_test commands for

rakelib/tests.rake

@@ -43,17 +43,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
     end
 
     cmd = django_admin(system, :test, 'test', test_id)
-    test_sh(run_under_coverage(cmd, system))
-end
-
-# Run documentation tests
-desc "Run documentation tests"
-task :test_docs do
-    # Be sure that sphinx can build docs w/o exceptions.
-    test_message = "If test fails, you shoud run '%s' and look at whole output and fix exceptions.
-(You shouldn't fix rst warnings and errors for this to pass, just get rid of exceptions.)"
-    puts (test_message  % ["rake doc[docs,verbose]"]).colorize( :light_green )
-    test_sh('rake builddocs')
+    test_sh(system, run_under_coverage(cmd, system))
 end
 
 task :clean_test_files do
@@ -113,7 +103,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
         args.with_defaults(:test_id => lib)
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
         cmd = "nosetests --id-file=#{test_ids} #{args.test_id}"
-        test_sh(run_under_coverage(cmd, lib))
+        test_sh(lib, run_under_coverage(cmd, lib))
     end
     TEST_TASK_DIRS << lib
 
@@ -140,9 +130,6 @@ namespace :test do
     task :python, [:test_id]
 end
 
-desc "Run all tests"
-task :test, [:test_id] => [:test_docs, 'test:python', 'i18n:test']
-
 desc "Build the html, xml, and diff coverage reports"
 task :coverage => :report_dirs do
 
@@ -176,3 +163,7 @@ task :coverage => :report_dirs do
         puts "\n"
     end
 end
+
+# Other Rake files append additional tests to the main test command.
+desc "Run all unit tests"
+task :test, [:test_id] => 'test:python'