Merge pull request #44 from edx/feature/cale/run-all-tests

Run all tests on jenkins

common/lib/xmodule/xmodule/static_content.py

@@ -58,7 +58,12 @@ def _ensure_dir(dir_):
 
 
 def _write_styles(selector, output_root, classes):
-    _ensure_dir(output_root)
+    """
+    Write the css fragments from all XModules in `classes`
+    into `output_root` as individual files, hashed by the contents to remove
+    duplicates
+    """
+    contents = {}
 
     css_fragments = defaultdict(set)
     for class_ in classes:
@@ -73,25 +78,34 @@ def _write_styles(selector, output_root, classes):
             hash=hashlib.md5(fragment).hexdigest(),
             type=filetype)
         # Prepend _ so that sass just includes the files into a single file
-        with open(output_root / '_' + fragment_name, 'w') as css_file:
-            css_file.write(fragment)
+        filename = '_' + fragment_name
+        contents[filename] = fragment
 
         for class_ in classes:
             css_imports[class_].add(fragment_name)
 
-    with open(output_root / '_module-styles.scss', 'w') as module_styles:
-
-        module_styles.write("@import 'bourbon/bourbon';\n")
-        module_styles.write("@import 'bourbon/addons/button';\n")
+    module_styles_lines = []
+    module_styles_lines.append("@import 'bourbon/bourbon';")
+    module_styles_lines.append("@import 'bourbon/addons/button';")
     for class_, fragment_names in css_imports.items():
-            imports = "\n".join('@import "{0}";'.format(name) for name in fragment_names)
-            module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}\n""".format(
-                class_=class_, imports=imports, selector=selector
+        module_styles_lines.append("""{selector}.xmodule_{class_} {{""".format(
+            class_=class_, selector=selector
         ))
+        module_styles_lines.extend('  @import "{0}";'.format(name) for name in fragment_names)
+        module_styles_lines.append('}')
+
+    contents['_module-styles.scss'] = '\n'.join(module_styles_lines)
+
+    _write_files(output_root, contents)
 
 
 def _write_js(output_root, classes):
-    _ensure_dir(output_root)
+    """
+    Write the javascript fragments from all XModules in `classes`
+    into `output_root` as individual files, hashed by the contents to remove
+    duplicates
+    """
+    contents = {}
 
     js_fragments = set()
     for class_ in classes:
@@ -100,18 +114,25 @@ def _write_js(output_root, classes):
             for idx, fragment in enumerate(module_js.get(filetype, [])):
                 js_fragments.add((idx, filetype, fragment))
 
-    module_js = []
     for idx, filetype, fragment in sorted(js_fragments):
-        path = output_root / "{idx:0=3d}-{hash}.{type}".format(
+        filename = "{idx:0=3d}-{hash}.{type}".format(
             idx=idx,
             hash=hashlib.md5(fragment).hexdigest(),
             type=filetype)
-        with open(path, 'w') as js_file:
-            js_file.write(fragment)
+        contents[filename] = fragment
+
+    _write_files(output_root, contents)
 
-        module_js.append(path)
+    return [output_root / filename for filename in contents.keys()]
+
+
+def _write_files(output_root, contents):
+    _ensure_dir(output_root)
+    for extra_file in set(output_root.files()) - set(contents.keys()):
+        extra_file.remove()
 
-    return module_js
+    for filename, file_content in contents.iteritems():
+        (output_root / filename).write_bytes(file_content)
 
 
 def main():
@@ -122,7 +143,6 @@ def main():
     args = docopt(main.__doc__)
     root = path(args['<output_root>'])
 
-    root.rmtree(ignore_errors=True)
     write_descriptor_js(root / 'descriptors/js')
     write_descriptor_styles(root / 'descriptors/css')
     write_module_js(root / 'modules/js')

doc/testing.md

@@ -141,21 +141,36 @@ Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you i
 
 ### Running Javascript Unit Tests
 
-These commands start a development server with jasmine testing enabled, and launch your default browser
-pointing to those tests
+To run all of the javascript unit tests, use
 
-    rake browse_jasmine_{lms,cms}
+    rake jasmine
 
-To run the tests headless, you must install [phantomjs](http://phantomjs.org/download.html), then run:
+If the `phantomjs` binary is on the path, or the `PHANTOMJS_PATH` environment variable is
+set to point to it, then the tests will be run headless. Otherwise, they will be run in
+your default browser
 
-    rake phantomjs_jasmine_{lms,cms}
+    export PATH=/path/to/phantomjs:$PATH
+    rake jasmine  # Runs headless
 
-If the `phantomjs` binary is not on the path, set the `PHANTOMJS_PATH` environment variable to point to it
+or
+
+    PHANTOMJS_PATH=/path/to/phantomjs rake jasmine  # Runs headless
+
+or
+
+    rake jasmine  # Runs in browser
+
+You can also force a run using phantomjs or the browser using the commands
+
+    rake jasmine:browser  # Runs in browser
+    rake jasmine:phantomjs  # Runs headless
+
+You can run tests for a specific subsystems as well
 
-    PHANTOMJS_PATH=/path/to/phantomjs rake phantomjs_jasmine_{lms,cms}
+    rake jasmine:lms  # Runs all lms javascript unit tests using the default method
+    rake jasmine:cms:browser  # Runs all cms javascript unit tests in the browser
 
-Once you have run the `rake` command, your browser should open to
-to `http://localhost/_jasmine/`, which displays the test results.
+Use `rake -T` to get a list of all available subsystems
 
 **Troubleshooting**: If you get an error message while running the `rake` task,
 try running `bundle install` to install the required ruby gems.

jenkins/test.sh

@@ -70,24 +70,12 @@ rake clobber
 rake pep8 > pep8.log || cat pep8.log
 rake pylint > pylint.log || cat pylint.log
 
-TESTS_FAILED=0
-
-# Run the python unit tests
-rake test_cms || TESTS_FAILED=1
-rake test_lms || TESTS_FAILED=1
-rake test_common/lib/capa || TESTS_FAILED=1
-rake test_common/lib/xmodule || TESTS_FAILED=1
-
-# Run the javascript unit tests
-rake phantomjs_jasmine_lms || TESTS_FAILED=1
-rake phantomjs_jasmine_cms || TESTS_FAILED=1
-rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1
-rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1
+# Run the unit tests (use phantomjs for javascript unit tests)
+rake test
 
 # Generate coverage reports
 rake coverage
 
-[ $TESTS_FAILED == '0' ]
 rake autodeploy_properties
 
 github_status state:success "passed"

rakefiles/assets.rake

@@ -13,6 +13,7 @@ def xmodule_cmd(watch=false, debug=false)
                   "--patterns='*.js;*.coffee;*.sass;*.scss;*.css' " +
                   "--recursive " +
                   "--command='#{xmodule_cmd}' " +
+                  "--wait " +
                   "common/lib/xmodule"
     else
         xmodule_cmd
@@ -27,15 +28,16 @@ def coffee_cmd(watch=false, debug=false)
         #
         # Ref: https://github.com/joyent/node/issues/2479
         #
-        # Rather than watching all of the directories in one command
-        # watch each static files subdirectory separately
-        cmds = []
-        ['lms/static/coffee', 'cms/static/coffee', 'common/static/coffee', 'common/static/xmodule'].each do |coffee_folder|
-            cmds << "node_modules/.bin/coffee --watch --compile #{coffee_folder}"
-        end
-        cmds
+        # So, instead, we use watchmedo, which works around the problem
+        "watchmedo shell-command " +
+                  "--command 'node_modules/.bin/coffee -c ${watch_src_path}' " +
+                  "--recursive " +
+                  "--patterns '*.coffee' " +
+                  "--ignore-directories " +
+                  "--wait " +
+                  "."
     else
-        'node_modules/.bin/coffee --compile */static'
+        'node_modules/.bin/coffee --compile .'
     end
 end
 
@@ -75,8 +77,8 @@ namespace :assets do
         $stdin.gets
     end
 
-    {:xmodule => :install_python_prereqs,
-     :coffee => :install_node_prereqs,
+    {:xmodule => [:install_python_prereqs],
+     :coffee => [:install_node_prereqs],
      :sass => [:install_ruby_prereqs, :preprocess]}.each_pair do |asset_type, prereq_tasks|
         desc "Compile all #{asset_type} assets"
         task asset_type => prereq_tasks do
@@ -105,7 +107,8 @@ namespace :assets do
                 $stdin.gets
             end
 
-            task :_watch => prereq_tasks do
+            # Fully compile before watching for changes
+            task :_watch => (prereq_tasks + ["assets:#{asset_type}:debug"]) do
                 cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true)
                 if cmd.kind_of?(Array)
                     cmd.each {|c| background_process(c)}
@@ -118,19 +121,18 @@ namespace :assets do
 
     multitask :sass => 'assets:xmodule'
     namespace :sass do
-        # In watch mode, sass doesn't immediately compile out of date files,
-        # so force a recompile first
-        # Also force xmodule files to be generated before we start watching anything
-        task :_watch => ['assets:sass:debug', 'assets:xmodule']
         multitask :debug => 'assets:xmodule:debug'
     end
 
     multitask :coffee => 'assets:xmodule'
     namespace :coffee do
-        # Force xmodule files to be generated before we start watching anything
-        task :_watch => 'assets:xmodule'
         multitask :debug => 'assets:xmodule:debug'
     end
+
+    namespace :xmodule do
+        # Only start the xmodule watcher after the coffee and sass watchers have already started
+        task :_watch => ['assets:coffee:_watch', 'assets:sass:_watch']
+    end
 end
 
 # This task does the real heavy lifting to gather all of the static

rakefiles/deprecated.rake

+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by)
+    task deprecated do
+        puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red)
+        sleep(5)
+        Rake::Task[deprecated_by].invoke
+    end
+end
+
+[:lms, :cms].each do |system|
+    deprecated("browse_jasmine_#{system}", "jasmine:#{system}:browser")
+    deprecated("phantomjs_jasmine_#{system}", "jasmine:#{system}:phantomjs")
+end
+
+Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
+    deprecated("browse_jasmine_#{lib}", "jasmine:#{lib}:browser")
+    deprecated("phantomjs_jasmine_#{lib}", "jasmine:#{lib}:phantomjs")
+end
+
+deprecated("browse_jasmine_discussion", "jasmine:common/static/coffee:browser")
+deprecated("phantomjs_jasmine_discussion", "jasmine:common/static/coffee:phantomjs")
\ No newline at end of file

rakefiles/helpers.rb

 require 'digest/md5'
 
+def find_executable(exec)
+    path = %x(which #{exec}).strip
+    $?.exitstatus == 0 ? path : nil
+end
 
 def select_executable(*cmds)
-    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
+    cmds.find_all{ |cmd| !find_executable(cmd).nil? }[0] || fail("No executables found from #{cmds.join(', ')}")
 end
 
 def django_admin(system, env, command, *args)
@@ -85,3 +89,31 @@ def environments(system)
         env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')
     end
 end
+
+$failed_tests = 0
+
+# Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure.
+# Otherwise, a final task will be added that will fail if any tests have failed
+def test_sh(*args)
+    sh(*args) do |ok, res|
+        if ok
+            return
+        end
+
+        if ENV['TESTS_FAIL_FAST']
+            fail("Test failed!")
+        else
+            $failed_tests += 1
+        end
+    end
+end
+
+# Add a task after all other tasks that fails if any tests have failed
+if !ENV['TESTS_FAIL_FAST']
+    task :fail_tests do
+        fail("#{$failed_tests} tests failed!") if $failed_tests > 0
+    end
+
+    Rake.application.top_level_tasks << :fail_tests
+end
+

rakefiles/jasmine.rake

@@ -3,6 +3,11 @@ require 'erb'
 require 'launchy'
 require 'net/http'
 
+PHANTOMJS_PATH = find_executable(ENV['PHANTOMJS_PATH'] || 'phantomjs')
+PREFERRED_METHOD = PHANTOMJS_PATH.nil? ? 'browser' : 'phantomjs'
+if PHANTOMJS_PATH.nil?
+    puts("phantomjs not found on path. Set $PHANTOMJS_PATH. Using browser for jasmine tests".blue)
+end
 
 def django_for_jasmine(system, django_reload)
     if !django_reload
@@ -35,18 +40,6 @@ def django_for_jasmine(system, django_reload)
 end
 
 def template_jasmine_runner(lib)
-    case lib
-    when /common\/lib\/.+/
-        coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"]
-    when /common\/static\/coffee/
-        coffee_files = Dir["#{lib}/**/*.coffee"]
-    else
-        puts('I do not know how to run jasmine tests for #{lib}')
-        exit
-    end
-    if !coffee_files.empty?
-        sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}")
-    end
     phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine")
     jasmine_reporters_path = File.expand_path("node_modules/jasmine-reporters")
     common_js_root = File.expand_path("common/static/js")
@@ -54,8 +47,8 @@ def template_jasmine_runner(lib)
 
     # Get arrays of spec and source files, ordered by how deep they are nested below the library
     # (and then alphabetically) and expanded from a relative to an absolute path
-    spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js")
-    src_glob = File.join("#{lib}", "**", "src", "**", "*.js")
+    spec_glob = File.join(lib, "**", "spec", "**", "*.js")
+    src_glob = File.join(lib, "**", "src", "**", "*.js")
     js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
     js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
 
@@ -68,74 +61,90 @@ def template_jasmine_runner(lib)
     yield File.expand_path(template_output)
 end
 
-def run_phantom_js(url)
-    phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
-    sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
+def jasmine_browser(url, wait=10)
+    # Jitter starting the browser so that the tests don't all try and
+    # start the browser simultaneously
+    sleep(rand(3))
+    sh("python -m webbrowser -t '#{url}'")
+    sleep(wait)
 end
 
-# Open jasmine tests for :system in the default browser. The :env
-# should (always?) be 'jasmine', but it's passed as an arg so that
-# the :assets dependency gets it.
-#
-# This task should be invoked via the wrapper below, so we don't
-# include a description to keep it from showing up in rake -T.
-task :browse_jasmine, [:system, :env] => :assets do |t, args|
-    django_for_jasmine(args.system, true) do |jasmine_url|
-        Launchy.open(jasmine_url)
-        puts "Press ENTER to terminate".red
-        $stdin.gets
-    end
-end
-
-# Use phantomjs to run jasmine tests from the console. The :env
-# should (always?) be 'jasmine', but it's passed as an arg so that
-# the :assets dependency gets it.
-#
-# This task should be invoked via the wrapper below, so we don't
-# include a description to keep it from showing up in rake -T.
-task :phantomjs_jasmine, [:system, :env] => :assets do |t, args|
-    django_for_jasmine(args.system, false) do |jasmine_url|
-        run_phantom_js(jasmine_url)
-    end
+def jasmine_phantomjs(url)
+    fail("phantomjs not found. Add it to your path, or set $PHANTOMJS_PATH") if PHANTOMJS_PATH.nil?
+    test_sh("#{PHANTOMJS_PATH} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
 end
 
 # Wrapper tasks for the real browse_jasmine and phantomjs_jasmine
 # tasks above. These have a nicer UI since there's no arg passing.
 [:lms, :cms].each do |system|
+    namespace :jasmine do
+        namespace system do
             desc "Open jasmine tests for #{system} in your default browser"
-    task "browse_jasmine_#{system}" do
-        Rake::Task[:browse_jasmine].invoke(system, 'jasmine')
+            task :browser do
+                Rake::Task[:assets].invoke(system, 'jasmine')
+                django_for_jasmine(system, true) do |jasmine_url|
+                    jasmine_browser(jasmine_url)
+                end
             end
 
             desc "Use phantomjs to run jasmine tests for #{system} from the console"
-    task "phantomjs_jasmine_#{system}" do
-        Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine')
+            task :phantomjs do
+                Rake::Task[:assets].invoke(system, 'jasmine')
+                phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
+                django_for_jasmine(system, false) do |jasmine_url|
+                    jasmine_phantomjs(jasmine_url)
+                end
+            end
         end
-end
 
-STATIC_JASMINE_TESTS = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
-STATIC_JASMINE_TESTS << 'common/static/coffee'
+        desc "Run jasmine tests for #{system} using #{PREFERRED_METHOD}"
+        task system => "jasmine:#{system}:#{PREFERRED_METHOD}"
 
-STATIC_JASMINE_TESTS.each do |lib|
-    desc "Open jasmine tests for #{lib} in your default browser"
-    task "browse_jasmine_#{lib}" do
-        template_jasmine_runner(lib) do |f|
-            sh("python -m webbrowser -t 'file://#{f}'")
-            puts "Press ENTER to terminate".red
-            $stdin.gets
+        task :phantomjs => "jasmine:#{system}:phantomjs"
+        multitask :browser => "jasmine:#{system}:browser"
+    end
+end
+
+static_js_dirs = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
+static_js_dirs << 'common/static/coffee'
+static_js_dirs.select!{|lib| !Dir["#{lib}/**/spec"].empty?}
+
+static_js_dirs.each do |dir|
+    namespace :jasmine do
+        namespace dir do
+            desc "Open jasmine tests for #{dir} in your default browser"
+            task :browser do
+                # We need to use either CMS or LMS to preprocess files. Use LMS by default
+                Rake::Task['assets:coffee'].invoke('lms', 'jasmine')
+                template_jasmine_runner(dir) do |f|
+                    jasmine_browser("file://#{f}")
                 end
             end
 
-    desc "Use phantomjs to run jasmine tests for #{lib} from the console"
-    task "phantomjs_jasmine_#{lib}" do
-        template_jasmine_runner(lib) do |f|
-            run_phantom_js(f)
+            desc "Use phantomjs to run jasmine tests for #{dir} from the console"
+            task :phantomjs do
+                # We need to use either CMS or LMS to preprocess files. Use LMS by default
+                Rake::Task[:assets].invoke('lms', 'jasmine')
+                template_jasmine_runner(dir) do |f|
+                    jasmine_phantomjs(f)
+                end
             end
         end
+
+        desc "Run jasmine tests for #{dir} using #{PREFERRED_METHOD}"
+        task dir => "jasmine:#{dir}:#{PREFERRED_METHOD}"
+
+        task :phantomjs => "jasmine:#{dir}:phantomjs"
+        multitask :browser => "jasmine:#{dir}:browser"
+    end
 end
 
-desc "Open jasmine tests for discussion in your default browser"
-task "browse_jasmine_discussion" => "browse_jasmine_common/static/coffee"
+desc "Run all jasmine tests using #{PREFERRED_METHOD}"
+task :jasmine => "jasmine:#{PREFERRED_METHOD}"
+
+['phantomjs', 'browser'].each do |method|
+    desc "Run all jasmine tests using #{method}"
+    task "jasmine:#{method}"
+end
 
-desc "Use phantomjs to run jasmine tests for discussion from the console"
-task "phantomjs_jasmine_discussion" => "phantomjs_jasmine_common/static/coffee"
+task :test => :jasmine

rakefiles/tests.rake

-
 # Set up the clean and clobber tasks
 CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
 
-$failed_tests = 0
-
 def run_under_coverage(cmd, root)
     cmd0, cmd_rest = cmd.split(" ", 2)
     # We use "python -m coverage" so that the proper python will run the importable coverage
@@ -17,12 +14,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
     dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
     test_id = dirs.join(' ') if test_id.nil? or test_id == ''
     cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id)
-    sh(run_under_coverage(cmd, system)) do |ok, res|
-        if !ok and stop_on_failure
-            abort "Test failed!"
-        end
-        $failed_tests += 1 unless ok
-    end
+    test_sh(run_under_coverage(cmd, system))
 end
 
 def run_acceptance_tests(system, report_dir, harvest_args)
@@ -38,7 +30,7 @@ def run_acceptance_tests(system, report_dir, harvest_args)
     end
     sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
     sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
-    sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
+    test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
 end
 
 
@@ -55,13 +47,13 @@ TEST_TASK_DIRS = []
 
     # Per System tasks
     desc "Run all django tests on our djangoapps for the #{system}"
-    task "test_#{system}", [:test_id, :stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
+    task "test_#{system}", [:test_id] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
 
     # Have a way to run the tests without running collectstatic -- useful when debugging without
     # messing with static files.
-    task "fasttest_#{system}", [:test_id, :stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args|
-        args.with_defaults(:stop_on_failure => 'true', :test_id => nil)
-        run_tests(system, report_dir, args.test_id, args.stop_on_failure)
+    task "fasttest_#{system}", [:test_id] => [report_dir, :install_prereqs, :predjango] do |t, args|
+        args.with_defaults(:test_id => nil)
+        run_tests(system, report_dir, args.test_id)
     end
 
     # Run acceptance tests
@@ -88,9 +80,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
     task "test_#{lib}"  => ["clean_test_files", report_dir] do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
         cmd = "nosetests #{lib}"
-        sh(run_under_coverage(cmd, lib)) do |ok, res|
-            $failed_tests += 1 unless ok
-        end
+        test_sh(run_under_coverage(cmd, lib))
     end
     TEST_TASK_DIRS << lib
 
@@ -109,17 +99,11 @@ TEST_TASK_DIRS.each do |dir|
     report_dir = report_dir_path(dir)
     directory report_dir
     task :report_dirs => [REPORT_DIR, report_dir]
+    task :test => "test_#{dir}"
 end
 
-task :test do
-    TEST_TASK_DIRS.each do |dir|
-        Rake::Task["test_#{dir}"].invoke(nil, false)
-    end
-
-    if $failed_tests > 0
-        abort "Tests failed!"
-    end
-end
+desc "Run all tests"
+task :test
 
 desc "Build the html, xml, and diff coverage reports"
 task :coverage => :report_dirs do