Merge pull request #7037 from edx/clytwynec/flaky_test_decorator

Clytwynec/flaky test decorator

common/test/acceptance/tests/studio/test_studio_library_container.py

@@ -2,6 +2,7 @@
 Acceptance tests for Library Content in LMS
 """
 import ddt
+from flaky import flaky
 import textwrap
 from unittest import skip
 
@@ -149,7 +150,7 @@ class StudioLibraryContainerTest(StudioLibraryTest, UniqueCourseTest):
         self.assertTrue(library_container.has_validation_error)
         self.assertIn(expected_text, library_container.validation_error_text)
 
-    @skip("TE-745 StudioLibraryContainerTest test_out_of_date_message fails intemittently")
+    @flaky  # TODO fix this, see TE-745
     def test_out_of_date_message(self):
         """
         Scenario: Given I have a library, a course and library content xblock in a course

pavelib/utils/test/suites/nose_suite.py

@@ -161,7 +161,7 @@ class LibTestSuite(NoseTestSuite):
     def cmd(self):
         cmd = (
             "nosetests --id-file={test_ids} {test_id} {test_opts} "
-            "--with-xunit --xunit-file={xunit_report} {extra}"
+            "--with-xunit --xunit-file={xunit_report} {extra} "
             "--verbosity={verbosity}".format(
                 test_ids=self.test_ids,
                 test_id=self.test_id,

requirements/edx/github.txt

@@ -11,6 +11,7 @@
 -e git+https://github.com/edx/django-pipeline.git@88ec8a011e481918fdc9d2682d4017c835acd8be#egg=django-pipeline
 -e git+https://github.com/edx/django-wiki.git@cd0b2b31997afccde519fe5b3365e61a9edb143f#egg=django-wiki
 -e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-2#egg=django-oauth2-provider
+-e git+https://github.com/clytwynec/flaky.git@clytwynec/fix-false-success-results#egg=flaky
 -e git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=mongodb_proxy
 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
 -e git+https://github.com/eventbrite/zendesk.git@d53fe0e81b623f084e91776bcf6369f8b7b63879#egg=zendesk

scripts/all-tests.sh

@@ -95,17 +95,17 @@ END
     "unit")
         case "$SHARD" in
             "lms")
-                paver test_system -s lms
+                paver test_system -s lms --extra_args="--with-flaky"
                 paver coverage
                 ;;
             "cms-js-commonlib")
-                paver test_system -s cms
+                paver test_system -s cms --extra_args="--with-flaky"
                 paver test_js --coverage --skip_clean
-                paver test_lib --skip_clean
+                paver test_lib --skip_clean --extra_args="--with-flaky"
                 paver coverage
                 ;;
             *)
-                paver test
+                paver test --extra_args="--with-flaky"
                 paver coverage
                 ;;
         esac
@@ -162,19 +162,19 @@ END
                 ;;
 
             "1")
-                paver test_bokchoy --extra_args="-a shard_1"
+                paver test_bokchoy --extra_args="-a shard_1 --with-flaky"
                 ;;
 
             "2")
-                paver test_bokchoy --extra_args="-a 'shard_2'"
+                paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky"
                 ;;
 
             "3")
-                paver test_bokchoy --extra_args="-a 'shard_3'"
+                paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky"
                 ;;
 
             "4")
-                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False"
+                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False --with-flaky"
                 ;;
 
             # Default case because if we later define another bok-choy shard on Jenkins