Merge pull request #15918 from edx/jmbowman/test_reliability_fixes

Fix tests that depend on execution order

cms/djangoapps/contentstore/tests/test_courseware_index.py

@@ -740,6 +740,12 @@ class TestTaskExecution(SharedModuleStoreTestCase):
             publish_item=False,
         )
 
+    @classmethod
+    def tearDownClass(cls):
+        SignalHandler.course_published.connect(listen_for_course_publish)
+        SignalHandler.library_updated.connect(listen_for_library_update)
+        super(TestTaskExecution, cls).tearDownClass()
+
     def test_task_indexing_course(self):
         """ Making sure that the receiver correctly fires off the task when invoked by signal """
         searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)

cms/djangoapps/contentstore/views/tests/test_course_index.py

@@ -23,9 +23,11 @@ from contentstore.views.course import (
     course_outline_initial_state,
     reindex_course_and_check_access
 )
+from contentstore.views.course import WAFFLE_NAMESPACE as COURSE_WAFFLE_NAMESPACE
 from contentstore.views.item import VisibilityState, create_xblock_info
 from course_action_state.managers import CourseRerunUIStateManager
 from course_action_state.models import CourseRerunState
+from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace
 from student.auth import has_course_author_access
 from student.roles import CourseStaffRole, GlobalStaff, LibraryUserRole
 from student.tests.factories import UserFactory
@@ -359,6 +361,10 @@ class TestCourseIndexArchived(CourseTestCase):
         for course in (self.course, self.active_course, self.archived_course):
             CourseStaffRole(course.id).add_users(self.staff)
 
+        # Make sure we've cached data which could change the query counts
+        # depending on test execution order
+        WaffleSwitchNamespace(name=COURSE_WAFFLE_NAMESPACE).is_enabled(u'enable_global_staff_optimization')
+
     def check_index_page_with_query_count(self, separate_archived_courses, org, mongo_queries, sql_queries):
         """
         Checks the index page, and ensures the number of database queries is as expected.

common/lib/capa/capa/safe_exec/tests/test_lazymod.py

@@ -34,15 +34,23 @@ class TestLazyMod(unittest.TestCase):
 
     def test_simple(self):
         # Import some stdlib module that has not been imported before
-        self.assertNotIn("colorsys", sys.modules)
-        colorsys = LazyModule("colorsys")
+        module_name = 'colorsys'
+        if module_name in sys.modules:
+            # May have been imported during test discovery, remove it again
+            del sys.modules[module_name]
+        assert module_name not in sys.modules
+        colorsys = LazyModule(module_name)
         hsv = colorsys.rgb_to_hsv(.3, .4, .2)
         self.assertEqual(hsv[0], 0.25)
 
     def test_dotted(self):
         # wsgiref is a module with submodules that is not already imported.
         # Any similar module would do. This test demonstrates that the module
-        # is not already im
-        self.assertNotIn("wsgiref.util", sys.modules)
-        wsgiref_util = LazyModule("wsgiref.util")
+        # is not already imported
+        module_name = 'wsgiref.util'
+        if module_name in sys.modules:
+            # May have been imported during test discovery, remove it again
+            del sys.modules[module_name]
+        assert module_name not in sys.modules
+        wsgiref_util = LazyModule(module_name)
         self.assertEqual(wsgiref_util.guess_scheme({}), "http")

common/lib/capa/capa/tests/test_html_render.py

@@ -7,6 +7,8 @@ import unittest
 import ddt
 import mock
 import os
+# Changes formatting of empty elements; import here to avoid test order dependence
+import xmodule.modulestore.xml  # pylint: disable=unused-import
 from capa.tests.helpers import test_capa_system, new_loncapa_problem
 from lxml import etree
 from openedx.core.djangolib.markup import HTML
@@ -303,7 +305,7 @@ class CapaHtmlRenderTest(unittest.TestCase):
 
         # Render the HTML
         the_html = problem.get_html()
-        self.assertRegexpMatches(the_html, r"<div>\s+</div>")
+        self.assertRegexpMatches(the_html, r"<div/>")
 
     def _create_test_file(self, path, content_str):
         test_fp = self.capa_system.filestore.open(path, "w")

common/lib/capa/capa/tests/test_targeted_feedback.py

@@ -5,6 +5,8 @@ i.e. those with the <multiplechoiceresponse> element
 
 import unittest
 import textwrap
+# Changes formatting of empty elements; import here to avoid test order dependence
+import xmodule.modulestore.xml  # pylint: disable=unused-import
 from capa.tests.helpers import test_capa_system, new_loncapa_problem, load_fixture
 
 
@@ -188,14 +190,14 @@ class CapaTargetedFeedbackTest(unittest.TestCase):
         problem.done = True
         problem.student_answers = {'1_2_1': 'choice_0'}
         the_html = problem.get_html()
-        self.assertRegexpMatches(the_html, r"<targetedfeedbackset>\s*</targetedfeedbackset>")
+        self.assertRegexpMatches(the_html, r"<targetedfeedbackset/>")
 
         # New problem with same XML -- try the correct choice.
         problem = new_loncapa_problem(xml_str)
         problem.done = True
         problem.student_answers = {'1_2_1': 'choice_2'}  # correct
         the_html = problem.get_html()
-        self.assertRegexpMatches(the_html, r"<targetedfeedbackset>\s*</targetedfeedbackset>")
+        self.assertRegexpMatches(the_html, r"<targetedfeedbackset/>")
 
     def test_targeted_feedback_no_solution_element(self):
         xml_str = textwrap.dedent("""
@@ -579,8 +581,7 @@ class CapaTargetedFeedbackTest(unittest.TestCase):
         # Q1 and Q2 have no feedback
         self.assertRegexpMatches(
             without_new_lines,
-            r'<targetedfeedbackset.*?>\s*</targetedfeedbackset>.*' +
-            r'<targetedfeedbackset.*?>\s*</targetedfeedbackset>'
+            r'<targetedfeedbackset.*?/>.*<targetedfeedbackset.*?/>'
         )
 
     def test_targeted_feedback_multiple_answer_1(self):
@@ -593,7 +594,7 @@ class CapaTargetedFeedbackTest(unittest.TestCase):
         self.assertRegexpMatches(
             without_new_lines,
             r'<targetedfeedbackset.*?>.*?explanation-id="feedback1".*?</targetedfeedbackset>.*' +
-            r'<targetedfeedbackset.*?>\s*</targetedfeedbackset>'
+            r'<targetedfeedbackset.*?/>'
         )
 
     def test_targeted_feedback_multiple_answer_2(self):

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -13,6 +13,12 @@ import uuid
 import ddt
 from contracts import contract
 from nose.plugins.attrib import attr
+# For the cache tests to work, we need to be using the Django default
+# settings (not our usual cms or lms test settings) and they need to
+# be configured before importing from django.core.cache
+from django.conf import settings
+if not settings.configured:
+    settings.configure()
 from django.core.cache import caches, InvalidCacheBackendError
 
 from openedx.core.lib import tempdir

openedx/core/djangoapps/content/course_structures/tests.py

@@ -22,6 +22,10 @@ class SignalDisconnectTestMixin(object):
         super(SignalDisconnectTestMixin, self).setUp()
         SignalHandler.course_published.disconnect(listen_for_course_publish)
 
+    def tearDown(self):
+        SignalHandler.course_published.connect(listen_for_course_publish)
+        super(SignalDisconnectTestMixin, self).tearDown()
+
 
 @attr(shard=2)
 class CourseStructureTaskTests(ModuleStoreTestCase):