Merge pull request #4593 from edx/clytwynec/shard_out_bokchoy_tests

Clytwynec/shard out bokchoy tests

common/test/acceptance/tests/test_discussion.py

@@ -3,6 +3,7 @@ Tests for discussion pages
 """
 
 from uuid import uuid4
+from nose.plugins.attrib import attr
 
 from .helpers import UniqueCourseTest
 from ..pages.lms.auto_auth import AutoAuthPage
@@ -122,6 +123,7 @@ class DiscussionResponsePaginationTestMixin(object):
         self.assertFalse(self.thread_page.has_add_response_button())
 
 
+@attr('shard_1')
 class DiscussionTabSingleThreadTest(UniqueCourseTest, DiscussionResponsePaginationTestMixin):
     """
     Tests for the discussion page displaying a single thread
@@ -141,6 +143,7 @@ class DiscussionTabSingleThreadTest(UniqueCourseTest, DiscussionResponsePaginati
         self.thread_page.visit()
 
 
+@attr('shard_1')
 class DiscussionCommentDeletionTest(UniqueCourseTest):
     """
     Tests for deleting comments displayed beneath responses in the single thread view.
@@ -184,6 +187,7 @@ class DiscussionCommentDeletionTest(UniqueCourseTest):
         page.delete_comment("comment_other_author")
 
 
+@attr('shard_1')
 class DiscussionCommentEditTest(UniqueCourseTest):
     """
     Tests for editing comments displayed beneath responses in the single thread view.
@@ -273,6 +277,7 @@ class DiscussionCommentEditTest(UniqueCourseTest):
         self.assertTrue(page.is_add_comment_visible("response1"))
 
 
+@attr('shard_1')
 class InlineDiscussionTest(UniqueCourseTest, DiscussionResponsePaginationTestMixin):
     """
     Tests for inline discussions
@@ -330,6 +335,7 @@ class InlineDiscussionTest(UniqueCourseTest, DiscussionResponsePaginationTestMix
         self.check_anonymous_to_peers(False)
 
 
+@attr('shard_1')
 class DiscussionUserProfileTest(UniqueCourseTest):
     """
     Tests for user profile page in discussion tab.
@@ -432,6 +438,8 @@ class DiscussionUserProfileTest(UniqueCourseTest):
     def test_151_threads(self):
         self.check_pages(151)
 
+
+@attr('shard_1')
 class DiscussionSearchAlertTest(UniqueCourseTest):
     """
     Tests for spawning and dismissing alerts related to user search actions and their results.
@@ -505,6 +513,7 @@ class DiscussionSearchAlertTest(UniqueCourseTest):
         ).wait_for_page()
 
 
+@attr('shard_1')
 class DiscussionSortPreferenceTest(UniqueCourseTest):
     """
     Tests for the discussion page displaying a single thread.

common/test/acceptance/tests/test_studio_acid_xblock.py

@@ -2,7 +2,7 @@
 Acceptance tests for Studio related to the acid xblock.
 """
 from unittest import skip
-
+from nose.plugins.attrib import attr
 from bok_choy.web_app_test import WebAppTest
 
 from ..pages.studio.auto_auth import AutoAuthPage
@@ -11,6 +11,7 @@ from ..pages.xblock.acid import AcidView
 from ..fixtures.course import CourseFixture, XBlockFixtureDesc
 
 
+@attr('shard_1')
 class XBlockAcidBase(WebAppTest):
     """
     Base class for tests that verify that XBlock integration is working correctly
@@ -121,6 +122,7 @@ class XBlockAcidNoChildTest(XBlockAcidBase):
         self.user = course_fix.user
 
 
+@attr('shard_1')
 class XBlockAcidParentBase(XBlockAcidBase):
     """
     Base class for tests that verify that parent XBlock integration is working correctly
@@ -178,6 +180,7 @@ class XBlockAcidEmptyParentTest(XBlockAcidParentBase):
         self.user = course_fix.user
 
 
+@attr('shard_1')
 class XBlockAcidChildTest(XBlockAcidParentBase):
     """
     Tests of an AcidBlock with children

common/test/acceptance/tests/test_studio_container.py

 """
 Acceptance tests for Studio related to the container page.
 """
+from nose.plugins.attrib import attr
 
 from ..pages.studio.overview import CourseOutlinePage
 from ..fixtures.course import XBlockFixtureDesc
@@ -9,10 +10,10 @@ from ..pages.studio.component_editor import ComponentEditorView
 from ..pages.studio.utils import add_discussion
 
 from unittest import skip
-
 from acceptance.tests.base_studio_test import StudioCourseTest
 
 
+@attr('shard_1')
 class ContainerBase(StudioCourseTest):
     """
     Base class for tests that do operations on the container page.
@@ -143,6 +144,7 @@ class NestedVerticalTest(ContainerBase):
         )
 
 
+@attr('shard_1')
 class DragAndDropTest(NestedVerticalTest):
     """
     Tests of reordering within the container page.
@@ -224,6 +226,7 @@ class DragAndDropTest(NestedVerticalTest):
         self.do_action_and_verify(add_new_components_and_rearrange, expected_ordering)
 
 
+@attr('shard_1')
 class AddComponentTest(NestedVerticalTest):
     """
     Tests of adding a component to the container page.
@@ -264,6 +267,7 @@ class AddComponentTest(NestedVerticalTest):
         self.add_and_verify(container_menu, expected_ordering)
 
 
+@attr('shard_1')
 class DuplicateComponentTest(NestedVerticalTest):
     """
     Tests of duplicating a component on the container page.
@@ -310,6 +314,7 @@ class DuplicateComponentTest(NestedVerticalTest):
         self.do_action_and_verify(duplicate_twice, expected_ordering)
 
 
+@attr('shard_1')
 class DeleteComponentTest(NestedVerticalTest):
     """
     Tests of deleting a component from the container page.
@@ -333,6 +338,7 @@ class DeleteComponentTest(NestedVerticalTest):
         self.delete_and_verify(group_a_item_1_delete_index, expected_ordering)
 
 
+@attr('shard_1')
 class EditContainerTest(NestedVerticalTest):
     """
     Tests of editing a container.

common/test/acceptance/tests/test_studio_general.py

@@ -4,6 +4,7 @@ Acceptance tests for Studio.
 
 from unittest import skip
 from bok_choy.web_app_test import WebAppTest
+from nose.plugins.attrib import attr
 
 from ..pages.studio.asset_index import AssetIndexPage
 from ..pages.studio.auto_auth import AutoAuthPage
@@ -27,6 +28,7 @@ from ..fixtures.course import XBlockFixtureDesc
 from acceptance.tests.base_studio_test import StudioCourseTest
 
 
+@attr('shard_1')
 class LoggedOutTest(WebAppTest):
     """
     Smoke test for pages in Studio that are visible when logged out.
@@ -46,6 +48,7 @@ class LoggedOutTest(WebAppTest):
             page.visit()
 
 
+@attr('shard_1')
 class LoggedInPagesTest(WebAppTest):
     """
     Tests that verify the pages in Studio that you can get to when logged
@@ -65,6 +68,7 @@ class LoggedInPagesTest(WebAppTest):
         self.dashboard_page.visit()
 
 
+@attr('shard_1')
 class CoursePagesTest(StudioCourseTest):
     """
     Tests that verify the pages in Studio that you can get to when logged
@@ -109,6 +113,7 @@ class CoursePagesTest(StudioCourseTest):
             page.visit()
 
 
+@attr('shard_1')
 class CourseSectionTest(StudioCourseTest):
     """
     Tests that verify the sections name editable only inside headers in Studio Course Outline that you can get to
@@ -156,6 +161,7 @@ class CourseSectionTest(StudioCourseTest):
         self.assertFalse(section_name_edit_form)
 
 
+@attr('shard_1')
 class DiscussionPreviewTest(StudioCourseTest):
     """
     Tests that Inline Discussions are rendered with a custom preview in Studio

common/test/acceptance/tests/test_studio_split_test.py

@@ -6,6 +6,7 @@ import json
 import os
 import math
 from unittest import skip, skipUnless
+from nose.plugins.attrib import attr
 
 from xmodule.partitions.partitions import Group, UserPartition
 from bok_choy.promise import Promise, EmptyPromise
@@ -65,6 +66,7 @@ class SplitTestMixin(object):
         Promise(missing_groups_button_not_present, "Add missing groups button should not be showing.").fulfill()
 
 
+@attr('shard_1')
 class SplitTest(ContainerBase, SplitTestMixin):
     """
     Tests for creating and editing split test instances in Studio.
@@ -175,6 +177,7 @@ class SplitTest(ContainerBase, SplitTestMixin):
         self.verify_groups(container, ['alpha'], [], verify_missing_groups_not_present=False)
 
 
+@attr('shard_1')
 @skipUnless(os.environ.get('FEATURE_GROUP_CONFIGURATIONS'), 'Tests Group Configurations feature')
 class SettingsMenuTest(StudioCourseTest):
     """
@@ -223,6 +226,7 @@ class SettingsMenuTest(StudioCourseTest):
         self.assertFalse(self.advanced_settings.q(css=link_css).present)
 
 
+@attr('shard_1')
 @skipUnless(os.environ.get('FEATURE_GROUP_CONFIGURATIONS'), 'Tests Group Configurations feature')
 class GroupConfigurationsTest(ContainerBase, SplitTestMixin):
     """

pavelib/bok_choy.py

@@ -20,6 +20,7 @@ __test__ = False  # do not collect
 @cmdopts([
     ('test_spec=', 't', 'Specific test to run'),
     ('fasttest', 'a', 'Skip some setup'),
+    ('extra_args=', 'e', 'adds as extra args to the test command'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity"),
@@ -39,7 +40,8 @@ def test_bokchoy(options):
     opts = {
         'test_spec': getattr(options, 'test_spec', None),
         'fasttest': getattr(options, 'fasttest', False),
-        'verbosity': getattr(options, 'verbosity', 2)
+        'verbosity': getattr(options, 'verbosity', 2),
+        'extra_args': getattr(options, 'extra_args', ''),
     }
 
     test_suite = BokChoyTestSuite('bok-choy', **opts)

pavelib/utils/test/suites/bokchoy_suite.py

@@ -29,6 +29,7 @@ class BokChoyTestSuite(TestSuite):
         self.fasttest = kwargs.get('fasttest', False)
         self.test_spec = kwargs.get('test_spec', None)
         self.verbosity = kwargs.get('verbosity', 2)
+        self.extra_args = kwargs.get('extra_args', '')
 
     def __enter__(self):
         super(BokChoyTestSuite, self).__enter__()
@@ -97,6 +98,7 @@ class BokChoyTestSuite(TestSuite):
             "--with-flaky",
             "--xunit-file={}".format(self.xunit_report),
             "--verbosity={}".format(self.verbosity),
+            self.extra_args,
         ]
 
         cmd = (" ").join(cmd)

scripts/all-tests.sh

@@ -23,9 +23,20 @@ set -e
 #       - "bok-choy": Run acceptance tests that use the bok-choy framework
 #
 #   `SHARD` is a number (1, 2, or 3) indicating which subset of the tests
-#       to build.  Currently, "lms-acceptance" has two shards (1 and 2),
-#       "cms-acceptance" has three shards (1, 2, and 3), and all the
-#       other test suites have one shard.
+#       to build.  Currently, "lms-acceptance" and "bok-choy" each have two
+#       shards (1 and 2), "cms-acceptance" has three shards (1, 2, and 3), 
+#       and all the other test suites have one shard.  
+# 
+#       For the "bok-choy", the tests are put into shard groups using the nose 
+#       'attr' decorator (e.g. "@attr('shard_1')").  Currently, anything with 
+#       the 'shard_1' attribute will run in the first shard.  All other bok-choy
+#       tests will run in shard 2.
+# 
+#       For the lettuce acceptance tests, ("lms-" and "cms-acceptance") they 
+#       are decorated with "@shard_{}" (e.g. @shard_1 for the first shard).
+#       The lettuce tests must have a shard specified to be run in jenkins,
+#       as there is no shard that runs unspecified tests.
+# 
 #
 #   Jenkins configuration:
 #
@@ -85,9 +96,9 @@ source $HOME/edx-venv/bin/activate
 case "$TEST_SUITE" in
 
     "quality")
-        rake pep8 > pep8.log || { cat pep8.log ; exit 1; }
-        rake pylint > pylint.log || { cat pylint.log; exit 1; }
-        rake quality
+        paver run_pep8 > pep8.log || { cat pep8.log ; exit 1; }
+        paver run_pylint > pylint.log || { cat pylint.log; exit 1; }
+        paver run_quality
 
         # Need to create an empty test result so the post-build
         # action doesn't fail the build.
@@ -101,21 +112,29 @@ END
         ;;
 
     "unit")
-        rake test
-        rake coverage
+        paver test
+        paver coverage
         ;;
 
     "lms-acceptance")
-        rake test:acceptance:lms["-v 3 --tag shard_${SHARD}"]
+        paver test_acceptance -s lms --extra_args="-v 3 --tag shard_${SHARD}"
         ;;
 
     "cms-acceptance")
-        rake test:acceptance:cms["-v 3 --tag shard_${SHARD}"]
+        paver test_acceptance -s cms --extra_args="-v 3 --tag shard_${SHARD}"
         ;;
 
     "bok-choy")
-        rake test:bok_choy
-        rake test:bok_choy:coverage
+        case "$SHARD" in
+            "1")
+                paver test_bokchoy --extra_args="-a shard_1"
+                ;;
+
+            "2")
+                paver test_bokchoy --extra_args="-a '!shard_1'"
+               ;;
+        esac
+        paver bokchoy_coverage
         ;;
 
 esac