Automatically collect HAR data for performance testing

common/test/acceptance/pages/studio/overview.py

@@ -20,7 +20,7 @@ class CourseOutlineItem(object):
     """
     BODY_SELECTOR = None
     EDIT_BUTTON_SELECTOR = '.xblock-field-value-edit'
-    NAME_SELECTOR = '.xblock-title .xblock-field-value'
+    NAME_SELECTOR = '.item-title'
     NAME_INPUT_SELECTOR = '.xblock-field-input'
     NAME_FIELD_WRAPPER_SELECTOR = '.xblock-title .wrapper-xblock-field'
     STATUS_MESSAGE_SELECTOR = '> div[class$="status"] .status-message'
@@ -30,7 +30,10 @@ class CourseOutlineItem(object):
         # CourseOutlineItem is also used as a mixin for CourseOutlinePage, which doesn't have a locator
         # Check for the existence of a locator so that errors when navigating to the course outline page don't show up
         # as errors in the repr method instead.
-        return "{}(<browser>, {!r})".format(self.__class__.__name__, self.locator if hasattr(self, 'locator') else None)
+        try:
+            return "{}(<browser>, {!r})".format(self.__class__.__name__, self.locator)
+        except AttributeError:
+            return "{}(<browser>)".format(self.__class__.__name__)
 
     def _bounded_selector(self, selector):
         """

common/test/acceptance/performance/test_studio_performance.py

+"""
+Single page performance tests for Studio.
+"""
+from bok_choy.performance import WebAppPerfReport, with_cache
+from ..pages.studio.auto_auth import AutoAuthPage
+from ..pages.studio.overview import CourseOutlinePage
+
+
+class StudioPagePerformanceTest(WebAppPerfReport):
+    """
+    Base class to capture studio performance with HTTP Archives.
+
+    To import courses for the bok choy tests, pass the --imports_dir=<course directory> argument to the paver command
+    where <course directory> contains the (un-archived) courses to be imported.
+    """
+    course_org = 'edX'
+    course_num = 'Open_DemoX'
+    course_run = 'edx_demo_course'
+
+    def setUp(self):
+        """
+        Authenticate as staff so we can view and edit courses.
+        """
+        super(StudioPagePerformanceTest, self).setUp()
+        AutoAuthPage(self.browser, staff=True).visit()
+
+    def record_visit_outline(self):
+        """
+        Produce a HAR for loading the course outline page.
+        """
+        course_outline_page = CourseOutlinePage(self.browser, self.course_org, self.course_num, self.course_run)
+        har_name = 'OutlinePage_{org}_{course}'.format(
+            org=self.course_org,
+            course=self.course_num
+        )
+        self.new_page(har_name)
+        course_outline_page.visit()
+        self.save_har(har_name)
+
+    def record_visit_unit(self, section_title, subsection_title, unit_title):
+        """
+        Produce a HAR for loading a unit page.
+        """
+        course_outline_page = CourseOutlinePage(self.browser, self.course_org, self.course_num, self.course_run).visit()
+        course_outline_unit = course_outline_page.section(section_title).subsection(subsection_title).toggle_expand().unit(unit_title)
+        har_name = 'UnitPage_{org}_{course}'.format(
+            org=self.course_org,
+            course=self.course_num
+        )
+        self.new_page(har_name)
+        course_outline_unit.go_to()
+        self.save_har(har_name)
+
+
+class StudioJusticePerformanceTest(StudioPagePerformanceTest):
+    """
+    Test performance on the HarvardX Justice course.
+    """
+    course_org = 'HarvardX'
+    course_num = 'ER22x'
+    course_run = '2013_Spring'
+
+    @with_cache
+    def test_visit_outline(self):
+        """Record visiting the Justice course outline page"""
+        self.record_visit_outline()
+
+    @with_cache
+    def test_visit_unit(self):
+        """Record visiting a Justice unit page"""
+        self.record_visit_unit(
+            'Lecture 1 - Doing the Right Thing',
+            'Discussion Prompt: Ethics of Torture',
+            'Discussion Prompt: Ethics of Torture'
+        )
+
+
+class StudioPub101PerformanceTest(StudioPagePerformanceTest):
+    """
+    Test performance on Andy's PUB101 outline page.
+    """
+    course_org = 'AndyA'
+    course_num = 'PUB101'
+    course_run = 'PUB101'
+
+    @with_cache
+    def test_visit_outline(self):
+        """Record visiting the PUB101 course outline page"""
+        self.record_visit_outline()
+
+    @with_cache
+    def test_visit_unit(self):
+        """Record visiting the PUB101 unit page"""
+        self.record_visit_unit('Released', 'Released', 'Released')

pavelib/bok_choy.py

@@ -42,6 +42,34 @@ def test_bokchoy(options):
         'fasttest': getattr(options, 'fasttest', False),
         'verbosity': getattr(options, 'verbosity', 2),
         'extra_args': getattr(options, 'extra_args', ''),
+        'test_dir': 'tests',
+    }
+
+    test_suite = BokChoyTestSuite('bok-choy', **opts)
+    test_suite.run()
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ('test_spec=', 't', 'Specific test to run'),
+    ('fasttest', 'a', 'Skip some setup'),
+    ('imports_dir=', 'd', 'Directory containing (un-archived) courses to be imported'),
+    make_option("--verbose", action="store_const", const=2, dest="verbosity"),
+    make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
+    make_option("-v", "--verbosity", action="count", dest="verbosity"),
+])
+def perf_report_bokchoy(options):
+    """
+    Generates a har file for with page performance info.
+    """
+    opts = {
+        'test_spec': getattr(options, 'test_spec', None),
+        'fasttest': getattr(options, 'fasttest', False),
+        'imports_dir': getattr(options, 'imports_dir', None),
+        'verbosity': getattr(options, 'verbosity', 2),
+        'test_dir': 'performance',
+        'ptests': True,
     }
 
     test_suite = BokChoyTestSuite('bok-choy', **opts)

pavelib/utils/test/suites/bokchoy_suite.py

@@ -21,7 +21,7 @@ class BokChoyTestSuite(TestSuite):
     """
     def __init__(self, *args, **kwargs):
         super(BokChoyTestSuite, self).__init__(*args, **kwargs)
-        self.test_dir = Env.BOK_CHOY_DIR / "tests"
+        self.test_dir = Env.BOK_CHOY_DIR / kwargs.get('test_dir', 'tests')
         self.log_dir = Env.BOK_CHOY_LOG_DIR
         self.report_dir = Env.BOK_CHOY_REPORT_DIR
         self.xunit_report = self.report_dir / "xunit.xml"
@@ -30,12 +30,19 @@ class BokChoyTestSuite(TestSuite):
         self.test_spec = kwargs.get('test_spec', None)
         self.verbosity = kwargs.get('verbosity', 2)
         self.extra_args = kwargs.get('extra_args', '')
+        self.ptests = kwargs.get('ptests', False)
+        self.har_dir = self.log_dir / 'hars'
+        self.imports_dir = kwargs.get('imports_dir', None)
 
     def __enter__(self):
         super(BokChoyTestSuite, self).__enter__()
 
         # Ensure that we have a directory to put logs and reports
         self.log_dir.makedirs_p()
+
+        if self.ptests:
+            self.har_dir.makedirs_p()
+
         self.report_dir.makedirs_p()
         test_utils.clean_reports_dir()
 
@@ -61,6 +68,9 @@ class BokChoyTestSuite(TestSuite):
             " common/test/db_fixtures/*.json"
         )
 
+        if self.imports_dir:
+            sh("./manage.py cms --settings=bok_choy import {}".format(self.imports_dir))
+
         # Ensure the test servers are available
         msg = colorize('green', "Starting test servers...")
         print(msg)
@@ -92,6 +102,7 @@ class BokChoyTestSuite(TestSuite):
         # screenshots and XUnit XML reports
         cmd = [
             "SCREENSHOT_DIR='{}'".format(self.log_dir),
+            "HAR_DIR='{}'".format(self.har_dir),
             "nosetests",
             test_spec,
             "--with-xunit",

requirements/edx/github.txt

@@ -24,7 +24,7 @@
 -e git+https://github.com/edx/django-waffle.git@823a102e48#egg=django-waffle
 -e git+https://github.com/edx/event-tracking.git@0.1.0#egg=event-tracking
 -e git+https://github.com/edx/edx-analytics-api-client.git@0.1.0#egg=analytics-client
--e git+https://github.com/edx/bok-choy.git@feb61863967134a378a7c912576cb31a94ba02bf#egg=bok_choy
+-e git+https://github.com/edx/bok-choy.git@9162c0bfb8e0eb1e2fa8e6df8dec12d181322a90#egg=bok_choy
 -e git+https://github.com/edx-solutions/django-splash.git@9965a53c269666a30bb4e2b3f6037c138aef2a55#egg=django-splash
 -e git+https://github.com/edx/acid-block.git@459aff7b63db8f2c5decd1755706c1a64fb4ebb1#egg=acid-xblock
 -e git+https://github.com/edx/edx-ora2.git@release-2014-07-28T12.09#egg=edx-ora2