ORA acceptance test cleanup

Made a few small optimizations to acceptance tests, added ability to profile
tests with an environment var, and updated .gitignore to ignore files
generated when running acceptance tests locally.

.gitignore

@@ -38,6 +38,7 @@ test_ora2db-journal
 test/custom_a11y_rules.js
 test/acceptance/screenshots/*.png
 test/logs/*
+test/acceptance/xunit-*
 
 # Mr Developer
 .mr.developer.cfg

requirements/test-acceptance.txt

 bok-choy==0.4.10
 ddt==1.0.0
 nose==1.3.7
+pyinstrument==0.13.1

scripts/jenkins-acceptance-tests.sh

@@ -29,6 +29,7 @@ if [ -z "$TEST_HOST" ]; then
     exit 1;
 fi
 
+# export ORA_PROFILING_ENABLED=True # uncomment this line to get profiling on jenkins runs
 export ORA_SANDBOX_URL="https://${BASIC_AUTH_USER}:${BASIC_AUTH_PASSWORD}@${TEST_HOST}"
 EXIT=0
 

test/acceptance/accessibility.py

@@ -135,7 +135,7 @@ class StaffAreaA11yTest(OpenAssessmentA11yTest):
         Check the accessibility of the learner information sections of the "Staff Tools" panel.
         """
         self.auto_auth_page.visit()
-        username = self.auto_auth_page.get_username()
+        username, _ = self.auto_auth_page.get_username_and_email()
 
         self.submission_page.visit().submit_response(self.SUBMISSION)
         self.assertTrue(self.submission_page.has_submitted)

test/acceptance/auto_auth.py

@@ -80,27 +80,12 @@ class AutoAuthPage(PageObject):
         match = re.search(r' user_id ([^$]+)$', message)
         return match.groups()[0] if match else None
 
-    def get_username(self):
-        """
-        Finds and returns the username of the current user.
-        """
-        username, _ = self._get_username_and_email()
-        return username
-
-    def get_email(self):
-        """
-        Finds and returns the email address of the current user.
-        """
-        _, email = self._get_username_and_email()
-        return email
-
-    def _get_username_and_email(self):
+    def get_username_and_email(self):
         """
         Finds and returns the username and email address of the current user.
         """
         message = self.q(css='BODY').text[0].strip()
-        match = re.search(r'Logged in user ([^$]+) with password ([^$]+) and user_id ([^$]+)$', message)
+        match = re.search(r'Logged in user (\S+) \(.*\) with password (\S+)', message)
         if not match:
             return None
-        username_and_email = match.groups()[0].split(' ')
-        return username_and_email[0], username_and_email[1]
+        return match.group(1), match.group(2)

test/acceptance/tests.py

@@ -6,6 +6,7 @@ import os
 import unittest
 import time
 from functools import wraps
+from pyinstrument import Profiler
 
 from nose.plugins.attrib import attr
 from bok_choy.web_app_test import WebAppTest
@@ -15,8 +16,10 @@ from pages import (
     SubmissionPage, AssessmentPage, GradePage, StaffAreaPage
 )
 
+# This value is generally used in jenkins, but not locally
+PROFILING_ENABLED = os.environ.get('ORA_PROFILING_ENABLED', False)
 
-def retry(tries=4, delay=3, backoff=2):
+def retry(tries=2, delay=4, backoff=2):
     """
     Retry decorator with exponential backoff.
 
@@ -103,6 +106,10 @@ class OpenAssessmentTest(WebAppTest):
         """
         super(OpenAssessmentTest, self).setUp()
 
+        if PROFILING_ENABLED:
+            self.profiler = Profiler(use_signal=False)
+            self.profiler.start()
+
         self.problem_loc = self.PROBLEM_LOCATIONS[problem_type]
         self.auto_auth_page = AutoAuthPage(self.browser, course_id=self.TEST_COURSE_ID, staff=staff)
         self.submission_page = SubmissionPage(self.browser, self.problem_loc)
@@ -112,6 +119,15 @@ class OpenAssessmentTest(WebAppTest):
         self.staff_asmnt_page = AssessmentPage('staff-assessment', self.browser, self.problem_loc)
         self.grade_page = GradePage(self.browser, self.problem_loc)
 
+    def log_to_file(self):
+        with open('{}-profile.log'.format(self.id()), 'w') as f:
+            f.write(self.profiler.output_text())
+
+    def tearDown(self):
+        if PROFILING_ENABLED:
+            self.profiler.stop()
+            self.log_to_file()
+
     def login_user(self, learner, email):
         """
         Logs in an already existing user.
@@ -132,7 +148,7 @@ class OpenAssessmentTest(WebAppTest):
         learner for which the self assessment was submitted.
         """
         self.auto_auth_page.visit()
-        username = self.auto_auth_page.get_username()
+        username, _ = self.auto_auth_page.get_username_and_email()
         self.submission_page.visit().submit_response(self.SUBMISSION)
         self.assertTrue(self.submission_page.has_submitted)
 
@@ -303,7 +319,7 @@ class StaffAssessmentTest(OpenAssessmentTest):
     def test_staff_assessment(self):
         # Set up user and navigate to submission page
         self.auto_auth_page.visit()
-        username = self.auto_auth_page.get_username()
+        username, _ = self.auto_auth_page.get_username_and_email()
         self.submission_page.visit()
 
         # Verify that staff grade step is shown initially
@@ -390,7 +406,7 @@ class PeerAssessmentTestStaffOverride(OpenAssessmentTest):
 
         # Create a submission for the third student (used for the remainder of the test).
         self.auto_auth_page.visit()
-        username = self.auto_auth_page.get_username()
+        username, _ = self.auto_auth_page.get_username_and_email()
         self.submission_page.visit().submit_response(self.SUBMISSION)
         # Staff Grade field should not be visible yet.
         self.assertFalse(self.staff_asmnt_page.is_browser_on_page())
@@ -472,12 +488,7 @@ class StaffAreaTest(OpenAssessmentTest):
 
     @retry()
     @attr('acceptance')
-    @ddt.data(
-        ("staff-tools", "MANAGE INDIVIDUAL LEARNERS"),
-        ("staff-info", "VIEW ASSIGNMENT STATISTICS"),
-    )
-    @ddt.unpack
-    def test_staff_area_panel(self, panel_name, button_label):
+    def test_staff_area_panel(self):
         """
         Scenario: the staff area panels should be shown correctly
 
@@ -495,17 +506,21 @@ class StaffAreaTest(OpenAssessmentTest):
         self.assertEqual(self.staff_area_page.selected_button_names, [])
         self.assertEqual(self.staff_area_page.visible_staff_panels, [])
 
-        # Click on the button and verify that the panel has opened
-        self.staff_area_page.click_staff_toolbar_button(panel_name)
-        self.assertEqual(self.staff_area_page.selected_button_names, [button_label])
-        visible_panels = self.staff_area_page.visible_staff_panels
-        self.assertEqual(1, len(visible_panels))
-        self.assertIn(u'openassessment__{button_name}'.format(button_name=panel_name), visible_panels[0])
-
-        # Click 'Close' and verify that the panel has been closed
-        self.staff_area_page.click_staff_panel_close_button(panel_name)
-        self.assertEqual(self.staff_area_page.selected_button_names, [])
-        self.assertEqual(self.staff_area_page.visible_staff_panels, [])
+        for panel_name, button_label in [
+            ("staff-tools", "MANAGE INDIVIDUAL LEARNERS"),
+            ("staff-info", "VIEW ASSIGNMENT STATISTICS"),
+        ]:
+            # Click on the button and verify that the panel has opened
+            self.staff_area_page.click_staff_toolbar_button(panel_name)
+            self.assertEqual(self.staff_area_page.selected_button_names, [button_label])
+            visible_panels = self.staff_area_page.visible_staff_panels
+            self.assertEqual(1, len(visible_panels))
+            self.assertIn(u'openassessment__{button_name}'.format(button_name=panel_name), visible_panels[0])
+
+            # Click 'Close' and verify that the panel has been closed
+            self.staff_area_page.click_staff_panel_close_button(panel_name)
+            self.assertEqual(self.staff_area_page.selected_button_names, [])
+            self.assertEqual(self.staff_area_page.visible_staff_panels, [])
 
     @retry()
     @attr('acceptance')
@@ -665,7 +680,7 @@ class StaffAreaTest(OpenAssessmentTest):
         """
         # View the problem-- no Staff Grade area.
         self.auto_auth_page.visit()
-        username = self.auto_auth_page.get_username()
+        username, _ = self.auto_auth_page.get_username_and_email()
         self.submission_page.visit()
         self.assertFalse(self.staff_asmnt_page.is_browser_on_page())
 
@@ -778,11 +793,10 @@ class FullWorkflowMixin(object):
             self.browser, password=self.TEST_PASSWORD, course_id=self.TEST_COURSE_ID, staff=True
         )
         auto_auth_page.visit()
-        username = auto_auth_page.get_username()
-        email = auto_auth_page.get_email()
+        username_email = auto_auth_page.get_username_and_email()
         self.submission_page.visit().submit_response(self.SUBMISSION)
 
-        return username, email
+        return username_email
 
     def do_submission_training_self_assessment(self):
         """