Merge pull request #16039 from edx/jeskew/paver_calls_tox_to_run_pytest

Paver changes to run tox testing for system & lib tests.

pavelib/tests.py

@@ -31,6 +31,10 @@ __test__ = False  # do not collect
     ("fail-fast", "x", "Fail suite on first failed test"),
     ("fasttest", "a", "Run without collectstatic"),
     make_option(
+        "--django_version", dest="django_version",
+        help="Run against which Django version (1.8 -or- 1.11)."
+    ),
+    make_option(
         "--eval-attr", dest="eval_attr",
         help="Only run tests matching given attribute expression."
     ),
@@ -66,23 +70,32 @@ def test_system(options, passthrough_options):
     """
     system = getattr(options, 'system', None)
     test_id = getattr(options, 'test_id', None)
+    django_version = getattr(options, 'django_version', None)
+
+    assert(system in (None, 'lms', 'cms'))
+    assert(django_version in (None, '1.8', '1.11'))
 
     if test_id:
+        # Testing a single test ID.
+        # Ensure the proper system for the test id.
         if not system:
             system = test_id.split('/')[0]
         if system in ['common', 'openedx']:
             system = 'lms'
-        options.test_system['test_id'] = test_id
-
-    if test_id or system:
         system_tests = [suites.SystemTestSuite(
             system,
             passthrough_options=passthrough_options,
             **options.test_system
         )]
     else:
+        # Testing a single system -or- both systems.
+        if system:
+            systems = [system]
+        else:
+            # No specified system or test_id, so run all tests of both systems.
+            systems = ['cms', 'lms']
         system_tests = []
-        for syst in ('cms', 'lms'):
+        for syst in systems:
             system_tests.append(suites.SystemTestSuite(
                 syst,
                 passthrough_options=passthrough_options,
@@ -108,6 +121,10 @@ def test_system(options, passthrough_options):
     ("failed", "f", "Run only failed tests"),
     ("fail-fast", "x", "Run only failed tests"),
     make_option(
+        "--django_version", dest="django_version",
+        help="Run against which Django version (1.8 -or- 1.11)."
+    ),
+    make_option(
         '-c', '--cov-args', default='',
         help='adds as args to coverage for the test run'
     ),
@@ -124,8 +141,12 @@ def test_lib(options, passthrough_options):
     """
     lib = getattr(options, 'lib', None)
     test_id = getattr(options, 'test_id', lib)
+    django_version = getattr(options, 'django_version', None)
+
+    assert(django_version in (None, '1.8', '1.11'))
 
     if test_id:
+        # Testing a single test id.
         if '/' in test_id:
             lib = '/'.join(test_id.split('/')[0:3])
         else:
@@ -137,6 +158,7 @@ def test_lib(options, passthrough_options):
             **options.test_lib
         )]
     else:
+        # Testing all common/lib test dirs - plus pavelib.
         lib_tests = [
             suites.LibTestSuite(
                 d,

pavelib/utils/envs.py

@@ -200,10 +200,12 @@ class Env(object):
 
     JS_REPORT_DIR = REPORT_DIR / 'javascript'
 
-    # Directories used for common/lib/ tests
+    # Directories used for common/lib/tests
+    IGNORED_TEST_DIRS = ('__pycache__', '.cache')
     LIB_TEST_DIRS = []
     for item in (REPO_ROOT / "common/lib").listdir():
-        if (REPO_ROOT / 'common/lib' / item).isdir():
+        dir_name = (REPO_ROOT / 'common/lib' / item)
+        if dir_name.isdir() and not dir_name.endswith(IGNORED_TEST_DIRS):
             LIB_TEST_DIRS.append(path("common/lib") / item.basename())
     LIB_TEST_DIRS.append(path("pavelib/paver_tests"))
 

pavelib/utils/test/suites/pytest_suite.py

@@ -25,6 +25,13 @@ class PytestSuite(TestSuite):
         self.failed_only = kwargs.get('failed_only', False)
         self.fail_fast = kwargs.get('fail_fast', False)
         self.run_under_coverage = kwargs.get('with_coverage', True)
+        django_version = kwargs.get('django_version', None)
+        if django_version is None:
+            self.django_toxenv = None
+        elif django_version == '1.11':
+            self.django_toxenv = 'py27-django111'
+        else:
+            self.django_toxenv = 'py27-django18'
         self.report_dir = Env.REPORT_DIR / self.root
 
         # If set, put reports for run in "unique" directories.
@@ -122,11 +129,15 @@ class SystemTestSuite(PytestSuite):
     @property
     def cmd(self):
 
-        cmd = [
-            'pytest',
+        if self.django_toxenv:
+            cmd = ['tox', '-e', self.django_toxenv, '--']
+        else:
+            cmd = ['pytest']
+        cmd.extend([
             '--ds={}'.format('{}.envs.{}'.format(self.root, self.settings)),
             '--junitxml={}'.format(self.report_dir / "nosetests.xml"),
-        ] + self.test_options_flags
+        ])
+        cmd.extend(self.test_options_flags)
         if self.verbosity < 1:
             cmd.append("--quiet")
         elif self.verbosity > 1:
@@ -180,7 +191,7 @@ class SystemTestSuite(PytestSuite):
             """
             Should this path be included in the pytest arguments?
             """
-            if path.endswith('__pycache__'):
+            if path.endswith(Env.IGNORED_TEST_DIRS):
                 return False
             return path.endswith('.py') or os.path.isdir(path)
 
@@ -205,12 +216,16 @@ class LibTestSuite(PytestSuite):
 
     @property
     def cmd(self):
-        cmd = [
-            "pytest",
+        if self.django_toxenv:
+            cmd = ['tox', '-e', self.django_toxenv, '--']
+        else:
+            cmd = ['pytest']
+        cmd.extend([
             "-p",
             "no:randomly",
             "--junitxml=".format(self.xunit_report),
-        ] + self.passthrough_options + self.test_options_flags
+        ])
+        cmd.extend(self.passthrough_options + self.test_options_flags)
         if self.verbosity < 1:
             cmd.append("--quiet")
         elif self.verbosity > 1:

requirements/edx/paver.txt

 # Requirements to run and test Paver
 Paver==1.2.4
-lazy==1.1
-path.py==8.2.1
-watchdog==0.8.3
-python-memcached
 libsass==0.10.0

requirements/edx/testing.txt

@@ -20,3 +20,5 @@ pytest-django==3.1.2
 pytest-forked==0.2
 pytest-randomly==1.2.1
 pytest-xdist==1.20.0
+tox==2.8.2
+tox-battery==0.5

tox.ini

@@ -15,6 +15,8 @@ deps =
     -rrequirements/edx/local.txt
     -rrequirements/edx/base.txt
     -rrequirements/edx/development.txt
+    # There's 1-2 tests which call a paver command...
+    -rrequirements/edx/paver.txt
     -rrequirements/edx/testing.txt
     -rrequirements/edx/post.txt