PLAT-1710 Support lettuce tests in Docker Devstack

cms/envs/acceptance_docker.py

+"""
+This config file extends the test environment configuration
+so that we can run the lettuce acceptance tests.
+"""
+
+# We intentionally define lots of variables that aren't used, and
+# want to import all variables from base settings files
+# pylint: disable=wildcard-import, unused-wildcard-import
+
+import os
+
+os.environ['EDXAPP_TEST_MONGO_HOST'] = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'edx.devstack.mongo')
+
+# noinspection PyUnresolvedReferences
+from .acceptance import *
+
+update_module_store_settings(
+    MODULESTORE,
+    doc_store_settings={
+        'db': 'acceptance_xmodule',
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
+        'collection': 'acceptance_modulestore_%s' % seed(),
+    },
+    module_store_options={
+        'default_class': 'xmodule.raw_module.RawDescriptor',
+        'fs_root': TEST_ROOT / "data",
+    },
+    default_store=os.environ.get('DEFAULT_STORE', 'draft'),
+)
+
+CONTENTSTORE = {
+    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
+    'DOC_STORE_CONFIG': {
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
+        'db': 'acceptance_xcontent_%s' % seed(),
+    },
+    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
+    'ADDITIONAL_OPTIONS': {
+        'trashcan': {
+            'bucket': 'trash_fs'
+        }
+    }
+}
+
+# Where to run: local, saucelabs, or grid
+LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'grid')
+SELENIUM_HOST = 'edx.devstack.{}'.format(LETTUCE_BROWSER)
+SELENIUM_PORT = os.environ.get('SELENIUM_PORT', '4444')
+
+SELENIUM_GRID = {
+    'URL': 'http://{}:{}/wd/hub'.format(SELENIUM_HOST, SELENIUM_PORT),
+    'BROWSER': LETTUCE_BROWSER,
+}
+
+# Point the URL used to test YouTube availability to our stub YouTube server
+LETTUCE_HOST = os.environ['BOK_CHOY_HOSTNAME']
+YOUTUBE['API'] = "http://{}:{}/get_youtube_api/".format(LETTUCE_HOST, YOUTUBE_PORT)
+YOUTUBE['METADATA_URL'] = "http://{}:{}/test_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT)
+YOUTUBE['TEXT_API']['url'] = "{}:{}/test_transcripts_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT)

common/djangoapps/terrain/stubs/lti.py

@@ -18,6 +18,7 @@ from uuid import uuid4
 import mock
 import oauthlib.oauth1
 import requests
+from django.conf import settings
 from http import StubHttpRequestHandler, StubHttpService
 from oauthlib.oauth1.rfc5849 import parameters, signature
 
@@ -29,7 +30,7 @@ class StubLtiHandler(StubHttpRequestHandler):
     DEFAULT_CLIENT_KEY = 'test_client_key'
     DEFAULT_CLIENT_SECRET = 'test_client_secret'
     DEFAULT_LTI_ENDPOINT = 'correct_lti_endpoint'
-    DEFAULT_LTI_ADDRESS = 'http://127.0.0.1:{port}/'
+    DEFAULT_LTI_ADDRESS = 'http://{host}:{port}/'
 
     def do_GET(self):
         """
@@ -71,7 +72,8 @@ class StubLtiHandler(StubHttpRequestHandler):
                         'sourcedId': self.post_dict.get('lis_result_sourcedid')
                     }
 
-                submit_url = '//{}:{}'.format(*self.server.server_address)
+                host = getattr(settings, 'LETTUCE_HOST', self.server.server_address[0])
+                submit_url = '//{}:{}'.format(host, self.server.server_address[1])
                 content = self._create_content(status_message, submit_url)
                 self.send_response(200, content)
 
@@ -290,8 +292,9 @@ class StubLtiHandler(StubHttpRequestHandler):
         """
         client_secret = unicode(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET))
 
+        host = getattr(settings, 'LETTUCE_HOST', '127.0.0.1')
         port = self.server.server_address[1]
-        lti_base = self.DEFAULT_LTI_ADDRESS.format(port=port)
+        lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port)
         lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT)
         url = lti_base + lti_endpoint
 

lms/djangoapps/courseware/features/events.py

 # pylint: disable=missing-docstring
 
+from django.conf import settings
 from lettuce import before, step, world
 from nose.tools import assert_equals, assert_in
 from pymongo import MongoClient
@@ -19,7 +20,7 @@ REQUIRED_EVENT_FIELDS = [
 
 @before.all
 def connect_to_mongodb():
-    world.mongo_client = MongoClient()
+    world.mongo_client = MongoClient(host=settings.MONGO_HOST, port=settings.MONGO_PORT_NUM)
     world.event_collection = world.mongo_client['track']['events']
 
 

lms/djangoapps/courseware/features/lti.py

@@ -153,9 +153,10 @@ def set_incorrect_lti_passport(_step):
 @step(r'the course has an LTI component with (.*) fields(?:\:)?$')  # , new_page is(.*), graded is(.*)
 def add_correct_lti_to_course(_step, fields):
     category = 'lti'
+    host = getattr(settings, 'LETTUCE_HOST', '127.0.0.1')
     metadata = {
         'lti_id': 'correct_lti_id',
-        'launch_url': 'http://127.0.0.1:{}/correct_lti_endpoint'.format(settings.LTI_PORT),
+        'launch_url': 'http://{}:{}/correct_lti_endpoint'.format(host, settings.LTI_PORT),
     }
 
     if fields.strip() == 'incorrect_lti_id':  # incorrect fields

lms/envs/acceptance_docker.py

+"""
+This config file extends the test environment configuration
+so that we can run the lettuce acceptance tests.
+"""
+
+# We intentionally define lots of variables that aren't used, and
+# want to import all variables from base settings files
+# pylint: disable=wildcard-import, unused-wildcard-import
+
+import os
+
+os.environ['EDXAPP_TEST_MONGO_HOST'] = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'edx.devstack.mongo')
+
+# noinspection PyUnresolvedReferences
+from .acceptance import *
+
+LETTUCE_HOST = os.environ['BOK_CHOY_HOSTNAME']
+SITE_NAME = '{}:{}'.format(LETTUCE_HOST, LETTUCE_SERVER_PORT)
+
+update_module_store_settings(
+    MODULESTORE,
+    doc_store_settings={
+        'db': 'acceptance_xmodule',
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
+        'collection': 'acceptance_modulestore_%s' % seed(),
+    },
+    module_store_options={
+        'fs_root': TEST_ROOT / "data",
+    },
+    default_store=os.environ.get('DEFAULT_STORE', 'draft'),
+)
+CONTENTSTORE = {
+    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
+    'DOC_STORE_CONFIG': {
+        'host': MONGO_HOST,
+        'port': MONGO_PORT_NUM,
+        'db': 'acceptance_xcontent_%s' % seed(),
+    }
+}
+
+TRACKING_BACKENDS.update({
+    'mongo': {
+        'ENGINE': 'track.backends.mongodb.MongoBackend',
+        'OPTIONS': {
+            'database': 'test',
+            'collection': 'events',
+            'host': [
+                'edx.devstack.mongo'
+            ],
+            'port': 27017
+        }
+    }
+})
+
+EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update({
+    'mongo': {
+        'ENGINE': 'eventtracking.backends.mongodb.MongoBackend',
+        'OPTIONS': {
+            'database': 'track',
+            'host': [
+                'edx.devstack.mongo'
+            ],
+            'port': 27017
+        }
+    }
+})
+
+# Where to run: local, saucelabs, or grid
+LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'grid')
+SELENIUM_HOST = 'edx.devstack.{}'.format(LETTUCE_BROWSER)
+SELENIUM_PORT = os.environ.get('SELENIUM_PORT', '4444')
+
+SELENIUM_GRID = {
+    'URL': 'http://{}:{}/wd/hub'.format(SELENIUM_HOST, SELENIUM_PORT),
+    'BROWSER': LETTUCE_BROWSER,
+}
+
+# Point the URL used to test YouTube availability to our stub YouTube server
+YOUTUBE['API'] = "http://{}:{}/get_youtube_api/".format(LETTUCE_HOST, YOUTUBE_PORT)
+YOUTUBE['METADATA_URL'] = "http://{}:{}/test_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT)
+YOUTUBE['TEXT_API']['url'] = "{}:{}/test_transcripts_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT)

pavelib/utils/test/suites/acceptance_suite.py

@@ -42,6 +42,7 @@ def setup_acceptance_db():
             # Since we are using SQLLite, we can reset the database by deleting it on disk.
             DBS[db].remove()
 
+    settings = 'acceptance_docker' if Env.USING_DOCKER else 'acceptance'
     if all(DB_CACHES[cache].isfile() for cache in DB_CACHES.keys()):
         # To speed up migrations, we check for a cached database file and start from that.
         # The cached database file should be checked into the repo
@@ -53,13 +54,13 @@ def setup_acceptance_db():
         # Run migrations to update the db, starting from its cached state
         for db_alias in sorted(DBS.keys()):
             # pylint: disable=line-too-long
-            sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
-            sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias))
+            sh("./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
+            sh("./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
     else:
         # If no cached database exists, syncdb before migrating, then create the cache
         for db_alias in sorted(DBS.keys()):
-            sh("./manage.py lms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
-            sh("./manage.py cms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias))
+            sh("./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
+            sh("./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
 
         # Create the cache if it doesn't already exist
         for db_alias in DBS.keys():
@@ -77,6 +78,7 @@ class AcceptanceTest(TestSuite):
         self.system = kwargs.get('system')
         self.default_store = kwargs.get('default_store')
         self.extra_args = kwargs.get('extra_args', '')
+        self.settings = 'acceptance_docker' if Env.USING_DOCKER else 'acceptance'
 
     def __enter__(self):
         super(AcceptanceTest, self).__enter__()
@@ -91,15 +93,16 @@ class AcceptanceTest(TestSuite):
     @property
     def cmd(self):
 
+        lettuce_host = ['LETTUCE_HOST={}'.format(Env.SERVER_HOST)] if Env.USING_DOCKER else []
         report_file = self.report_dir / "{}.xml".format(self.system)
         report_args = ["--xunit-file {}".format(report_file)]
-        return [
+        return lettuce_host + [
             # set DBUS_SESSION_BUS_ADDRESS to avoid hangs on Chrome
             "DBUS_SESSION_BUS_ADDRESS=/dev/null",
             "DEFAULT_STORE={}".format(self.default_store),
             "./manage.py",
             self.system,
-            "--settings=acceptance",
+            "--settings={}".format(self.settings),
             "harvest",
             "--traceback",
             "--debug-mode",
@@ -112,7 +115,7 @@ class AcceptanceTest(TestSuite):
         """
         Internal helper method to manage asset compilation
         """
-        args = [self.system, '--settings=acceptance']
+        args = [self.system, '--settings={}'.format(self.settings)]
         call_task('pavelib.assets.update_assets', args=args)
 
 

requirements/edx/github.txt

@@ -68,7 +68,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c
 git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
 
 # Used for testing
-git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
+git+https://github.com/edx/lettuce.git@31b0dfd865766243e9b563ec65fae9122edf7975#egg=lettuce==0.2.23+edx.1
 
 # Why a DRF fork? See: https://openedx.atlassian.net/browse/PLAT-1581
 git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3