Skip to content

E
edx-platform
Commit 5a6b8cd3 by Jeremy Bowman
Options

PLAT-1228 Support bok-choy in Docker Devstack

parent 1ad2b66e
Showing with 660 additions and 34 deletions
cms/envs/bok_choy.py
......@@ -21,7 +21,7 @@ from openedx.core.release import RELEASE_LINE
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy'
os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()
from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import
......
cms/envs/bok_choy_docker.auth.json 0 → 100644
{
"ANALYTICS_API_KEY": "",
"AWS_ACCESS_KEY_ID": "",
"AWS_SECRET_ACCESS_KEY": "",
"CELERY_BROKER_PASSWORD": "celery",
"CELERY_BROKER_USER": "celery",
"CONTENTSTORE": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
"OPTIONS": {
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
}
},
"DATABASES": {
"default": {
"ENGINE": "django.db.backends.mysql",
"HOST": "edx.devstack.mysql",
"NAME": "edxtest",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
},
"student_module_history": {
"ENGINE": "django.db.backends.mysql",
"HOST": "edx.devstack.mysql",
"NAME": "student_module_history_test",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
}
},
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"MODULESTORE": {
"default": {
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"mappings": {},
"stores": [
{
"NAME": "draft",
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"edx.devstack.mongo"
],
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string"
}
},
{
"NAME": "xml",
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "** OVERRIDDEN **",
"default_class": "xmodule.hidden_module.HiddenDescriptor"
}
}
]
}
}
},
"DJFS": {
"type": "s3fs",
"bucket": "test",
"prefix": "test",
"aws_access_key_id": "test",
"aws_secret_access_key": "test"
},
"SECRET_KEY": "",
"XQUEUE_INTERFACE": {
"basic_auth": [
"edx",
"edx"
],
"django_auth": {
"password": "password",
"username": "lms"
},
"url": "http://localhost:18040"
},
"ZENDESK_API_KEY": "",
"ZENDESK_USER": ""
}
cms/envs/bok_choy_docker.env.json 0 → 100644
{
"ANALYTICS_SERVER_URL": "",
"BOOK_URL": "",
"BUGS_EMAIL": "bugs@example.com",
"BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com",
"CACHES": {
"celery": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_celery",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"default": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "sandbox_default",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"general": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "sandbox_general",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"mongo_metadata_inheritance": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_mongo_metadata_inheritance",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"staticfiles": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_static_files",
"LOCATION": [
"edx.devstack.memcached:11211"
]
}
},
"CELERY_ALWAYS_EAGER": true,
"CELERY_BROKER_HOSTNAME": "localhost",
"CELERY_BROKER_TRANSPORT": "amqp",
"CERT_QUEUE": "certificates",
"CMS_BASE": "** OVERRIDDEN **",
"CODE_JAIL": {
"limits": {
"REALTIME": 3,
"VMEM": 0
}
},
"COMMENTS_SERVICE_KEY": "password",
"COMMENTS_SERVICE_URL": "http://localhost:4567",
"CONTACT_EMAIL": "info@example.com",
"DEFAULT_FEEDBACK_EMAIL": "feedback@example.com",
"DEFAULT_FROM_EMAIL": "registration@example.com",
"EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
"SOCIAL_SHARING_SETTINGS": {
"CUSTOM_COURSE_URLS": true
},
"FEATURES": {
"AUTH_USE_OPENID_PROVIDER": true,
"CERTIFICATES_HTML_VIEW": true,
"ENABLE_DISCUSSION_SERVICE": true,
"ENABLE_GRADE_DOWNLOADS": true,
"ENTRANCE_EXAMS": true,
"MILESTONES_APP": true,
"PREVIEW_LMS_BASE": "preview.localhost:8003",
"ENABLE_CONTENT_LIBRARIES": true,
"ENABLE_SPECIAL_EXAMS": true,
"SHOW_HEADER_LANGUAGE_SELECTOR": true,
"ENABLE_EXTENDED_COURSE_DETAILS": true,
"CUSTOM_COURSES_EDX": true
},
"FEEDBACK_SUBMISSION_EMAIL": "",
"GITHUB_REPO_ROOT": "** OVERRIDDEN **",
"JWT_AUTH": {
"JWT_SECRET_KEY": "super-secret-key"
},
"GRADES_DOWNLOAD": {
"BUCKET": "edx-grades",
"ROOT_PATH": "/tmp/edx-s3/grades",
"STORAGE_TYPE": "localfs"
},
"LMS_BASE": "** OVERRIDDEN **",
"LMS_ROOT_URL": "** OVERRIDDEN **",
"LOCAL_LOGLEVEL": "INFO",
"LOGGING_ENV": "sandbox",
"LOG_DIR": "** OVERRIDDEN **",
"MEDIA_URL": "/media/",
"MKTG_URL_LINK_MAP": {},
"PLATFORM_NAME": "édX",
"SERVER_EMAIL": "devops@example.com",
"SESSION_COOKIE_DOMAIN": null,
"SITE_NAME": "localhost",
"STATIC_URL_BASE": "/static/",
"SYSLOG_SERVER": "",
"TECH_SUPPORT_EMAIL": "technical@example.com",
"TIME_ZONE": "America/New_York",
"WIKI_ENABLED": true,
"OAUTH_OIDC_ISSUER": "https://www.example.com/oauth2"
}
cms/envs/bok_choy_docker.py 0 → 100644
# -*- coding: utf-8 -*-
"""
Settings for Bok Choy tests that are used when running Studio in Docker-based devstack.
"""
# noinspection PyUnresolvedReferences
from .bok_choy import * # pylint: disable=wildcard-import
CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT'])
LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT'])
LMS_ROOT_URL = 'http://{}'.format(LMS_BASE)
# Docker does not support the syslog socket at /dev/log. Rely on the console.
LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = {
'class': 'logging.NullHandler',
}
LOGGING['loggers']['tracking']['handlers'] = ['console']
common/test/acceptance/fixtures/__init__.py
import os
HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', '8031')
LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', '8003')
# Get the URL of the Studio instance under test
STUDIO_BASE_URL = os.environ.get('studio_url', 'http://localhost:8031')
STUDIO_BASE_URL = os.environ.get('studio_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
# Get the URL of the LMS instance under test
LMS_BASE_URL = os.environ.get('lms_url', 'http://localhost:8003')
LMS_BASE_URL = os.environ.get('lms_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))
# Get the URL of the XQueue stub used in the test
XQUEUE_STUB_URL = os.environ.get('xqueue_url', 'http://localhost:8040')
......
common/test/acceptance/pages/common/__init__.py
import os
HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
LMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8003)
# Get the URL of the instance under test
BASE_URL = os.environ.get('test_url', 'http://localhost:8003')
BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))
# The URL used for user auth in testing
AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
common/test/acceptance/pages/common/auto_auth.py
......@@ -8,7 +8,9 @@ import urllib
from bok_choy.page_object import XSS_INJECTION, PageObject, unguarded
# The URL used for user auth in testing
AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
class AutoAuthPage(PageObject):
......
common/test/acceptance/pages/lms/__init__.py
import os
# Get the URL of the instance under test
BASE_URL = os.environ.get('test_url', 'http://localhost:8003')
HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', 8003)
BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))
common/test/acceptance/pages/studio/__init__.py
import os
# Get the URL of the instance under test
BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
lms/envs/bok_choy.py
......@@ -25,7 +25,7 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy'
os.environ['CONFIG_ROOT'] = CONFIG_ROOT
from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import
......
lms/envs/bok_choy_docker.auth.json 0 → 100644
{
"ANALYTICS_API_KEY": "",
"AWS_ACCESS_KEY_ID": "",
"AWS_SECRET_ACCESS_KEY": "",
"CC_PROCESSOR_NAME": "CyberSource2",
"CC_PROCESSOR": {
"CyberSource2": {
"SECRET_KEY": "abcd123",
"ACCESS_KEY": "abcd123",
"PROFILE_ID": "edx",
"PURCHASE_ENDPOINT": "/shoppingcart/payment_fake"
}
},
"CELERY_BROKER_PASSWORD": "celery",
"CELERY_BROKER_USER": "celery",
"CONTENTSTORE": {
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
"OPTIONS": {
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
}
},
"DATABASES": {
"default": {
"ENGINE": "django.db.backends.mysql",
"HOST": "edx.devstack.mysql",
"NAME": "edxtest",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
},
"student_module_history": {
"ENGINE": "django.db.backends.mysql",
"HOST": "edx.devstack.mysql",
"NAME": "student_module_history_test",
"PASSWORD": "",
"PORT": "3306",
"USER": "root"
}
},
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"TRACKING_BACKENDS": {
"mongo": {
"ENGINE": "track.backends.mongodb.MongoBackend",
"OPTIONS": {
"database": "test",
"collection": "events",
"host": [
"edx.devstack.mongo"
],
"port": 27017
}
}
},
"EVENT_TRACKING_BACKENDS": {
"mongo": {
"ENGINE": "eventtracking.backends.mongodb.MongoBackend",
"OPTIONS": {
"database": "test",
"collection": "events",
"host": [
"edx.devstack.mongo"
],
"port": 27017
}
}
},
"MODULESTORE": {
"default": {
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"mappings": {},
"stores": [
{
"NAME": "draft",
"DOC_STORE_CONFIG": {
"collection": "modulestore",
"db": "test",
"host": [
"edx.devstack.mongo"
],
"port": 27017
},
"ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "** OVERRIDDEN **",
"host": [
"edx.devstack.mongo"
],
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string"
}
},
{
"NAME": "xml",
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "** OVERRIDDEN **",
"default_class": "xmodule.hidden_module.HiddenDescriptor"
}
}
]
}
}
},
"SECRET_KEY": "",
"DJFS": {
"type": "s3fs",
"bucket": "test",
"prefix": "test",
"aws_access_key_id": "test",
"aws_secret_access_key": "test"
},
"XQUEUE_INTERFACE": {
"basic_auth": [
"edx",
"edx"
],
"django_auth": {
"password": "password",
"username": "lms"
},
"url": "** OVERRIDDEN **"
},
"ZENDESK_API_KEY": "",
"ZENDESK_USER": ""
}
lms/envs/bok_choy_docker.env.json 0 → 100644
{
"ANALYTICS_SERVER_URL": "",
"ANALYTICS_DASHBOARD_URL": "",
"BOOK_URL": "",
"BUGS_EMAIL": "bugs@example.com",
"BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com",
"CACHES": {
"celery": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_celery",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"default": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "sandbox_default",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"general": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "sandbox_general",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"mongo_metadata_inheritance": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_mongo_metadata_inheritance",
"LOCATION": [
"edx.devstack.memcached:11211"
]
},
"staticfiles": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"KEY_PREFIX": "integration_static_files",
"LOCATION": [
"edx.devstack.memcached:11211"
]
}
},
"CELERY_BROKER_HOSTNAME": "localhost",
"CELERY_BROKER_TRANSPORT": "amqp",
"CERT_QUEUE": "certificates",
"CMS_BASE": "** OVERRIDDEN **",
"CODE_JAIL": {
"limits": {
"REALTIME": 3,
"VMEM": 0
}
},
"COMMENTS_SERVICE_KEY": "password",
"COMMENTS_SERVICE_URL": "http://localhost:4567",
"CONTACT_EMAIL": "info@example.com",
"DEFAULT_FEEDBACK_EMAIL": "feedback@example.com",
"DEFAULT_FROM_EMAIL": "registration@example.com",
"EMAIL_BACKEND": "django.core.mail.backends.dummy.EmailBackend",
"SOCIAL_SHARING_SETTINGS": {
"CUSTOM_COURSE_URLS": true,
"DASHBOARD_FACEBOOK": true,
"CERTIFICATE_FACEBOOK": true,
"CERTIFICATE_FACEBOOK_TEXT": "Testing facebook feature:",
"DASHBOARD_TWITTER": true,
"DASHBOARD_TWITTER_TEXT": "Testing feature:"
},
"FEATURES": {
"AUTH_USE_OPENID_PROVIDER": true,
"CERTIFICATES_HTML_VIEW": true,
"CERTIFICATES_INSTRUCTOR_GENERATION": true,
"ENABLE_PAYMENT_FAKE": true,
"ENABLE_VERIFIED_CERTIFICATES": true,
"ENABLE_DISCUSSION_SERVICE": true,
"ENABLE_GRADE_DOWNLOADS": true,
"ENABLE_THIRD_PARTY_AUTH": true,
"ENABLE_COMBINED_LOGIN_REGISTRATION": true,
"PREVIEW_LMS_BASE": "preview.localhost:8003",
"ALLOW_AUTOMATED_SIGNUPS": true,
"AUTOMATIC_AUTH_FOR_TESTING": true,
"MODE_CREATION_FOR_TESTING": true,
"EXPOSE_CACHE_PROGRAMS_ENDPOINT": true,
"AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING": true,
"ENABLE_COURSE_DISCOVERY": true,
"ENABLE_SPECIAL_EXAMS": true,
"SHOW_HEADER_LANGUAGE_SELECTOR": true,
"CUSTOM_COURSES_EDX": true
},
"FEEDBACK_SUBMISSION_EMAIL": "",
"GITHUB_REPO_ROOT": "** OVERRIDDEN **",
"JWT_AUTH": {
"JWT_SECRET_KEY": "super-secret-key"
},
"LMS_BASE": "** OVERRIDDEN **",
"LMS_ROOT_URL": "** OVERRIDDEN **",
"LOCAL_LOGLEVEL": "INFO",
"LOGGING_ENV": "sandbox",
"LOG_DIR": "** OVERRIDDEN **",
"MEDIA_URL": "/media/",
"MKTG_URL_LINK_MAP": {
"ABOUT": "about",
"PRIVACY": "privacy",
"TOS": "tos",
"WHAT_IS_VERIFIED_CERT": "verified-certificate",
"COURSES": "courses",
"CONTACT": "contact",
"NEWS": "news",
"HONOR": "honor",
"CAREERS": "careers",
"HELP_CENTER": "help-center",
"BLOG": "blog",
"PRESS": "press",
"DONATE": "donate",
"ROOT": "root",
"SITEMAP.XML": "sitemap_xml"
},
"PLATFORM_NAME": "édX",
"REGISTRATION_EXTENSION_FORM": "openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm",
"REGISTRATION_EXTRA_FIELDS": {
"level_of_education": "optional",
"gender": "optional",
"year_of_birth": "optional",
"mailing_address": "optional",
"goals": "optional",
"honor_code": "required",
"terms_of_service": "hidden",
"city": "hidden",
"country": "required"
},
"SERVER_EMAIL": "devops@example.com",
"SESSION_COOKIE_DOMAIN": null,
"SITE_NAME": "localhost:8003",
"STATIC_URL_BASE": "/static/",
"SUPPORT_SITE_LINK": "https://support.example.com",
"PASSWORD_RESET_SUPPORT_LINK": "https://support.example.com/password-reset-help.html",
"ACTIVATION_EMAIL_SUPPORT_LINK": "https://support.example.com/activation-email-help.html",
"SYSLOG_SERVER": "",
"TECH_SUPPORT_EMAIL": "technical@example.com",
"THIRD_PARTY_AUTH_BACKENDS": [
"social.backends.google.GoogleOAuth2",
"social.backends.linkedin.LinkedinOAuth2",
"social.backends.facebook.FacebookOAuth2",
"third_party_auth.dummy.DummyBackend",
"third_party_auth.saml.SAMLAuthBackend"
],
"TIME_ZONE": "America/New_York",
"WIKI_ENABLED": true
}
lms/envs/bok_choy_docker.py 0 → 100644
# -*- coding: utf-8 -*-
"""
Settings for Bok Choy tests that are used when running Studio in Docker-based devstack.
"""
# noinspection PyUnresolvedReferences
from .bok_choy import * # pylint: disable=wildcard-import
CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT'])
LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT'])
LMS_ROOT_URL = 'http://{}'.format(LMS_BASE)
# Docker does not support the syslog socket at /dev/log. Rely on the console.
LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = {
'class': 'logging.NullHandler',
}
LOGGING['loggers']['tracking']['handlers'] = ['console']
pavelib/utils/envs.py
......@@ -65,13 +65,20 @@ class Env(object):
# Directory that videos are served from
VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video"
# Detect if in a Docker container, and if so which one
SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0')
USING_DOCKER = SERVER_HOST != '0.0.0.0'
SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy'
BOK_CHOY_SERVERS = {
'lms': {
'port': 8003,
'host': SERVER_HOST,
'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'),
'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log"
},
'cms': {
'port': 8031,
'host': SERVER_HOST,
'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'),
'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log"
}
}
......@@ -123,8 +130,10 @@ class Env(object):
}
# Mongo databases that will be dropped before/after the tests run
BOK_CHOY_MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost'
BOK_CHOY_MONGO_DATABASE = "test"
BOK_CHOY_CACHE = memcache.Client(['0.0.0.0:11211'], debug=0)
BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0'
BOK_CHOY_CACHE = memcache.Client(['{}:11211'.format(BOK_CHOY_CACHE_HOST)], debug=0)
# Test Ids Directory
TEST_DIR = REPO_ROOT / ".testids"
......
pavelib/utils/test/bokchoy_utils.py
......@@ -44,11 +44,12 @@ def start_servers(options):
cmd = (
"DEFAULT_STORE={default_store} "
"coverage run --rcfile={coveragerc} -m "
"manage {service} --settings bok_choy runserver "
"manage {service} --settings {settings} runserver "
"{address} --traceback --noreload".format(
default_store=options.default_store,
coveragerc=coveragerc,
service=service,
settings=Env.SETTINGS,
address=address,
)
)
......@@ -83,7 +84,7 @@ def wait_for_server(server, port):
attempts = 0
server_ok = False
while attempts < 30:
while attempts < 120:
try:
connection = httplib.HTTPConnection(server, port, timeout=10)
connection.request('GET', '/')
......@@ -107,7 +108,7 @@ def wait_for_test_servers():
"""
for service, info in Env.BOK_CHOY_SERVERS.iteritems():
ready = wait_for_server("0.0.0.0", info['port'])
ready = wait_for_server(info['host'], info['port'])
if not ready:
msg = colorize(
"red",
......@@ -123,7 +124,7 @@ def is_mongo_running():
"""
# The mongo command will connect to the service,
# failing with a non-zero exit code if it cannot connect.
output = os.popen('mongo --eval "print(\'running\')"').read()
output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.BOK_CHOY_MONGO_HOST)).read()
return output and "running" in output
......@@ -155,7 +156,8 @@ def clear_mongo():
Clears mongo database.
"""
sh(
"mongo {} --eval 'db.dropDatabase()' > /dev/null".format(
"mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format(
Env.BOK_CHOY_MONGO_HOST,
Env.BOK_CHOY_MONGO_DATABASE,
)
)
......@@ -191,6 +193,9 @@ def check_mysql():
"""
Check that mysql is running
"""
if 'BOK_CHOY_HOSTNAME' in os.environ:
# mysql should be running in a separate Docker container
return
if not is_mysql_running():
msg = colorize('red', "MySQL is not running locally.")
print msg
......
pavelib/utils/test/suites/bokchoy_suite.py
......@@ -48,9 +48,10 @@ def load_bok_choy_data(options):
print 'Loading data from json fixtures in db_fixtures directory'
sh(
"DEFAULT_STORE={default_store}"
" ./manage.py lms --settings bok_choy loaddata --traceback"
" ./manage.py lms --settings {settings} loaddata --traceback"
" common/test/db_fixtures/*.json".format(
default_store=options.default_store,
settings=Env.SETTINGS
)
)
......@@ -76,9 +77,10 @@ def load_courses(options):
sh(
"DEFAULT_STORE={default_store}"
" ./manage.py cms --settings=bok_choy import {import_dir}".format(
" ./manage.py cms --settings={settings} import {import_dir}".format(
default_store=options.default_store,
import_dir=options.imports_dir
import_dir=options.imports_dir,
settings=Env.SETTINGS
)
)
else:
......@@ -261,7 +263,7 @@ class BokChoyTestSuite(TestSuite):
# Clean up data we created in the databases
msg = colorize('green', "Cleaning up databases...")
print msg
sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS))
clear_mongo()
@property
......
pavelib/utils/test/utils.py
......@@ -10,6 +10,8 @@ from paver.easy import cmdopts, sh, task
from pavelib.utils.envs import Env
from pavelib.utils.timer import timed
from bok_choy.browser import browser
MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
MINIMUM_FIREFOX_VERSION = 28.0
......@@ -78,6 +80,33 @@ def check_firefox_version():
"""
Check that firefox is the correct version.
"""
if 'BOK_CHOY_HOSTNAME' in os.environ:
# Firefox is running in a separate Docker container; get its version via Selenium
driver = browser()
capabilities = driver.capabilities
if capabilities['browserName'] == 'firefox':
firefox_version_regex = re.compile(r'^\d+\.\d+')
version_key = 'browserVersion' if 'browserVersion' in 'capabilities' else 'version'
try:
firefox_ver = float(firefox_version_regex.search(capabilities[version_key]).group(0))
except AttributeError:
firefox_ver = 0.0
else:
firefox_ver = 0.0
driver.close()
if firefox_ver < MINIMUM_FIREFOX_VERSION:
raise Exception(
'Required firefox version not found.\n'
'Expected: {expected_version}; Actual: {actual_version}.\n\n'
'Make sure that the edx.devstack.firefox container is up-to-date and running\n'
'\t{expected_version}'.format(
actual_version=firefox_ver,
expected_version=MINIMUM_FIREFOX_VERSION
)
)
return
# Firefox will be run as a local process
expected_firefox_ver = "Mozilla Firefox " + str(MINIMUM_FIREFOX_VERSION)
firefox_ver_string = subprocess.check_output("firefox --version", shell=True).strip()
firefox_version_regex = re.compile(r"Mozilla Firefox (\d+.\d+)")
......
requirements/edx/base.txt
......@@ -154,7 +154,7 @@ sqlparse>=0.2.0,<0.3.0
# Used for testing
before_after==0.1.3
bok-choy==0.7.0
bok-choy==0.7.1
chrono==1.0.2
ddt==0.8.0
django-crum==0.5
......
scripts/reset-test-db.sh
......@@ -27,6 +27,14 @@ set -e
DB_CACHE_DIR="common/test/db_cache"
if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then
MYSQL_HOST=""
SETTINGS="bok_choy"
else
MYSQL_HOST="--host=edx.devstack.mysql"
SETTINGS="bok_choy_docker"
fi
declare -A databases
declare -a database_order
databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
......@@ -34,7 +42,7 @@ database_order=("default" "student_module_history")
# Ensure the test database exists.
for db in "${database_order[@]}"; do
echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root
echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root
# Clear out the test database
#
......@@ -43,7 +51,7 @@ for db in "${database_order[@]}"; do
# or a jenkins worker environment) that already ran tests on another commit that had
# different migrations that created, dropped, or altered tables.
echo "Issuing a reset_db command to the $db bok_choy MySQL database."
./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db
./manage.py lms --settings $SETTINGS reset_db --traceback --noinput --router $db
# If there are cached database schemas/data, load them
if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then
......@@ -61,19 +69,19 @@ if [[ -z $REBUILD_CACHE ]]; then
for db in "${database_order[@]}"; do
# Load the schema, then the data (including the migration history)
echo "Loading the schema from the filesystem into the $db MySQL DB."
mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
echo "Loading the fixture data from the filesystem into the $db MySQL DB."
./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
./manage.py lms --settings $SETTINGS loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
# Migrations are stored in the default database
echo "Loading the migration data from the filesystem into the $db MySQL DB."
mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
# Re-run migrations to ensure we are up-to-date
echo "Running the lms migrations on the $db bok_choy DB."
./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
echo "Running the cms migrations on the $db bok_choy DB."
./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput
done
......@@ -86,18 +94,18 @@ else
for db in "${database_order[@]}"; do
# Re-run migrations on the test database
echo "Issuing a migrate command to the $db bok_choy MySQL database for the lms django apps."
./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
echo "Issuing a migrate command to the $db bok_choy MySQL database for the cms django apps."
./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput
# Dump the schema and data to the cache
echo "Using the dumpdata command to save the $db fixture data to the filesystem."
./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog
./manage.py lms --settings $SETTINGS dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog
echo "Saving the schema of the $db bok_choy DB to the filesystem."
mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
mysqldump $MYSQL_HOST -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
# dump_data does not dump the django_migrations table so we do it separately.
echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem."
mysqldump -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
mysqldump $MYSQL_HOST -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
done
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment