Skip to content

E
edx-analytics-dashboard
Commit 036616b3 by Dennis Jen Committed by GitHub
Options

Merge branch 'master' into thallada/course-list-metadata

parents 9a3ec156 35ff02ef
Showing with 124 additions and 130 deletions
.travis.yml
language:
- python
- node_js
python: "2.7"
node_js: "5.2.0"
sudo: false
python:
- "2.7"
sudo: required
services:
- docker
env:
# Make sure to update this string on every Insights or Data API release
- DATA_API_VERSION="0.20.0-rc.3"
DATA_API_VERSION: "0.20.0-rc.3"
DOCKER_COMPOSE_VERSION: "1.9.0"
before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
addons:
firefox: "46.0"
apt:
packages:
- language-pack-en
install:
- make develop
- make migrate
- ./scripts/install_analytics_data_api.sh $DATA_API_VERSION
- pip install coveralls
# Install a newer version of docker-compose
# Remove once dockers default is this version: https://docs.travis-ci.com/user/docker/#Using-Docker-Compose
- sudo rm /usr/local/bin/docker-compose
- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
# Start up the relevant services
- docker-compose -f .travis/docker-compose-travis.yml up -d
- docker exec analytics_api make -C /edx/app/analytics_api/analytics_api test.requirements
- docker exec analytics_api make -C /edx/app/analytics_api/analytics_api travis
script:
- make static_no_compress
- make validate_translations
- make validate
- make generate_fake_translations
- make accept
- make a11y
- docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_tests.sh
after_success:
- coveralls
- bash ./scripts/build-stats-to-datadog.sh
- pip install -U codecov
- docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh
- codecov
deploy:
- provider: s3
access_key_id: $S3_ACCESS_KEY_ID
......
.travis/docker-compose-travis.yml 0 → 100644
version: "2.1"
services:
es:
image: elasticsearch:1.5.2
analytics_api:
image: edxops/analytics_api:${DATA_API_VERSION:-latest}
container_name: analytics_api
environment:
ELASTICSEARCH_LEARNERS_HOST: 'http://es:9200/'
ELASTICSEARCH_LEARNERS_INDEX: 'learner'
ELASTICSEARCH_LEARNERS_UPDATE_INDEX: 'index_update'
command: /edx/app/analytics_api/venvs/analytics_api/bin/python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:80
insights:
image: edxops/insights:latest
container_name: insights_testing
volumes:
- ..:/edx/app/insights/edx_analytics_dashboard
command: tail -f /dev/null
environment:
# Pull these from the host environment.
TRAVIS_BRANCH:
TRAVIS_PULL_REQUEST:
DATADOG_API_KEY:
# Rest of the environment variables for testing.
API_SERVER_URL: http://analytics_api/api/v0
API_AUTH_TOKEN: edx
LMS_HOSTNAME: lms
LMS_PASSWORD: pass
LMS_USERNAME: user
ENABLE_AUTO_AUTH: "True"
ENABLE_OAUTH_TESTS: "False"
ENABLE_ERROR_PAGE_TESTS: "False"
DISPLAY_LEARNER_ANALYTICS: "True"
depends_on:
- "es"
- "analytics_api"
.travis/run_coverage.sh 0 → 100755
#!/bin/bash -xe
. /edx/app/insights/venvs/insights/bin/activate
. /edx/app/insights/nodeenvs/insights/bin/activate
cd /edx/app/insights/insights
coverage xml
bash ./scripts/build-stats-to-datadog.sh
.travis/run_tests.sh 0 → 100755
#!/bin/bash -xe
. /edx/app/insights/venvs/insights/bin/activate
. /edx/app/insights/nodeenvs/insights/bin/activate
apt update
apt install -y xvfb language-pack-en firefox # gettext
# Need firefox 46 specifically, later versions don't work with Karma(frontend testing library).
curl -O https://ftp.mozilla.org/pub/firefox/releases/46.0/linux-x86_64/en-US/firefox-46.0.tar.bz2
tar xvf firefox-46.0.tar.bz2
mv -f firefox /opt
mv -f /usr/bin/firefox /usr/bin/firefox_default
ln -s /opt/firefox/firefox /usr/bin/firefox
cd /edx/app/insights/edx_analytics_dashboard
export PATH=$PATH:$PWD/node_modules/.bin
# Make it so bower can run without sudo.
# https://github.com/GeoNode/geonode/pull/1070
echo '{ "allow_root": true }' > /root/.bowerrc
make develop
make migrate
# Compile assets and run validation
make static_no_compress
make validate_translations
make validate
make generate_fake_translations
# The following tests need insights running. We have to do it here
# because we need to wait till all the requirements have been installed
# otherwise the server will startup with potentially the wrong libraries.
/edx/bin/python.insights /edx/bin/manage.insights runserver 0.0.0.0:9000 --noreload --traceback > dashboard.log 2>&1 &
xvfb-run make accept
xvfb-run make a11y
Makefile
......@@ -35,13 +35,16 @@ test_python_no_compress: clean
--cover-package=analytics_dashboard --cover-package=common --cover-branches --cover-html --cover-html-dir=$(COVERAGE)/html/ \
--with-ignore-docstrings --cover-xml --cover-xml-file=$(COVERAGE)/coverage.xml
test_compress:
test_compress: static_no_compress
python manage.py compress --settings=analytics_dashboard.settings.test
test_python: test_compress test_python_no_compress
accept:
./scripts/runTests.sh acceptance_tests
ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
./manage.py waffle_flag enable_learner_analytics on --create --everyone
endif
nosetests -v acceptance_tests -e NUM_PROCESSES=1 --exclude-dir=acceptance_tests/course_validation
# local acceptance tests are typically run with by passing in environment variables on the commandline
# e.g. API_SERVER_URL="http://localhost:9001/api/v0" API_AUTH_TOKEN="edx" make accept_local
......@@ -49,7 +52,10 @@ accept_local:
nosetests -v acceptance_tests --exclude-dir=acceptance_tests/course_validation
a11y:
BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=phantomjs ./scripts/runTests.sh a11y_tests
ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
./manage.py waffle_flag enable_learner_analytics on --create --everyone
endif
BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=phantomjs nosetests -v a11y_tests -e NUM_PROCESSES=1 --exclude-dir=acceptance_tests/course_validation
course_validation:
python -m acceptance_tests.course_validation.generate_report
......
analytics_dashboard/settings/local.py
......@@ -22,3 +22,7 @@ DATABASES = {
}
}
########## END DATABASE CONFIGURATION
########## DATA API CONFIGURATION
DATA_API_URL = os.getenv("API_SERVER_URL", DATA_API_URL)
########## END DATA API CONFIGURATION
package.json
......@@ -16,10 +16,10 @@
"eslint-config-edx": "^1.2.0",
"gulp": "^3.8.8",
"gulp-eslint": "^2.0.0",
"gulp-karma": "0.0.4",
"gulp-karma": "0.0.5",
"jasmine-core": "^2.4.1",
"jscs": "^1.10.0",
"karma": "^0.12.16",
"karma": "^1.3.0",
"karma-coverage": "^0.2.6",
"karma-chrome-launcher": "^0.2.3",
"karma-jasmine": "^0.3.6",
......
scripts/install_analytics_data_api.sh deleted 100755 → 0
#!/usr/bin/env bash
git clone https://github.com/edx/edx-analytics-data-api.git
cd edx-analytics-data-api
if [ -z $1 ]; then
git checkout master
else
git checkout $1
fi
virtualenv venv
source venv/bin/activate
make requirements
make test.install_elasticsearch
deactivate
cd -
scripts/runTests.sh deleted 100755 → 0
#!/usr/bin/env bash
TEST_DIR=${1?arg missing, specify a directory to find tests}
# this stops the django servers
stopServers() {
kill $(ps aux | grep "[m]anage.py" | awk '{print $2}')
}
echo "Stop any already running servers..."
stopServers
echo "Setting test related environment variables..."
export API_SERVER_URL=http://127.0.0.1:9001/api/v0
export API_AUTH_TOKEN=edx
export LMS_HOSTNAME=lms
export LMS_PASSWORD=pass
export LMS_USERNAME=user
export ENABLE_AUTO_AUTH=True
export ENABLE_OAUTH_TESTS=False
export ENABLE_ERROR_PAGE_TESTS=False
export DISPLAY_LEARNER_ANALYTICS=True
echo "Preparing Analytics Data API..."
cd edx-analytics-data-api/
source venv/bin/activate
DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.local" make develop
DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.local" make travis
deactivate
cd -
mkdir -p logs
echo "Enabling waffle flags..."
if [[ "${DISPLAY_LEARNER_ANALYTICS}" = "True" ]]; then
./manage.py waffle_flag enable_learner_analytics on --create --everyone
fi
echo "Starting Analytics Data API Server..."
./scripts/run_analytics_data_api.sh
echo "Starting Analytics Dashboard Server..."
./manage.py runserver 9000 --noreload --traceback > logs/dashboard.log 2>&1 &
echo "Running $TEST_DIR tests..."
nosetests -v $TEST_DIR -e NUM_PROCESSES=1 --exclude-dir=acceptance_tests/course_validation
# capture the exit code from the test. Anything more than 0 indicates failed cases.
EXIT_CODE=$?
echo "Shutting down server..."
stopServers
if [[ "$EXIT_CODE" = "0" ]]; then
echo "All tests passed..."
else
echo "Failed tests..."
echo -e "\033[33;34m Server Logs for Analytics Data API Server... \033[0m "
cat logs/api.log
echo -e "\033[33;34m Server logs for Analytics Dashboard Server... \033[0m "
cat logs/dashboard.log
fi
exit $EXIT_CODE
scripts/run_analytics_data_api.sh deleted 100755 → 0
#!/usr/bin/env bash
cd edx-analytics-data-api
# Use the analytics data api virtualenv
source venv/bin/activate
# Kick off elasticsearch.
make test.run_elasticsearch
# This is unfortunate, but recommended by Travis in order to make sure
# ElasticSearch is running by the time the Analytics Data API accesses
# it during testing.
sleep 10
# Create the ElasticSearch indicies and mappings.
export ELASTICSEARCH_LEARNERS_HOST='http://localhost:9223'
export ELASTICSEARCH_LEARNERS_INDEX='learner'
export ELASTICSEARCH_LEARNERS_UPDATE_INDEX='index_update'
export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.local"
./manage.py create_elasticsearch_learners_indices
# Run the analytics data api server.
./manage.py runserver 9001 --noreload > ../logs/api.log 2>&1 &
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment