Skip to content

E
edx-ora2
Commit 00605095 by Eric Fischer
Options

Modernize ORA, commit 1 of 2 

Contains installation and testing changes for the django 1.11 upgrade
preparedness initiative. Splits tests out into seperate shards on travis.
parent 5a07f1f0
Showing with 155 additions and 398 deletions
.travis.yml
addons:
apt:
packages:
- aspell
- aspell-en
- g++
- gcc
- git
- gfortran
- libblas3gf
- libblas-dev
- liblapack3gf
- liblapack-dev
- libatlas-base-dev
- libfontconfig1
- libmysqlclient-dev
- libxml2-dev
- libxslt1-dev
- nodejs
- python2.7
- python2.7-dev
- python-pip
- python-software-properties
language: python
sudo: false
python:
- "2.7"
env:
- TOXENV=django18
- TOXENV=django111
matrix:
include:
- python: 2.7
env: TOXENV=quality
- python: 2.7
env: TOXENV=js
before_install:
- "pip install -U pip"
cache:
directories:
- $HOME/.cache/pip
install:
- "pip install coveralls"
- "make install"
before_script:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- "pip install tox"
script:
- "make verify-generated-files"
- "make test"
- "python manage.py makemessages -l eo"
- tox
branches:
only:
- master
- ora-staff-grading
after_success:
coveralls
MANIFEST.in
include LICENSE
include AUTHORS
include README.rst
include setup.cfg
include openassessment/xblock/static/css/*.css
include openassessment/xblock/static/css/lib/backgrid/*.css
include openassessment/xblock/static/js/openassessment*.min.js
include openassessment/xblock/static/js/lib/backgrid/*.js
include requirements/*.txt
recursive-include openassessment/xblock/static/js/src *.js
recursive-include openassessment/templates *.html *.underscore
recursive-include openassessment/locale *.po
recursive-include openassessment/locale *.mo
global-exclude */test*
global-exclude */test?/*
Makefile
all: install test
.PHONY: install test
# not used by travis
install-system:
sudo apt-get update -qq
sudo xargs -a apt-packages.txt apt-get install -qq --fix-missing
# not used by travis
install-node:
sudo apt-get install -qq nodejs
install-wheels:
./scripts/install-wheels.sh
##################
# Install commands
##################
install-python:
./scripts/install-python.sh
pip install -r requirements/django.txt
pip install -r requirements/base.txt --only-binary=lxml,libsass
install-js:
npm install
install-nltk-data:
./scripts/download-nltk-data.sh
install-test:
pip install -r requirements/test.txt
install: install-python install-js install-test javascript sass
##############################
# Generate js/css output files
##############################
STATIC_JS = openassessment/xblock/static/js
STATIC_CSS = openassessment/xblock/static/css
update-npm-requirements:
npm update --silent
cp ./node_modules/backgrid/lib/backgrid*.js $(STATIC_JS)/lib/backgrid/
cp ./node_modules/backgrid/lib/backgrid*.css $(STATIC_CSS)/lib/backgrid/
javascript: update-npm-requirements
node_modules/.bin/uglifyjs $(STATIC_JS)/src/oa_shared.js $(STATIC_JS)/src/*.js $(STATIC_JS)/src/lms/*.js $(STATIC_JS)/lib/backgrid/backgrid.min.js -c warnings=false > "$(STATIC_JS)/openassessment-lms.min.js"
node_modules/.bin/uglifyjs $(STATIC_JS)/src/oa_shared.js $(STATIC_JS)/src/*.js $(STATIC_JS)/src/studio/*.js $(STATIC_JS)/lib/backgrid/backgrid.min.js -c warnings=false > "$(STATIC_JS)/openassessment-studio.min.js"
......@@ -38,27 +34,17 @@ sass:
verify-generated-files:
@git diff --quiet || (echo 'Modifications exist locally! Run `make javascript` or `make sass` to update bundled files.'; exit 1)
install-test:
pip install -q -r requirements/test.txt
install-sys-requirements: install-system install-node
npm config set loglevel warn
install-dev:
pip install -q -r requirements/dev.txt
install: install-wheels install-python install-js install-nltk-data install-test install-dev javascript sass
################
#Tests and checks
################
quality:
./node_modules/.bin/jshint $(STATIC_JS)/src -c .jshintrc --verbose
./node_modules/jscs/bin/jscs $(STATIC_JS)/src --verbose
./scripts/run-pep8.sh
./scripts/run-pylint.sh
test: quality test-python test-js
test-python:
./scripts/test-python.sh
coverage run manage.py test openassessment
render-templates:
./scripts/render-templates.sh
......@@ -69,15 +55,13 @@ test-js: render-templates
test-js-debug: render-templates
./scripts/js-debugger.sh
test-sandbox: test-acceptance test-a11y
test: quality test-python test-js
# acceptance and a11y tests require a functioning sandbox, and do not run on travis
test-acceptance:
./scripts/test-acceptance.sh tests
test-a11y:
./scripts/test-acceptance.sh accessibility
update-npm-requirements:
npm update --silent
cp ./node_modules/backgrid/lib/backgrid*.js $(STATIC_JS)/lib/backgrid/
cp ./node_modules/backgrid/lib/backgrid*.css $(STATIC_CSS)/lib/backgrid/
test-sandbox: test-acceptance test-a11y
README.rst
......@@ -7,7 +7,7 @@ Open Response Assessment |build-status| |coverage-status|
Installation, Tests, and other Developer Tasks
==============================================
EdX engineers follow the `guides on our wiki <https://openedx.atlassian.net/wiki/display/EDUCATOR/ORA+FAQ>`_.
EdX engineers follow the `guides on our wiki <https://openedx.atlassian.net/wiki/spaces/EDUCATOR/pages/9765004/ORA+Developer+Guide>`_.
License
=======
......
manage.py
#!/usr/bin/env python
import sys
import os
import sys
if __name__ == "__main__":
......
pylintrc
......@@ -55,7 +55,7 @@
[MASTER]
ignore =
persistent = yes
load-plugins = edx_lint.pylint,pylint_django,pylint_celery
load-plugins = edx_lint.pylint,pylint_django
[MESSAGES CONTROL]
disable =
......
requirements/base.txt
# Include the requirements we're caching as "wheel" archives
# to speed up the test builds.
-r wheels.txt
# edX Internal Requirements
git+https://github.com/edx/XBlock.git@xblock-0.4.12#egg=XBlock==0.4.12
edx-submissions>=2.0.6,<3.0.0
git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
git+https://github.com/edx/XBlock.git@xblock-1.0.1#egg=XBlock==1.0.1
git+https://github.com/edx/xblock-sdk.git@v0.1.4#egg=xblock-sdk==0.1.4
# Third Party Requirements
boto>=2.32.1,<3.0.0
python-swiftclient==3.1.0
celery==3.1.18
defusedxml==0.4.1
django<1.9a0 # Resolves known bug on gemnasium. See TNL-6266
django-extensions==1.5.9
django-model-utils==2.3.1
git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
dogapi==1.2.1
edx-submissions==2.0.6
jsonfield==1.0.3
lazy==1.1
libsass==0.10.0
loremipsum==1.0.5
path.py==8.2.1
python-dateutil==2.1
python-memcached==1.48
boto3>=1.4.4,<2.0.0
python-swiftclient>=3.1.0,<4.0.0
defusedxml>=0.4.1,<1.0.0
django-model-utils>=2.3.1,<3.0.0
django-nose>=1.4.1,<2.0.0
dogapi>=1.2.1,<2.0.0
jsonfield>=1.0.3,<2.0.0
lazy>=1.1,<2.0
libsass>=0.10.0,<1.0.0
loremipsum>=1.0.5,<2.0.0
lxml>=3.4.4,<4.0.0
path.py>=8.2.1,<9.0.0
python-dateutil>=2.1,<3.0
pytz
voluptuous==0.8.5
# AI grading
git+https://github.com/edx/ease.git@release-2015-07-14#egg=ease==0.1.3
git+https://github.com/edx/nltk.git@2.0.6#egg=nltk==2.0.6
voluptuous>=0.10.5,<1.0.0
requirements/dev.txt deleted 100644 → 0
-r test.txt
# Debug tools
bpython==0.13
django-debug-panel==0.8.1
django-debug-toolbar==1.3.2
django-pdb==0.3.2
sqlparse==0.1.10
# runserver_plus
Werkzeug==0.9.6
git+https://github.com/edx/xblock-sdk.git@0.1.1#egg=xblock-sdk==0.1.1
\ No newline at end of file
requirements/perf.txt deleted 100644 → 0
locustio==0.7.0
loremipsum==1.0.5
pyzmq==14.0.1
requirements/quality.txt 0 → 100644
pep8==1.7.0
git+https://github.com/edx/edx-lint.git@0.5.4#egg=edx_lint==0.5.4
git+https://github.com/edx/i18n-tools.git@v0.3.9#egg=i18n_tools==0.3.9
requirements/test-acceptance.txt
bok-choy==0.5.3
ddt==1.0.0
nose==1.3.7
pyinstrument==0.13.1
selenium==2.53.1
pyinstrument==0.13.2
selenium==2.53.1 # when upgrading to 3.x, will need to ensure geckodriver is installed
requirements/test.txt
-r base.txt
-r test-acceptance.txt
-r quality.txt
coverage==4.0.2
django-nose==1.4.1
freezegun==0.1.11
mock==1.0.1
moto==0.3.1
pep8==1.7.0
coverage==4.4.1
ddt==1.0.0
factory_boy==2.8.1
freezegun==0.3.9
mock==2.0.0
moto==1.0.1
nose==1.3.7
tox==2.7.0
git+https://github.com/edx/django-pyfs.git@1.0.3#egg=django-pyfs==1.0.3
git+https://github.com/edx/i18n-tools.git@56f048af9b6868613c14aeae760548834c495011#egg=i18n_tools
git+https://github.com/edx/edx-lint.git@v0.4.1#egg=edx_lint==0.4.1
git+https://github.com/edx/django-pyfs.git@1.0.6#egg=django-pyfs==1.0.6
requirements/wheels.txt deleted 100644 → 0
# These packages require compilation, so they take a long time to install.
# To speed up the test builds, we create binary "wheel" archives.
# http://pip.readthedocs.org/en/latest/reference/pip_wheel.html
lxml==3.4.4
numpy==1.6.2
scikit-learn==0.12.1
scipy==0.14.0
scripts/install-python.sh deleted 100755 → 0
#!/usr/bin/env bash
cd `dirname $BASH_SOURCE` && cd ..
echo "Installing Python requirements..."
pip install -q -r requirements/base.txt --exists-action w
echo "Installing the OpenAssessment XBlock..."
cat <<EOF | python -
import pkg_resources
import sys
try:
pkg_resources.require('ora2')
except pkg_resources.DistributionNotFound:
sys.exit(1)
EOF
ORA2_MISSING=$?
if [[ $ORA2_MISSING -eq 1 ]]; then
pip install -q -e .
echo "Installed."
else
echo "Already installed."
fi
scripts/install-wheels.sh deleted 100755 → 0
#!/usr/bin/env bash
cd `dirname $BASH_SOURCE` && cd ..
# Install "wheel" archives of the requirements for running the test suite.
# http://pip.readthedocs.org/en/latest/reference/pip_wheel.html
# This runs in Travis to install pre-built binary packages, which
# means the builds are faster and more reliable.
pip install --upgrade pip
pip install wheel
# Ensure that numpy is installed first; otherwise scipy won't be able to install
pip install --use-wheel numpy==1.6.2
# Then install everything else
pip install --use-wheel -r requirements/wheels.txt
scripts/render-templates.sh
......@@ -3,5 +3,5 @@
cd `dirname $BASH_SOURCE` && cd ..
echo "Generating HTML fixtures for JavaScript tests..."
export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-"settings.test_with_coverage"}
export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-"settings.base"}
./scripts/render_templates.py openassessment/xblock/static/js/fixtures/templates.json
scripts/render_templates.py
......@@ -100,7 +100,7 @@ def render_templates(root_dir, template_json):
template = get_template(template_dict['template'])
context = parse_dates(template_dict['context'])
print "Rendering template: {}".format(template_dict['template'])
rendered = template.render(Context(context))
rendered = template.render(context)
output_path = os.path.join(root_dir, template_dict['output'])
try:
......
scripts/run-pep8.sh
#!/usr/bin/env bash
MAX_PEP8_VIOLATIONS=106
MAX_PEP8_VIOLATIONS=60
mkdir -p test/logs
PEP8_VIOLATIONS=test/logs/pep8.txt
......
scripts/run-pylint.sh
#!/usr/bin/env bash
MAX_PYLINT_VIOLATIONS=438
MAX_PYLINT_VIOLATIONS=400
mkdir -p test/logs
PYLINT_VIOLATIONS=test/logs/pylint.txt
......
scripts/test-python.sh deleted 100755 → 0
#!/usr/bin/env bash
cd `dirname $BASH_SOURCE` && cd ..
# Cleanup uploaded files from previous test runs (AI classifiers)
git clean -xfd "./storage/test/"
echo "Running Python tests..."
export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-"settings.test_with_coverage"}
python manage.py test $1
settings/base.py
......@@ -3,10 +3,8 @@ Base settings for ORA2.
"""
import os
from celery import Celery
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = (
('admin', 'admin'),
......@@ -70,29 +68,18 @@ STATIC_ROOT = ''
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
)
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = ')68&amp;-c!+og)cy$o9pju_$c707+fett&amp;ph%t%gqgu-@5)!cl$cr'
# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'APP_DIRS': True,
'OPTIONS': {
'debug': DEBUG,
},
},
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
......@@ -120,7 +107,7 @@ INSTALLED_APPS = (
'django.contrib.admindocs',
# Third party
'django_extensions',
'django_nose',
# XBlock
'workbench',
......@@ -149,14 +136,6 @@ CACHES = {
BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
# Celery initialization and configuration
class Config:
# We run Celery in "always eager" mode in the test suite and local dev,
# which executes tasks synchronously instead of using the task queue.
CELERY_ALWAYS_EAGER = True
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
LOCALE_PATHS = [os.path.join(BASE_DIR, "openassessment", "locale")]
APP = Celery()
APP.config_from_object(Config)
APP.autodiscover_tasks(lambda: INSTALLED_APPS)
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
settings/dev.py deleted 100644 → 0
"""
Dev-specific Django settings.
"""
# Inherit from base settings
from .base import * # pylint:disable=W0614,W0401
INSTALLED_APPS += (
'django_pdb', # Allows post-mortem debugging on exceptions
'debug_toolbar',
'debug_panel',
)
MIDDLEWARE_CLASSES += (
'django_pdb.middleware.PdbMiddleware', # Needed to enable shell-on-crash behavior
'debug_panel.middleware.DebugPanelMiddleware',
)
# We need to use explicit discovery or we'll have problems with syncdb and
# displaying the admin site. See:
# http://django-debug-toolbar.readthedocs.org/en/1.0/installation.html#explicit-setup
DEBUG_TOOLBAR_PATCH_SETTINGS = False
INTERNAL_IPS = ('127.0.0.1',)
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'TIMEOUT': 60 * 60 * 8
}
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'apps_info': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': 'logs/apps_info.log',
'formatter': 'simple',
},
'apps_debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'logs/apps_debug.log',
'formatter': 'simple',
},
'trace': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/trace.log',
'formatter': 'simple',
'maxBytes': 1000000,
'backupCount': 2,
},
'events': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': 'logs/events.log',
'formatter': 'simple',
},
'errors': {
'level': 'ERROR',
'class': 'logging.FileHandler',
'filename': 'logs/errors.log',
'formatter': 'simple',
}
},
'formatters': {
'simple': {
'format': '%(asctime)s %(name)s [%(levelname)s] %(message)s'
}
},
'loggers': {
'': {
'handlers': ['trace', 'errors'],
'propagate': True,
},
'openassessment': {
'handlers': ['apps_debug', 'apps_info'],
'propagate': True,
},
'submissions': {
'handlers': ['apps_debug', 'apps_info'],
'propagate': True,
},
'workbench.runtime': {
'handlers': ['apps_debug', 'apps_info', 'events'],
'propogate': True,
}
},
}
# Store uploaded files in a dev-specific directory
MEDIA_ROOT = os.path.join(BASE_DIR, 'storage/dev')
settings/test.py deleted 100644 → 0
"""
Test-specific Django settings.
"""
# Inherit from base settings
from .base import * # pylint:disable=W0614,W0401
TEST_APPS = (
'openassessment',
'openassessment.assessment',
'openassessment.workflow',
'openassessment.xblock',
)
# Configure nose
NOSE_ARGS = [
"-a !acceptance",
]
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': 'test_ora2db',
'TEST_NAME': 'test_ora2db',
},
'read_replica': {
'ENGINE': 'django.db.backends.sqlite3',
'TEST_MIRROR': 'default',
},
}
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
# Install test-specific Django apps
INSTALLED_APPS += ('django_nose',)
# Store uploaded files in a test-specific directory
MEDIA_ROOT = os.path.join(BASE_DIR, 'storage/test')
settings/test_with_coverage.py deleted 100644 → 0
"""
Django settings for running tests with coverage.
"""
# Inherit from the test settings
from .test import * # pylint:disable=W0614,W0401
# Configure nose so that tests are run with coverage
NOSE_ARGS = [
"-a !acceptance",
'--with-coverage',
'--cover-package=' + ",".join(TEST_APPS),
'--cover-branches',
'--cover-erase',
]
import warnings
from django.core.cache import CacheKeyWarning
warnings.simplefilter("ignore", CacheKeyWarning)
setup.cfg 0 → 100644
[isort]
line_length = 120
known_third_party = ddt,mock
known_django = django
known_djangoapp = jsonfield,model_utils
known_edx =
known_first_party = openassessment
sections = FUTURE,STDLIB,THIRDPARTY,DJANGO,DJANGOAPP,EDX,FIRSTPARTY,LOCALFOLDER
force_sort_within_sections = true
setup.py
#!/usr/bin/env python
from setuptools import setup, find_packages
from setuptools import find_packages, setup
def is_requirement(line):
"""
......@@ -33,7 +34,7 @@ def load_requirements(*requirements_paths):
setup(
name='ora2',
version='1.4.9',
version='2.0.0',
author='edX',
url='http://github.com/edx/edx-ora2',
description='edx-ora2',
......@@ -47,7 +48,7 @@ setup(
],
packages=find_packages(exclude=['*.test', '*.tests']),
include_package_data=True,
install_requires=load_requirements('requirements/base.txt', 'requirements/wheels.txt'),
install_requires=load_requirements('requirements/base.txt', "requirements/django.txt"),
tests_require=load_requirements('requirements/test.txt'),
entry_points={
'xblock.v1': [
......
tox.ini 0 → 100644
[tox]
envlist = py27-django{18,111}
[testenv]
whitelist_externals = make
passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH
deps =
coveralls
-rrequirements/base.txt
-rrequirements/test.txt
django18: django>=1.8,<1.9
django111: django>=1.11,<2.0
commands =
make test-python
[testenv:js]
whitelist_externals = make
passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH
deps =
-rrequirements/base.txt
commands =
make install-js
make javascript
make sass
make verify-generated-files
make test-js
[testenv:quality]
whitelist_externals =
make
npm
passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH
deps =
-rrequirements/base.txt
-rrequirements/quality.txt
commands =
npm install jshint --no-save
npm install jscs --no-save
make quality
python manage.py makemessages -l eo
urls.py
import workbench.urls
from django.conf import settings
from django.conf.urls import include, patterns, url
from django.views.i18n import javascript_catalog
from django.conf.urls import include, url
from django.contrib import admin
from django.views.i18n import javascript_catalog
import openassessment.assessment.urls
import openassessment.fileupload.urls
import workbench.urls
# Packages to include in the JavaScript i18n strings
JS_INFO_DICT = {
'packages': ('openassessment.xblock',),
}
urlpatterns = patterns(
'',
urlpatterns = [
# Django built-in
url(r'^admin/', include(admin.site.urls)),
......@@ -24,11 +24,11 @@ urlpatterns = patterns(
url(r'^peer/evaluations/', include(openassessment.assessment.urls)),
# JavaScript i18n
(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', JS_INFO_DICT),
url(r'^jsi18n/$', javascript_catalog, JS_INFO_DICT),
# File upload to local filesystem
url(r'^openassessment/storage', include(openassessment.fileupload.urls)),
)
]
# We need to do explicit setup of the Django debug toolbar because autodiscovery
# causes problems when you mix debug toolbar >= 1.0 + django < 1.7, and the
......@@ -36,6 +36,4 @@ urlpatterns = patterns(
# http://django-debug-toolbar.readthedocs.org/en/1.0/installation.html#explicit-setup
if settings.DEBUG:
import debug_toolbar
urlpatterns += patterns('',
url(r'^__debug__/', include(debug_toolbar.urls)),
)
urlpatterns.append(url(r'^__debug__/', include(debug_toolbar.urls)))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment