Skip to content

E
edx-ora2
Commit 9907db42 by Will Daly
Options

Update the Vagrant configuration to use: 

* gunicorn
* nginx
* mysql
* memcached
* celery flower
* upstart

Set file encoding for Python interpreter to fix a test failure in the Vagrant VM.
Configure Vagrant log files to go to logs/vagrant.
parent 8f9dd4c8
Showing with 307 additions and 28 deletions
.gitignore
......@@ -61,7 +61,7 @@ apps/openassessment/xblock/static/js/fixtures/*.html
storage/*
# logging
logs/*.log*
logs/*/*.log*
# Vagrant
.vagrant
README.rst
......@@ -112,6 +112,48 @@ Disable quality violations on a line or file:
# pylint: disable=W0123,E4567
Vagrant
=======
This repository includes a Vagrant configuration file, which is useful for testing
ORA2 in an environment that is closer to production:
* Uses `gunicorn <http://gunicorn.org/>`_ to serve the workbench application.
Unlike Django ``runserver``, gunicorn will process requests in parallel.
* Uses `mysql <http://www.mysql.com/>`_ as the database, which (unlike
`sqlite <http://www.sqlite.org/>`_) allows for simultaneous writes.
* Serves static files using `nginx <http://wiki.nginx.org/Main>`_ instead
of Django `staticfiles <https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/>`_.
* Runs multiple `celery workers <http://celery.readthedocs.org/en/latest/>`_.
* Uses `memcached <http://memcached.org/>`_.
* Installs `EASE <https://github.com/edx/ease>`_ for AI grading, including
its many requirements.
To use the Vagrant VM:
1) `Install Vagrant <https://docs.vagrantup.com/v2/installation/>`_.
2) ``vagrant up`` to start and provision the Vagrant VM.
3) Visit `http://192.168.44.10/workbench/ <http://192.168.44.10/workbench/>`_
4) You should see the workbench index page load.
After making a change to the code in the ``edx-ora2`` directory,
you must restart the services on the Vagrant VM:
1) ``vagrant ssh`` to ssh into the Vagrant VM.
2) ``./update.sh`` to restart the services, run database migrations, and collect static assets.
3) Visit `http://192.168.44.10/workbench/ <http://192.168.44.10/workbench/>`_
By default, the Vagrant VM also includes a monitoring tool for Celery tasks called `Flower <https://github.com/mher/flower>`_.
To use the tool, visit: `http://192.168.44.10:5555 <http://192.168.44.10:5555>`_
The log files from the Vagrant VM are located in ``edx-ora2/logs/vagrant``, which is shared with the host machine.
License
=======
......
Vagrantfile
......@@ -11,24 +11,21 @@ echo "Updating apt packages..."
apt-get update -y
echo "Installing basic system requirements..."
apt-get install -y curl git vim libxml2-dev libxslt1-dev
apt-get install -y curl git vim libxml2-dev libxslt1-dev memcached nginx
echo "Installing mysql server..."
DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server-5.5
echo "CREATE DATABASE IF NOT EXISTS workbench" | mysql -u root
echo "Installing Python system requirements..."
apt-get install -y python2.7 python2.7-dev python-pip python-software-properties
apt-get install -y python2.7 python2.7-dev python-pip python-software-properties python-mysqldb libmysqlclient-dev
pip install virtualenv
echo "Installing FireFox and xvfb (for JavaScript tests)..."
add-apt-repository "ppa:ubuntu-mozilla-security/ppa"
apt-get install -y firefox dbus-x11 xvfb
cat > /etc/init/xvfb.conf <<XVFB_UPSTART
description "Xvfb X Server"
start on (net-device-up and local-filesystems and runlevel [2345])
stop on runlevel [016]
exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
respawn
respawn limit 15 5
XVFB_UPSTART
cat /home/vagrant/edx-ora2/vagrant/xvfb.conf > /etc/init/xvfb.conf
start xvfb || true
echo "Installing RabbitMQ..."
......@@ -42,6 +39,11 @@ add-apt-repository ppa:chris-lea/node.js
apt-get update -y
apt-get install -y nodejs
# Stop all Python upstart jobs
sudo stop workbench || true
sudo stop celery || true
sudo stop flower || true
su vagrant <<EOF
set -e
......@@ -51,31 +53,62 @@ su vagrant <<EOF
source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
echo "Configuring login script..."
cat > /home/vagrant/.bash_profile <<LOGIN_SCRIPT
source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
export NLTK_DATA=/home/vagrant/data/nltk_data
export DISPLAY=:1
LOGIN_SCRIPT
cat /home/vagrant/edx-ora2/vagrant/bash_profile > /home/vagrant/.bash_profile
echo "Installing EASE..."
if [ ! -d /home/vagrant/ease ]; then
git clone https://github.com/edx/ease.git /home/vagrant/ease
fi
cat /home/vagrant/ease/apt-packages.txt | xargs sudo apt-get -y install
pip install -r /home/vagrant/ease/pre-requirements.txt
pip install -r /home/vagrant/ease/requirements.txt
cd /home/vagrant/ease && pip install -r pre-requirements.txt
cd /home/vagrant/ease && python setup.py install
echo "Downloading NLTK corpus..."
mkdir -p /home/vagrant/data
curl -o /home/vagrant/data/nltk.tmp.tar.tz http://edx-static.s3.amazonaws.com/nltk/nltk-data-20131113.tar.gz
cd /home/vagrant/data && tar zxf /home/vagrant/data/nltk.tmp.tar.tz
cd /home/vagrant/ease && ./download-nltk-corpus.sh
echo "Installing gunicorn..."
pip install gunicorn
echo "Instally Python MySQL library..."
pip install MySQL-python
echo "Installing celery flower..."
pip install flower
echo "Install edx-ora2..."
cd /home/vagrant/edx-ora2 && ./scripts/install.sh
echo "Update the database..."
cd /home/vagrant/edx-ora2 && python manage.py syncdb --migrate --noinput --settings settings.vagrant
echo "Collect static assets..."
mkdir -p /home/vagrant/static
cd /home/vagrant/edx-ora2 && python manage.py collectstatic --noinput --settings settings.vagrant
echo "Creating the update script..."
cp /home/vagrant/edx-ora2/vagrant/update.sh /home/vagrant/update.sh
EOF
echo "Creating upstart script for workbench..."
cat /home/vagrant/edx-ora2/vagrant/workbench_upstart.conf > /etc/init/workbench.conf
start workbench || true
echo "Create upstart script for Celery workers..."
cat /home/vagrant/edx-ora2/vagrant/celery_upstart.conf > /etc/init/celery.conf
start celery || true
echo "Create upstart script for Celery flower..."
cat /home/vagrant/edx-ora2/vagrant/flower_upstart.conf > /etc/init/flower.conf
start flower || true
echo "Configure nginx"
cat /home/vagrant/edx-ora2/vagrant/nginx.conf > /etc/nginx/sites-enabled/workbench.conf
echo "Restart nginx"
sudo service nginx stop || true
sudo service nginx start || true
END
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
......@@ -83,10 +116,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "precise64"
config.vm.box_url = "http://files.vagrantup.com/precise64.box"
config.vm.network "forwarded_port", guest: 80, host: 8082
config.vm.network "private_network", ip: "192.168.44.10"
config.vm.synced_folder ".", "/home/vagrant/edx-ora2"
config.vm.provision "shell", inline: $script
config.vm.provider :virtualbox do |vb|
# Increase memory and CPU
vb.customize ["modifyvm", :id, "--memory", "2048"]
vb.customize ["modifyvm", :id, "--cpus", "2"]
# Allow DNS to work for Ubuntu 12.10 host
# http://askubuntu.com/questions/238040/how-do-i-fix-name-service-for-vagrant-client
vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
end
config.vm.provision "shell", inline: $script
end
settings/dev.py
......@@ -42,19 +42,19 @@ LOGGING = {
'apps_info': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': 'logs/apps_info.log',
'filename': 'logs/dev/apps_info.log',
'formatter': 'simple',
},
'apps_debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'logs/apps_debug.log',
'filename': 'logs/dev/apps_debug.log',
'formatter': 'simple',
},
'trace': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/trace.log',
'filename': 'logs/dev/trace.log',
'formatter': 'simple',
'maxBytes': 1000000,
'backupCount': 2,
......@@ -62,13 +62,13 @@ LOGGING = {
'events': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': 'logs/events.log',
'filename': 'logs/dev/events.log',
'formatter': 'simple',
},
'errors': {
'level': 'ERROR',
'class': 'logging.FileHandler',
'filename': 'logs/errors.log',
'filename': 'logs/dev/errors.log',
'formatter': 'simple',
}
},
......
settings/vagrant.py 0 → 100644
"""
Settings for running workbench in Vagrant.
To mimic production, the Vagrant setup uses:
* gunicorn (run multiple server processes)
* memcached
* mysql
* rabbitmq
"""
# Inherit from base settings
from .base import * # pylint:disable=W0614,W0401
VAGRANT_HOME = "/home/vagrant"
REPO_ROOT = u"{home}/edx-ora2".format(home=VAGRANT_HOME)
DEBUG = False
INSTALLED_APPS += ('gunicorn',)
STATIC_ROOT = u"{home}/static".format(home=VAGRANT_HOME)
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'workbench',
'USER': 'root',
'PASSWORD': '',
'HOST': '',
'PORT': '',
}
}
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'TIMEOUT': 60 * 60 * 8
}
}
LOG_ROOT = u"{repo}/logs/vagrant".format(repo=REPO_ROOT)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'apps_info': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': u'{}/apps_info.log'.format(LOG_ROOT),
'formatter': 'simple',
},
'apps_debug': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': u'{}/apps_debug.log'.format(LOG_ROOT),
'formatter': 'simple',
},
'trace': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': u'{}/trace.log'.format(LOG_ROOT),
'formatter': 'simple',
'maxBytes': 1000000,
'backupCount': 2,
},
'events': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': u'{}/events.log'.format(LOG_ROOT),
'formatter': 'simple',
},
'errors': {
'level': 'ERROR',
'class': 'logging.FileHandler',
'filename': u'{}/errors.log'.format(LOG_ROOT),
'formatter': 'simple',
}
},
'formatters': {
'simple': {
'format': '%(asctime)s %(name)s [%(levelname)s] %(message)s'
}
},
'loggers': {
'': {
'handlers': ['trace', 'errors'],
'propagate': True,
},
'openassessment': {
'handlers': ['apps_debug', 'apps_info'],
'propagate': True,
},
'submissions': {
'handlers': ['apps_debug', 'apps_info'],
'propagate': True,
},
'workbench.runtime': {
'handlers': ['apps_debug', 'apps_info', 'events'],
'propogate': True,
}
},
}
# Store uploaded files in a dev-specific directory
MEDIA_ROOT = os.path.join(BASE_DIR, 'storage/dev')
# AI algorithm configuration
ORA2_AI_ALGORITHMS = {
'fake': 'openassessment.assessment.worker.algorithm.FakeAIAlgorithm',
'ease': 'openassessment.assessment.worker.algorithm.EaseAIAlgorithm'
}
# Celery Broker
CELERY_BROKER_TRANSPORT = "amqp"
CELERY_BROKER_HOSTNAME = "localhost:5672//"
CELERY_BROKER_USER = "guest"
CELERY_BROKER_PASSWORD = "guest"
BROKER_URL = "{0}://{1}:{2}@{3}".format(
CELERY_BROKER_TRANSPORT,
CELERY_BROKER_USER,
CELERY_BROKER_PASSWORD,
CELERY_BROKER_HOSTNAME,
)
vagrant/bash_profile 0 → 100644
source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
export DISPLAY=:1
export LC_ALL="en_US.UTF-8"
vagrant/celery_upstart.conf 0 → 100644
description "Celery workers"
start on vagrant-mounted
stop on runlevel [016]
exec /home/vagrant/.virtualenvs/edx-ora2/bin/python /home/vagrant/edx-ora2/manage.py celery --settings=settings.vagrant worker --concurrency=2
respawn
respawn limit 15 5
vagrant/flower_upstart.conf 0 → 100644
description "Celery flower"
start on vagrant-mounted
stop on runlevel [016]
exec /home/vagrant/.virtualenvs/edx-ora2/bin/celery flower
respawn
respawn limit 15 5
vagrant/nginx.conf 0 → 100644
server {
listen 80;
server_name 192.168.44.10;
charset utf-8;
client_max_body_size 75M;
location /static {
alias /home/vagrant/static;
}
location / {
proxy_pass http://127.0.0.1:8000/;
}
}
vagrant/update.sh 0 → 100755
#!/usr/bin/env bash
echo "Stopping services..."
sudo service nginx stop || true
sudo stop workbench || true
sudo stop celery || true
sudo stop flower || true
echo "Updating ORA2..."
cd /home/vagrant/edx-ora2 && ./scripts/install.sh
echo "Updating the database..."
cd /home/vagrant/edx-ora2 && python manage.py syncdb --migrate --noinput --settings settings.vagrant
echo "Collecting static assets..."
cd /home/vagrant/edx-ora2 && python manage.py collectstatic --noinput --settings settings.vagrant
echo "Restarting services..."
sudo start workbench || true
sudo start celery || true
sudo start flower || true
sudo service nginx start || true
vagrant/workbench_upstart.conf 0 → 100644
description "Workbench"
start on vagrant-mounted
stop on runlevel [016]
exec /home/vagrant/.virtualenvs/edx-ora2/bin/python /home/vagrant/edx-ora2/manage.py run_gunicorn --log-level=DEBUG --workers=4 -b 127.0.0.1:8000 --settings settings.vagrant
respawn
respawn limit 15 5
vagrant/xvfb.conf 0 → 100644
description "Xvfb X Server"
start on (net-device-up and local-filesystems and runlevel [2345])
stop on runlevel [016]
exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
respawn
respawn limit 15 5
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment