Merge pull request #384 from edx/will/vagrant-like-prod

Will/vagrant like prod

.gitignore

@@ -61,7 +61,7 @@ apps/openassessment/xblock/static/js/fixtures/*.html
 storage/*
 
 # logging
-logs/*.log*
+logs/*/*.log*
 
 # Vagrant
 .vagrant

README.rst

@@ -112,6 +112,48 @@ Disable quality violations on a line or file:
     # pylint: disable=W0123,E4567
 
 
+Vagrant
+=======
+
+This repository includes a Vagrant configuration file, which is useful for testing
+ORA2 in an environment that is closer to production:
+
+* Uses `gunicorn <http://gunicorn.org/>`_ to serve the workbench application.
+  Unlike Django ``runserver``, gunicorn will process requests in parallel.
+
+* Uses `mysql <http://www.mysql.com/>`_ as the database, which (unlike
+  `sqlite <http://www.sqlite.org/>`_) allows for simultaneous writes.
+
+* Serves static files using `nginx <http://wiki.nginx.org/Main>`_ instead
+  of Django `staticfiles <https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/>`_.
+
+* Runs multiple `celery workers <http://celery.readthedocs.org/en/latest/>`_.
+
+* Uses `memcached <http://memcached.org/>`_.
+
+* Installs `EASE <https://github.com/edx/ease>`_ for AI grading, including
+  its many requirements.
+
+To use the Vagrant VM:
+
+1) `Install Vagrant <https://docs.vagrantup.com/v2/installation/>`_.
+2) ``vagrant up`` to start and provision the Vagrant VM.
+3) Visit `http://192.168.44.10 <http://192.168.44.10>`_
+4) You should see the workbench index page load.
+
+After making a change to the code in the ``edx-ora2`` directory,
+you must restart the services on the Vagrant VM:
+
+1) ``vagrant ssh`` to ssh into the Vagrant VM.
+2) ``./update.sh`` to restart the services, run database migrations, and collect static assets.
+3) Visit `http://192.168.44.10 <http://192.168.44.10>`_
+
+By default, the Vagrant VM also includes a monitoring tool for Celery tasks called `Flower <https://github.com/mher/flower>`_.
+To use the tool, visit: `http://192.168.44.10:5555 <http://192.168.44.10:5555>`_
+
+The log files from the Vagrant VM are located in ``edx-ora2/logs/vagrant``, which is shared with the host machine.
+
+
 License
 =======
 

Vagrantfile

@@ -11,24 +11,21 @@ echo "Updating apt packages..."
 apt-get update -y
 
 echo "Installing basic system requirements..."
-apt-get install -y curl git vim libxml2-dev libxslt1-dev
+apt-get install -y curl git vim libxml2-dev libxslt1-dev memcached nginx
+
+echo "Installing mysql server..."
+DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server-5.5
+echo "CREATE DATABASE IF NOT EXISTS workbench DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" | mysql -u root
 
 echo "Installing Python system requirements..."
-apt-get install -y python2.7 python2.7-dev python-pip python-software-properties
+apt-get install -y python2.7 python2.7-dev python-pip python-software-properties python-mysqldb libmysqlclient-dev
 pip install virtualenv
 
 echo "Installing FireFox and xvfb (for JavaScript tests)..."
 add-apt-repository "ppa:ubuntu-mozilla-security/ppa"
 apt-get install -y firefox dbus-x11 xvfb
 
-cat > /etc/init/xvfb.conf <<XVFB_UPSTART
-    description     "Xvfb X Server"
-    start on (net-device-up and local-filesystems and runlevel [2345])
-    stop on runlevel [016]
-    exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
-    respawn
-    respawn limit 15 5
-XVFB_UPSTART
+cat /home/vagrant/edx-ora2/vagrant/xvfb.conf > /etc/init/xvfb.conf
 start xvfb || true
 
 echo "Installing RabbitMQ..."
@@ -42,6 +39,11 @@ add-apt-repository ppa:chris-lea/node.js
 apt-get update -y
 apt-get install -y nodejs
 
+# Stop all Python upstart jobs
+sudo stop workbench || true
+sudo stop celery || true
+sudo stop flower || true
+
 su vagrant <<EOF
     set -e
 
@@ -51,31 +53,62 @@ su vagrant <<EOF
     source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
 
     echo "Configuring login script..."
-    cat > /home/vagrant/.bash_profile <<LOGIN_SCRIPT
-        source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
-        export NLTK_DATA=/home/vagrant/data/nltk_data
-        export DISPLAY=:1
-LOGIN_SCRIPT
+    cat /home/vagrant/edx-ora2/vagrant/bash_profile > /home/vagrant/.bash_profile
 
     echo "Installing EASE..."
     if [ ! -d /home/vagrant/ease ]; then
         git clone https://github.com/edx/ease.git /home/vagrant/ease
     fi
     cat /home/vagrant/ease/apt-packages.txt | xargs sudo apt-get -y install
-    pip install -r /home/vagrant/ease/pre-requirements.txt
-    pip install -r /home/vagrant/ease/requirements.txt
+    cd /home/vagrant/ease && pip install -r pre-requirements.txt
     cd /home/vagrant/ease && python setup.py install
 
     echo "Downloading NLTK corpus..."
-    mkdir -p /home/vagrant/data
-    curl -o /home/vagrant/data/nltk.tmp.tar.tz http://edx-static.s3.amazonaws.com/nltk/nltk-data-20131113.tar.gz
-    cd /home/vagrant/data && tar zxf /home/vagrant/data/nltk.tmp.tar.tz
+    cd /home/vagrant/ease && ./download-nltk-corpus.sh
+
+    echo "Installing gunicorn..."
+    pip install gunicorn
+
+    echo "Instally Python MySQL library..."
+    pip install MySQL-python
+
+    echo "Installing celery flower..."
+    pip install flower
 
     echo "Install edx-ora2..."
     cd /home/vagrant/edx-ora2 && ./scripts/install.sh
 
+    echo "Update the database..."
+    cd /home/vagrant/edx-ora2 && python manage.py syncdb --migrate --noinput --settings settings.vagrant
+
+    echo "Collect static assets..."
+    mkdir -p /home/vagrant/static
+    cd /home/vagrant/edx-ora2 && python manage.py collectstatic --noinput --settings settings.vagrant
+
+    echo "Creating the update script..."
+    cp /home/vagrant/edx-ora2/vagrant/update.sh /home/vagrant/update.sh
+
 EOF
 
+echo "Creating upstart script for workbench..."
+cat /home/vagrant/edx-ora2/vagrant/workbench_upstart.conf > /etc/init/workbench.conf
+start workbench || true
+
+echo "Create upstart script for Celery workers..."
+cat /home/vagrant/edx-ora2/vagrant/celery_upstart.conf > /etc/init/celery.conf
+start celery || true
+
+echo "Create upstart script for Celery flower..."
+cat /home/vagrant/edx-ora2/vagrant/flower_upstart.conf > /etc/init/flower.conf
+start flower || true
+
+echo "Configure nginx"
+cat /home/vagrant/edx-ora2/vagrant/nginx.conf > /etc/nginx/sites-enabled/workbench.conf
+
+echo "Restart nginx"
+sudo service nginx stop || true
+sudo service nginx start || true
+
 END
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
@@ -83,10 +116,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "precise64"
   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 
-  config.vm.network "forwarded_port", guest: 80, host: 8082
   config.vm.network "private_network", ip: "192.168.44.10"
   config.vm.synced_folder ".", "/home/vagrant/edx-ora2"
 
-  config.vm.provision "shell", inline: $script
 
+  config.vm.provider :virtualbox do |vb|
+    # Increase memory and CPU
+    vb.customize ["modifyvm", :id, "--memory", "2048"]
+    vb.customize ["modifyvm", :id, "--cpus", "2"]
+
+    # Allow DNS to work for Ubuntu 12.10 host
+    # http://askubuntu.com/questions/238040/how-do-i-fix-name-service-for-vagrant-client
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+  end
+
+  config.vm.provision "shell", inline: $script
 end

settings/dev.py

@@ -42,19 +42,19 @@ LOGGING = {
         'apps_info': {
             'level': 'INFO',
             'class': 'logging.FileHandler',
-            'filename': 'logs/apps_info.log',
+            'filename': 'logs/dev/apps_info.log',
             'formatter': 'simple',
         },
         'apps_debug': {
             'level': 'DEBUG',
             'class': 'logging.FileHandler',
-            'filename': 'logs/apps_debug.log',
+            'filename': 'logs/dev/apps_debug.log',
             'formatter': 'simple',
         },
         'trace': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/trace.log',
+            'filename': 'logs/dev/trace.log',
             'formatter': 'simple',
             'maxBytes': 1000000,
             'backupCount': 2,
@@ -62,13 +62,13 @@ LOGGING = {
         'events': {
             'level': 'INFO',
             'class': 'logging.FileHandler',
-            'filename': 'logs/events.log',
+            'filename': 'logs/dev/events.log',
             'formatter': 'simple',
         },
         'errors': {
             'level': 'ERROR',
             'class': 'logging.FileHandler',
-            'filename': 'logs/errors.log',
+            'filename': 'logs/dev/errors.log',
             'formatter': 'simple',
         }
     },

settings/vagrant.py

+"""
+Settings for running workbench in Vagrant.
+To mimic production, the Vagrant setup uses:
+
+    * gunicorn (run multiple server processes)
+    * memcached
+    * mysql
+    * rabbitmq
+
+"""
+
+# Inherit from base settings
+from .base import *  # pylint:disable=W0614,W0401
+
+VAGRANT_HOME = "/home/vagrant"
+REPO_ROOT = u"{home}/edx-ora2".format(home=VAGRANT_HOME)
+
+DEBUG = False
+
+INSTALLED_APPS += ('gunicorn',)
+
+STATIC_ROOT = u"{home}/static".format(home=VAGRANT_HOME)
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'workbench',
+        'USER': 'root',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': '',
+    }
+}
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': '127.0.0.1:11211',
+        'TIMEOUT': 60 * 60 * 8
+    }
+}
+
+LOG_ROOT = u"{repo}/logs/vagrant".format(repo=REPO_ROOT)
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'apps_info': {
+            'level': 'INFO',
+            'class': 'logging.FileHandler',
+            'filename': u'{}/apps_info.log'.format(LOG_ROOT),
+            'formatter': 'simple',
+        },
+        'apps_debug': {
+            'level': 'DEBUG',
+            'class': 'logging.FileHandler',
+            'filename': u'{}/apps_debug.log'.format(LOG_ROOT),
+            'formatter': 'simple',
+        },
+        'trace': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': u'{}/trace.log'.format(LOG_ROOT),
+            'formatter': 'simple',
+            'maxBytes': 1000000,
+            'backupCount': 2,
+        },
+        'events': {
+            'level': 'INFO',
+            'class': 'logging.FileHandler',
+            'filename': u'{}/events.log'.format(LOG_ROOT),
+            'formatter': 'simple',
+        },
+        'errors': {
+            'level': 'ERROR',
+            'class': 'logging.FileHandler',
+            'filename': u'{}/errors.log'.format(LOG_ROOT),
+            'formatter': 'simple',
+        }
+    },
+    'formatters': {
+        'simple': {
+            'format': '%(asctime)s %(name)s [%(levelname)s] %(message)s'
+        }
+    },
+    'loggers': {
+        '': {
+            'handlers': ['trace', 'errors'],
+            'propagate': True,
+        },
+        'openassessment': {
+            'handlers': ['apps_debug', 'apps_info'],
+            'propagate': True,
+        },
+        'submissions': {
+            'handlers': ['apps_debug', 'apps_info'],
+            'propagate': True,
+        },
+        'workbench.runtime': {
+            'handlers': ['apps_debug', 'apps_info', 'events'],
+            'propogate': True,
+        }
+    },
+}
+
+
+
+# Store uploaded files in a dev-specific directory
+MEDIA_ROOT = os.path.join(BASE_DIR, 'storage/dev')
+
+# AI algorithm configuration
+ORA2_AI_ALGORITHMS = {
+    'fake': 'openassessment.assessment.worker.algorithm.FakeAIAlgorithm',
+    'ease': 'openassessment.assessment.worker.algorithm.EaseAIAlgorithm'
+}
+
+# Celery Broker
+CELERY_BROKER_TRANSPORT = "amqp"
+CELERY_BROKER_HOSTNAME = "localhost:5672//"
+CELERY_BROKER_USER = "guest"
+CELERY_BROKER_PASSWORD = "guest"
+
+BROKER_URL = "{0}://{1}:{2}@{3}".format(
+    CELERY_BROKER_TRANSPORT,
+    CELERY_BROKER_USER,
+    CELERY_BROKER_PASSWORD,
+    CELERY_BROKER_HOSTNAME,
+)

urls.py

@@ -20,7 +20,7 @@ urlpatterns = patterns(
     url(r'^admin/', include(admin.site.urls)),
 
     # Provided by XBlock
-    url(r'^workbench/', include(workbench.urls)),
+    url(r'^/?', include(workbench.urls)),
 
     # edx-ora2 apps
     url(r'^submissions/', include(submissions.urls)),

vagrant/bash_profile

+source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
+export DISPLAY=:1
+export LC_ALL="en_US.UTF-8"

vagrant/celery_upstart.conf

+description "Celery workers"
+start on vagrant-mounted
+stop on runlevel [016]
+exec /home/vagrant/.virtualenvs/edx-ora2/bin/python /home/vagrant/edx-ora2/manage.py celery --settings=settings.vagrant worker --concurrency=2
+respawn
+respawn limit 15 5

vagrant/flower_upstart.conf

+description "Celery flower"
+start on vagrant-mounted
+stop on runlevel [016]
+exec /home/vagrant/.virtualenvs/edx-ora2/bin/celery flower
+respawn
+respawn limit 15 5

vagrant/nginx.conf

+server {
+    listen  80;
+    server_name 192.168.44.10;
+    charset utf-8;
+    client_max_body_size 75M;
+    location /static {
+        alias /home/vagrant/static;
+    }
+    location / {
+        proxy_pass http://127.0.0.1:8000/;
+    }
+}
+

vagrant/update.sh

+#!/usr/bin/env bash
+
+echo "Stopping services..."
+sudo service nginx stop || true
+sudo stop workbench || true
+sudo stop celery || true
+sudo stop flower || true
+
+echo "Updating ORA2..."
+cd /home/vagrant/edx-ora2 && ./scripts/install.sh
+
+echo "Updating the database..."
+cd /home/vagrant/edx-ora2 && python manage.py syncdb --migrate --noinput --settings settings.vagrant
+
+echo "Collecting static assets..."
+cd /home/vagrant/edx-ora2 && python manage.py collectstatic --noinput --settings settings.vagrant
+
+echo "Restarting services..."
+sudo start workbench || true
+sudo start celery || true
+sudo start flower || true
+sudo service nginx start || true

vagrant/workbench_upstart.conf

+description     "Workbench"
+start on vagrant-mounted
+stop on runlevel [016]
+exec /home/vagrant/.virtualenvs/edx-ora2/bin/python /home/vagrant/edx-ora2/manage.py run_gunicorn --log-level=DEBUG --workers=10 -b 127.0.0.1:8000 --settings settings.vagrant
+respawn
+respawn limit 15 5

vagrant/xvfb.conf

+description     "Xvfb X Server"
+start on (net-device-up and local-filesystems and runlevel [2345])
+stop on runlevel [016]
+exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
+respawn
+respawn limit 15 5