Merge pull request #332 from edx/will/celery-config

Install and configure Celery.

README.rst

@@ -46,6 +46,22 @@ to start the server on port 8001:
     ./scripts/workbench.sh 8001
 
 
+Celery Workers
+==============
+
+Some of the OpenAssessment APIs execute tasks asynchronously using `celery <http://docs.celeryproject.org>`_.
+The tasks are executed by worker processes.
+
+First, you will need to `install RabbitMQ <http://http://www.rabbitmq.com/download.html>`_.
+
+Once RabbitMQ is installed, you can start a worker process locally:
+
+.. code:: bash
+
+    ./scripts/celery-worker.sh
+
+
+
 Running Tests
 =============
 

requirements/base.txt

@@ -4,9 +4,11 @@ git+https://github.com/edx/xblock-sdk.git@643900aadcb18aaeb7fe67271ca9dbf36e463e
 
 # Third Party Requirements
 boto==2.13.3
+celery==3.0.19
 defusedxml==0.4.1
 dogapi==1.2.1
 django>=1.4,<1.5
+django-celery==3.0.17
 django-extensions==1.2.5
 django-model-utils==1.4.0
 djangorestframework==2.3.5

scripts/celery-worker.sh

+#!/usr/bin/env bash
+
+##################################################
+#
+#   celery-worker.sh
+#
+#   Start a worker instance for local development.
+#
+#   Usage:
+#
+#       ./celery-worker.sh
+#
+###################################################
+
+cd `dirname $BASH_SOURCE` && cd ..
+
+# Check whether RabbitMQ is installed
+if [ -z `which rabbitmqctl` ]; then
+    echo "Please install RabbitMQ: http://www.rabbitmq.com/download.html"
+    exit 1;
+fi
+
+# Install Python and JS dependencies
+./scripts/install.sh
+
+# Configure Django settings
+export DJANGO_SETTINGS_MODULE="settings.dev"
+
+# Update the database
+echo "Updating the database..."
+python manage.py syncdb --migrate -v 0
+
+# Start the RabbitMQ server (ignore errors if it's already started)
+echo "Starting RabbitMQ server..."
+rabbitmq-server -detached || true
+
+# Start a RabbitMQ node
+echo "Starting RabbitMQ node..."
+rabbitmqctl start_app
+
+# Start the worker
+echo "Starting worker..."
+python manage.py celery worker

settings/base.py

-# Django settings for Tim project.
+"""
+Base settings for ORA2.
+"""
 
 import os
 import sys
@@ -7,7 +9,7 @@ import sys
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 APPS_DIR = os.path.join(BASE_DIR, "apps")
 
-sys.path.append(APPS_DIR)  # So it can find the gradebook apps dir
+sys.path.append(APPS_DIR)  # So it can find the submissions apps dir
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
@@ -125,6 +127,7 @@ INSTALLED_APPS = (
 
     # Third party
     'django_extensions',
+    'djcelery',
     'south',
 
     # XBlock
@@ -152,4 +155,10 @@ CACHES = {
 
 EDX_ORA2 = {
 
-}
\ No newline at end of file
+}
+
+# Celery configuration
+# Note: Version 3.1 of Celery includes Django support, but since we're using
+# version 3.0 (same as edx-platform), we need to use an external library.
+import djcelery
+djcelery.setup_loader()

settings/dev.py

@@ -2,7 +2,7 @@
 Dev-specific Django settings.
 """
 # Inherit from base settings
-from .base import *
+from .base import *  # pylint:disable=W0614,W0401
 
 INSTALLED_APPS += (
     'django_pdb',            # Allows post-mortem debugging on exceptions

settings/test.py

@@ -39,3 +39,8 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 INSTALLED_APPS += ('django_nose',)
 
 EDX_ORA2["EVENT_LOGGER"] = "openassessment.workflow.test.events.fake_event_logger"
+
+
+# We run Celery in "always eager" mode in the test suite,
+# which executes tasks synchronously instead of using the task queue.
+CELERY_ALWAYS_EAGER = True