Merge pull request #7842 from edx/feanil/update_celery

Update to the latest version of celery, and django-celery.

cms/__init__.py

+"""
+Celery needs to be loaded when the cms modules are so that task
+registration and discovery can work correctly.
+"""
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import APP as CELERY_APP

cms/celery.py

+"""
+Import celery, load its settings from the django settings
+and auto discover tasks in all installed django apps.
+
+Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html
+"""
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+
+from django.conf import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')
+
+APP = Celery('proj')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+APP.config_from_object('django.conf:settings')
+APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

cms/envs/aws.py

@@ -60,8 +60,7 @@ BROKER_POOL_LIMIT = 0
 BROKER_CONNECTION_TIMEOUT = 1
 
 # For the Result Store, use the django cache named 'celery'
-CELERY_RESULT_BACKEND = 'cache'
-CELERY_CACHE_BACKEND = 'celery'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 # When the broker is behind an ELB, use a heartbeat to refresh the
 # connection and to detect if it has been dropped.

cms/envs/test.py

@@ -187,9 +187,7 @@ simplefilter('ignore')
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True
-CELERY_RESULT_BACKEND = 'cache'
-BROKER_TRANSPORT = 'memory'
-
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 ########################### Server Ports ###################################
 

cms/envs/yaml_config.py

@@ -75,8 +75,7 @@ BROKER_POOL_LIMIT = 0
 BROKER_CONNECTION_TIMEOUT = 1
 
 # For the Result Store, use the django cache named 'celery'
-CELERY_RESULT_BACKEND = 'cache'
-CELERY_CACHE_BACKEND = 'celery'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 # When the broker is behind an ELB, use a heartbeat to refresh the
 # connection and to detect if it has been dropped.

lms/__init__.py

+"""
+Celery needs to be loaded when the cms modules are so that task
+registration and discovery can work correctly.
+"""
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import APP as CELERY_APP

lms/celery.py

+"""
+Import celery, load its settings from the django settings
+and auto discover tasks in all installed django apps.
+
+Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html
+"""
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+
+from django.conf import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')
+
+APP = Celery('proj')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+APP.config_from_object('django.conf:settings')
+APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

lms/djangoapps/bulk_email/tasks.py

@@ -773,7 +773,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
     # needs to be returned back to Celery.  If it fails, we return the existing
     # exception.
     try:
-        send_course_email.retry(
+        retry_task = send_course_email.retry(
             args=[
                 entry_id,
                 email_id,
@@ -786,6 +786,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
             max_retries=max_retries,
             throw=True,
         )
+        raise retry_task
     except RetryTaskError as retry_error:
         # If the retry call is successful, update with the current progress:
         log.exception(u'Task %s: email with id %d caused send_course_email task to retry.',
@@ -800,7 +801,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
         log.exception(u'Task %s: email with id %d caused send_course_email task to fail to retry. To list: %s',
                       task_id, email_id, [i['email'] for i in to_list])
         num_failed = len(to_list)
-        subtask_status.increment(subtask_status, failed=num_failed, state=FAILURE)
+        subtask_status.increment(failed=num_failed, state=FAILURE)
         return subtask_status, retry_exc
 
 

lms/djangoapps/bulk_email/tests/test_tasks.py

@@ -104,15 +104,9 @@ class TestBulkEmailInstructorTask(InstructorTaskCourseTestCase):
         return instructor_task
 
     def _run_task_with_mock_celery(self, task_class, entry_id, task_id):
-        """Submit a task and mock how celery provides a current_task."""
-        mock_current_task = Mock()
-        mock_current_task.max_retries = settings.BULK_EMAIL_MAX_RETRIES
-        mock_current_task.default_retry_delay = settings.BULK_EMAIL_DEFAULT_RETRY_DELAY
+        """Mock was not needed for some tests, testing to see if it's needed at all."""
         task_args = [entry_id, {}]
-
-        with patch('bulk_email.tasks._get_current_task') as mock_get_task:
-            mock_get_task.return_value = mock_current_task
-            return task_class.apply(task_args, task_id=task_id).get()
+        return task_class.apply(task_args, task_id=task_id).get()
 
     def test_email_missing_current_task(self):
         task_entry = self._create_input_entry()

lms/envs/aws.py

@@ -64,8 +64,7 @@ BROKER_POOL_LIMIT = 0
 BROKER_CONNECTION_TIMEOUT = 1
 
 # For the Result Store, use the django cache named 'celery'
-CELERY_RESULT_BACKEND = 'cache'
-CELERY_CACHE_BACKEND = 'celery'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 # When the broker is behind an ELB, use a heartbeat to refresh the
 # connection and to detect if it has been dropped.

lms/envs/bok_choy.py

@@ -55,8 +55,7 @@ MEDIA_URL = "/static/uploads/"
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True
-CELERY_RESULT_BACKEND = 'cache'
-BROKER_TRANSPORT = 'memory'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 ###################### Grade Downloads ######################
 GRADES_DOWNLOAD = {

lms/envs/test.py

@@ -299,8 +299,7 @@ GIT_REPO_DIR = TEST_ROOT / "course_repos"
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True
-CELERY_RESULT_BACKEND = 'cache'
-BROKER_TRANSPORT = 'memory'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 ######################### MARKETING SITE ###############################
 

lms/envs/yaml_config.py

@@ -112,8 +112,7 @@ BROKER_POOL_LIMIT = 0
 BROKER_CONNECTION_TIMEOUT = 1
 
 # For the Result Store, use the django cache named 'celery'
-CELERY_RESULT_BACKEND = 'cache'
-CELERY_CACHE_BACKEND = 'celery'
+CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
 
 # When the broker is behind an ELB, use a heartbeat to refresh the
 # connection and to detect if it has been dropped.

requirements/edx/base.txt

@@ -9,13 +9,13 @@ beautifulsoup==3.2.1
 bleach==1.4
 html5lib==0.999
 boto==2.32.1
-celery==3.0.19
+celery==3.1.18
 cssselect==0.9.1
 dealer==2.0.4
 defusedxml==0.4.1
 distribute>=0.6.28, <0.7
 django-babel-underscore==0.1.0
-django-celery==3.0.17
+django-celery==3.1.16
 django-countries==2.1.2
 django-extensions==1.2.5
 django-filter==0.6.0

requirements/edx/github.txt

@@ -36,7 +36,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c
 -e git+https://github.com/edx/bok-choy.git@82d2f4b72e807b10d112179c0a4abd810a001b82#egg=bok_choy
 -e git+https://github.com/edx-solutions/django-splash.git@7579d052afcf474ece1239153cffe1c89935bc4f#egg=django-splash
 -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock
--e git+https://github.com/edx/edx-ora2.git@release-2015-05-07T14.06#egg=edx-ora2
+-e git+https://github.com/edx/edx-ora2.git@release-2015-05-08T16.15#egg=edx-ora2
 -e git+https://github.com/edx/edx-submissions.git@e2361932b9bce061a018a31bb3929e9cade80f49#egg=edx-submissions
 -e git+https://github.com/edx/opaque-keys.git@1254ed4d615a428591850656f39f26509b86d30a#egg=opaque-keys
 -e git+https://github.com/edx/ease.git@c6dee053eae6b3ac4fdf6be11fa7a9f8265540aa#egg=ease