Skip to content

N
notifier
Commit 2afd72cd by stv
Options

Create new digest email for flagged forum posts 

This feature will be used to send a daily digest email with a list of
flagged forum posts to the courses' moderators.

This exposes a new management command: `forums_digest_flagged`.

The command can optionally be configured to run at regular intervals by
setting FORUM_DIGEST_TASK_INTERVAL_FLAGGED to be >= 1 (minutes)
(default=0/disabled). Setting this value to 1440 will run the command
daily at midnight. If enabled, this command is scheduled alongside the
`forums_digest` command via `./manage.py scheduler`.

This command is intended to pull the list of threads via the Heroku
toolbelt, which must be installed. This command can be updated by
changing the value of COMMAND_FETCH_FLAGGED_FORUM_POSTS.

Note: Moderators cannot currently unsubscribe from this digest.
parent b23c857b
Showing with 733 additions and 5 deletions
AUTHORS
...@@ -3,3 +3,4 @@ Greg Price <gprice@edx.org> ...@@ -3,3 +3,4 @@ Greg Price <gprice@edx.org>
Marco Morales <marcotuts@gmail.com> Marco Morales <marcotuts@gmail.com>
David Adams <dcadams@stanford.edu> David Adams <dcadams@stanford.edu>
Sarina Canelake <sarina@edx.org> Sarina Canelake <sarina@edx.org>
Steven Burch <stv@stanford.edu>
notifier/digest.py
...@@ -236,3 +236,32 @@ def render_digest(user, digest, title, description): ...@@ -236,3 +236,32 @@ def render_digest(user, digest, title, description):
html = get_template('digest-email.html').render(context) html = get_template('digest-email.html').render(context)
return (text, html) return (text, html)
@statsd.timed('notifier.render_digest_flagged.elapsed')
def render_digest_flagged(message):
"""
Generate plaintext rendering of digest material, suitable for emailing.
Args:
message (dict): with the following keys:
course_id (str): identifier of the course
recipient (dict): user info
posts (list): post URLs
Returns:
str: plaintext email body
"""
logger.info("rendering email message: {%s}", message['recipient'])
context = Context({
'title': settings.FORUM_DIGEST_EMAIL_TITLE_FLAGGED,
'description': message['course_id'],
'thread_count': len(message['posts']),
'logo_image_url': settings.LOGO_IMAGE_URL,
'course_id': message['course_id'],
'posts': message['posts'],
})
with _activate_user_lang(message['recipient']):
text = get_template('digest-email-flagged.txt').render(context)
html = get_template('digest-email-flagged.html').render(context)
return (text, html)
notifier/management/commands/forums_digest_flagged.py 0 → 100644
from django.core.management.base import BaseCommand
import logging
from optparse import make_option
from notifier.tasks import do_forums_digests_flagged
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
This Command is used to send a digest of flagged posts to forum moderators.
"""
help = "Send a digest list of flagged forum posts to each moderator"
option_list = BaseCommand.option_list + (
make_option('--courses-file',
action='store',
dest='courses_file',
default=None,
help='send digests for the specified courses only' +
' (defaults to fetching course list via Heroku)'),
)
def handle(self, *args, **options):
"""
Handle a request to send a digest of flagged posts to forum moderators.
"""
input_file = options.get('courses_file')
do_forums_digests_flagged(input_file)
notifier/management/commands/scheduler.py
...@@ -3,6 +3,7 @@ from django.conf import settings ...@@ -3,6 +3,7 @@ from django.conf import settings
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from notifier.tasks import do_forums_digests from notifier.tasks import do_forums_digests
from notifier.tasks import do_forums_digests_flagged
# N.B. standalone=True means that sched.start() will block until forcibly stopped. # N.B. standalone=True means that sched.start() will block until forcibly stopped.
sched = Scheduler(standalone=True) sched = Scheduler(standalone=True)
...@@ -11,19 +12,31 @@ sched = Scheduler(standalone=True) ...@@ -11,19 +12,31 @@ sched = Scheduler(standalone=True)
def digest_job(): def digest_job():
do_forums_digests.delay() do_forums_digests.delay()
def digest_job_flagged():
"""
Schedule this task via cron job
"""
do_forums_digests_flagged()
class Command(BaseCommand): class Command(BaseCommand):
help = """Start the notifier scheduler. Important environment settings are: help = """Start the notifier scheduler. Important environment settings are:
BROKER_URL BROKER_URL
Celery broker URL. Point this where your celery workers look for tasks. Celery broker URL. Point this where your celery workers look for tasks.
FORUM_DIGEST_TASK_INTERVAL (optional) FORUM_DIGEST_TASK_INTERVAL (optional)
Number of minutes between digests (int). Default is 1440. The value must Number of minutes between digests (int). Default is 1440. The value must
be a factor of 1440. If 1440, the forums digest job will fire at midnight be a factor of 1440. If 1440, the forums digest job will fire at midnight
daily. daily.
FORUM_DIGEST_TASK_INTERVAL_FLAGGED (optional)
Number of minutes between digests (int). Default is 0, which disables
it. The value must be a factor of 1440. If 1440, the forums digest
job will fire at midnight daily.
""" """
def handle(self, *args, **options): def handle(self, *args, **options):
if settings.FLAGGED_FORUM_DIGEST_TASK_INTERVAL > 0:
sched.add_cron_job(digest_job_flagged, **settings.DIGEST_CRON_SCHEDULE_FLAGGED)
sched.start() sched.start()
notifier/settings.py
...@@ -32,6 +32,10 @@ TEST_RUNNER = 'django_coverage.coverage_runner.CoverageRunner' ...@@ -32,6 +32,10 @@ TEST_RUNNER = 'django_coverage.coverage_runner.CoverageRunner'
FORUM_DIGEST_EMAIL_SENDER = os.getenv('FORUM_DIGEST_EMAIL_SENDER', 'notifications@example.org') FORUM_DIGEST_EMAIL_SENDER = os.getenv('FORUM_DIGEST_EMAIL_SENDER', 'notifications@example.org')
FORUM_DIGEST_EMAIL_SUBJECT = os.getenv('FORUM_DIGEST_EMAIL_SUBJECT', 'Daily Discussion Digest') FORUM_DIGEST_EMAIL_SUBJECT = os.getenv('FORUM_DIGEST_EMAIL_SUBJECT', 'Daily Discussion Digest')
FORUM_DIGEST_EMAIL_TITLE = os.getenv('FORUM_DIGEST_EMAIL_TITLE', 'Discussion Digest') FORUM_DIGEST_EMAIL_TITLE = os.getenv('FORUM_DIGEST_EMAIL_TITLE', 'Discussion Digest')
FORUM_DIGEST_EMAIL_TITLE_FLAGGED = os.getenv(
'FORUM_DIGEST_EMAIL_TITLE_FLAGGED',
'Flagged Posts Digest'
)
FORUM_DIGEST_EMAIL_DESCRIPTION = os.getenv( FORUM_DIGEST_EMAIL_DESCRIPTION = os.getenv(
'FORUM_DIGEST_EMAIL_DESCRIPTION', 'FORUM_DIGEST_EMAIL_DESCRIPTION',
'A digest of unread content from course discussions you are following.' 'A digest of unread content from course discussions you are following.'
...@@ -103,6 +107,7 @@ FORUM_DIGEST_TASK_MAX_RETRIES = 2 ...@@ -103,6 +107,7 @@ FORUM_DIGEST_TASK_MAX_RETRIES = 2
FORUM_DIGEST_TASK_RETRY_DELAY = 300 FORUM_DIGEST_TASK_RETRY_DELAY = 300
# set the interval (in minutes) at which the top-level digest task is triggered # set the interval (in minutes) at which the top-level digest task is triggered
FORUM_DIGEST_TASK_INTERVAL = int(os.getenv('FORUM_DIGEST_TASK_INTERVAL', 1440)) FORUM_DIGEST_TASK_INTERVAL = int(os.getenv('FORUM_DIGEST_TASK_INTERVAL', 1440))
FORUM_DIGEST_TASK_INTERVAL_FLAGGED = int(os.getenv('FORUM_DIGEST_TASK_INTERVAL_FLAGGED', 0))
LOGGING = { LOGGING = {
...@@ -176,6 +181,16 @@ if FORUM_DIGEST_TASK_INTERVAL==1440: ...@@ -176,6 +181,16 @@ if FORUM_DIGEST_TASK_INTERVAL==1440:
else: else:
DIGEST_CRON_SCHEDULE = {'minute': '*/{}'.format(FORUM_DIGEST_TASK_INTERVAL) } DIGEST_CRON_SCHEDULE = {'minute': '*/{}'.format(FORUM_DIGEST_TASK_INTERVAL) }
# set up schedule for flagged forum digest job
if FORUM_DIGEST_TASK_INTERVAL_FLAGGED == 1440:
# in the production case, make the 24 hour cycle happen at a
# predetermined time of day (midnight UTC)
DIGEST_CRON_SCHEDULE_FLAGGED = {'hour': 0}
else:
DIGEST_CRON_SCHEDULE_FLAGGED = {
'minute': '*/{}'.format(FORUM_DIGEST_TASK_INTERVAL_FLAGGED),
}
DAILY_TASK_MAX_RETRIES = 2 DAILY_TASK_MAX_RETRIES = 2
DAILY_TASK_RETRY_DELAY = 60 DAILY_TASK_RETRY_DELAY = 60
...@@ -202,3 +217,5 @@ LOCALE_PATHS = (os.path.join(os.path.dirname(os.path.dirname(__file__)), 'locale ...@@ -202,3 +217,5 @@ LOCALE_PATHS = (os.path.join(os.path.dirname(os.path.dirname(__file__)), 'locale
# Parameterize digest logo image url # Parameterize digest logo image url
LOGO_IMAGE_URL = os.getenv('LOGO_IMAGE_URL', "{}/static/images/header-logo.png".format(LMS_URL_BASE)) LOGO_IMAGE_URL = os.getenv('LOGO_IMAGE_URL', "{}/static/images/header-logo.png".format(LMS_URL_BASE))
COMMAND_FETCH_FLAGGED_FORUM_POSTS = "cat notifier/tests/fixtures/flagged.list"
notifier/tasks.py
...@@ -4,6 +4,8 @@ Celery tasks for generating and sending digest emails. ...@@ -4,6 +4,8 @@ Celery tasks for generating and sending digest emails.
from contextlib import closing from contextlib import closing
from datetime import datetime, timedelta from datetime import datetime, timedelta
import logging import logging
import re
import subprocess
from boto.ses.exceptions import SESMaxSendingRateExceededError from boto.ses.exceptions import SESMaxSendingRateExceededError
import celery import celery
...@@ -12,8 +14,10 @@ from django.core.mail import EmailMultiAlternatives ...@@ -12,8 +14,10 @@ from django.core.mail import EmailMultiAlternatives
from notifier.connection_wrapper import get_connection from notifier.connection_wrapper import get_connection
from notifier.digest import render_digest from notifier.digest import render_digest
from notifier.digest import render_digest_flagged
from notifier.pull import generate_digest_content from notifier.pull import generate_digest_content
from notifier.user import get_digest_subscribers, UserServiceException from notifier.user import get_digest_subscribers, UserServiceException
from notifier.user import get_moderators
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -65,6 +69,48 @@ def generate_and_send_digests(users, from_dt, to_dt): ...@@ -65,6 +69,48 @@ def generate_and_send_digests(users, from_dt, to_dt):
# raise right away, since we don't support partial retry # raise right away, since we don't support partial retry
raise raise
@celery.task(rate_limit=settings.FORUM_DIGEST_TASK_RATE_LIMIT, max_retries=settings.FORUM_DIGEST_TASK_MAX_RETRIES)
def generate_and_send_digests_flagged(messages):
"""
This task generates and sends flagged forum digest emails to multiple users
in a single background operation.
Args:
messages (gen): contains dicts with the following keys:
course_id (str): identifier of the course
recipient (dict): a single user dict
posts (list): a list of post URLs
"""
with closing(get_connection()) as cx:
msgs = []
for message in messages:
text, html = render_digest_flagged(message)
msg = EmailMultiAlternatives(
settings.FORUM_DIGEST_EMAIL_SUBJECT,
text,
settings.FORUM_DIGEST_EMAIL_SENDER,
[message['recipient']['email']],
)
msg.attach_alternative(html, "text/html")
msgs.append(msg)
if not msgs:
return
try:
cx.send_messages(msgs)
except SESMaxSendingRateExceededError as e:
# we've tripped the per-second send rate limit. we generally
# rely on the django_ses auto throttle to prevent this,
# but in case we creep over, we can re-queue and re-try this task
# - if and only if none of the messages in our batch were
# sent yet.
# this implementation is also non-ideal in that the data will be
# fetched from the comments service again in the event of a retry.
if not any((getattr(msg, 'extra_headers', {}).get('status') == 200 for msg in msgs)):
raise generate_and_send_digests_flagged.retry(exc=e)
else:
# raise right away, since we don't support partial retry
raise
def _time_slice(minutes, now=None): def _time_slice(minutes, now=None):
""" """
Returns the most recently-elapsed time slice of the specified length (in Returns the most recently-elapsed time slice of the specified length (in
...@@ -111,6 +157,130 @@ def _time_slice(minutes, now=None): ...@@ -111,6 +157,130 @@ def _time_slice(minutes, now=None):
return (dt_start, dt_end) return (dt_start, dt_end)
@celery.task(max_retries=settings.DAILY_TASK_MAX_RETRIES, default_retry_delay=settings.DAILY_TASK_RETRY_DELAY) @celery.task(max_retries=settings.DAILY_TASK_MAX_RETRIES, default_retry_delay=settings.DAILY_TASK_RETRY_DELAY)
def do_forums_digests_flagged(input_file=None):
"""
Generate and batch send digest emails for each thread specified in the input_file
"""
def get_input(input_file):
"""
Yield lines of input text to be processed
Args:
input_file (str): text file containing URLs of flagged thread posts
Returns:
gen: string lines of text
"""
if input_file is not None:
generator = get_input_file(input_file)
else:
generator = get_input_command()
return generator
def get_input_command():
"""
Get input text from an executed command, e.g.
heroku run rake flags:flagged --app=APP_NAME_HERE
or
bundle exec rake flags:flagged 2>/dev/null
"""
command = settings.COMMAND_FETCH_FLAGGED_FORUM_POSTS.split(' ')
return_value = subprocess.check_output(command)
for line in return_value.split('\n'):
yield line
def get_input_file(input_file):
"""
Yield each line of text from the input file.
Args:
input_file (str): text file containing URLs of flagged thread posts
Returns:
gen: string lines of text
"""
with open(input_file, 'r') as fin:
for i in fin:
yield i.strip()
def get_posts(input_file):
"""
Parse posts and moderators from the corresponding text file
Args:
input_file (str): text file containing URLs of flagged thread posts
Returns:
dict: key=course_id, value={
posts (list): strings of post URLs
moderators (gen): users listed as moderators for specified course
course_id (str): course identifier
}
"""
output = {}
for line in get_input(input_file):
match = re.search('^https?:\/\/\S+\/courses\/((?:[^\/]+\/){3})(\S*)', line)
if match:
course_id = match.group(1)
course_id = course_id.strip()
course_id = course_id[0:-1]
thread_id = match.group(2)
url = '{0}/courses/{1}/{2}'.format(settings.LMS_URL_BASE, course_id, thread_id)
if not output.has_key(course_id):
output[course_id] = {}
output[course_id]['posts'] = list()
output[course_id]['moderators'] = get_moderators(course_id)
output[course_id]['course_id'] = course_id
output[course_id]['posts'].append(url)
return output
def get_messages(input_file):
"""
Yield message dicts to be used for sending digest emails to moderators
Args:
input_file (str): text file containing URLs of flagged thread posts
Returns:
dict gen:
course_id (str): course identifier
posts (list): strings of post URLs
recipient (dict): a single user
"""
posts = get_posts(input_file)
for course_id in posts:
for user in posts[course_id]['moderators']:
yield {
'course_id': course_id,
'posts': posts[course_id]['posts'],
'recipient': user,
}
def batch(messages):
"""
Generate and send messages in batches
Args:
messages (gen): contains dicts with the following keys:
course_id (str): course identifier
posts (list): strings of post URLs
recipient (dict): a single user
"""
batch = []
for message in messages:
batch.append(message)
if len(batch) == settings.FORUM_DIGEST_TASK_BATCH_SIZE:
generate_and_send_digests_flagged.delay(batch)
batch = []
# get the remainder if any
if batch:
generate_and_send_digests_flagged.delay(batch)
messages = get_messages(input_file)
batch(messages)
@celery.task(max_retries=settings.DAILY_TASK_MAX_RETRIES, default_retry_delay=settings.DAILY_TASK_RETRY_DELAY)
def do_forums_digests(): def do_forums_digests():
def batch_digest_subscribers(): def batch_digest_subscribers():
......
notifier/templates/digest-email-flagged.html 0 → 100644
{% load i18n %}
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>{{ title }}</title>
<style type="text/css">
.ReadMsgBody { width: 100%; background-color: #ebebeb;}
.ExternalClass {width: 100%; background-color: #ebebeb;}
.ExternalClass, .ExternalClass p, .ExternalClass span, .ExternalClass font, .ExternalClass td, .ExternalClass div {line-height:100%;}
body {-webkit-text-size-adjust:none; -ms-text-size-adjust:none;}
body {margin:0; padding:0;}
table {border-spacing:0;}
table td {border-collapse:collapse;}
.yshortcuts a {border-bottom: none !important;}
/* Constrain email width for small screens */
@media screen and (max-width: 600px) {
table[class="container"] {
width: 95% !important;
}
}
/* Give content more room on mobile */
@media screen and (max-width: 480px) {
td[class="container-padding"] {
padding-left: 12px !important;
padding-right: 12px !important;
}
}
</style>
</head>
<body style="margin:0; padding:10px 0;" bgcolor="#ebebeb" leftmargin="0" topmargin="0" marginwidth="0" marginheight="0">
<br>
<!-- 100% wrapper (grey background) -->
<table border="0" width="100%" height="100%" cellpadding="0" cellspacing="0" bgcolor="#ebebeb">
<tr>
<td align="center" valign="top" bgcolor="#ebebeb" style="background-color: #ebebeb">
<!-- 600px container (white background) -->
<table border="0" width="600" cellpadding="0" cellspacing="0" class="container" bgcolor="#ffffff" style="border-radius: 5px;">
<tr>
<td align="left" class="container-padding" bgcolor="#ffffff" style="background-color: #ffffff; border-radius: 5px; box-shadow: 0 1px 2px rgba(0,0,0,0.2); padding-left: 30px; padding-right: 30px; font-size: 14px; line-height: 20px; font-family: Open Sans, Helvetica, sans-serif; color: #333333;">
<br>
<!-- ### BEGIN CONTENT ### -->
<table class="email-header" cellpadding="0" cellspacing="0" border="0">
<tbody>
<tr>
<td class="edx-logo" valign="middle">
<!-- TODO: is this the proper logo image? -->
<img src="{{ logo_image_url }}" alt="Logo" border="0" hspace="0" vspace="0" style="vertical-align:top;" class="logo">
</td>
<td valign="top" style="padding-left: 20px">
<p style="margin: 0; color: #5597DD">{{ title }}</p>
<p style="margin: 0; font-size: 12px; color: #777777">{{ description }}</p>
</td>
</tr>
</tbody>
</table>
<br>
You currently have {{thread_count}} flagged discussion {% blocktrans count thread_count=thread_count %}thread{% plural %}threads{% endblocktrans %} in the {{course_id}} course.
<br><br>
As a forum moderator, you can delete posts with the "delete" button for each post, or clear the flag by clicking the "Misuse Reported" flag. Please review and take action upon the following posts:
<br><br>
<table class="course-table" cellpadding="0" cellspacing="0" border="0" style="width:100%; margin-bottom: 30px;">
<tbody>
<tr>
<td class="course-name" valign="middle">
<div style="font-size: 16px; line-height: 24px;">
Flagged Posts
</div>
</td>
</tr>
{% for post in posts %}
<tr>
<td>
<table class="course-thread" cellpadding="0" cellspacing="0" border="0" style="width: 100%;">
<tbody>
<tr>
<td class="course-thread-title" valign="middle">
<a href="{{ post }}" class="course-thread-link" style="display: block; border-bottom: 1px solid #cccccc; padding-bottom: 4px; margin-top: 15px; font-size: 16px; font-weight:bold; text-decoration: none; color: #5597DD">
<span>{{ post|escape }}</span>
</a>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<br><br>
{% if postal_address %}
<div class="postal-address" style="font-size: 10px; margin-top: 10px; text-align:center; color: #777777;">
{{postal_address}}
</div>
{% endif %}
<br>
<!-- ### END CONTENT ### -->
</td>
</tr>
</table>
<!--/600px container -->
</td>
</tr>
</table>
<!--/100% wrapper-->
<br>
<br>
</body>
</html>
notifier/templates/digest-email-flagged.txt 0 → 100644
{% load i18n %}
{% autoescape off %}
* {{ title }} *
{{ description }}
---
You currently have {{thread_count}} flagged discussion {% blocktrans count thread_count=thread_count %}thread{% plural %}threads{% endblocktrans %} in the {{course_id}} course.
As a forum moderator, you can delete posts with the "delete" button for each post, or clear the flag by clicking the "Misuse Reported" flag. Please review and take action upon the following posts:
Flagged Posts
---
{% for post in posts %}
{{ post }}
{% endfor %}
{% endautoescape %}
notifier/tests/test_commands.py
...@@ -5,15 +5,18 @@ import json ...@@ -5,15 +5,18 @@ import json
from os.path import dirname, join from os.path import dirname, join
from django.conf import settings from django.conf import settings
from django.core import management
from django.test import TestCase from django.test import TestCase
from django.test.utils import override_settings from django.test.utils import override_settings
from mock import patch, Mock from mock import patch, Mock
from notifier.management.commands import forums_digest from notifier.management.commands import forums_digest
from notifier.tests.test_tasks import usern
class CommandsTestCase(TestCase):
class CommandsTestCase(TestCase):
""" """
Test notifier management commands
""" """
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, @override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
...@@ -21,3 +24,15 @@ class CommandsTestCase(TestCase): ...@@ -21,3 +24,15 @@ class CommandsTestCase(TestCase):
BROKER_BACKEND='memory',) BROKER_BACKEND='memory',)
def test_forums_digest(self): def test_forums_digest(self):
pass pass
def test_forums_digest_flagged(self):
"""
Test typical use of command
"""
return_value = [usern(i) for i in xrange(10)]
with patch('notifier.tasks.get_moderators', return_value=return_value) as m:
input_file = "./notifier/tests/fixtures/flagged.list"
management.call_command('forums_digest_flagged',
courses_file=input_file,
)
notifier/tests/test_digest.py
...@@ -5,6 +5,9 @@ from django.test import TestCase ...@@ -5,6 +5,9 @@ from django.test import TestCase
from mock import patch from mock import patch
from notifier.digest import Digest, DigestCourse, DigestItem, DigestThread, render_digest from notifier.digest import Digest, DigestCourse, DigestItem, DigestThread, render_digest
from notifier.digest import _get_thread_url
from notifier.digest import render_digest_flagged
from notifier.tests.test_tasks import usern
from notifier.user import LANGUAGE_PREFERENCE_KEY from notifier.user import LANGUAGE_PREFERENCE_KEY
TEST_COURSE_ID = "test_org/test_num/test_course" TEST_COURSE_ID = "test_org/test_num/test_course"
...@@ -58,6 +61,28 @@ class DigestThreadTestCase(TestCase): ...@@ -58,6 +61,28 @@ class DigestThreadTestCase(TestCase):
self._test_unicode_data(u"This post contains %s string interpolation #{syntax}", u"This post...") self._test_unicode_data(u"This post contains %s string interpolation #{syntax}", u"This post...")
class RenderDigestFlaggedTestCase(TestCase):
"""
Test rendering of messages for digests of flagged posts
"""
def test_posts(self):
"""
Test that rendered messages contain the correct text
"""
posts = [
_get_thread_url(TEST_COURSE_ID, TEST_COMMENTABLE, i)
for i in xrange(5)
]
message = {
"course_id": TEST_COURSE_ID,
"recipient": usern(1),
"posts": posts,
}
rendered_text, rendered_html = render_digest_flagged(message)
for post in posts:
self.assertIn(post, rendered_text)
@patch("notifier.digest.THREAD_TITLE_MAXLEN", 17) @patch("notifier.digest.THREAD_TITLE_MAXLEN", 17)
class RenderDigestTestCase(TestCase): class RenderDigestTestCase(TestCase):
def set_digest(self, thread_title): def set_digest(self, thread_title):
......
notifier/tests/test_tasks.py
...@@ -15,6 +15,8 @@ from django.test.utils import override_settings ...@@ -15,6 +15,8 @@ from django.test.utils import override_settings
from mock import patch, Mock from mock import patch, Mock
from notifier.tasks import generate_and_send_digests, do_forums_digests from notifier.tasks import generate_and_send_digests, do_forums_digests
from notifier.tasks import generate_and_send_digests_flagged
from notifier.tasks import do_forums_digests_flagged
from notifier.pull import Parser from notifier.pull import Parser
from notifier.user import UserServiceException, DIGEST_NOTIFICATION_PREFERENCE_KEY from notifier.user import UserServiceException, DIGEST_NOTIFICATION_PREFERENCE_KEY
...@@ -30,6 +32,29 @@ usern = lambda n: { ...@@ -30,6 +32,29 @@ usern = lambda n: {
}, },
} }
def make_messages(count_messages=5, count_posts=10):
"""
Create sample messages for testing
Args:
count_messages (int): number of total messages to be made
count_posts (int): number of posts per message
Returns:
messages (list): contains dicts of messages
"""
return [
{
'course_id': 'org/course/run',
'recipient': usern(i),
'posts': [
'post{0}'.format(j)
for j in xrange(count_posts)
],
}
for i in xrange(count_messages)
]
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, @override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_ALWAYS_EAGER=True, CELERY_ALWAYS_EAGER=True,
...@@ -139,6 +164,92 @@ class TasksTestCase(TestCase): ...@@ -139,6 +164,92 @@ class TasksTestCase(TestCase):
# should have raised # should have raised
self.fail('task did not retry twice before giving up') self.fail('task did not retry twice before giving up')
def test_generate_and_send_digests_empty_list_flagged(self):
"""
Test that empty lists of messages are handled
"""
messages = {}
task_result = generate_and_send_digests_flagged.delay(messages)
self.assertTrue(task_result.successful())
def test_generate_and_send_digests_partial_retry_flagged(self):
"""
Test that partial retries are not attempted
"""
def side_effect(msgs):
msgs[0].extra_headers['status'] = 200
raise SESMaxSendingRateExceededError(400, 'Throttling')
messages = make_messages()
mock_backend = Mock(
name='mock_backend',
send_messages=Mock(
side_effect=side_effect,
),
)
with patch('notifier.connection_wrapper.dj_get_connection', return_value=mock_backend) as p:
# execute task - should fail, retry twice and still fail, then
# give up
try:
generate_and_send_digests_flagged.delay(messages)
except SESMaxSendingRateExceededError as e:
self.assertEqual(mock_backend.send_messages.call_count, 1)
else:
# should have raised
self.fail('task did not retry twice before giving up')
def test_generate_and_send_digests_retry_limit_flagged(self):
"""
Test that retries are attempted
"""
messages = make_messages()
# setting this here because override_settings doesn't seem to
# work on celery task configuration decorators
expected_num_tries = 1 + settings.FORUM_DIGEST_TASK_MAX_RETRIES
mock_backend = Mock(name='mock_backend', send_messages=Mock(
side_effect=SESMaxSendingRateExceededError(400, 'Throttling')))
with patch('notifier.connection_wrapper.dj_get_connection', return_value=mock_backend) as p2:
# execute task - should fail, retry twice and still fail, then
# give up
try:
task_result = generate_and_send_digests_flagged.delay(messages)
except SESMaxSendingRateExceededError as e:
self.assertEqual(
mock_backend.send_messages.call_count,
expected_num_tries,
)
else:
# should have raised
self.fail('task did not retry twice before giving up')
@override_settings(FORUM_DIGEST_TASK_BATCH_SIZE=9)
def test_do_forums_digests_flagged(self):
"""
Test that we can send forum digests for flagged posts
"""
data = [{
'course_id': 'org/course/run',
'recipient': usern(x),
'posts': [
'http://local.lan/courses/{0}'.format(n)
for n in xrange(10)
],
} for x in xrange(10)]
return_value = [usern(i) for i in xrange(10)]
with patch('notifier.tasks.get_moderators', return_value=return_value) as m:
task_result = do_forums_digests_flagged.delay()
self.assertTrue(task_result.successful())
@override_settings(FORUM_DIGEST_TASK_BATCH_SIZE=10)
def test_do_forums_digests_empty_flagged(self):
"""
Test that we handle empty lists
"""
input_file = "./notifier/tests/fixtures/flagged.empty"
task_result = do_forums_digests_flagged.delay(input_file=input_file)
self.assertTrue(task_result.successful())
@override_settings(FORUM_DIGEST_TASK_BATCH_SIZE=10) @override_settings(FORUM_DIGEST_TASK_BATCH_SIZE=10)
def test_do_forums_digests(self): def test_do_forums_digests(self):
# patch _time_slice # patch _time_slice
......
notifier/tests/test_user.py
...@@ -6,6 +6,7 @@ from django.test.utils import override_settings ...@@ -6,6 +6,7 @@ from django.test.utils import override_settings
from mock import MagicMock, Mock, patch from mock import MagicMock, Mock, patch
from notifier.user import get_digest_subscribers, DIGEST_NOTIFICATION_PREFERENCE_KEY from notifier.user import get_digest_subscribers, DIGEST_NOTIFICATION_PREFERENCE_KEY
from notifier.user import get_moderators
TEST_API_KEY = 'ZXY123!@#$%' TEST_API_KEY = 'ZXY123!@#$%'
...@@ -24,6 +25,152 @@ mkresult = lambda n: { ...@@ -24,6 +25,152 @@ mkresult = lambda n: {
} }
mkexpected = lambda d: dict([(key, val) for (key, val) in d.items() if key != "url"]) mkexpected = lambda d: dict([(key, val) for (key, val) in d.items() if key != "url"])
@override_settings(US_API_KEY=TEST_API_KEY)
class RoleTestCase(TestCase):
"""
Test forum roles for moderators
"""
def setUp(self):
"""
Setup common test state
"""
self.course_id = "org/course/run"
self.expected_api_url = "test_server_url/user_api/v1/forum_roles/Moderator/users/"
self.expected_headers = {'X-EDX-API-Key': TEST_API_KEY}
self.expected_params = {
"page_size": 3,
"page": 1,
"course_id": self.course_id,
}
@override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3)
def test_get_users_empty(self):
"""
Test that an empty moderator list can be retrieved
"""
expected_empty = {
"count": 0,
"next": None,
"previous": None,
"results": [],
}
with patch('requests.get', return_value=Mock(json=expected_empty)) as p:
result = list(get_moderators(self.course_id))
p.assert_called_once_with(
self.expected_api_url,
params=self.expected_params,
headers=self.expected_headers,
)
@override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3)
def test_get_users_single_page(self):
"""
Test that a moderator list can be retrieved
"""
expected = {
"count": 3,
"next": None,
"previous": None,
"results": [
mkresult(i) for i in xrange(3)
],
}
with patch('requests.get', return_value=Mock(json=expected)) as p:
result = get_moderators(self.course_id)
result = list(result)
p.assert_called_once_with(
self.expected_api_url,
params=self.expected_params,
headers=self.expected_headers
)
self.assertEqual(result, expected['results'])
self.assertEqual(expected['count'], len(result))
@override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3, US_HTTP_AUTH_USER='someuser', US_HTTP_AUTH_PASS='somepass')
def test_get_users_basic_auth(self):
"""
Test that basic auth works
"""
expected = {
"count": 3,
"next": None,
"previous": None,
"results": [
mkresult(i) for i in xrange(10)
],
}
with patch('requests.get', return_value=Mock(json=expected)) as p:
result = get_moderators(self.course_id)
result = list(result)
p.assert_called_once_with(
self.expected_api_url,
params=self.expected_params,
headers=self.expected_headers,
auth=('someuser', 'somepass'),
)
self.assertEqual(result, expected['results'])
@override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3)
def test_get_users_multi_page(self):
"""
Test that a moderator list can be paged
"""
expected_pages = [
{
"count": 5,
"next": "not none",
"previous": None,
"results": [
mkresult(i) for i in xrange(1, 4)
],
},
{
"count": 5,
"next": None,
"previous": "not none",
"results": [
mkresult(i) for i in xrange(4, 6)
],
},
]
def side_effect(*a, **kw):
return expected_pages.pop(0)
mock = Mock()
with patch('requests.get', return_value=mock) as p:
result = []
mock.json = expected_pages[0]
users = get_moderators(self.course_id)
result.append(users.next())
p.assert_called_once_with(
self.expected_api_url,
params=self.expected_params,
headers=self.expected_headers)
result.append(users.next())
result.append(users.next()) # result 3, end of page
self.assertEqual(
[
mkexpected(mkresult(i)) for i in xrange(1, 4)
],
result
)
# still should only have called requests.get() once
self.assertEqual(1, p.call_count)
p.reset_mock() # reset call count
self.expected_params['page'] = 2
mock.json = expected_pages[1]
self.assertEqual(mkexpected(mkresult(4)), users.next())
p.assert_called_once_with(
self.expected_api_url,
params=self.expected_params,
headers=self.expected_headers)
self.assertEqual(mkexpected(mkresult(5)), users.next())
self.assertEqual(1, p.call_count)
self.assertRaises(StopIteration, users.next)
@override_settings(US_API_KEY=TEST_API_KEY) @override_settings(US_API_KEY=TEST_API_KEY)
class UserTestCase(TestCase): class UserTestCase(TestCase):
...@@ -82,7 +229,6 @@ class UserTestCase(TestCase): ...@@ -82,7 +229,6 @@ class UserTestCase(TestCase):
mkexpected(mkresult(2)), mkexpected(mkresult(2)),
mkexpected(mkresult(3))], res) mkexpected(mkresult(3))], res)
@override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3) @override_settings(US_URL_BASE="test_server_url", US_RESULT_PAGE_SIZE=3)
def test_get_digest_subscribers_multi_page(self): def test_get_digest_subscribers_multi_page(self):
""" """
......
notifier/user.py
...@@ -55,7 +55,7 @@ def get_digest_subscribers(): ...@@ -55,7 +55,7 @@ def get_digest_subscribers():
'page_size': settings.US_RESULT_PAGE_SIZE, 'page_size': settings.US_RESULT_PAGE_SIZE,
'page': 1 'page': 1
} }
logger.info('calling user api for digest subscribers') logger.info('calling user api for digest subscribers')
while True: while True:
with dog_stats_api.timer('notifier.get_digest_subscribers.time'): with dog_stats_api.timer('notifier.get_digest_subscribers.time'):
...@@ -67,6 +67,34 @@ def get_digest_subscribers(): ...@@ -67,6 +67,34 @@ def get_digest_subscribers():
break break
params['page'] += 1 params['page'] += 1
def get_moderators(course_id):
"""
Generator function that calls the edX user API and yields an email address
for each user listed as a moderator for the specified course.
Args:
course_id (str): course identifier
Return:
users (gen): generator of users
"""
api_url = settings.US_URL_BASE + '/user_api/v1/forum_roles/Moderator/users/'
params = {
'page_size': settings.US_RESULT_PAGE_SIZE,
'page': 1,
'course_id': course_id,
}
logger.info('calling user api for forum moderators')
while True:
with dog_stats_api.timer('notifier.get_moderators.time'):
data = _http_get(api_url, params=params, headers=_headers(), **_auth()).json
for result in data['results']:
if result.has_key('url'):
del result['url']
yield result
if data['next'] is None:
break
params['page'] += 1
def get_user(user_id): def get_user(user_id):
api_url = '{}/user_api/v1/users/{}/'.format(settings.US_URL_BASE, user_id) api_url = '{}/user_api/v1/users/{}/'.format(settings.US_URL_BASE, user_id)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment