Remove dogstatsd-python in favor of dogapi

Also added datadog application to commom django apps and set it to use
a local statsd server by default.

cms/envs/aws.py

@@ -147,7 +147,12 @@ MODULESTORE = AUTH_TOKENS['MODULESTORE']
 CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 
 # Datadog for events!
-DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
+DATADOG = AUTH_TOKENS.get("DATADOG", {})
+DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {}))
+
+# TODO: deprecated (compatibility with previous settings)
+if 'DATADOG_API' in AUTH_TOKENS:
+    DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']
 
 # Celery Broker
 CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")

cms/envs/common.py

@@ -358,6 +358,9 @@ INSTALLED_APPS = (
     # Tracking
     'track',
 
+    # Monitoring
+    'datadog',
+
     # For asset pipelining
     'mitxmako',
     'pipeline',

common/djangoapps/datadog/startup.py

 from django.conf import settings
-from dogapi import dog_http_api, dog_stats_api
+
+from dogapi import dog_stats_api, dog_http_api
+
 
 def run():
     """
     Initialize connection to datadog during django startup.
 
-    Expects the datadog api key in the DATADOG_API settings key
+    Can be configured using a dictionary named DATADOG in the django
+    project settings.
+
     """
-    if hasattr(settings, 'DATADOG_API'):
-        dog_http_api.api_key = settings.DATADOG_API
-        dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
+
+    # By default use the statsd agent
+    options = {'statsd': True}
+
+    if hasattr(settings, 'DATADOG'):
+        options.update(settings.DATADOG)
+
+    # Not all arguments are documented.
+    # Look at the source code for details.
+    dog_stats_api.start(**options)
+
+    dog_http_api.api_key = options.get('api_key')

common/djangoapps/student/management/commands/pearson_import_conf_zip.py

 import csv
-
-from zipfile import ZipFile, is_zipfile
 from time import strptime, strftime
-
 from datetime import datetime
+from zipfile import ZipFile, is_zipfile
+
 from dogapi import dog_http_api
+from pytz import UTC
 
 from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
 
+import django_startup
+
 from student.models import TestCenterUser, TestCenterRegistration
-from pytz import UTC
+
+
+django_startup.autostartup()
 
 
 class Command(BaseCommand):
 
-    dog_http_api.api_key = settings.DATADOG_API
     args = '<input zip file>'
     help = """
     Import Pearson confirmation files and update TestCenterUser

common/djangoapps/student/management/commands/pearson_transfer.py

-import os
 from optparse import make_option
+import os
 from stat import S_ISDIR
 
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.core.management import call_command
+import boto
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
-import boto
 
-dog_http_api.api_key = settings.DATADOG_API
-dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import BaseCommand, CommandError
+
+import django_startup
+
+
+django_startup.autostartup()
 
 
 class Command(BaseCommand):

common/djangoapps/student/management/commands/tests/test_pearson.py

@@ -303,15 +303,13 @@ class PearsonTransferTestCase(PearsonTestCase):
     '''
 
     def test_transfer_config(self):
-        with self.settings(DATADOG_API='FAKE_KEY'):
-            # TODO: why is this failing with the wrong error message?!
-            stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
-            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
-        with self.settings(DATADOG_API='FAKE_KEY'):
-            stderrmsg = get_command_error_text('pearson_transfer')
-            self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
+        self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+
+        stderrmsg = get_command_error_text('pearson_transfer')
+        self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
+
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'LOCAL_IMPORT': self.import_dir}):
             stderrmsg = get_command_error_text('pearson_transfer')
             self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
@@ -319,8 +317,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_export_missing_dest_dir(self):
         raise SkipTest()
         create_multiple_registrations('export_missing_dest')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'SFTP_EXPORT': 'this/does/not/exist',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -336,8 +333,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_export(self):
         raise SkipTest()
         create_multiple_registrations("transfer_export")
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_EXPORT': self.export_dir,
+        with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
                                     'SFTP_EXPORT': 'results/topvue',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -354,8 +350,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_import_missing_source_dir(self):
         raise SkipTest()
         create_multiple_registrations('import_missing_src')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+        with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
                                     'SFTP_IMPORT': 'this/does/not/exist',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,
@@ -371,8 +366,7 @@ class PearsonTransferTestCase(PearsonTestCase):
     def test_transfer_import(self):
         raise SkipTest()
         create_multiple_registrations('import_missing_src')
-        with self.settings(DATADOG_API='FAKE_KEY',
-                           PEARSON={'LOCAL_IMPORT': self.import_dir,
+        with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
                                     'SFTP_IMPORT': 'results',
                                     'SFTP_HOSTNAME': SFTP_HOSTNAME,
                                     'SFTP_USERNAME': SFTP_USERNAME,

common/djangoapps/student/views.py

@@ -59,7 +59,7 @@ from bulk_email.models import Optout
 
 import track.views
 
-from statsd import statsd
+from dogapi import dog_stats_api
 from pytz import UTC
 
 log = logging.getLogger("mitx.student")
@@ -388,10 +388,12 @@ def change_enrollment(request):
             )
 
         org, course_num, run = course_id.split("/")
-        statsd.increment("common.student.enrollment",
-                         tags=["org:{0}".format(org),
-                               "course:{0}".format(course_num),
-                               "run:{0}".format(run)])
+        dog_stats_api.increment(
+            "common.student.enrollment",
+            tags=["org:{0}".format(org),
+                  "course:{0}".format(course_num),
+                  "run:{0}".format(run)]
+        )
 
         CourseEnrollment.enroll(user, course.id)
 
@@ -402,10 +404,12 @@ def change_enrollment(request):
             CourseEnrollment.unenroll(user, course_id)
 
             org, course_num, run = course_id.split("/")
-            statsd.increment("common.student.unenrollment",
-                             tags=["org:{0}".format(org),
-                                   "course:{0}".format(course_num),
-                                   "run:{0}".format(run)])
+            dog_stats_api.increment(
+                "common.student.unenrollment",
+                tags=["org:{0}".format(org),
+                      "course:{0}".format(course_num),
+                      "run:{0}".format(run)]
+            )
 
             return HttpResponse()
         except CourseEnrollment.DoesNotExist:
@@ -471,7 +475,7 @@ def login_user(request, error=""):
 
         redirect_url = try_change_enrollment(request)
 
-        statsd.increment("common.student.successful_login")
+        dog_stats_api.increment("common.student.successful_login")
         response = HttpResponse(json.dumps({'success': True, 'redirect_url': redirect_url}))
 
         # set the login cookie for the edx marketing site
@@ -740,7 +744,7 @@ def create_account(request, post_override=None):
 
     redirect_url = try_change_enrollment(request)
 
-    statsd.increment("common.student.account_created")
+    dog_stats_api.increment("common.student.account_created")
 
     response_params = {'success': True,
                        'redirect_url': redirect_url}

common/lib/capa/capa/safe_exec/safe_exec.py

@@ -4,7 +4,7 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec
 from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
 from codejail.safe_exec import json_safe, SafeExecException
 from . import lazymod
-from statsd import statsd
+from dogapi import dog_stats_api
 
 import hashlib
 
@@ -70,7 +70,7 @@ def update_hash(hasher, obj):
         hasher.update(repr(obj))
 
 
-@statsd.timed('capa.safe_exec.time')
+@dog_stats_api.timed('capa.safe_exec.time')
 def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False):
     """
     Execute python code safely.

lms/djangoapps/bulk_email/tasks.py

@@ -6,6 +6,7 @@ import math
 import re
 import time
 
+from dogapi import dog_stats_api
 from smtplib import SMTPServerDisconnected, SMTPDataError, SMTPConnectError
 
 from django.conf import settings
@@ -15,8 +16,6 @@ from django.http import Http404
 from celery import task, current_task
 from celery.utils.log import get_task_logger
 from django.core.urlresolvers import reverse
-from statsd import statsd
-from dogapi import dog_stats_api
 
 from bulk_email.models import (
     CourseEmail, Optout, CourseEmailTemplate,
@@ -192,7 +191,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t
                 with dog_stats_api.timer('course_email.single_send.time.overall', tags=[_statsd_tag(course_title)]):
                     connection.send_messages([email_msg])
 
-                statsd.increment('course_email.sent', tags=[_statsd_tag(course_title)])
+                dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
 
                 log.info('Email with id %s sent to %s', email_id, email)
                 num_sent += 1
@@ -205,7 +204,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t
                     # This will fall through and not retry the message, since it will be popped
                     log.warning('Email with id %s not delivered to %s due to error %s', email_id, email, exc.smtp_error)
 
-                    statsd.increment('course_email.error', tags=[_statsd_tag(course_title)])
+                    dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
 
                     num_error += 1
 

lms/djangoapps/courseware/module_render.py

@@ -13,7 +13,7 @@ from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 
 from requests.auth import HTTPBasicAuth
-from statsd import statsd
+from dogapi import dog_stats_api
 
 from capa.xqueue_interface import XQueueInterface
 from mitxmako.shortcuts import render_to_string
@@ -332,7 +332,7 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
         if grade_bucket_type is not None:
             tags.append('type:%s' % grade_bucket_type)
 
-        statsd.increment("lms.courseware.question_answered", tags=tags)
+        dog_stats_api.increment("lms.courseware.question_answered", tags=tags)
 
     # TODO (cpennington): When modules are shared between courses, the static
     # prefix is going to have to be specific to the module, not the directory

lms/djangoapps/shoppingcart/models.py

@@ -18,7 +18,7 @@ from courseware.courses import get_course_about_section
 from mitxmako.shortcuts import render_to_string
 from student.views import course_from_id
 from student.models import CourseEnrollment
-from statsd import statsd
+from dogapi import dog_stats_api
 from xmodule.modulestore.django import modulestore
 from xmodule.course_module import CourseDescriptor
 
@@ -300,10 +300,12 @@ class PaidCourseRegistration(OrderItem):
 
         log.info("Enrolled {0} in paid course {1}, paid ${2}".format(self.user.email, self.course_id, self.line_cost))
         org, course_num, run = self.course_id.split("/")
-        statsd.increment("shoppingcart.PaidCourseRegistration.purchased_callback.enrollment",
-                         tags=["org:{0}".format(org),
-                               "course:{0}".format(course_num),
-                               "run:{0}".format(run)])
+        dog_stats_api.increment(
+            "shoppingcart.PaidCourseRegistration.purchased_callback.enrollment",
+            tags=["org:{0}".format(org),
+                  "course:{0}".format(course_num),
+                  "run:{0}".format(run)]
+        )
 
 
 class CertificateItem(OrderItem):

lms/envs/aws.py

@@ -229,7 +229,12 @@ PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")
 PEARSON = AUTH_TOKENS.get("PEARSON")
 
 # Datadog for events!
-DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
+DATADOG = AUTH_TOKENS.get("DATADOG", {})
+DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {}))
+
+# TODO: deprecated (compatibility with previous settings)
+if 'DATADOG_API' in AUTH_TOKENS:
+    DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']
 
 # Analytics dashboard server
 ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL")

lms/envs/common.py

@@ -790,7 +790,7 @@ INSTALLED_APPS = (
     'external_auth',
     'django_openid_auth',
 
-    #For the wiki
+    # For the wiki
     'wiki',  # The new django-wiki from benjaoming
     'django_notify',
     'course_wiki',  # Our customizations
@@ -801,7 +801,7 @@ INSTALLED_APPS = (
     'wiki.plugins.notifications',
     'course_wiki.plugins.markdownedx',
 
-    # foldit integration
+    # Foldit integration
     'foldit',
 
     # For testing
@@ -814,11 +814,14 @@ INSTALLED_APPS = (
     'django_comment_common',
     'notes',
 
+    # Monitoring
+    'datadog',
+
     # User API
     'rest_framework',
     'user_api',
 
-    # shopping cart
+    # Shopping cart
     'shoppingcart',
 
     # Notification preferences setting

requirements/edx/base.txt

@@ -70,7 +70,6 @@ watchdog==0.6.0
 
 # Metrics gathering and monitoring
 dogapi==1.2.1
-dogstatsd-python==0.2.1
 newrelic==1.13.1.31
 
 # Used for documentation gathering