Refactor tracking and xqueue initialization to no longer require a request. Get…

Refactor tracking and xqueue initialization to no longer require a request.  Get tracking logs to work.  Add transaction annotations.

Make sure to read only once from result objects when getting result status,
so that values are consistent.

common/djangoapps/track/views.py

 import json
 import logging
-import os
 import pytz
 import datetime
 import dateutil.parser
 
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
-from django.http import Http404
 from django.shortcuts import redirect
 from django.conf import settings
 from mitxmako.shortcuts import render_to_response
@@ -95,6 +93,46 @@ def server_track(request, event_type, event, page=None):
     log_event(event)
 
 
+def task_track(request_info, task_info, event_type, event, page=None):
+    """
+    Outputs tracking information for events occuring within celery tasks.
+
+    The `event_type` is a string naming the particular event being logged,
+    while `event` is a dict containing whatever additional contextual information
+    is desired.
+
+    The `request_info` is a dict containing information about the original
+    task request.  Relevant keys are `username`, `ip`, `agent`, and `host`.
+
+    In addition, a `task_info` dict provides more information to be stored with
+    the `event` dict.
+
+    The `page` parameter is optional, and allows the name of the page to
+    be provided.
+    """
+
+    # supplement event information with additional information
+    # about the task in which it is running.
+    full_event = dict(event, **task_info)
+
+    # All fields must be specified, in case the tracking information is
+    # also saved to the TrackingLog model.  Get values from the task-level
+    # information, or just add placeholder values.
+    event = {
+        "username": request_info.get('username', 'unknown'),
+        "ip": request_info.get('ip', 'unknown'),
+        "event_source": "task",
+        "event_type": event_type,
+        "event": full_event,
+        "agent": request_info.get('agent', 'unknown'),
+        "page": page,
+        "time": datetime.datetime.utcnow().isoformat(),
+        "host": request_info.get('host', 'unknown')
+        }
+
+    log_event(event)
+
+
 @login_required
 @ensure_csrf_cookie
 def view_tracking_log(request, args=''):

lms/djangoapps/courseware/module_render.py

@@ -121,7 +121,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, model_
 
 
 def get_module(user, request, location, model_data_cache, course_id,
-               position=None, not_found_ok = False, wrap_xmodule_display=True,
+               position=None, not_found_ok=False, wrap_xmodule_display=True,
                grade_bucket_type=None, depth=0):
     """
     Get an instance of the xmodule class identified by location,
@@ -161,10 +161,45 @@ def get_module(user, request, location, model_data_cache, course_id,
         return None
 
 
+def get_xqueue_callback_url_prefix(request):
+    """
+    Calculates default prefix based on request, but allows override via settings
+
+    This is separated so that it can be called by the LMS before submitting
+    background tasks to run.  The xqueue callbacks should go back to the LMS,
+    not to the worker.
+    """
+    default_xqueue_callback_url_prefix = '{proto}://{host}'.format(
+            proto=request.META.get('HTTP_X_FORWARDED_PROTO', 'https' if request.is_secure() else 'http'),
+            host=request.get_host()
+        )
+    return settings.XQUEUE_INTERFACE.get('callback_url', default_xqueue_callback_url_prefix)
+
+
 def get_module_for_descriptor(user, request, descriptor, model_data_cache, course_id,
                 position=None, wrap_xmodule_display=True, grade_bucket_type=None):
     """
-    Actually implement get_module.  See docstring there for details.
+    Implements get_module, extracting out the request-specific functionality.
+
+    See get_module() docstring for further details.
+    """
+    track_function = make_track_function(request)
+    xqueue_callback_url_prefix = get_xqueue_callback_url_prefix(request)
+
+    return get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
+                                              track_function, xqueue_callback_url_prefix,
+                                              position=position,
+                                              wrap_xmodule_display=wrap_xmodule_display,
+                                              grade_bucket_type=grade_bucket_type)
+
+
+def get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
+                track_function, xqueue_callback_url_prefix,
+                position=None, wrap_xmodule_display=True, grade_bucket_type=None):
+    """
+    Actually implement get_module, without requiring a request.
+
+    See get_module() docstring for further details.
     """
 
     # allow course staff to masquerade as student
@@ -186,19 +221,13 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours
 
     def make_xqueue_callback(dispatch='score_update'):
         # Fully qualified callback URL for external queueing system
-        xqueue_callback_url = '{proto}://{host}'.format(
-            host=request.get_host(),
-            proto=request.META.get('HTTP_X_FORWARDED_PROTO', 'https' if request.is_secure() else 'http')
-        )
-        xqueue_callback_url = settings.XQUEUE_INTERFACE.get('callback_url',xqueue_callback_url)	# allow override
-
-        xqueue_callback_url += reverse('xqueue_callback',
-                                      kwargs=dict(course_id=course_id,
-                                                  userid=str(user.id),
-                                                  id=descriptor.location.url(),
-                                                  dispatch=dispatch),
-                                      )
-        return xqueue_callback_url
+        relative_xqueue_callback_url = reverse('xqueue_callback',
+                                               kwargs=dict(course_id=course_id,
+                                                           userid=str(user.id),
+                                                           id=descriptor.location.url(),
+                                                           dispatch=dispatch),
+                                       )
+        return xqueue_callback_url_prefix + relative_xqueue_callback_url
 
     # Default queuename is course-specific and is derived from the course that
     #   contains the current module.
@@ -211,20 +240,20 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours
               'waittime': settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
              }
 
-    #This is a hacky way to pass settings to the combined open ended xmodule
-    #It needs an S3 interface to upload images to S3
-    #It needs the open ended grading interface in order to get peer grading to be done
-    #this first checks to see if the descriptor is the correct one, and only sends settings if it is
+    # This is a hacky way to pass settings to the combined open ended xmodule
+    # It needs an S3 interface to upload images to S3
+    # It needs the open ended grading interface in order to get peer grading to be done
+    # this first checks to see if the descriptor is the correct one, and only sends settings if it is
 
-    #Get descriptor metadata fields indicating needs for various settings
+    # Get descriptor metadata fields indicating needs for various settings
     needs_open_ended_interface = getattr(descriptor, "needs_open_ended_interface", False)
     needs_s3_interface = getattr(descriptor, "needs_s3_interface", False)
 
-    #Initialize interfaces to None
+    # Initialize interfaces to None
     open_ended_grading_interface = None
     s3_interface = None
 
-    #Create interfaces if needed
+    # Create interfaces if needed
     if needs_open_ended_interface:
         open_ended_grading_interface = settings.OPEN_ENDED_GRADING_INTERFACE
         open_ended_grading_interface['mock_peer_grading'] = settings.MOCK_PEER_GRADING
@@ -240,8 +269,13 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours
         """
         Delegate to get_module.  It does an access check, so may return None
         """
-        return get_module_for_descriptor(user, request, descriptor,
-                                         model_data_cache, course_id, position)
+        # TODO: fix this so that make_xqueue_callback uses the descriptor passed into
+        # inner_get_module, not the parent's callback.  Add it as an argument....
+        return get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
+                                                  track_function, make_xqueue_callback,
+                                                  position=position,
+                                                  wrap_xmodule_display=wrap_xmodule_display,
+                                                  grade_bucket_type=grade_bucket_type)
 
     def xblock_model_data(descriptor):
         return DbModel(
@@ -291,7 +325,7 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours
     # TODO (cpennington): When modules are shared between courses, the static
     # prefix is going to have to be specific to the module, not the directory
     # that the xml was loaded from
-    system = ModuleSystem(track_function=make_track_function(request),
+    system = ModuleSystem(track_function=track_function,
                           render_template=render_to_string,
                           ajax_url=ajax_url,
                           xqueue=xqueue,

lms/djangoapps/instructor/views.py

@@ -220,7 +220,10 @@ def instructor_dashboard(request, course_id):
         try:
             course_task_log_entry = task_queue.submit_regrade_problem_for_all_students(request, course_id, problem_url)
             if course_task_log_entry is None:
-                msg += '<font="red">Failed to create a background task for regrading "{0}".</font>'.format(problem_url)
+                msg += '<font color="red">Failed to create a background task for regrading "{0}".</font>'.format(problem_url)
+            else:
+                track_msg = 'regrade problem {problem} for all students in {course}'.format(problem=problem_url, course=course_id)
+                track.views.server_track(request, track_msg, {}, page='idashboard')
         except Exception as e:
             log.error("Encountered exception from regrade: {0}".format(e))
             msg += '<font="red">Failed to create a background task for regrading "{0}": {1}.</font>'.format(problem_url, e)

lms/envs/test.py

@@ -188,3 +188,8 @@ PASSWORD_HASHERS = (
     'django.contrib.auth.hashers.MD5PasswordHasher',
     # 'django.contrib.auth.hashers.CryptPasswordHasher',
 )
+
+################################# CELERY ######################################
+
+# By default don't use a worker, execute tasks as if they were local functions
+CELERY_ALWAYS_EAGER = True