Merge pull request #648 from MITx/feature/ichuang/psychometrics

Interrim psychometrics with real-time IRT plots and histograms (see note)

Ike is committing to implement the SOA API which discussed in person, in which a separate djangoapp would receive the tracking logs from the LMS, and also be queried by the LMS to perform psychometrics analyses.  Initially this psychometrics djangoapp might share some tables like StudentModule with the LMS, but the plan would be to separate them when possible.  

Ike will implement this for edX by November.

common/djangoapps/external_auth/admin.py

@@ -5,4 +5,8 @@ django admin pages for courseware model
 from external_auth.models import *
 from django.contrib import admin
 
-admin.site.register(ExternalAuthMap)
+class ExternalAuthMapAdmin(admin.ModelAdmin):
+    search_fields = ['external_id','user__username']
+    date_hierarchy = 'dtcreated'
+
+admin.site.register(ExternalAuthMap, ExternalAuthMapAdmin)

common/djangoapps/track/admin.py

+'''
+django admin pages for courseware model
+'''
+
+from track.models import *
+from django.contrib import admin
+
+admin.site.register(TrackingLog)

common/lib/xmodule/xmodule/capa_module.py

@@ -507,8 +507,12 @@ class CapaModule(XModule):
         #       'success' will always be incorrect
         event_info['correct_map'] = correct_map.get_dict()
         event_info['success'] = success
+	event_info['attempts'] = self.attempts
         self.system.track_function('save_problem_check', event_info)
 
+	if hasattr(self.system,'psychometrics_handler'):	# update PsychometricsData using callback
+		self.system.psychometrics_handler(self.get_instance_state())
+
         # render problem into HTML
         html = self.get_problem_html(encapsulate=False)
 

common/lib/xmodule/xmodule/seq_module.py

@@ -75,7 +75,7 @@ class SequenceModule(XModule):
         contents = []
         for child in self.get_display_items():
             progress = child.get_progress()
-            contents.append({
+            childinfo = {
                 'content': child.get_html(),
                 'title': "\n".join(
                     grand_child.display_name.strip()
@@ -85,7 +85,10 @@ class SequenceModule(XModule):
                 'progress_status': Progress.to_js_status_str(progress),
                 'progress_detail': Progress.to_js_detail_str(progress),
                 'type': child.get_icon_class(),
-            })
+            }
+            if childinfo['title']=='':
+                childinfo['title'] = child.metadata.get('display_name','')
+            contents.append(childinfo)
 
         params = {'items': contents,
                   'element_id': self.location.html_id(),

lms/djangoapps/courseware/module_render.py

@@ -16,6 +16,7 @@ from capa.xqueue_interface import XQueueInterface
 from courseware.access import has_access
 from mitxmako.shortcuts import render_to_string
 from models import StudentModule, StudentModuleCache
+from psychometrics.psychoanalyze import make_psychometrics_data_update_handler
 from static_replace import replace_urls
 from xmodule.errortracker import exc_info_to_str
 from xmodule.exceptions import NotFoundError
@@ -230,6 +231,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
     system.set('DEBUG', settings.DEBUG)
+    if settings.MITX_FEATURES.get('ENABLE_PSYCHOMETRICS') and instance_module is not None:
+        system.set('psychometrics_handler',		# set callback for updating PsychometricsData
+                   make_psychometrics_data_update_handler(instance_module))
 
     try:
         module = descriptor.xmodule_constructor(system)(instance_state, shared_state)

lms/djangoapps/instructor/views.py

@@ -27,6 +27,7 @@ from django.views.decorators.cache import cache_control
 from courseware import grades
 from courseware.access import has_access, get_access_group_name
 from courseware.courses import (get_course_with_access, get_courses_by_university)
+from psychometrics import psychoanalyze
 from student.models import UserProfile
 
 from student.models import UserTestGroup, CourseEnrollment
@@ -51,7 +52,18 @@ def instructor_dashboard(request, course_id):
     instructor_access = has_access(request.user, course, 'instructor')   # an instructor can manage staff lists
 
     msg = ''
-    # msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
+    #msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
+
+    problems = []
+    plots = []
+
+    # the instructor dashboard page is modal: grades, psychometrics, admin
+    # keep that state in request.session (defaults to grades mode)
+    idash_mode = request.POST.get('idash_mode','')
+    if idash_mode:
+        request.session['idash_mode'] = idash_mode
+    else:
+        idash_mode = request.session.get('idash_mode','Grades')
 
     def escape(s):
         """escape HTML special characters in string"""
@@ -149,6 +161,9 @@ def instructor_dashboard(request, course_id):
         track.views.server_track(request, 'dump-answer-dist-csv', {}, page='idashboard')
         return return_csv('answer_dist_%s.csv' % course_id, get_answers_distribution(request, course_id))
 
+    #----------------------------------------
+    # Admin
+
     elif 'List course staff' in action:
         group = get_staff_group(course)
         msg += 'Staff group = %s' % group.name
@@ -187,14 +202,31 @@ def instructor_dashboard(request, course_id):
             user.groups.remove(group)
             track.views.server_track(request, 'remove-staff %s' % user, {}, page='idashboard')
 
-    # For now, mostly a static page
+    #----------------------------------------
+    # psychometrics
+
+    elif action == 'Generate Histogram and IRT Plot':
+        problem = request.POST['Problem']
+        nmsg, plots = psychoanalyze.generate_plots_for_problem(problem)
+        msg += nmsg
+        track.views.server_track(request, 'psychometrics %s' % problem, {}, page='idashboard')
+
+    if idash_mode=='Psychometrics':
+        problems = psychoanalyze.problems_with_psychometric_data(course_id)
+
+    #----------------------------------------
+    # context for rendering
     context = {'course': course,
                'staff_access': True,
                'admin_access': request.user.is_staff,
                'instructor_access': instructor_access,
                'datatable': datatable,
                'msg': msg,
+               'modeflag': {idash_mode: 'selectedmode'},
+               'problems': problems,		# psychometrics
+               'plots': plots,			# psychometrics
                'course_errors': modulestore().get_item_errors(course.location),
+               'djangopid' : os.getpid(),
                }
 
     return render_to_response('courseware/instructor_dashboard.html', context)

lms/djangoapps/lms_migration/migrate.py

@@ -35,7 +35,17 @@ def getip(request):
         ip = request.META.get('REMOTE_ADDR','None')
     return ip
 
-def manage_modulestores(request,reload_dir=None):
+
+def get_commit_id(course):
+    return course.metadata.get('GIT_COMMIT_ID','No commit id')
+    # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id')
+
+
+def set_commit_id(course,commit_id):
+    course.metadata['GIT_COMMIT_ID'] = commit_id
+    # setattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID', new_commit_id)
+
+def manage_modulestores(request, reload_dir=None, commit_id=None):
     '''
     Manage the static in-memory modulestores.
 
@@ -52,8 +62,9 @@ def manage_modulestores(request,reload_dir=None):
     ip = getip(request)
 
     if LOCAL_DEBUG:
-        html += '<h3>IP address: %s ' % ip
-        html += '<h3>User: %s ' % request.user
+        html += '<h3>IP address: %s <h3>' % ip
+        html += '<h3>User: %s </h3>' % request.user
+        html += '<h3>My pid: %s</h3>' % os.getpid()
         log.debug('request from ip=%s, user=%s' % (ip,request.user))
 
     if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
@@ -66,14 +77,36 @@ def manage_modulestores(request,reload_dir=None):
             return HttpResponse(html, status=403)
 
     #----------------------------------------
-    # reload course if specified
+    # reload course if specified; handle optional commit_id
 
     if reload_dir is not None:
         if reload_dir not in def_ms.courses:
             html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</h2>' % reload_dir
         else:
+            # reloading based on commit_id is needed when running mutiple worker threads,
+            # so that a given thread doesn't reload the same commit multiple times
+            current_commit_id = get_commit_id(def_ms.courses[reload_dir])
+            log.debug('commit_id="%s"' % commit_id)
+            log.debug('current_commit_id="%s"' % current_commit_id)
+
+            if (commit_id is not None) and (commit_id==current_commit_id):
+                html += "<h2>Already at commit id %s for %s</h2>" % (commit_id, reload_dir)
+                track.views.server_track(request,
+                                         'reload %s skipped already at %s (pid=%s)' % (reload_dir,
+                                                                                       commit_id,
+                                                                                       os.getpid(),
+                                                                                       ),
+                                         {}, page='migrate')
+            else:
                 html += '<h2>Reloaded course directory "%s"</h2>' % reload_dir
                 def_ms.try_load_course(reload_dir)
+                gdir = settings.DATA_DIR / reload_dir 
+                new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1]
+                set_commit_id(def_ms.courses[reload_dir], new_commit_id)
+                html += '<p>commit_id=%s</p>' % new_commit_id
+                track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir,
+                                                                                      new_commit_id,
+                                                                                      os.getpid()), {}, page='migrate')
 
     #----------------------------------------
 
@@ -94,6 +127,8 @@ def manage_modulestores(request,reload_dir=None):
         html += '<hr width="100%"/>'
         html += '<h2>Course: %s (%s)</h2>' % (course.display_name,cdir)
 
+        html += '<p>commit_id=%s</p>' % get_commit_id(course)
+
         for field in dumpfields:
             data = getattr(course,field)
             html += '<h3>%s</h3>' % field

lms/djangoapps/psychometrics/admin.py

+'''
+django admin pages for courseware model
+'''
+
+from psychometrics.models import *
+from django.contrib import admin
+
+admin.site.register(PsychometricData)

lms/djangoapps/psychometrics/management/commands/init_psychometrics.py

+#!/usr/bin/python
+#
+# generate pyschometrics data from tracking logs and student module data
+
+import os, sys, string
+import datetime
+import json
+
+from courseware.models import *
+from track.models import *
+from psychometrics.models import *
+from xmodule.modulestore import Location
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+#db = "ocwtutor"	# for debugging
+#db = "default"
+
+db = getattr(settings,'DATABASE_FOR_PSYCHOMETRICS','default')
+
+
+class Command(BaseCommand):
+    help = "initialize PsychometricData tables from StudentModule instances (and tracking data, if in SQL)."
+    help += "Note this is done for all courses for which StudentModule instances exist."
+
+    def handle(self, *args, **options):
+
+        # delete all pmd
+
+        #PsychometricData.objects.all().delete()
+        #PsychometricData.objects.using(db).all().delete()
+
+        smset = StudentModule.objects.using(db).exclude(max_grade=None)
+        
+        for sm in smset:
+            url = sm.module_state_key
+            location = Location(url)
+            if not location.category=="problem":
+                continue
+            try:
+                state = json.loads(sm.state)
+                done = state['done']
+            except:
+                print "Oops, failed to eval state for %s (state=%s)" % (sm,sm.state)
+                continue
+            
+            if done:			# only keep if problem completed
+                try:
+                    pmd = PsychometricData.objects.using(db).get(studentmodule=sm)
+                except PsychometricData.DoesNotExist:
+                    pmd = PsychometricData(studentmodule=sm)
+        
+                pmd.done = done
+                pmd.attempts = state['attempts']
+        
+                # get attempt times from tracking log
+                uname = sm.student.username
+                tset = TrackingLog.objects.using(db).filter(username=uname, event_type__contains='save_problem_check')        
+                tset = tset.filter(event_source='server')
+                tset = tset.filter(event__contains="'%s'" % url)
+                checktimes = [x.dtcreated for x in tset]
+                pmd.checktimes = checktimes
+                if not len(checktimes)==pmd.attempts:
+                    print "Oops, mismatch in number of attempts and check times for %s" % pmd
+        
+                #print pmd
+                pmd.save(using=db)
+        
+        print "%d PMD entries" % PsychometricData.objects.using(db).all().count()

lms/djangoapps/psychometrics/models.py

+#
+# db model for psychometrics data
+#
+# this data is collected in real time
+#
+
+from django.db import models
+from courseware.models import StudentModule
+
+class PsychometricData(models.Model):
+    """
+    This data is a table linking student, module, and module performance,
+    including number of attempts, grade, max grade, and time of checks.
+
+    Links to instances of StudentModule, but only those for capa problems.
+
+    Note that StudentModule.module_state_key is nominally a Location instance (url string).
+    That means it is of the form {tag}://{org}/{course}/{category}/{name}[@{revision}]
+    and for capa problems, category = "problem".
+
+    checktimes is extracted from tracking logs, or added by capa module via psychometrics callback.
+    """
+
+    studentmodule = models.ForeignKey(StudentModule, db_index=True, unique=True)   # contains student, module_state_key, course_id
+
+    done = models.BooleanField(default=False)
+    attempts = models.IntegerField(default=0)			# extracted from studentmodule.state
+    checktimes = models.TextField(null=True, blank=True)	# internally stored as list of datetime objects
+
+    # keep in mind
+    # grade = studentmodule.grade
+    # max_grade = studentmodule.max_grade
+    # student = studentmodule.student
+    # course_id = studentmodule.course_id
+    # location = studentmodule.module_state_key
+    
+    def __unicode__(self):
+        sm = self.studentmodule
+        return "[PsychometricData] %s url=%s, grade=%s, max=%s, attempts=%s, ct=%s" % (sm.student,
+                                                                                       sm.module_state_key,
+                                                                                       sm.grade,
+                                                                                       sm.max_grade,
+                                                                                       self.attempts,
+                                                                                       self.checktimes)
+    

lms/envs/common.py

@@ -71,6 +71,8 @@ MITX_FEATURES = {
     'ENABLE_DISCUSSION' : False,
     'ENABLE_DISCUSSION_SERVICE': True,
 
+    'ENABLE_PSYCHOMETRICS': False,	# real-time psychometrics (eg item response theory analysis in instructor dashboard)
+
     'ENABLE_SQL_TRACKING_LOGS': False,
     'ENABLE_LMS_MIGRATION': False,
     'ENABLE_MANUAL_GIT_RELOAD': False,
@@ -619,6 +621,7 @@ INSTALLED_APPS = (
     'util',
     'certificates',
     'instructor',
+    'psychometrics',
     
     #For the wiki
     'wiki', # The new django-wiki from benjaoming

lms/envs/dev.py

@@ -20,6 +20,8 @@ MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains-
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
 MITX_FEATURES['FORCE_UNIVERSITY_DOMAIN'] = None		# show all university courses if in dev (ie don't use HTTP_HOST)
 MITX_FEATURES['ENABLE_MANUAL_GIT_RELOAD'] = True
+MITX_FEATURES['ENABLE_PSYCHOMETRICS'] = True    # real-time psychometrics (eg item response theory analysis in instructor dashboard)
+
 
 WIKI_ENABLED = True
 

lms/templates/courseware/instructor_dashboard.html

@@ -4,6 +4,8 @@
 
 <%block name="headextra">
   <%static:css group='course'/>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
 </%block>
 
 <%include file="/courseware/course_navigation.html" args="active_page='instructor'" />
@@ -31,37 +33,91 @@ table.stat_table td {
 	border-color: #666666;
 	background-color: #ffffff;
 }
+
+a.selectedmode { background-color: yellow; }
+
 </style>
 
+<script language="JavaScript" type="text/javascript">
+function goto( mode)
+{
+  document.idashform.idash_mode.value = mode;
+  document.idashform.submit() ;
+}
+</script>
+
 <section class="container">
 <div class="instructor-dashboard-wrapper">
+
   <section class="instructor-dashboard-content">
     <h1>Instructor Dashboard</h1>
 
-    <form method="POST">
+    <h2>[ <a href="#" onclick="goto('Grades');" class="${modeflag.get('Grades')}">Grades</a> | 
+          %if settings.MITX_FEATURES.get('ENABLE_PSYCHOMETRICS'):
+            <a href="#" onclick="goto('Psychometrics');" class="${modeflag.get('Psychometrics')}">Psychometrics</a> | 
+          %endif
+          <a href="#" onclick="goto('Admin');" class="${modeflag.get('Admin')}">Admin</a> ]
+    </h2>
+
+  <div style="text-align:right" id="djangopid">${djangopid}</div>
+
+    <form name="idashform" method="POST">
     <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+    <input type="hidden" name="idash_mode" value="">
 
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Grades'):
     <p>
     <a href="${reverse('gradebook', kwargs=dict(course_id=course.id))}">Gradebook</a>
+    </p>
 
     <p>
     <a href="${reverse('grade_summary', kwargs=dict(course_id=course.id))}">Grade summary</a>
+    </p>
 
     <p>
     <input type="submit" name="action" value="Dump list of enrolled students">
+    </p>
 
     <p>
     <input type="submit" name="action" value="Dump Grades for all students in this course">
     <input type="submit" name="action" value="Download CSV of all student grades for this course">
+    </p>
 
     <p>
     <input type="submit" name="action" value="Dump all RAW grades for all students in this course">
     <input type="submit" name="action" value="Download CSV of all RAW grades">
+    </p>
 
     <p>
     <input type="submit" name="action" value="Download CSV of answer distributions">
+    </p>
+ %endif
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Psychometrics'):
+
+    <p>Select a problem and an action:
+    </p>
+
+    <p>
+    <select name="Problem">
+    %for problem, count in sorted(problems.items(), key=lambda x: x[0]):
+        <option value="${problem}">${problem} [${count}]</option>
+    %endfor
+    </select>
+    </p>
+    <p>
+    <input type="submit" name="action" value="Generate Histogram and IRT Plot">
+    </p>
+
+    <p></p>
 
-%if instructor_access:
+%endif
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Admin'):
+  %if instructor_access:
     <hr width="40%" style="align:left">
     <p>
     <input type="submit" name="action" value="List course staff members">
@@ -71,14 +127,18 @@ table.stat_table td {
     <hr width="40%" style="align:left">
   %endif
 
-%if settings.MITX_FEATURES['ENABLE_MANUAL_GIT_RELOAD'] and admin_access:
+  %if settings.MITX_FEATURES['ENABLE_MANUAL_GIT_RELOAD'] and admin_access:
     <p>
     <input type="submit" name="action" value="Reload course from XML files">
     <input type="submit" name="action" value="GIT pull and Reload course">
+  %endif
 %endif
 
     </form>
 
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Psychometrics') is None:
+
     <br/>
     <br/>
     <p>
@@ -99,14 +159,45 @@ table.stat_table td {
         %endfor
       </table>
     </p>
+%endif
+
+##-----------------------------------------------------------------------------  
+%if modeflag.get('Psychometrics'):
+
+    %for plot in plots:
+      <br/>
+      <h3>${plot['title']}</h3>
+      <br/>
+      <p>${plot['info']}</p>
+      <br/>
+      <div id="plot_${plot['id']}" style="width:600px;height:300px;"></div>
+      <script type="text/javascript">
+          $(function () {
+              ${plot['data']}
+              $.plot($("#plot_${plot['id']}"), ${plot['cmd']}  );
+          });
+      </script>
+      <br/>
+      <br/>
+    %endfor
+
+%endif
+ 
+##-----------------------------------------------------------------------------  
+## always show msg
 
 %if msg:
     <p>${msg}</p>
 %endif
 
-% if course_errors is not UNDEFINED:
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Admin'):
+ % if course_errors is not UNDEFINED:
     <h2>Course errors</h2>
     <div id="course-errors">
+    %if not course_errors:
+          None
+    %else:
       <ul>
         % for (summary, err) in course_errors:
         <li>${summary | h}
@@ -118,8 +209,10 @@ table.stat_table td {
         </li>
         % endfor
         </ul>
+      %endif
       </div>
-% endif
+  % endif
+%endif      
 
   </section>
 </div>

lms/templates/portal/course_about.html

@@ -76,7 +76,11 @@
 	             <div id="register_message"></div>
             %endif
         %else:
-            <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up or <a href="#login-modal" rel="leanModal">Log In</a> to enroll.'>Register for ${course.number}</a>
+            <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up
+% if not settings.MITX_FEATURES['DISABLE_LOGIN_BUTTON']:
+		     or <a href="#login-modal" rel="leanModal">Log In</a>
+% endif
+		     to enroll.'>Register for ${course.number}</a>
         %endif
         </div>
 

lms/urls.py

@@ -237,6 +237,7 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
     urlpatterns += (
         url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
         url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
+        url(r'^migrate/reload/(?P<reload_dir>[^/]+)/(?P<commit_id>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
         url(r'^gitreload$', 'lms_migration.migrate.gitreload'),
         url(r'^gitreload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.gitreload'),
         )