Skip to content

I
insights
Commit f2857d46 by VikParuchuri
Options

Merge pull request #12 from MITx/vik/add-htsql 

Vik/add htsql
parents 8d249071 be3bcf53
Showing with 1091 additions and 528 deletions
anserv/an_evt/views.py
...@@ -76,6 +76,16 @@ def handle_probe(request, cls=None, category=None, details = None): ...@@ -76,6 +76,16 @@ def handle_probe(request, cls=None, category=None, details = None):
l = [error_message.format(details,request_handlers)] l = [error_message.format(details,request_handlers)]
return HttpResponse("\n".join(l), mimetype='text/text') return HttpResponse("\n".join(l), mimetype='text/text')
def list_all_endpoints(request):
if not request.user.is_authenticated():
return HttpResponseRedirect(reverse('django.contrib.auth.views.login'))
endpoints = []
for cls in request_handlers:
for category in request_handlers[cls]:
for details in request_handlers[cls][category]:
endpoints.append({'type' : cls, 'category' : category, 'name' : details})
return HttpResponse(json.dumps(endpoints))
def handle_request(request, cls, category, name, **kwargs): def handle_request(request, cls, category, name, **kwargs):
''' Generic code from handle_view and handle_query ''' ''' Generic code from handle_view and handle_query '''
args = dict() args = dict()
......
anserv/aws.py
...@@ -20,3 +20,10 @@ DEBUG = False ...@@ -20,3 +20,10 @@ DEBUG = False
TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(days=1) TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(days=1)
ROOT_URLCONF = 'urls' ROOT_URLCONF = 'urls'
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
'LOCATION': 'django_cache'
}
}
anserv/aws_xml.py
...@@ -21,6 +21,13 @@ TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(days=1) ...@@ -21,6 +21,13 @@ TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(days=1)
ROOT_URLCONF = 'urls' ROOT_URLCONF = 'urls'
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
'LOCATION': 'django_cache'
}
}
anserv/dashboard/static/dashboard.html
<!doctype html> {% extends "base.html" %}
{% block title %}
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Dashboard - edX</title> <title>Dashboard - edX</title>
{% endblock %}
{% block additional_js %}
<meta charset="utf-8" />
<link rel="stylesheet" href="http://code.jquery.com/ui/1.9.2/themes/base/jquery-ui.css" /> <link rel="stylesheet" href="http://code.jquery.com/ui/1.9.2/themes/base/jquery-ui.css" />
<link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.2.2/css/bootstrap-combined.min.css" rel="stylesheet">
<script src="http://code.jquery.com/jquery-1.8.3.js"></script>
<script src="http://code.jquery.com/ui/1.9.2/jquery-ui.js"></script>
<script src="http://demo.presinet.com/jscript/jquery.flot.patched-multi.js"></script>
<script> <script>
/* this is the contents of the multi plugin */ /* this is the contents of the multi plugin */
// You can yank the plugin and patches from here: http://code.google.com/p/flot/issues/detail?id=159 // You can yank the plugin and patches from here: http://code.google.com/p/flot/issues/detail?id=159
...@@ -379,6 +375,7 @@ ...@@ -379,6 +375,7 @@
for(i = 0 ; i< split_data.length; i++){ for(i = 0 ; i< split_data.length; i++){
$("#sidebar_ul").append('<li><a href="#" onclick="load_global_rcontent(\'' + split_data[i] + '\');">'+tidy_str(split_data[i])+'</li>'); $("#sidebar_ul").append('<li><a href="#" onclick="load_global_rcontent(\'' + split_data[i] + '\');">'+tidy_str(split_data[i])+'</li>');
} }
$("#username_text").attr('placeholder', "")
}); });
}); });
...@@ -392,14 +389,15 @@ ...@@ -392,14 +389,15 @@
for(i = 0 ; i< split_data.length; i++){ for(i = 0 ; i< split_data.length; i++){
$("#sidebar_ul").append('<li><a href="#" onclick="load_user_rcontent(\'' + split_data[i] + '\');">'+tidy_str(split_data[i])+'</li>'); $("#sidebar_ul").append('<li><a href="#" onclick="load_user_rcontent(\'' + split_data[i] + '\');">'+tidy_str(split_data[i])+'</li>');
} }
$("#username_text").attr('placeholder', "Username")
}); });
}); });
$('#global_tab').click(); $('#global_tab').click();
}); });
</script> </script>
</head> {% endblock %}
<body> {% block content %}
<div class="container-fluid"><br /><br /> <div class="container-fluid"><br /><br />
...@@ -407,7 +405,7 @@ ...@@ -407,7 +405,7 @@
<li class="active"><a href="#" id="global_tab">Global</a></li> <li class="active"><a href="#" id="global_tab">Global</a></li>
<li><a href="#" id="user_tab">User</a></li> <li><a href="#" id="user_tab">User</a></li>
<li><form class="navbar-search pull-left"> <li><form class="navbar-search pull-left">
<input type="text" id="username_text" class="search-query" placeholder="Username"> <input type="text" id="username_text" class="search-query" placeholder="">
</form></li> </form></li>
</ul> </ul>
...@@ -436,7 +434,7 @@ ...@@ -436,7 +434,7 @@
</div><!--/.fluid-container--> </div><!--/.fluid-container-->
<!-- <!--
<div id="tabs"> <div id="tabs">
<ul> <ul>
<li><a href="#global_tab2">Global</a></li> <li><a href="#global_tab2">Global</a></li>
<li><a href="#user_tab2">User</a></li> <li><a href="#user_tab2">User</a></li>
...@@ -444,20 +442,19 @@ ...@@ -444,20 +442,19 @@
<div id="global_tab"> <div id="global_tab">
<table> <table>
<tr> <tr>
<td> <td>
<select id="globalselect" name="globalselect" size=10 width=150 style="width:150px"> <select id="globalselect" name="globalselect" size=10 width=150 style="width:150px">
</select> </select>
</td><td><div id="global_data"></div></td></tr> </td><td><div id="global_data"></div></td></tr>
</table> </table>
</div> </div>
<div id="user_tab"> <div id="user_tab">
<table> <table>
<tr><td> <select id="userselect" name="userselect" size=10 width=150 style="width:150px"> </select> <br><input id="user_input" style="width:150px"> <tr><td> <select id="userselect" name="userselect" size=10 width=150 style="width:150px"> </select> <br><input id="user_input" style="width:150px">
</td><td><div id="user_data"></div></td></tr> </td><td><div id="user_data"></div></td></tr>
</table> </table>
</div> </div>
</div> </div>
--> -->
</body> {% endblock %}
</html>
anserv/dashboard/views.py
...@@ -10,3 +10,9 @@ def dashboard(request): ...@@ -10,3 +10,9 @@ def dashboard(request):
else: else:
return HttpResponseRedirect(reverse("django.contrib.auth.views.login")) return HttpResponseRedirect(reverse("django.contrib.auth.views.login"))
def new_dashboard(request):
if request.user.is_authenticated():
return render(request, 'new_dashboard/new_dashboard.html')
else:
return HttpResponseRedirect(reverse("django.contrib.auth.views.login"))
anserv/modules/course_stats/course_stats.py
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.error("BLAH")
from modules.decorators import view, query, event_handler, memoize_query from modules.decorators import view, query, event_handler, memoize_query
#from an_evt.models import StudentBookAccesses #from an_evt.models import StudentBookAccesses
from django.contrib.auth.models import User from django.contrib.auth.models import User
from collections import Counter
import json import json
from django.conf import settings from django.conf import settings
...@@ -16,14 +16,13 @@ from django.contrib.auth.models import User ...@@ -16,14 +16,13 @@ from django.contrib.auth.models import User
import csv import csv
from pymongo import MongoClient from pymongo import MongoClient
connection = MongoClient() connection = MongoClient()
import django.template.loader
log=logging.getLogger(__name__) log=logging.getLogger(__name__)
import re import re
import os import os
from django.http import HttpResponse from django.http import HttpResponse
import courseware
from courseware.models import StudentModule from courseware.models import StudentModule
from mitxmako.shortcuts import render_to_response, render_to_string from mitxmako.shortcuts import render_to_response, render_to_string
...@@ -87,34 +86,112 @@ def new_course_enrollment_view(fs, db, params): ...@@ -87,34 +86,112 @@ def new_course_enrollment_view(fs, db, params):
@query('global', 'available_courses') @query('global', 'available_courses')
def courses_available_query(fs, db, params): def courses_available_query(fs, db, params):
r = [c['course_id'] for c in StudentModule.objects.all().values('course_id').distinct()] collection = connection['modules_tasks']['student_course_stats']
course_data = collection.find({}, {'course' : 1})
r = [c['course'] for c in course_data]
return r return r
@query('course', 'student_grades') @view('course', 'student_grades')
def course_grades_query(fs,db,course, params): def course_grades_view(fs, db, course, params):
collection = connection['modules_tasks']['student_problem_stats'] """
View student course-level grades
fs - filesystem
db - mongo collection
course - string course id
"""
type="course_grades"
return course_grades_view_base(fs, db, course, type,params)
@view('course', 'student_problem_grades')
def problem_grades_view(fs, db, course, params):
"""
View student exercise-level grades
fs - filesystem
db - mongo collection
course - string course id
"""
type="problem_grades"
return course_grades_view_base(fs, db, course, type,params)
def course_grades_view_base(fs, db, course, type,params):
"""
Base logic to generate charts for course grade views.
fs - filesystem
db - mongo collection
course - string course id
type - either "course_grades" which returns weighted scores, or "problem_grades" which returns unweighted.
"""
y_label = "Count"
if type=="course_grades":
query_data = course_grades_query(fs,db,course, params)
x_label = "Weighted Score"
else:
query_data = problem_grades_query(fs,db,course, params)
x_label = "Unweighted Score"
json_data = query_data['json']
results = json_data['results']
headers = results[0].keys()
excluded_keys = ['student']
headers = [h for h in headers if h not in excluded_keys]
charts = []
for header in headers:
fixed_name = re.sub(" ","_",header).lower()
header_data = [round(float(j[header])*10,1)/10 for j in results]
counter = Counter(header_data)
counter_keys = counter.keys()
counter_keys.sort()
counter_list = [[float(c),int(counter[c])] for c in counter_keys]
tick_data = [float(c) for c in counter_keys]
min_val = min(tick_data + [0])
max_val = max(tick_data + [1])
context_dict = {'graph_name' : fixed_name, 'chart_data' : counter_list, 'graph_title' : header, 'tick_data' : tick_data, 'x_min' : min_val, 'x_max' : max_val, 'x_label' : x_label, 'y_label' : y_label}
rendered_data = django.template.loader.render_to_string("grade_distribution/student_grade_distribution.html",context_dict)
charts.append(rendered_data)
chart_string = " ".join(charts)
return HttpResponse(chart_string)
def course_grades_query_base(fs,db,course, params, type="course"):
"""
Base logic to query all student grades for a given course. Returns a dictionary.
fs- file system
db- mongo collection
course - string course id
type - either "course" or "problem". "course" will return weighted grades, "problem" unweighted.
"""
types = {
'course' : ['student_course_stats', 'student_grades'],
'problem' : ['student_problem_stats', 'student_problem_grades']
}
type_list = types[type]
collection = connection['modules_tasks'][type_list[0]]
course_name = re.sub("[/:]","_",course) course_name = re.sub("[/:]","_",course)
json_data = list(collection.find({'course' : course})) json_data = list(collection.find({'course' : course}))
if len(json_data)<1: if len(json_data)<1:
return {'success' : False, 'message' : "Cannot find the course in the list or data not available.", 'courses' : courses_available_query(fs,db,params)} return {'success' : False, 'message' : "Cannot find the course in the list or data not available." , 'courses' : courses_available_query(fs,db,params)}
json_data = json_data[0] json_data = json_data[0]
json_data = {k:json_data[k] for k in json_data if k in ["course", "updated", "results"]} json_data = {k:json_data[k] for k in json_data if k in ["course", "updated", "results"]}
csv_file = "{0}/{1}_{2}.csv".format(settings.PROTECTED_DATA_URL,"student_grades",course_name) csv_file = "{0}/{1}_{2}.csv".format(settings.PROTECTED_DATA_URL,type_list[1],course_name)
return {'csv' : csv_file, 'json' : json_data, 'success' : True} return {'csv' : csv_file, 'json' : json_data, 'success' : True}
@query('course', 'student_grades')
def course_grades_query(fs,db,course, params):
"""
Query all student weighted grades for a given course
fs- file system
db- mongo collection
course - string course id
"""
return course_grades_query_base(fs,db,course,params,type="course")
@query('course', 'student_problem_grades') @query('course', 'student_problem_grades')
def problem_grades_query(fs,db,course, params): def problem_grades_query(fs,db,course, params):
collection = connection['modules_tasks']['student_course_stats'] """
course_name = re.sub("[/:]","_",course) Query all student unweighted grades for a given course
json_data = list(collection.find({'course' : course})) fs- file system
db- mongo collection
if len(json_data)<1: course - string course id
return {'success' : False, 'message' : "Cannot find the course in the list or data not available." , 'courses' : courses_available_query(fs,db,params)} """
json_data = json_data[0] return course_grades_query_base(fs,db,course,params,type="problem")
json_data = {k:json_data[k] for k in json_data if k in ["course", "updated", "results"]}
csv_file = "{0}/{1}_{2}.csv".format(settings.PROTECTED_DATA_URL,"student_problem_grades",course_name)
return {'csv' : csv_file, 'json' : json_data}
anserv/modules/student_course_stats/tasks.py 0 → 100644
from celery import task
from modules.decorators import memoize_query, query
from modules.mixpanel.mixpanel import EventTracker
import logging
from celery.task import periodic_task
from modules import common
from django.conf import settings
import sys
import io
from dateutil import parser
log=logging.getLogger(__name__)
#If we are importing the MITx modules, then full functionality will be enabled here.
if settings.IMPORT_MITX_MODULES:
from courseware import grades
from courseware.courses import get_course_with_access
from courseware.model_data import ModelDataCache, LmsKeyValueStore
#If not, we will retain the studentmodule, which will give minimal functionality
from courseware.models import StudentModule
from django.contrib.auth.models import User
import re
import csv
from django.http import HttpResponse
import json
from celery import current_task
import datetime
from django.utils.timezone import utc
from django.core.cache import cache
import time
#Locks are set by long-running tasks to ensure that they are not duplicated.
LOCK_EXPIRE = 24 * 60 * 60 # 1 day
class RequestDict(object):
"""
Mocks the request object. Needed for the MITx grading functions to work.
"""
def __init__(self, user):
self.META = {}
self.POST = {}
self.GET = {}
self.user = user
self.path = None
def get_db_and_fs_cron(f):
"""
Gets the correct fs and db for a given input function
f - a function signature
fs - A filesystem object
db - A mongo database collection
"""
import an_evt.views
db = an_evt.views.get_database(f)
fs = an_evt.views.get_filesystem(f)
return fs,db
@periodic_task(run_every=settings.TIME_BETWEEN_DATA_REGENERATION)
def regenerate_student_course_data():
"""
Generates the data for a given student's performance in a course.
This function is a periodic task (cron job) that runs at a specified interval,
pulls a list of courses, and for each course sends messages to the appropriate tasks.
"""
if not settings.IMPORT_MITX_MODULES:
log.error("Cannot import mitx modules and thus cannot regenerate student course data.")
return
log.debug("Regenerating course data.")
user = User.objects.all()[0]
request = RequestDict(user)
all_courses = [c['course_id'] for c in StudentModule.objects.values('course_id').distinct()]
all_courses = list(set(all_courses))
log.debug(all_courses)
for course in all_courses:
for type in ['course', 'problem']:
STUDENT_TASK_TYPES[type].delay(request,course)
@task
def get_student_course_stats(request, course):
"""
Regenerates student stats for a course (weighted section grades)
Stores the grades to a mongo (json) collection, and to csv files
request - a mock request (using RequestDict)
course - a string course id
"""
course_name = re.sub("[/:]","_",course)
log.info(course_name)
lock_id = "regenerate_student_course_data-lock-{0}-{1}-{2}".format(course,"student_course_grades", course_name)
log.info(lock_id)
acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
release_lock = lambda: cache.delete(lock_id)
if acquire_lock():
try:
fs, db = get_db_and_fs_cron(get_student_course_stats)
collection = db['student_course_stats']
courseware_summaries, users_in_course_ids = get_student_course_stats_base(request,course, "grades")
rows = []
for z in xrange(0,len(courseware_summaries)):
row = {'student' : users_in_course_ids[z], 'overall_percent' : courseware_summaries[z]["percent"]}
row.update({c['category'] : c['percent'] for c in courseware_summaries[z]["grade_breakdown"]})
rows.append(row)
file_name = "student_grades_{0}.csv".format(course_name)
try:
return_csv(fs,file_name, rows)
except:
log.exception("Could not generate csv file.")
file_name = "no_file_generated"
write_to_collection(collection, rows, course)
finally:
release_lock()
return json.dumps({'result_data' : rows, 'result_file' : "{0}/{1}".format(settings.PROTECTED_DATA_URL, file_name)})
return {}
@task
def get_student_problem_stats(request,course):
"""
Regenerates student stats for a course (unweighted exercise grades)
Stores the grades to a mongo (json) collection, and to csv files
request - a mock request (using RequestDict)
course - a string course id
"""
course_name = re.sub("[/:]","_",course)
log.info(course_name)
lock_id = "regenerate_student_course_data-lock-{0}-{1}-{2}".format(course,"student_problem_grades", course_name)
log.info(lock_id)
acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
release_lock = lambda: cache.delete(lock_id)
if acquire_lock():
try:
fs, db = get_db_and_fs_cron(get_student_course_stats)
collection = db['student_problem_stats']
courseware_summaries, users_in_course_ids = get_student_course_stats_base(request,course, "grades")
rows = []
for z in xrange(0,len(courseware_summaries)):
log.info(courseware_summaries[z])
row = {'student' : users_in_course_ids[z]}
#row.update({'problem_data' : courseware_summaries[z]})
row.update({c['label'] : c['percent'] for c in courseware_summaries[z]["section_breakdown"]})
rows.append(row)
file_name = "student_problem_grades_{0}.csv".format(course_name)
try:
return_csv(fs,file_name, rows)
except:
log.exception("Could not generate csv file.")
file_name = "no_file_generated"
write_to_collection(collection, rows, course)
finally:
release_lock()
return json.dumps({'result_data' : rows, 'result_file' : "{0}/{1}".format(settings.PROTECTED_DATA_URL, file_name)})
return {}
def get_student_course_stats_base(request,course, type="grades"):
"""
Called by get_student_course_stats and get_student_problem_stats
Gets a list of users in a course, and then computes grades for them
request - a mock request (using RequestDict)
course - a string course id
type - whether to get student weighted grades or unweighted grades. If "grades" will get weighted
"""
fs, db = get_db_and_fs_cron(get_student_course_stats)
course_obj = get_course_with_access(request.user, course, 'load', depth=None)
users_in_course = StudentModule.objects.filter(course_id=course).values('student').distinct()
users_in_course_ids = [u['student'] for u in users_in_course]
log.debug("Users in course count: {0}".format(len(users_in_course_ids)))
courseware_summaries = []
for i in xrange(0,len(users_in_course_ids)):
try:
user = users_in_course_ids[i]
current_task.update_state(state='PROGRESS', meta={'current': i, 'total': len(users_in_course_ids)})
student = User.objects.using('default').prefetch_related("groups").get(id=int(user))
model_data_cache = None
if type=="grades":
grade_summary = grades.grade(student, request, course_obj, model_data_cache)
else:
grade_summary = grades.progress_summary(student, request, course_obj, model_data_cache)
courseware_summaries.append(grade_summary)
except:
log.exception("Could not generate data for {0}".format(users_in_course_ids[i]))
return courseware_summaries, users_in_course_ids
def return_csv(fs, filename, results):
"""
Given a filesystem and a list of results, will write the results to a file
fs - filesystem object
filename - the name of the csv file to write
results - a list of dictionaries
"""
if len(results)<1:
return
output = fs.open(filename, 'w')
writer = csv.writer(output, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
row_keys = results[0].keys()
writer.writerow(row_keys)
for datarow in results:
encoded_row = []
for key in row_keys:
encoded_row+=[unicode(datarow[key]).encode('utf-8')]
writer.writerow(encoded_row)
output.close()
return True
def write_to_collection(collection, results, course):
"""
Given a collection and results, writes the results to the collection
collection - a mongo collection
results - a list of dictionaries
course - string course id
"""
if len(results)<1:
return
now = datetime.datetime.utcnow().replace(tzinfo=utc)
now_string = str(now)
mongo_results = {'updated' : now_string, 'course' : course, 'results' : results}
sba = list(collection.find({'course' : course}))
if len(sba)>0:
collection.update({'course':course}, mongo_results, True)
else:
collection.insert(mongo_results)
#Used by regenerate_student_course_data to find and call tasks
STUDENT_TASK_TYPES = {
'course' : get_student_course_stats,
'problem' : get_student_problem_stats
}
\ No newline at end of file
anserv/modules/tasks.py
"""
This module has examples periodic tasks (akin to cron) and normal tasks (delayed jobs).
It can also contain global tasks.
"""
from celery import task from celery import task
from decorators import memoize_query, query from decorators import memoize_query, query
from mixpanel.mixpanel import EventTracker from mixpanel.mixpanel import EventTracker
import logging import logging
from celery.task import periodic_task from celery.task import periodic_task
from modules import common from modules import common
from django.conf import settings
import sys
import io
from dateutil import parser
log=logging.getLogger(__name__)
if settings.IMPORT_MITX_MODULES:
from courseware import grades
from courseware.courses import get_course_with_access
import courseware
from courseware.models import StudentModule
from django.contrib.auth.models import User
import re
import csv
from django.http import HttpResponse
import json
from celery import current_task
import datetime import datetime
from django.utils.timezone import utc
from django.core.cache import cache
import time
LOCK_EXPIRE = 24 * 60 * 60 # 1 day
class RequestDict(object): log=logging.getLogger(__name__)
def __init__(self, user):
self.META = {}
self.POST = {}
self.GET = {}
self.user = user
self.path = None
@task() @task()
def track_event_mixpanel_batch(event_list): def track_event_mixpanel_batch(event_list):
"""
An example of a task. This will take in a list of events and stream them to mixpanel.
Tasks run as delayed jobs, and are processed by celery workers running on the backend.
"""
for list_start in xrange(0,len(event_list),50): for list_start in xrange(0,len(event_list),50):
event_tracker = EventTracker() event_tracker = EventTracker()
event_tracker.track(event_list[list_start:(list_start+50)],event_list=True) event_tracker.track(event_list[list_start:(list_start+50)],event_list=True)
...@@ -48,152 +26,30 @@ def track_event_mixpanel_batch(event_list): ...@@ -48,152 +26,30 @@ def track_event_mixpanel_batch(event_list):
@memoize_query @memoize_query
#@periodic_task(run_every=2) #@periodic_task(run_every=2)
def foo(): def foo():
"""
An example of a periodic task. Uncomment the periodic_task decorator to run this every 10 seconds.
"""
fs,db = get_db_and_fs_cron(foo) fs,db = get_db_and_fs_cron(foo)
print "Test" print "Test"
@memoize_query @memoize_query
#@periodic_task(run_every=10) #@periodic_task(run_every=datetime.timedelta(seconds=10))
def foo2(): def foo2():
"""
An example of a periodic task. Uncomment the periodic_task decorator to run this every 10 seconds.
"""
fs,db = get_db_and_fs_cron(foo2) fs,db = get_db_and_fs_cron(foo2)
print "Another Test" print "Another Test"
def get_db_and_fs_cron(f): def get_db_and_fs_cron(f):
"""
Gets the correct fs and db for a given input function
f - a function signature
fs - A filesystem object
db - A mongo database collection
"""
import an_evt.views import an_evt.views
db = an_evt.views.get_database(f) db = an_evt.views.get_database(f)
fs = an_evt.views.get_filesystem(f) fs = an_evt.views.get_filesystem(f)
return fs,db return fs,db
@periodic_task(run_every=settings.TIME_BETWEEN_DATA_REGENERATION)
def regenerate_student_course_data():
if not settings.IMPORT_MITX_MODULES:
log.error("Cannot import mitx modules and thus cannot regenerate student course data.")
return
log.debug("Regenerating course data.")
user = User.objects.all()[0]
request = RequestDict(user)
all_courses = [c['course_id'] for c in StudentModule.objects.values('course_id').distinct()]
all_courses = list(set(all_courses))
log.debug(all_courses)
for course in all_courses:
for type in ['course', 'problem']:
STUDENT_TASK_TYPES[type].delay(request,course)
@task
def get_student_course_stats(request, course):
course_name = re.sub("[/:]","_",course)
log.info(course_name)
lock_id = "regenerate_student_course_data-lock-{0}-{1}-{2}".format(course,"student_course_grades", course_name)
log.info(lock_id)
acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
release_lock = lambda: cache.delete(lock_id)
if acquire_lock():
try:
fs, db = get_db_and_fs_cron(get_student_course_stats)
collection = db['student_course_stats']
courseware_summaries, users_in_course_ids = get_student_course_stats_base(request,course,course_name, "grades")
rows = []
for z in xrange(0,len(courseware_summaries)):
row = {'student' : users_in_course_ids[z], 'overall_percent' : courseware_summaries[z]["percent"]}
row.update({c['category'] : c['percent'] for c in courseware_summaries[z]["grade_breakdown"]})
rows.append(row)
file_name = "student_grades_{0}.csv".format(course_name)
try:
return_csv(fs,file_name, rows)
except:
log.exception("Could not generate csv file.")
file_name = "no_file_generated"
write_to_collection(collection, rows, course)
finally:
release_lock()
return json.dumps({'result_data' : rows, 'result_file' : "{0}/{1}".format(settings.PROTECTED_DATA_URL, file_name)})
return {}
@task
def get_student_problem_stats(request,course):
course_name = re.sub("[/:]","_",course)
log.info(course_name)
lock_id = "regenerate_student_course_data-lock-{0}-{1}-{2}".format(course,"student_problem_grades", course_name)
log.info(lock_id)
acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
release_lock = lambda: cache.delete(lock_id)
if acquire_lock():
try:
fs, db = get_db_and_fs_cron(get_student_course_stats)
collection = db['student_problem_stats']
courseware_summaries, users_in_course_ids = get_student_course_stats_base(request,course,course_name, "grades")
rows = []
for z in xrange(0,len(courseware_summaries)):
log.info(courseware_summaries[z])
row = {'student' : users_in_course_ids[z]}
#row.update({'problem_data' : courseware_summaries[z]})
row.update({c['label'] : c['percent'] for c in courseware_summaries[z]["section_breakdown"]})
rows.append(row)
file_name = "student_problem_grades_{0}.csv".format(course_name)
try:
return_csv(fs,file_name, rows)
except:
log.exception("Could not generate csv file.")
file_name = "no_file_generated"
write_to_collection(collection, rows, course)
finally:
release_lock()
return json.dumps({'result_data' : rows, 'result_file' : "{0}/{1}".format(settings.PROTECTED_DATA_URL, file_name)})
return {}
def get_student_course_stats_base(request,course,course_name, type="grades"):
fs, db = get_db_and_fs_cron(get_student_course_stats)
course_obj = get_course_with_access(request.user, course, 'load', depth=None)
users_in_course = StudentModule.objects.filter(course_id=course).values('student').distinct()
users_in_course_ids = [u['student'] for u in users_in_course]
log.debug("Users in course count: {0}".format(len(users_in_course_ids)))
courseware_summaries = []
for i in xrange(0,len(users_in_course_ids)):
try:
user = users_in_course_ids[i]
current_task.update_state(state='PROGRESS', meta={'current': i, 'total': len(users_in_course_ids)})
student = User.objects.using('default').prefetch_related("groups").get(id=int(user))
model_data_cache = None
if type=="grades":
grade_summary = grades.grade(student, request, course_obj, model_data_cache)
else:
grade_summary = grades.progress_summary(student, request, course_obj, model_data_cache)
courseware_summaries.append(grade_summary)
except:
log.exception("Could not generate data for {0}".format(users_in_course_ids[i]))
return courseware_summaries, users_in_course_ids
def return_csv(fs, filename, results):
if len(results)<1:
return
output = fs.open(filename, 'w')
writer = csv.writer(output, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
row_keys = results[0].keys()
writer.writerow(row_keys)
for datarow in results:
encoded_row = []
for key in row_keys:
encoded_row+=[unicode(datarow[key]).encode('utf-8')]
writer.writerow(encoded_row)
output.close()
return True
def write_to_collection(collection, results, course):
if len(results)<1:
return
now = datetime.datetime.utcnow().replace(tzinfo=utc)
now_string = str(now)
mongo_results = {'updated' : now_string, 'course' : course, 'results' : results}
sba = list(collection.find({'course' : course}))
if len(sba)>0:
collection.update({'course':course}, mongo_results, True)
else:
collection.insert(mongo_results)
STUDENT_TASK_TYPES = {
'course' : get_student_course_stats,
'problem' : get_student_problem_stats
}
anserv/modules/user_stats/user_stats.py
...@@ -13,7 +13,6 @@ log=logging.getLogger(__name__) ...@@ -13,7 +13,6 @@ log=logging.getLogger(__name__)
import re import re
import sys import sys
import courseware
from courseware.models import StudentModule from courseware.models import StudentModule
from mitxmako.shortcuts import render_to_response, render_to_string from mitxmako.shortcuts import render_to_response, render_to_string
......
anserv/settings.py
...@@ -5,25 +5,27 @@ import sys ...@@ -5,25 +5,27 @@ import sys
from path import path from path import path
import datetime import datetime
# I really don't like this structure. TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(minutes=1)
# 1) We should have this be local to the module, as much as possible
# 2) At this point, at any given point in time, the courseware install script #Initialize celery
# has perhaps a 50/50 chance of working, and over half a gig of import djcelery
# requirements. By requiring it (or de-facto requiring it), we're dumping djcelery.setup_loader()
# that on anyone who wants to build an analytics module.
# BASE_DIR = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
IMPORT_MITX_MODULES = False
ROOT_PATH = path(__file__).dirname()
REPO_PATH = ROOT_PATH.dirname()
ENV_ROOT = REPO_PATH.dirname()
IMPORT_MITX_MODULES = True
if IMPORT_MITX_MODULES: if IMPORT_MITX_MODULES:
MITX_PATH = os.path.abspath("../../mitx/") MITX_PATH = os.path.abspath("../../mitx/")
DJANGOAPPS_PATH = "{0}/{1}/{2}".format(MITX_PATH, "lms", "djangoapps") DJANGOAPPS_PATH = "{0}/{1}/{2}".format(MITX_PATH, "lms", "djangoapps")
LMS_LIB_PATH = "{0}/{1}/{2}".format(MITX_PATH, "lms", "lib") LMS_LIB_PATH = "{0}/{1}/{2}".format(MITX_PATH, "lms", "lib")
COMMON_PATH = "{0}/{1}/{2}".format(MITX_PATH, "common", "djangoapps") COMMON_PATH = "{0}/{1}/{2}".format(MITX_PATH, "common", "djangoapps")
MITX_LIB_PATHS = [MITX_PATH, DJANGOAPPS_PATH, LMS_LIB_PATH, COMMON_PATH]
sys.path.append(MITX_PATH) sys.path += MITX_LIB_PATHS
sys.path.append(DJANGOAPPS_PATH)
sys.path.append(LMS_LIB_PATH)
sys.path.append(COMMON_PATH)
IMPORT_GIT_MODULES = False IMPORT_GIT_MODULES = False
GIT_CLONE_URL = "git@github.com:MITx/{0}.git" GIT_CLONE_URL = "git@github.com:MITx/{0}.git"
...@@ -33,21 +35,7 @@ if IMPORT_MITX_MODULES: ...@@ -33,21 +35,7 @@ if IMPORT_MITX_MODULES:
#Needed for MITX imports to work #Needed for MITX imports to work
from mitx_settings import * from mitx_settings import *
else: else:
sys.path.append("datalib") sys.path.append(ROOT_PATH / "mitx_libraries")
TIME_BETWEEN_DATA_REGENERATION = datetime.timedelta(minutes=1)
#Initialize celery
import djcelery
djcelery.setup_loader()
BASE_DIR = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
ROOT_PATH = path(__file__).dirname()
REPO_PATH = ROOT_PATH.dirname()
ENV_ROOT = REPO_PATH.dirname()
DUMMY_MODE = False DUMMY_MODE = False
...@@ -88,20 +76,12 @@ DATABASES = { ...@@ -88,20 +76,12 @@ DATABASES = {
DATABASE_ROUTERS = ['an_evt.router.DatabaseRouter'] DATABASE_ROUTERS = ['an_evt.router.DatabaseRouter']
# CACHES = {
# 'default': {
# 'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
# 'LOCATION': 'django_cache'
# }
# }
CACHES = { CACHES = {
'default': { 'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-snowflake' 'LOCATION': 'analytics-experiments'
} }
} }
LOG_READ_DIRECTORY = "../../analytics-logs/" LOG_READ_DIRECTORY = "../../analytics-logs/"
LOG_POST_URL = "http://127.0.0.1:9022/event" LOG_POST_URL = "http://127.0.0.1:9022/event"
...@@ -211,8 +191,7 @@ INSTALLED_APPS = ( ...@@ -211,8 +191,7 @@ INSTALLED_APPS = (
'south', 'south',
'frontend', 'frontend',
'pipeline', 'pipeline',
# 'staticfiles', 'staticfiles',
# 'static_replace',
'pipeline', 'pipeline',
) )
...@@ -297,13 +276,38 @@ PIPELINE_JS = { ...@@ -297,13 +276,38 @@ PIPELINE_JS = {
'js/backbone.js', 'js/backbone.js',
'js/backbone.validations.js' 'js/backbone.validations.js'
'js/jquery.cookie.js', 'js/jquery.cookie.js',
'js/bootstrap.js',
'js/jquery-ui-1.10.2.custom.js',
'js/jquery.flot.patched-multi.js',
'js/jquery.flot.tooltip.js',
'js/jquery.flot.axislabels.js',
], ],
'output_filename': 'js/util.js', 'output_filename': 'js/util.js',
} },
'new_dashboard' : {
'source_filenames': [
'js/new_dashboard/load_analytics.js'
],
'output_filename': 'js/new_dashboard.js',
},
} }
PIPELINE_CSS = { PIPELINE_CSS = {
'bootstrap': {
'source_filenames': [
'css/bootstrap.css',
'css/bootstrap-responsive.css',
'css/bootstrap-extensions.css',
],
'output_filename': 'css/bootstrap.css',
},
'util_css' : {
'source_filenames': [
'css/jquery-ui-1.10.2.custom.css',
],
'output_filename': 'css/util_css.css',
} }
}
PIPELINE_DISABLE_WRAPPER = True PIPELINE_DISABLE_WRAPPER = True
PIPELINE_YUI_BINARY = "yui-compressor" PIPELINE_YUI_BINARY = "yui-compressor"
...@@ -314,6 +318,8 @@ PIPELINE_JS_COMPRESSOR = None ...@@ -314,6 +318,8 @@ PIPELINE_JS_COMPRESSOR = None
PIPELINE_COMPILE_INPLACE = True PIPELINE_COMPILE_INPLACE = True
PIPELINE = True PIPELINE = True
CELERY_IMPORTS = ('modules.student_course_stats.tasks',)
override_settings = os.path.join(BASE_DIR, "override_settings.py") override_settings = os.path.join(BASE_DIR, "override_settings.py")
if os.path.isfile(override_settings): if os.path.isfile(override_settings):
execfile(override_settings) execfile(override_settings)
anserv/static_replace/__init__.py deleted 100644 → 0
import logging
import re
from staticfiles.storage import staticfiles_storage
from staticfiles import finders
from django.conf import settings
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.contentstore.content import StaticContent
log = logging.getLogger(__name__)
def _url_replace_regex(prefix):
"""
Match static urls in quotes that don't end in '?raw'.
To anyone contemplating making this more complicated:
http://xkcd.com/1171/
"""
return r"""
(?x) # flags=re.VERBOSE
(?P<quote>\\?['"]) # the opening quotes
(?P<prefix>{prefix}) # the prefix
(?P<rest>.*?) # everything else in the url
(?P=quote) # the first matching closing quote
""".format(prefix=prefix)
def try_staticfiles_lookup(path):
"""
Try to lookup a path in staticfiles_storage. If it fails, return
a dead link instead of raising an exception.
"""
try:
url = staticfiles_storage.url(path)
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
path, str(err)))
# Just return the original path; don't kill everything.
url = path
return url
def replace_course_urls(text, course_id):
"""
Replace /course/$stuff urls with /courses/$course_id/$stuff urls
text: The text to replace
course_module: A CourseDescriptor
returns: text with the links replaced
"""
def replace_course_url(match):
quote = match.group('quote')
rest = match.group('rest')
return "".join([quote, '/courses/' + course_id + '/', rest, quote])
return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
def replace_static_urls(text, data_directory, course_namespace=None):
"""
Replace /static/$stuff urls either with their correct url as generated by collectstatic,
(/static/$md5_hashed_stuff) or by the course-specific content static url
/static/$course_data_dir/$stuff, or, if course_namespace is not None, by the
correct url in the contentstore (c4x://)
text: The source text to do the substitution in
data_directory: The directory in which course data is stored
course_namespace: The course identifier used to distinguish static content for this course in studio
"""
def replace_static_url(match):
original = match.group(0)
prefix = match.group('prefix')
quote = match.group('quote')
rest = match.group('rest')
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware_copy specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url(rest, course_namespace)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
return re.sub(
_url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
replace_static_url,
text
)
anserv/static_replace/management/commands/clear_collectstatic_cache.py deleted 100644 → 0
###
### Script for importing courseware_copy from XML format
###
from django.core.management.base import NoArgsCommand
from django.core.cache import get_cache
class Command(NoArgsCommand):
help = \
'''Import the specified data directory into the default ModuleStore'''
def handle_noargs(self, **options):
staticfiles_cache = get_cache('staticfiles')
staticfiles_cache.clear()
anserv/static_replace/test/test_static_replace.py deleted 100644 → 0
import re
from nose.tools import assert_equals, assert_true, assert_false
from static_replace import (replace_static_urls, replace_course_urls,
_url_replace_regex)
from mock import patch, Mock
from xmodule.modulestore import Location
from xmodule.modulestore.mongo import MongoModuleStore
from xmodule.modulestore.xml import XMLModuleStore
DATA_DIRECTORY = 'data_dir'
COURSE_ID = 'org/course/run'
NAMESPACE = Location('org', 'course', 'run', None, None)
STATIC_SOURCE = '"/static/file.png"'
def test_multi_replace():
course_source = '"/course/file.png"'
assert_equals(
replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY),
replace_static_urls(replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY), DATA_DIRECTORY)
)
assert_equals(
replace_course_urls(course_source, COURSE_ID),
replace_course_urls(replace_course_urls(course_source, COURSE_ID), COURSE_ID)
)
@patch('static_replace.staticfiles_storage')
def test_storage_url_exists(mock_storage):
mock_storage.exists.return_value = True
mock_storage.url.return_value = '/static/file.png'
assert_equals('"/static/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
mock_storage.exists.called_once_with('file.png')
mock_storage.url.called_once_with('data_dir/file.png')
@patch('static_replace.staticfiles_storage')
def test_storage_url_not_exists(mock_storage):
mock_storage.exists.return_value = False
mock_storage.url.return_value = '/static/data_dir/file.png'
assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
mock_storage.exists.called_once_with('file.png')
mock_storage.url.called_once_with('file.png')
@patch('static_replace.StaticContent')
@patch('static_replace.modulestore')
def test_mongo_filestore(mock_modulestore, mock_static_content):
mock_modulestore.return_value = Mock(MongoModuleStore)
mock_static_content.convert_legacy_static_url.return_value = "c4x://mock_url"
# No namespace => no change to path
assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
# Namespace => content url
assert_equals(
'"' + mock_static_content.convert_legacy_static_url.return_value + '"',
replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY, NAMESPACE)
)
mock_static_content.convert_legacy_static_url.assert_called_once_with('file.png', NAMESPACE)
@patch('static_replace.settings')
@patch('static_replace.modulestore')
@patch('static_replace.staticfiles_storage')
def test_data_dir_fallback(mock_storage, mock_modulestore, mock_settings):
mock_modulestore.return_value = Mock(XMLModuleStore)
mock_storage.url.side_effect = Exception
mock_storage.exists.return_value = True
assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
mock_storage.exists.return_value = False
assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
def test_raw_static_check():
"""
Make sure replace_static_urls leaves alone things that end in '.raw'
"""
path = '"/static/foo.png?raw"'
assert_equals(path, replace_static_urls(path, DATA_DIRECTORY))
text = 'text <tag a="/static/js/capa/protex/protex.nocache.js?raw"/><div class="'
assert_equals(path, replace_static_urls(path, text))
def test_regex():
yes = ('"/static/foo.png"',
'"/static/foo.png"',
"'/static/foo.png'")
no = ('"/not-static/foo.png"',
'"/static/foo', # no matching quote
)
regex = _url_replace_regex('/static/')
for s in yes:
print 'Should match: {0!r}'.format(s)
assert_true(re.match(regex, s))
for s in no:
print 'Should not match: {0!r}'.format(s)
assert_false(re.match(regex, s))
anserv/templates/base.html
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
<head> <head>
<!--[if lt IE 9]>
<script type="text/javascript">
var html5 = { 'elements': 'abbr article aside audio bdi data
datalist details figcaption figure footer header hgroup mark meter nav
output progress section summary time video' };
</script>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></
script>
<![endif]-->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
{% load compressed %}
{% compressed_js 'util' %}
{% compressed_css 'bootstrap' %}
{% compressed_css 'util_css' %}
{% block additional_js %}
{% endblock %}
{% block title %}
<title>Analytics</title> <title>Analytics</title>
{% endblock %}
<meta name="csrf-token" content="{{csrf_token}}"> <meta name="csrf-token" content="{{csrf_token}}">
<div id="navbar"> </head>
<body>
<div id="navbar" class="navbar navbar-fixed-top">
<div class="navbar-inner">
<div class="container">
{% block nav %} {% block nav %}
<a href="/">Home</a> <a href="/" class="brand">edX Analytics</a>
<ul class="nav">
{% if user.is_authenticated %} {% if user.is_authenticated %}
<li class="">
<a href="/frontend/logout">Logout</a> <a href="/frontend/logout">Logout</a>
</li>
{% else %} {% else %}
<li class="">
<a href="/frontend/login">Login</a> <a href="/frontend/login">Login</a>
</li>
{% endif %} {% endif %}
</ul>
{% endblock %} {% endblock %}
</div> </div>
</head> </div>
<body> </div>
<header class="subhead">
<div class="container">
{% block header %}
{% endblock %}
</div>
</header>
<div class="container">
{% block content %} {% block content %}
{% endblock %} {% endblock %}
</div>
</body> </body>
</html> </html>
\ No newline at end of file
anserv/templates/grade_distribution/student_grade_distribution.html 0 → 100644
<div class="chart-wrapper">
<div class="chart-title">
{{ graph_title }}
</div>
<div class="chart">
<div id="{{graph_name}}" style="width:700px;height:400px"></div>
</div>
</div>
<script>
function render{{graph_name}}()
{
var graph_name = "{{graph_name}}";
var chart_data = {{chart_data}};
var ticks = {{tick_data}};
$.plot($("#" + graph_name), [
{
data: chart_data,
}],
{
grid: {
hoverable: true
},
xaxis: {
mode: null,
axisLabel: "{{x_label}}",
min: {{x_min}},
max: {{x_max}}
},
yaxis: {
mode: null,
axisLabel: "{{y_label}}"
},
series: {
lines: { show: true },
points: {
radius: 3,
show: true,
fill: true
}
},
tooltip: true,
tooltipOpts: {
content: "Score: %x.3 Count: %y",
shifts: {
x: -60,
y: 25
}
}
}
);
}
render{{graph_name}}();
</script>
\ No newline at end of file
anserv/templates/new_dashboard/new_dashboard.html 0 → 100644
{% extends "base.html" %}
{% block additional_js %}
{% load compressed %}
{% compressed_js 'new_dashboard' %}
{% endblock %}
{% block content %}
<script>
function scrape_courses()
{
$.getJSON('/query/global/available_courses', function(data){
var i;
var course_dropdown_tag = "#dropdown_select_course";
$(course_dropdown_tag).empty();
for(i = 0 ; i< data.length; i++){
$(course_dropdown_tag).append('<li><a class="course_select" href="#" course_name="'+ data[i] +'" >' + data[i] +'</a></li>');
}
setup_course_links();
});
}
function tidy_str(str)
{
return str.replace(/_/g," ").replace(/\w\S*/g, function(txt){return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();});
}
function load_analytics(url)
{
$.get(url, function(data) {
var split_data = data.split('\n');
var i;
var sidebar_tag = '#sidebar_ul';
$(sidebar_tag).empty();
for(i = 0 ; i< split_data.length; i++){
$(sidebar_tag).append('<li><a class="analytic_select" href="#" analytic_name="'+ split_data[i] +'" >' + tidy_str(split_data[i]) +'</a></li>');
}
query_analytic();
});
}
function query_analytic()
{
$(".analytic_select").click(function(event){
target = $(event.target)
course_name = $('a.dropdown-toggle').attr('current_course')
analytic_name = target.attr('analytic_name')
var content_tag = '#rcontent';
$(content_tag).empty();
query_url = '/view/course/' + analytic_name + '?course=' + course_name
$.get(query_url, function(data){
$(content_tag).append(data)
});
});
}
function setup_course_links()
{
$(".course_select").click(function(event){
target = $(event.target)
change_name = $('a.dropdown-toggle')
change_name.text(target.attr('course_name'))
change_name.attr('current_course', target.attr('course_name'))
});
}
$(function() {
$("#course_tab").click(function(){
scrape_courses();
load_analytics('/probe/view/course');
$('#course_name_box')[0].style.display = "none";
$('#course_name_dropdown')[0].style.display = "block";
});
$('#course_tab').click();
});
</script>
<div class="container-fluid">
<ul class="nav nav-tabs">
<li class="active"><a href="#" id="course_tab">Course</a></li>
<li class="course_select_link dropdown" id="course_name_dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Select Course</a>
<ul class="dropdown-menu" role="menu" aria-labelledby="dLabel" id="dropdown_select_course">
</ul>
</li>
<li class="param_textbox" id="course_name_box"><form class="navbar-search pull-left">
<input type="text" id="coursename_text" class="search-query" placeholder="Course name">
</form></li>
</ul>
<div class="row-fluid">
<div class="span3">
<div class="well sidebar-nav">
<ul class="nav nav-list" id="sidebar_ul">
</ul>
</div><!--/.well -->
</div><!--/span-->
<div class="span9" id="rcontent">
</div><!--/span-->
</div><!--/row-->
</div>
{% endblock %}
\ No newline at end of file
anserv/templates/registration/login.html
{% extends "base.html" %} {% extends "base.html" %}
{% block header %}
<h1>Login to Analytics</h1>
{% endblock %}
{% block content %} {% block content %}
{%if form.errors %} {%if form.errors %}
<p> Invalid username/password combination, please try again </p> <p> Invalid username/password combination, please try again </p>
{% endif %} {% endif %}
<div class="row">
<h1>Login to Analytics</h1> <div class="span12">
<form action="{% url django.contrib.auth.views.login %}" method="post"> <form action="{% url django.contrib.auth.views.login %}" method="post" class="form-horizontal">
{% csrf_token %} {% csrf_token %}
{{form.username.label_tag}}{{form.username}} {{form.username.label_tag}}{{form.username}}
{{form.password.label_tag}}{{form.password}} {{form.password.label_tag}}{{form.password}}
<input type="submit" id="submit" name="submit" value="Sign in" /> <input type="submit" id="submit" name="submit" value="Sign in" />
<input type="hidden" name="next" value="{{ next }}" /> <input type="hidden" name="next" value="{{ next }}" />
</form> </form>
</div>
</div>
{% endblock %} {% endblock %}
\ No newline at end of file
anserv/urls.py
from django.conf.urls.defaults import patterns, include, url from django.conf.urls.defaults import patterns, include, url
from django.conf import settings from django.conf import settings
from django.contrib.auth.decorators import login_required
# Uncomment the next two lines to enable the admin: # Uncomment the next two lines to enable the admin:
# from django.contrib import admin # from django.contrib import admin
...@@ -15,12 +16,14 @@ urlpatterns = patterns('', ...@@ -15,12 +16,14 @@ urlpatterns = patterns('',
url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_query'), url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_query'),
url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_0-9+]+)$', 'an_evt.views.handle_query'), url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_0-9+]+)$', 'an_evt.views.handle_query'),
url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_0-9+]+)/([A-Za-z_0-9+]+)$', 'an_evt.views.handle_query'), url(r'^query/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_0-9+]+)/([A-Za-z_0-9+]+)$', 'an_evt.views.handle_query'),
url(r'^schema$', 'an_evt.views.list_all_endpoints'),
url(r'^probe$', 'an_evt.views.handle_probe'), url(r'^probe$', 'an_evt.views.handle_probe'),
url(r'^probe/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'), url(r'^probe/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'),
url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'), url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'),
url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'), url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'),
url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'), url(r'^probe/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)/([A-Za-z_+]+)$', 'an_evt.views.handle_probe'),
url(r'^dashboard$', 'dashboard.views.dashboard'), url(r'^dashboard$', 'dashboard.views.dashboard'),
url(r'^new_dashboard$', 'dashboard.views.new_dashboard'),
url(r'^sns', 'sns.views.sns'), url(r'^sns', 'sns.views.sns'),
# url(r'^anserv/', include('anserv.foo.urls')), # url(r'^anserv/', include('anserv.foo.urls')),
......
anserv/urls.py~ deleted 100644 → 0
from django.conf.urls.defaults import patterns, include, url
# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()
urlpatterns = patterns('',
# Examples:
# url(r'^$', 'anserv.views.home', name='home'),
# url(r'^anserv/', include('anserv.foo.urls')),
# Uncomment the admin/doc line below to enable admin documentation:
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
# Uncomment the next line to enable the admin:
# url(r'^admin/', include(admin.site.urls)),
)
apt-packages.txt
python-pip
python-matplotlib python-matplotlib
python-scipy python-scipy
emacs emacs
......
css_js_src/css/bootstrap-extensions.css 0 → 100644
.center.navbar .nav,
.center.navbar .nav > li {
float:none;
display:inline-block;
*display:inline; /* ie7 fix */
*zoom:1; /* hasLayout ie7 trigger */
vertical-align: top;
}
.center .navbar-inner {
text-align:center;
}
.center .dropdown-menu {
text-align: left;
}
body { padding-top: 40px; }
@media screen and (max-width: 768px) {
body { padding-top: 0px; }
}
\ No newline at end of file
css_js_src/css/bootstrap.css 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
css_js_src/css/bootstrap.min.css 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
css_js_src/js/jquery-ui-1.10.2.custom.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
css_js_src/js/jquery.flot.axislabels.js 0 → 100644
/*
CAxis Labels Plugin for flot. :P
Copyright (c) 2010 Xuan Luo
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
(function ($) {
var options = { };
function init(plot) {
// This is kind of a hack. There are no hooks in Flot between
// the creation and measuring of the ticks (setTicks, measureTickLabels
// in setupGrid() ) and the drawing of the ticks and plot box
// (insertAxisLabels in setupGrid() ).
//
// Therefore, we use a trick where we run the draw routine twice:
// the first time to get the tick measurements, so that we can change
// them, and then have it draw it again.
var secondPass = false;
plot.hooks.draw.push(function (plot, ctx) {
if (!secondPass) {
// MEASURE AND SET OPTIONS
$.each(plot.getAxes(), function(axisName, axis) {
var opts = axis.options // Flot 0.7
|| plot.getOptions()[axisName]; // Flot 0.6
if (!opts || !opts.axisLabel)
return;
var w, h;
if (opts.axisLabelUseCanvas != false)
opts.axisLabelUseCanvas = true;
if (opts.axisLabelUseCanvas) {
// canvas text
if (!opts.axisLabelFontSizePixels)
opts.axisLabelFontSizePixels = 14;
if (!opts.axisLabelFontFamily)
opts.axisLabelFontFamily = 'sans-serif';
// since we currently always display x as horiz.
// and y as vertical, we only care about the height
w = opts.axisLabelFontSizePixels;
h = opts.axisLabelFontSizePixels;
} else {
// HTML text
var elem = $('<div class="axisLabels" style="position:absolute;">' + opts.axisLabel + '</div>');
plot.getPlaceholder().append(elem);
w = elem.outerWidth(true);
h = elem.outerHeight(true);
elem.remove();
}
if (axisName.charAt(0) == 'x')
axis.labelHeight += h;
else
axis.labelWidth += w;
opts.labelHeight = axis.labelHeight;
opts.labelWidth = axis.labelWidth;
});
// re-draw with new label widths and heights
secondPass = true;
plot.setupGrid();
plot.draw();
} else {
// DRAW
$.each(plot.getAxes(), function(axisName, axis) {
var opts = axis.options // Flot 0.7
|| plot.getOptions()[axisName]; // Flot 0.6
if (!opts || !opts.axisLabel)
return;
if (opts.axisLabelUseCanvas) {
// canvas text
var ctx = plot.getCanvas().getContext('2d');
ctx.save();
ctx.font = opts.axisLabelFontSizePixels + 'px ' +
opts.axisLabelFontFamily;
var width = ctx.measureText(opts.axisLabel).width;
var height = opts.axisLabelFontSizePixels;
var x, y;
if (axisName.charAt(0) == 'x') {
x = plot.getPlotOffset().left + plot.width()/2 - width/2;
y = plot.getCanvas().height;
} else {
x = height * 0.72;
y = plot.getPlotOffset().top + plot.height()/2 - width/2;
}
ctx.translate(x, y);
ctx.rotate((axisName.charAt(0) == 'x') ? 0 : -Math.PI/2);
ctx.fillText(opts.axisLabel, 0, 0);
ctx.restore();
} else {
// HTML text
plot.getPlaceholder().find('#' + axisName + 'Label').remove();
var elem = $('<div id="' + axisName + 'Label" " class="axisLabels" style="position:absolute;">' + opts.axisLabel + '</div>');
if (axisName.charAt(0) == 'x') {
elem.css('left', plot.getPlotOffset().left + plot.width()/2 - elem.outerWidth()/2 + 'px');
elem.css('bottom', '0px');
} else {
elem.css('top', plot.getPlotOffset().top + plot.height()/2 - elem.outerHeight()/2 + 'px');
elem.css('left', '0px');
}
plot.getPlaceholder().append(elem);
}
});
secondPass = false;
}
});
}
$.plot.plugins.push({
init: init,
options: options,
name: 'axisLabels',
version: '1.0'
});
})(jQuery);
css_js_src/js/jquery.flot.tooltip.js 0 → 100644
/*
* jquery.flot.tooltip
*
* description: easy-to-use tooltips for Flot charts
* version: 0.6.1
* author: Krzysztof Urbas @krzysu [myviews.pl]
* website: https://github.com/krzysu/flot.tooltip
*
* build on 2013-03-24
* released under MIT License, 2012
*/
(function ($) {
// plugin options, default values
var defaultOptions = {
tooltip: false,
tooltipOpts: {
content: "%s | X: %x | Y: %y",
// allowed templates are:
// %s -> series label,
// %x -> X value,
// %y -> Y value,
// %x.2 -> precision of X value,
// %p -> percent
xDateFormat: null,
yDateFormat: null,
shifts: {
x: 10,
y: 20
},
defaultTheme: true,
// callbacks
onHover: function(flotItem, $tooltipEl) {}
}
};
// object
var FlotTooltip = function(plot) {
// variables
this.tipPosition = {x: 0, y: 0};
this.init(plot);
};
// main plugin function
FlotTooltip.prototype.init = function(plot) {
var that = this;
plot.hooks.bindEvents.push(function (plot, eventHolder) {
// get plot options
that.plotOptions = plot.getOptions();
// if not enabled return
if (that.plotOptions.tooltip === false || typeof that.plotOptions.tooltip === 'undefined') return;
// shortcut to access tooltip options
that.tooltipOptions = that.plotOptions.tooltipOpts;
// create tooltip DOM element
var $tip = that.getDomElement();
// bind event
$( plot.getPlaceholder() ).bind("plothover", function (event, pos, item) {
if (item) {
var tipText;
// convert tooltip content template to real tipText
tipText = that.stringFormat(that.tooltipOptions.content, item);
$tip.html( tipText )
.css({
left: that.tipPosition.x + that.tooltipOptions.shifts.x,
top: that.tipPosition.y + that.tooltipOptions.shifts.y
})
.show();
// run callback
if(typeof that.tooltipOptions.onHover === 'function') {
that.tooltipOptions.onHover(item, $tip);
}
}
else {
$tip.hide().html('');
}
});
eventHolder.mousemove( function(e) {
var pos = {};
pos.x = e.pageX;
pos.y = e.pageY;
that.updateTooltipPosition(pos);
});
});
};
/**
* get or create tooltip DOM element
* @return jQuery object
*/
FlotTooltip.prototype.getDomElement = function() {
var $tip;
if( $('#flotTip').length > 0 ){
$tip = $('#flotTip');
}
else {
$tip = $('<div />').attr('id', 'flotTip');
$tip.appendTo('body').hide().css({position: 'absolute'});
if(this.tooltipOptions.defaultTheme) {
$tip.css({
'background': '#fff',
'z-index': '100',
'padding': '0.4em 0.6em',
'border-radius': '0.5em',
'font-size': '0.8em',
'border': '1px solid #111'
});
}
}
return $tip;
};
// as the name says
FlotTooltip.prototype.updateTooltipPosition = function(pos) {
this.tipPosition.x = pos.x;
this.tipPosition.y = pos.y;
};
/**
* core function, create tooltip content
* @param {string} content - template with tooltip content
* @param {object} item - Flot item
* @return {string} real tooltip content for current item
*/
FlotTooltip.prototype.stringFormat = function(content, item) {
var percentPattern = /%p\.{0,1}(\d{0,})/;
var seriesPattern = /%s/;
var xPattern = /%x\.{0,1}(\d{0,})/;
var yPattern = /%y\.{0,1}(\d{0,})/;
// if it is a function callback get the content string
if( typeof(content) === 'function' ) {
content = content(item.series.data[item.dataIndex][0], item.series.data[item.dataIndex][1]);
}
// percent match for pie charts
if( typeof (item.series.percent) !== 'undefined' ) {
content = this.adjustValPrecision(percentPattern, content, item.series.percent);
}
// series match
if( typeof(item.series.label) !== 'undefined' ) {
content = content.replace(seriesPattern, item.series.label);
}
// time mode axes with custom dateFormat
if(this.isTimeMode('xaxis', item) && this.isXDateFormat(item)) {
content = content.replace(xPattern, this.timestampToDate(item.series.data[item.dataIndex][0], this.tooltipOptions.xDateFormat));
}
if(this.isTimeMode('yaxis', item) && this.isYDateFormat(item)) {
content = content.replace(yPattern, this.timestampToDate(item.series.data[item.dataIndex][1], this.tooltipOptions.yDateFormat));
}
// set precision if defined
if( typeof item.series.data[item.dataIndex][0] === 'number' ) {
content = this.adjustValPrecision(xPattern, content, item.series.data[item.dataIndex][0]);
}
if( typeof item.series.data[item.dataIndex][1] === 'number' ) {
content = this.adjustValPrecision(yPattern, content, item.series.data[item.dataIndex][1]);
}
// if no value customization, use tickFormatter by default
if(typeof item.series.xaxis.tickFormatter !== 'undefined') {
content = content.replace(xPattern, item.series.xaxis.tickFormatter(item.series.data[item.dataIndex][0], item.series.xaxis));
}
if(typeof item.series.yaxis.tickFormatter !== 'undefined') {
content = content.replace(yPattern, item.series.yaxis.tickFormatter(item.series.data[item.dataIndex][1], item.series.yaxis));
}
return content;
};
// helpers just for readability
FlotTooltip.prototype.isTimeMode = function(axisName, item) {
return false;
};
FlotTooltip.prototype.isXDateFormat = function(item) {
return (typeof this.tooltipOptions.xDateFormat !== 'undefined' && this.tooltipOptions.xDateFormat !== null);
};
FlotTooltip.prototype.isYDateFormat = function(item) {
return (typeof this.tooltipOptions.yDateFormat !== 'undefined' && this.tooltipOptions.yDateFormat !== null);
};
//
FlotTooltip.prototype.timestampToDate = function(tmst, dateFormat) {
var theDate = new Date(tmst);
return $.plot.formatDate(theDate, dateFormat);
};
//
FlotTooltip.prototype.adjustValPrecision = function(pattern, content, value) {
var precision;
if( content.match(pattern) !== null ) {
if(RegExp.$1 !== '') {
precision = RegExp.$1;
value = value.toFixed(precision);
// only replace content if precision exists
content = content.replace(pattern, value);
}
}
return content;
};
//
var init = function(plot) {
new FlotTooltip(plot);
};
// define Flot plugin
$.plot.plugins.push({
init: init,
options: defaultOptions,
name: 'tooltip',
version: '0.6.1'
});
})(jQuery);
install.txt 0 → 100644
git clone git@github.com:MITx/analytics-experiments.git
cd analytics-experiments
sudo xargs -a apt-packages.txt apt-get install
sudo aptitude remove python-virtualenv python-pip
sudo easy_install pip virtualenv
git checkout vik/add-htsql
pip install virtualenv
sudo mkdir /opt/edx
sudo chown CURRENT_USER /opt/edx
virtualenv /opt/edx
source /opt/edx/bin/activate
pip install -r requirements.txt
mkdir /opt/wwc/db
python manage.py syncdb
python manage.py syncdb --database=local
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment