Skip to content

E
edx-analytics-data-api
Commit fbaef361 by Clinton Blackburn
Options

Removed Course Model 

The pipeline does not yet support creating a courses table.
parent cd98dd07
Showing with 77 additions and 141 deletions
Makefile
......@@ -50,7 +50,7 @@ syncdb:
$(foreach db_name,$(DATABASES),./manage.py syncdb --migrate --noinput --database=$(db_name);)
loaddata: syncdb
python manage.py loaddata courses education_levels single_course_activity problem_response_answer_distribution --database=analytics
python manage.py loaddata education_levels single_course_activity problem_response_answer_distribution --database=analytics
python manage.py generate_fake_enrollment_data
demo: clean requirements loaddata
......
analytics_data_api/fixtures/courses.json deleted 100644 → 0
[
{
"pk": 1,
"model": "v0.course",
"fields": {
"course_id": "edX/DemoX/Demo_Course"
}
}
]
analytics_data_api/fixtures/single_course_activity.json
......@@ -3,7 +3,7 @@
"pk": 40,
"model": "v0.CourseActivityByWeek",
"fields": {
"course": ["edX/DemoX/Demo_Course"],
"course_id": "edX/DemoX/Demo_Course",
"interval_start": "2014-05-24T00:00:00Z",
"activity_type": "posted_forum",
"count": 100,
......@@ -14,7 +14,7 @@
"pk": 106,
"model": "v0.CourseActivityByWeek",
"fields": {
"course": ["edX/DemoX/Demo_Course"],
"course_id": "edX/DemoX/Demo_Course",
"interval_start": "2014-05-24T00:00:00Z",
"activity_type": "attempted_problem",
"count": 200,
......@@ -25,7 +25,7 @@
"pk": 201,
"model": "v0.CourseActivityByWeek",
"fields": {
"course": ["edX/DemoX/Demo_Course"],
"course_id": "edX/DemoX/Demo_Course",
"interval_start": "2014-05-24T00:00:00Z",
"activity_type": "any",
"count": 300,
......@@ -36,7 +36,7 @@
"pk": 725,
"model": "v0.CourseActivityByWeek",
"fields": {
"course": ["edX/DemoX/Demo_Course"],
"course_id": "edX/DemoX/Demo_Course",
"interval_start": "2014-05-24T00:00:00Z",
"activity_type": "played_video",
"count": 400,
......
analytics_data_api/management/commands/generate_fake_enrollment_data.py
# pylint: disable=line-too-long
import datetime
import random
......@@ -23,7 +25,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
days = 120
course = models.Course.objects.first()
course_id = 'edX/DemoX/Demo_Course'
start_date = datetime.date(year=2014, month=1, day=1)
genders = {
......@@ -65,29 +67,29 @@ class Command(BaseCommand):
models.CourseEnrollmentByCountry]:
model.objects.all().delete()
# Create new data data
# Create new data
daily_total = 1500
for i in range(days):
daily_total = get_count(daily_total)
date = start_date + datetime.timedelta(days=i)
models.CourseEnrollmentDaily.objects.create(course=course, date=date, count=daily_total)
models.CourseEnrollmentDaily.objects.create(course_id=course_id, date=date, count=daily_total)
for gender, ratio in genders.iteritems():
count = int(ratio * daily_total)
models.CourseEnrollmentByGender.objects.create(course=course, date=date, count=count, gender=gender)
models.CourseEnrollmentByGender.objects.create(course_id=course_id, date=date, count=count, gender=gender)
for short_name, ratio in education_levels.iteritems():
education_level = models.EducationLevel.objects.get(short_name=short_name)
count = int(ratio * daily_total)
models.CourseEnrollmentByEducation.objects.create(course=course, date=date, count=count,
models.CourseEnrollmentByEducation.objects.create(course_id=course_id, date=date, count=count,
education_level=education_level)
for country_code, ratio in countries.iteritems():
count = int(ratio * daily_total)
models.CourseEnrollmentByCountry.objects.create(course=course, date=date, count=count,
models.CourseEnrollmentByCountry.objects.create(course_id=course_id, date=date, count=count,
country_code=country_code)
for birth_year, ratio in birth_years.iteritems():
count = int(ratio * daily_total)
models.CourseEnrollmentByBirthYear.objects.create(course=course, date=date, count=count,
models.CourseEnrollmentByBirthYear.objects.create(course_id=course_id, date=date, count=count,
birth_year=birth_year)
analytics_data_api/v0/managers.py deleted 100644 → 0
from django.db import models
class CourseManager(models.Manager):
def get_by_natural_key(self, course_id):
return self.get(course_id=course_id)
analytics_data_api/v0/models.py
from collections import namedtuple
from django.db import models
from iso3166 import countries
from analytics_data_api.v0.managers import CourseManager
class Course(models.Model):
course_id = models.CharField(unique=True, max_length=255)
objects = CourseManager() # pylint: disable=no-value-for-parameter
class Meta(object):
db_table = 'courses'
class CourseActivityByWeek(models.Model):
......@@ -19,7 +9,7 @@ class CourseActivityByWeek(models.Model):
class Meta(object):
db_table = 'course_activity'
course = models.ForeignKey(Course, null=False)
course_id = models.CharField(max_length=255)
interval_start = models.DateTimeField()
interval_end = models.DateTimeField()
activity_type = models.CharField(db_index=True, max_length=255)
......@@ -28,11 +18,11 @@ class CourseActivityByWeek(models.Model):
@classmethod
def get_most_recent(cls, course_id, activity_type):
"""Activity for the week that was mostly recently computed."""
return cls.objects.filter(course__course_id=course_id, activity_type=activity_type).latest('interval_end')
return cls.objects.filter(course_id=course_id, activity_type=activity_type).latest('interval_end')
class BaseCourseEnrollment(models.Model):
course = models.ForeignKey(Course, null=False)
course_id = models.CharField(max_length=255)
date = models.DateField(null=False, db_index=True)
count = models.IntegerField(null=False)
created = models.DateTimeField(auto_now_add=True)
......@@ -40,14 +30,14 @@ class BaseCourseEnrollment(models.Model):
class Meta(object):
abstract = True
get_latest_by = 'date'
index_together = [('course', 'date',)]
index_together = [('course_id', 'date',)]
class CourseEnrollmentDaily(BaseCourseEnrollment):
class Meta(BaseCourseEnrollment.Meta):
db_table = 'course_enrollment_daily'
ordering = ('date', 'course')
unique_together = [('course', 'date',)]
ordering = ('date', 'course_id')
unique_together = [('course_id', 'date',)]
class CourseEnrollmentByBirthYear(BaseCourseEnrollment):
......@@ -55,8 +45,8 @@ class CourseEnrollmentByBirthYear(BaseCourseEnrollment):
class Meta(BaseCourseEnrollment.Meta):
db_table = 'course_enrollment_birth_year_daily'
ordering = ('date', 'course', 'birth_year')
unique_together = [('course', 'date', 'birth_year')]
ordering = ('date', 'course_id', 'birth_year')
unique_together = [('course_id', 'date', 'birth_year')]
class EducationLevel(models.Model):
......@@ -75,8 +65,8 @@ class CourseEnrollmentByEducation(BaseCourseEnrollment):
class Meta(BaseCourseEnrollment.Meta):
db_table = 'course_enrollment_education_level_daily'
ordering = ('date', 'course', 'education_level')
unique_together = [('course', 'date', 'education_level')]
ordering = ('date', 'course_id', 'education_level')
unique_together = [('course_id', 'date', 'education_level')]
class CourseEnrollmentByGender(BaseCourseEnrollment):
......@@ -84,8 +74,8 @@ class CourseEnrollmentByGender(BaseCourseEnrollment):
class Meta(BaseCourseEnrollment.Meta):
db_table = 'course_enrollment_gender_daily'
ordering = ('date', 'course', 'gender')
unique_together = [('course', 'date', 'gender')]
ordering = ('date', 'course_id', 'gender')
unique_together = [('course_id', 'date', 'gender')]
class ProblemResponseAnswerDistribution(models.Model):
......@@ -119,5 +109,5 @@ class CourseEnrollmentByCountry(BaseCourseEnrollment):
class Meta(BaseCourseEnrollment.Meta):
db_table = 'course_enrollment_location_current'
ordering = ('date', 'course', 'country_code')
unique_together = [('course', 'date', 'country_code')]
ordering = ('date', 'course_id', 'country_code')
unique_together = [('course_id', 'date', 'country_code')]
analytics_data_api/v0/serializers.py
......@@ -3,16 +3,7 @@ from rest_framework import serializers
from analytics_data_api.v0 import models
class CourseIdMixin(object):
def get_course_id(self, obj):
return obj.course.course_id
class RequiredSerializerMethodField(serializers.SerializerMethodField):
required = True
class CourseActivityByWeekSerializer(serializers.ModelSerializer, CourseIdMixin):
class CourseActivityByWeekSerializer(serializers.ModelSerializer):
"""
Representation of CourseActivityByWeek that excludes the id field.
......@@ -20,8 +11,6 @@ class CourseActivityByWeekSerializer(serializers.ModelSerializer, CourseIdMixin)
particular record is likely to change unexpectedly so we avoid exposing it.
"""
course_id = RequiredSerializerMethodField('get_course_id')
class Meta(object):
model = models.CourseActivityByWeek
fields = ('interval_start', 'interval_end', 'activity_type', 'count', 'course_id')
......@@ -51,8 +40,7 @@ class ProblemResponseAnswerDistributionSerializer(serializers.ModelSerializer):
)
class BaseCourseEnrollmentModelSerializer(serializers.ModelSerializer, CourseIdMixin):
course_id = RequiredSerializerMethodField('get_course_id')
class BaseCourseEnrollmentModelSerializer(serializers.ModelSerializer):
date = serializers.DateField(format=settings.DATE_FORMAT)
......
analytics_data_api/v0/tests/test_managers.py deleted 100644 → 0
from django.core.exceptions import ObjectDoesNotExist
from django.test import TestCase
from django_dynamic_fixture import G
from analytics_data_api.v0.models import Course
class CourseManagerTests(TestCase):
def test_get_by_natural_key(self):
course_id = 'edX/DemoX/Demo_Course'
self.assertRaises(ObjectDoesNotExist, Course.objects.get_by_natural_key, course_id)
course = G(Course, course_id=course_id)
self.assertEqual(course, Course.objects.get_by_natural_key(course_id))
analytics_data_api/v0/tests/test_views.py
......@@ -4,7 +4,6 @@
import StringIO
import csv
import datetime
import random
from django.conf import settings
from django_dynamic_fixture import G
......@@ -18,19 +17,19 @@ from analyticsdataserver.tests import TestCaseWithAuthentication
class CourseActivityLastWeekTest(TestCaseWithAuthentication):
# pylint: disable=line-too-long
def setUp(self):
super(CourseActivityLastWeekTest, self).setUp()
self.course_id = 'edX/DemoX/Demo_Course'
self.course = G(models.Course, course_id=self.course_id)
interval_start = '2014-05-24T00:00:00Z'
interval_end = '2014-06-01T00:00:00Z'
G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
G(models.CourseActivityByWeek, course_id=self.course_id, interval_start=interval_start, interval_end=interval_end,
activity_type='posted_forum', count=100)
G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
G(models.CourseActivityByWeek, course_id=self.course_id, interval_start=interval_start, interval_end=interval_end,
activity_type='attempted_problem', count=200)
G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
G(models.CourseActivityByWeek, course_id=self.course_id, interval_start=interval_start, interval_end=interval_end,
activity_type='any', count=300)
G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
G(models.CourseActivityByWeek, course_id=self.course_id, interval_start=interval_start, interval_end=interval_end,
activity_type='played_video', count=400)
def test_activity(self):
......@@ -87,27 +86,18 @@ class CourseEnrollmentViewTestCase(object):
path = None
order_by = []
def _get_non_existent_course_id(self):
course_id = random.randint(100, 9999)
if not models.Course.objects.filter(course_id=course_id).exists():
return course_id
return self._get_non_existent_course_id()
def get_expected_response(self, *args):
raise NotImplementedError
def test_get_not_found(self):
""" Requests made against non-existent courses should return a 404 """
course_id = self._get_non_existent_course_id()
self.assertFalse(self.model.objects.filter(course__course_id=course_id).exists())
course_id = 'edX/DemoX/Non_Existent_Course'
response = self.authenticated_get('/api/v0/courses/%s%s' % (course_id, self.path))
self.assertEquals(response.status_code, 404)
def test_get(self):
# Validate the basic response status
response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course.course_id, self.path,))
response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course_id, self.path,))
self.assertEquals(response.status_code, 200)
# Validate the data is correct and sorted chronologically
......@@ -115,7 +105,7 @@ class CourseEnrollmentViewTestCase(object):
self.assertEquals(response.data, expected)
def test_get_csv(self):
path = '/api/v0/courses/%s%s' % (self.course.course_id, self.path,)
path = '/api/v0/courses/%s%s' % (self.course_id, self.path,)
csv_content_type = 'text/csv'
response = self.authenticated_get(path, HTTP_ACCEPT=csv_content_type)
......@@ -143,17 +133,15 @@ class CourseEnrollmentViewTestCase(object):
self.assertIntervalFilteringWorks(expected, self.date, self.date + datetime.timedelta(days=1))
def assertIntervalFilteringWorks(self, expected_response, start_date, end_date):
course = self.course
# If start date is after date of existing data, no data should be returned
date = (start_date + datetime.timedelta(days=30)).strftime(settings.DATE_FORMAT)
response = self.authenticated_get('/api/v0/courses/%s%s?start_date=%s' % (course.course_id, self.path, date))
response = self.authenticated_get('/api/v0/courses/%s%s?start_date=%s' % (self.course_id, self.path, date))
self.assertEquals(response.status_code, 200)
self.assertListEqual([], response.data)
# If end date is before date of existing data, no data should be returned
date = (start_date - datetime.timedelta(days=30)).strftime(settings.DATE_FORMAT)
response = self.authenticated_get('/api/v0/courses/%s%s?end_date=%s' % (course.course_id, self.path, date))
response = self.authenticated_get('/api/v0/courses/%s%s?end_date=%s' % (self.course_id, self.path, date))
self.assertEquals(response.status_code, 200)
self.assertListEqual([], response.data)
......@@ -161,7 +149,7 @@ class CourseEnrollmentViewTestCase(object):
start_date = start_date.strftime(settings.DATE_FORMAT)
end_date = end_date.strftime(settings.DATE_FORMAT)
response = self.authenticated_get(
'/api/v0/courses/%s%s?start_date=%s&end_date=%s' % (course.course_id, self.path, start_date, end_date))
'/api/v0/courses/%s%s?start_date=%s&end_date=%s' % (self.course_id, self.path, start_date, end_date))
self.assertEquals(response.status_code, 200)
self.assertListEqual(response.data, expected_response)
......@@ -173,20 +161,20 @@ class CourseEnrollmentByBirthYearViewTests(TestCaseWithAuthentication, CourseEnr
@classmethod
def setUpClass(cls):
cls.course = G(models.Course)
cls.course_id = 'edX/DemoX/Demo_Course'
cls.date = datetime.date(2014, 1, 1)
G(cls.model, course=cls.course, date=cls.date, birth_year=1956)
G(cls.model, course=cls.course, date=cls.date, birth_year=1986)
G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=10), birth_year=1956)
G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=10), birth_year=1986)
G(cls.model, course_id=cls.course_id, date=cls.date, birth_year=1956)
G(cls.model, course_id=cls.course_id, date=cls.date, birth_year=1986)
G(cls.model, course_id=cls.course_id, date=cls.date - datetime.timedelta(days=10), birth_year=1956)
G(cls.model, course_id=cls.course_id, date=cls.date - datetime.timedelta(days=10), birth_year=1986)
def get_expected_response(self, *args):
return [
{'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
{'course_id': str(ce.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
'birth_year': ce.birth_year} for ce in args]
def test_get(self):
response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course.course_id, self.path,))
response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course_id, self.path,))
self.assertEquals(response.status_code, 200)
expected = self.get_expected_response(*self.model.objects.filter(date=self.date))
......@@ -206,16 +194,16 @@ class CourseEnrollmentByEducationViewTests(TestCaseWithAuthentication, CourseEnr
def setUpClass(cls):
cls.el1 = G(models.EducationLevel, name='Doctorate', short_name='doctorate')
cls.el2 = G(models.EducationLevel, name='Top Secret', short_name='top_secret')
cls.course = G(models.Course)
cls.course_id = 'edX/DemoX/Demo_Course'
cls.date = datetime.date(2014, 1, 1)
G(cls.model, course=cls.course, date=cls.date, education_level=cls.el1)
G(cls.model, course=cls.course, date=cls.date, education_level=cls.el2)
G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=2),
G(cls.model, course_id=cls.course_id, date=cls.date, education_level=cls.el1)
G(cls.model, course_id=cls.course_id, date=cls.date, education_level=cls.el2)
G(cls.model, course_id=cls.course_id, date=cls.date - datetime.timedelta(days=2),
education_level=cls.el2)
def get_expected_response(self, *args):
return [
{'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
{'course_id': str(ce.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
'education_level': {'name': ce.education_level.name, 'short_name': ce.education_level.short_name}} for
ce in args]
......@@ -227,15 +215,15 @@ class CourseEnrollmentByGenderViewTests(TestCaseWithAuthentication, CourseEnroll
@classmethod
def setUpClass(cls):
cls.course = G(models.Course)
cls.course_id = 'edX/DemoX/Demo_Course'
cls.date = datetime.date(2014, 1, 1)
G(cls.model, course=cls.course, gender='m', date=cls.date, count=34)
G(cls.model, course=cls.course, gender='f', date=cls.date, count=45)
G(cls.model, course=cls.course, gender='f', date=cls.date - datetime.timedelta(days=2), count=45)
G(cls.model, course_id=cls.course_id, gender='m', date=cls.date, count=34)
G(cls.model, course_id=cls.course_id, gender='f', date=cls.date, count=45)
G(cls.model, course_id=cls.course_id, gender='f', date=cls.date - datetime.timedelta(days=2), count=45)
def get_expected_response(self, *args):
return [
{'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
{'course_id': str(ce.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
'gender': ce.gender} for ce in args]
......@@ -276,14 +264,14 @@ class CourseEnrollmentViewTests(TestCaseWithAuthentication, CourseEnrollmentView
@classmethod
def setUpClass(cls):
cls.course = G(models.Course)
cls.course_id = 'edX/DemoX/Demo_Course'
cls.date = datetime.date(2014, 1, 1)
G(cls.model, course=cls.course, date=cls.date, count=203)
G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=5), count=203)
G(cls.model, course_id=cls.course_id, date=cls.date, count=203)
G(cls.model, course_id=cls.course_id, date=cls.date - datetime.timedelta(days=5), count=203)
def get_expected_response(self, *args):
return [
{'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT)}
{'course_id': str(ce.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT)}
for ce in args]
......@@ -292,16 +280,16 @@ class CourseEnrollmentByLocationViewTests(TestCaseWithAuthentication, CourseEnro
model = models.CourseEnrollmentByCountry
def get_expected_response(self, *args):
args = sorted(args, key=lambda item: (item.date, item.course.course_id, item.country.code))
args = sorted(args, key=lambda item: (item.date, item.course_id, item.country.code))
return [
{'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
{'course_id': str(ce.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
'country': {'code': ce.country.code, 'name': ce.country.name}} for ce in args]
@classmethod
def setUpClass(cls):
cls.course = G(models.Course)
cls.course_id = 'edX/DemoX/Demo_Course'
cls.date = datetime.date(2014, 1, 1)
cls.country = countries.get('US')
G(cls.model, course=cls.course, country_code='US', count=455, date=cls.date)
G(cls.model, course=cls.course, country_code='CA', count=356, date=cls.date)
G(cls.model, course=cls.course, country_code='IN', count=12, date=cls.date - datetime.timedelta(days=29))
G(cls.model, course_id=cls.course_id, country_code='US', count=455, date=cls.date)
G(cls.model, course_id=cls.course_id, country_code='CA', count=356, date=cls.date)
G(cls.model, course_id=cls.course_id, country_code='IN', count=12, date=cls.date - datetime.timedelta(days=29))
analytics_data_api/v0/views/courses.py
......@@ -5,7 +5,6 @@ from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Max
from django.http import Http404
from rest_framework import generics
from rest_framework.generics import get_object_or_404
from analytics_data_api.v0 import models, serializers
......@@ -53,8 +52,11 @@ class CourseActivityMostRecentWeekView(generics.RetrieveAPIView):
class BaseCourseEnrollmentView(generics.ListAPIView):
def get_course_or_404(self):
return get_object_or_404(models.Course, course_id=self.kwargs.get('course_id'))
def verify_course_exists_or_404(self, course_id):
if self.model.objects.filter(course_id=course_id).exists():
return True
raise Http404
def apply_date_filtering(self, queryset):
if 'start_date' in self.request.QUERY_PARAMS or 'end_date' in self.request.QUERY_PARAMS:
......@@ -77,8 +79,9 @@ class BaseCourseEnrollmentView(generics.ListAPIView):
return queryset
def get_queryset(self):
course = self.get_course_or_404()
queryset = self.model.objects.filter(course=course)
course_id = self.kwargs.get('course_id')
self.verify_course_exists_or_404(course_id)
queryset = self.model.objects.filter(course_id=course_id)
queryset = self.apply_date_filtering(queryset)
return queryset
......@@ -185,9 +188,4 @@ class CourseEnrollmentByLocationView(BaseCourseEnrollmentView):
"""
serializer_class = serializers.CourseEnrollmentByCountrySerializer
def get_queryset(self):
course = self.get_course_or_404()
queryset = models.CourseEnrollmentByCountry.objects.filter(course=course)
queryset = self.apply_date_filtering(queryset)
return queryset
model = models.CourseEnrollmentByCountry
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment