Skip to content

E
edx-analytics-data-api
Commit 3809ba5b by Dennis Jen
Options

Update DRF to 3.4.6

parent fee33f4c
Showing with 161 additions and 126 deletions
analytics_data_api/v0/models.py
......@@ -234,7 +234,7 @@ class Video(BaseVideo):
class RosterUpdate(DocType):
date = Date()
date = Date(format=settings.DATE_FORMAT)
# pylint: disable=old-style-class
class Meta:
......@@ -265,8 +265,8 @@ class RosterEntry(DocType):
attempt_ratio_order = Integer()
discussion_contributions = Integer()
videos_watched = Integer()
enrollment_date = Date()
last_updated = Date()
enrollment_date = Date(format=settings.DATE_FORMAT)
last_updated = Date(format=settings.DATE_FORMAT)
# pylint: disable=old-style-class
class Meta:
......@@ -460,7 +460,7 @@ class ModuleEngagement(models.Model):
course_id = models.CharField(db_index=True, max_length=255)
username = models.CharField(max_length=255)
date = models.DateTimeField()
date = models.DateField()
# This will be one of "problem", "video" or "discussion"
entity_type = models.CharField(max_length=255)
# For problems this will be the usage key, for videos it will be the html encoded module ID,
......
analytics_data_api/v0/serializers.py
from collections import OrderedDict
from urlparse import urljoin
from django.conf import settings
from rest_framework import pagination, serializers
from rest_framework.response import Response
from analytics_data_api.constants import (
engagement_events,
enrollment_modes,
genders,
learner,
)
from analytics_data_api.v0 import models
......@@ -23,7 +25,7 @@ class CourseActivityByWeekSerializer(serializers.ModelSerializer):
particular record is likely to change unexpectedly so we avoid exposing it.
"""
activity_type = serializers.SerializerMethodField('get_activity_type')
activity_type = serializers.SerializerMethodField()
def get_activity_type(self, obj):
"""
......@@ -187,13 +189,7 @@ class SequentialOpenDistributionSerializer(ModelSerializerWithCreatedField):
)
class DefaultIfNoneMixin(object):
def default_if_none(self, value, default=0):
return value if value is not None else default
class BaseCourseEnrollmentModelSerializer(DefaultIfNoneMixin, ModelSerializerWithCreatedField):
class BaseCourseEnrollmentModelSerializer(ModelSerializerWithCreatedField):
date = serializers.DateField(format=settings.DATE_FORMAT)
......@@ -207,27 +203,29 @@ class CourseEnrollmentDailySerializer(BaseCourseEnrollmentModelSerializer):
class CourseEnrollmentModeDailySerializer(BaseCourseEnrollmentModelSerializer):
""" Representation of course enrollment, broken down by mode, for a single day and course. """
audit = serializers.SerializerMethodField()
credit = serializers.SerializerMethodField()
honor = serializers.SerializerMethodField()
professional = serializers.SerializerMethodField()
verified = serializers.SerializerMethodField()
def get_default_fields(self):
# pylint: disable=super-on-old-class
fields = super(CourseEnrollmentModeDailySerializer, self).get_default_fields()
# Create a field for each enrollment mode
for mode in ENROLLMENT_MODES:
fields[mode] = serializers.IntegerField(required=True, default=0)
def get_audit(self, obj):
return obj.get('audit', 0)
# Create a transform method for each field
setattr(self, 'transform_%s' % mode, self._transform_mode)
def get_honor(self, obj):
return obj.get('honor', 0)
fields['cumulative_count'] = serializers.IntegerField(required=True, default=0)
def get_credit(self, obj):
return obj.get('credit', 0)
return fields
def get_professional(self, obj):
return obj.get('professional', 0)
def _transform_mode(self, _obj, value):
return self.default_if_none(value, 0)
def get_verified(self, obj):
return obj.get('verified', 0)
class Meta(object):
model = models.CourseEnrollmentDaily
model = models.CourseEnrollmentModeDaily
# Declare the dynamically-created fields here as well so that they will be picked up by Swagger.
fields = ['course_id', 'date', 'count', 'cumulative_count', 'created'] + ENROLLMENT_MODES
......@@ -256,21 +254,23 @@ class CourseEnrollmentByCountrySerializer(BaseCourseEnrollmentModelSerializer):
class CourseEnrollmentByGenderSerializer(BaseCourseEnrollmentModelSerializer):
def get_default_fields(self):
# pylint: disable=super-on-old-class
fields = super(CourseEnrollmentByGenderSerializer, self).get_default_fields()
# Create a field for each gender
for gender in genders.ALL:
fields[gender] = serializers.IntegerField(required=True, default=0)
female = serializers.ReadOnlyField()
male = serializers.ReadOnlyField()
other = serializers.ReadOnlyField()
unknown = serializers.ReadOnlyField()
def get_female(self, obj):
return obj.get('female', None)
# Create a transform method for each field
setattr(self, 'transform_%s' % gender, self._transform_gender)
def get_male(self, obj):
return obj.get('male', None)
return fields
def get_other(self, obj):
return obj.get('other', None)
def _transform_gender(self, _obj, value):
return self.default_if_none(value, 0)
def get_unknown(self, obj):
return obj.get('unknown', None)
class Meta(object):
model = models.CourseEnrollmentByGender
......@@ -330,27 +330,34 @@ class VideoTimelineSerializer(ModelSerializerWithCreatedField):
class LastUpdatedSerializer(serializers.Serializer):
last_updated = serializers.DateField(source='date', format=settings.DATE_FORMAT)
last_updated = serializers.DateTimeField(source='date', format=settings.DATE_FORMAT)
class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin):
username = serializers.CharField(source='username')
enrollment_mode = serializers.CharField(source='enrollment_mode')
name = serializers.CharField(source='name')
account_url = serializers.SerializerMethodField('get_account_url')
email = serializers.CharField(source='email')
segments = serializers.Field(source='segments')
engagements = serializers.SerializerMethodField('get_engagements')
enrollment_date = serializers.DateField(source='enrollment_date', format=settings.DATE_FORMAT)
cohort = serializers.CharField(source='cohort')
def transform_segments(self, _obj, value):
# returns null instead of empty strings
return value or []
class LearnerSerializer(serializers.Serializer):
username = serializers.CharField()
enrollment_mode = serializers.CharField()
name = serializers.CharField()
account_url = serializers.SerializerMethodField()
email = serializers.CharField()
segments = serializers.SerializerMethodField()
engagements = serializers.SerializerMethodField()
enrollment_date = serializers.DateTimeField(format=settings.DATE_FORMAT)
cohort = serializers.SerializerMethodField()
def get_segments(self, obj):
# using hasattr() instead because DocType.get() is overloaded and makes a request
if hasattr(obj, 'segments'):
# json parsing will fail unless in unicode
return [unicode(segment) for segment in obj.segments]
else:
return []
def transform_cohort(self, _obj, value):
# returns null instead of empty strings
return value or None
def get_cohort(self, obj):
# using hasattr() instead because DocType.get() is overloaded and makes a request
if hasattr(obj, 'cohort') and len(obj.cohort) > 0:
return obj.cohort
else:
return None
def get_account_url(self, obj):
if settings.LMS_USER_ACCOUNT_BASE_URL:
......@@ -358,6 +365,9 @@ class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin):
else:
return None
def default_if_none(self, value, default=0):
return value if value is not None else default
def get_engagements(self, obj):
"""
Add the engagement totals.
......@@ -376,45 +386,45 @@ class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin):
return engagements
class EdxPaginationSerializer(pagination.PaginationSerializer):
class EdxPaginationSerializer(pagination.PageNumberPagination):
"""
Adds values to the response according to edX REST API Conventions.
"""
count = serializers.Field(source='paginator.count')
num_pages = serializers.Field(source='paginator.num_pages')
page_size_query_param = 'page_size'
page_size = learner.LEARNER_API_DEFAULT_LIST_PAGE_SIZE
max_page_size = 100 # TODO -- tweak during load testing
class ElasticsearchDSLSearchSerializer(EdxPaginationSerializer):
def __init__(self, *args, **kwargs):
"""Make sure that the elasticsearch query is executed."""
# Because the elasticsearch-dsl search object has a different
# API from the queryset object that's expected by the django
# Paginator object, we have to manually execute the query.
# Note that the `kwargs['instance']` is the Page object, and
# `kwargs['instance'].object_list` is actually an
# elasticsearch-dsl search object.
kwargs['instance'].object_list = kwargs['instance'].object_list.execute()
super(ElasticsearchDSLSearchSerializer, self).__init__(*args, **kwargs)
def get_paginated_response(self, data):
# The output is more readable with num_pages included not at the end, but
# inefficient to insert into an OrderedDict, so the response is copied from
# rest_framework.pagination with the addition of "num_pages".
return Response(OrderedDict([
('count', self.page.paginator.count),
('num_pages', self.page.paginator.num_pages),
('next', self.get_next_link()),
('previous', self.get_previous_link()),
('results', data)
]))
class EngagementDaySerializer(DefaultIfNoneMixin, serializers.Serializer):
class EngagementDaySerializer(serializers.Serializer):
date = serializers.DateField(format=settings.DATE_FORMAT)
problems_attempted = serializers.IntegerField(required=True, default=0)
problems_completed = serializers.IntegerField(required=True, default=0)
discussion_contributions = serializers.IntegerField(required=True, default=0)
videos_viewed = serializers.IntegerField(required=True, default=0)
problems_attempted = serializers.SerializerMethodField()
problems_completed = serializers.SerializerMethodField()
discussion_contributions = serializers.SerializerMethodField()
videos_viewed = serializers.SerializerMethodField()
def transform_problems_attempted(self, _obj, value):
return self.default_if_none(value, 0)
def get_problems_attempted(self, obj):
return obj.get('problems_attempted', 0)
def transform_problems_completed(self, _obj, value):
return self.default_if_none(value, 0)
def get_problems_completed(self, obj):
return obj.get('problems_completed', 0)
def transform_discussion_contributions(self, _obj, value):
return self.default_if_none(value, 0)
def get_discussion_contributions(self, obj):
return obj.get('discussion_contributions', 0)
def transform_videos_viewed(self, _obj, value):
return self.default_if_none(value, 0)
def get_videos_viewed(self, obj):
return obj.get('videos_viewed', 0)
class DateRangeSerializer(serializers.Serializer):
......@@ -429,9 +439,9 @@ class EnagementRangeMetricSerializer(serializers.Serializer):
represented as arrays. If any one of the ranges is not defined, it is not
included in the serialized output.
"""
class_rank_bottom = serializers.SerializerMethodField('get_class_rank_bottom')
class_rank_average = serializers.SerializerMethodField('get_class_rank_average')
class_rank_top = serializers.SerializerMethodField('get_class_rank_top')
class_rank_bottom = serializers.SerializerMethodField()
class_rank_average = serializers.SerializerMethodField()
class_rank_top = serializers.SerializerMethodField()
def get_class_rank_average(self, obj):
return self._transform_range(obj['average'])
......@@ -447,10 +457,10 @@ class EnagementRangeMetricSerializer(serializers.Serializer):
class CourseLearnerMetadataSerializer(serializers.Serializer):
enrollment_modes = serializers.Field(source='es_data.enrollment_modes')
segments = serializers.Field(source='es_data.segments')
cohorts = serializers.Field(source='es_data.cohorts')
engagement_ranges = serializers.SerializerMethodField('get_engagement_ranges')
enrollment_modes = serializers.ReadOnlyField(source='es_data.enrollment_modes')
segments = serializers.ReadOnlyField(source='es_data.segments')
cohorts = serializers.ReadOnlyField(source='es_data.cohorts')
engagement_ranges = serializers.SerializerMethodField()
def get_engagement_ranges(self, obj):
query_set = obj['engagement_ranges']
......
analytics_data_api/v0/tests/views/test_courses.py
......@@ -210,10 +210,11 @@ class CourseActivityLastWeekTest(DemoCourseMixin, TestCaseWithAuthentication):
@staticmethod
def get_activity_record(**kwargs):
datetime_format = "%Y-%m-%dT%H:%M:%SZ"
default = {
'course_id': DEMO_COURSE_ID,
'interval_start': datetime.datetime(2014, 1, 1, 0, 0, tzinfo=pytz.utc),
'interval_end': datetime.datetime(2014, 1, 8, 0, 0, tzinfo=pytz.utc),
'interval_start': datetime.datetime(2014, 1, 1, 0, 0, tzinfo=pytz.utc).strftime(datetime_format),
'interval_end': datetime.datetime(2014, 1, 8, 0, 0, tzinfo=pytz.utc).strftime(datetime_format),
'activity_type': 'any',
'count': 300,
}
......@@ -339,6 +340,9 @@ class CourseEnrollmentByGenderViewTests(CourseEnrollmentViewTestCaseMixin, Defau
super(CourseEnrollmentByGenderViewTests, self).setUp()
self.generate_data()
def tearDown(self):
self.destroy_data()
def serialize_enrollment(self, enrollment):
return {
'created': enrollment.created.strftime(settings.DATETIME_FORMAT),
......@@ -606,8 +610,8 @@ class CourseProblemsListViewTests(DemoCourseMixin, TestCaseWithAuthentication):
# Create multiple objects here to test the grouping. Add a model with a different module_id to break up the
# natural order and ensure the view properly sorts the objects before grouping.
module_id = 'i4x://test/problem/1'
alt_module_id = 'i4x://test/problem/2'
module_id = u'i4x://test/problem/1'
alt_module_id = u'i4x://test/problem/2'
created = datetime.datetime.utcnow()
alt_created = created + datetime.timedelta(seconds=2)
date_time_format = '%Y-%m-%d %H:%M:%S'
......@@ -624,21 +628,21 @@ class CourseProblemsListViewTests(DemoCourseMixin, TestCaseWithAuthentication):
'module_id': module_id,
'total_submissions': 150,
'correct_submissions': 50,
'part_ids': [o1.part_id, o3.part_id],
'part_ids': unicode([o1.part_id, o3.part_id]),
'created': alt_created.strftime(settings.DATETIME_FORMAT)
},
{
'module_id': alt_module_id,
'total_submissions': 100,
'correct_submissions': 100,
'part_ids': [o2.part_id],
'created': created.strftime(settings.DATETIME_FORMAT)
'part_ids': unicode([o2.part_id]),
'created': unicode(created.strftime(settings.DATETIME_FORMAT))
}
]
response = self._get_data(self.course_id)
self.assertEquals(response.status_code, 200)
self.assertListEqual(response.data, expected)
self.assertListEqual([dict(d) for d in response.data], expected)
def test_get_404(self):
"""
......@@ -669,8 +673,8 @@ class CourseProblemsAndTagsListViewTests(DemoCourseMixin, TestCaseWithAuthentica
# Create multiple objects here to test the grouping. Add a model with a different module_id to break up the
# natural order and ensure the view properly sorts the objects before grouping.
module_id = 'i4x://test/problem/1'
alt_module_id = 'i4x://test/problem/2'
module_id = u'i4x://test/problem/1'
alt_module_id = u'i4x://test/problem/2'
tags = {
'difficulty': ['Easy', 'Medium', 'Hard'],
......@@ -695,26 +699,26 @@ class CourseProblemsAndTagsListViewTests(DemoCourseMixin, TestCaseWithAuthentica
'module_id': module_id,
'total_submissions': 11,
'correct_submissions': 4,
'tags': {
'difficulty': 'Easy',
'learning_outcome': 'Learned a few things',
},
'tags': unicode({
u'difficulty': u'Easy',
u'learning_outcome': u'Learned a few things',
}),
'created': alt_created.strftime(settings.DATETIME_FORMAT)
},
{
'module_id': alt_module_id,
'total_submissions': 4,
'correct_submissions': 0,
'tags': {
'learning_outcome': 'Learned everything',
},
'tags': unicode({
u'learning_outcome': u'Learned everything',
}),
'created': created.strftime(settings.DATETIME_FORMAT)
}
]
response = self._get_data(self.course_id)
self.assertEquals(response.status_code, 200)
self.assertListEqual(sorted(response.data), sorted(expected))
self.assertListEqual(sorted([dict(d) for d in response.data]), sorted(expected))
def test_get_404(self):
"""
......
analytics_data_api/v0/tests/views/test_learners.py
......@@ -205,8 +205,7 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
returned.
"""
self.assertEqual(response.status_code, 200)
payload = json.loads(response.content)
returned_learners = payload['results']
returned_learners = json.loads(response.content)['results']
if expected_learners is None:
self.assertEqual(returned_learners, list())
else:
......
analytics_data_api/v0/views/__init__.py
......@@ -12,7 +12,7 @@ class CourseViewMixin(object):
course_id = None
def get(self, request, *args, **kwargs):
self.course_id = self.kwargs.get('course_id', request.QUERY_PARAMS.get('course_id', None))
self.course_id = self.kwargs.get('course_id', request.query_params.get('course_id', None))
if not self.course_id:
raise CourseNotSpecifiedError()
......
analytics_data_api/v0/views/courses.py
......@@ -15,6 +15,8 @@ from analytics_data_api.constants import enrollment_modes
from analytics_data_api.utils import dictfetchall
from analytics_data_api.v0 import models, serializers
from utils import raise_404_if_none
class BaseCourseView(generics.ListAPIView):
start_date = None
......@@ -25,8 +27,8 @@ class BaseCourseView(generics.ListAPIView):
def get(self, request, *args, **kwargs):
self.course_id = self.kwargs.get('course_id')
start_date = request.QUERY_PARAMS.get('start_date')
end_date = request.QUERY_PARAMS.get('end_date')
start_date = request.query_params.get('start_date')
end_date = request.query_params.get('end_date')
timezone = utc
self.start_date = self.parse_date(start_date, timezone)
......@@ -46,6 +48,7 @@ class BaseCourseView(generics.ListAPIView):
def apply_date_filtering(self, queryset):
raise NotImplementedError
@raise_404_if_none
def get_queryset(self):
queryset = self.model.objects.filter(course_id=self.course_id)
queryset = self.apply_date_filtering(queryset)
......@@ -232,9 +235,9 @@ class CourseActivityMostRecentWeekView(generics.RetrieveAPIView):
""" Retrieve the activity type from the query string. """
# Support the old label param
activity_type = self.request.QUERY_PARAMS.get('label', None)
activity_type = self.request.query_params.get('label', None)
activity_type = activity_type or self.request.QUERY_PARAMS.get('activity_type', self.DEFAULT_ACTIVITY_TYPE)
activity_type = activity_type or self.request.query_params.get('activity_type', self.DEFAULT_ACTIVITY_TYPE)
activity_type = self._format_activity_type(activity_type)
return activity_type
......@@ -633,6 +636,7 @@ class ProblemsListView(BaseCourseView):
serializer_class = serializers.ProblemSerializer
allow_empty = False
@raise_404_if_none
def get_queryset(self):
# last_response_count is the number of submissions for the problem part and must
# be divided by the number of problem parts to get the problem submission rather
......@@ -709,6 +713,7 @@ class ProblemsAndTagsListView(BaseCourseView):
allow_empty = False
model = models.ProblemsAndTags
@raise_404_if_none
def get_queryset(self):
queryset = self.model.objects.filter(course_id=self.course_id)
items = queryset.all()
......
analytics_data_api/v0/views/learners.py
......@@ -5,9 +5,6 @@ import logging
from rest_framework import generics, status
from analytics_data_api.constants import (
learner
)
from analytics_data_api.v0.exceptions import (
LearnerEngagementTimelineNotFoundError,
LearnerNotFoundError,
......@@ -21,7 +18,7 @@ from analytics_data_api.v0.models import (
)
from analytics_data_api.v0.serializers import (
CourseLearnerMetadataSerializer,
ElasticsearchDSLSearchSerializer,
EdxPaginationSerializer,
EngagementDaySerializer,
LastUpdatedSerializer,
LearnerSerializer,
......@@ -187,14 +184,12 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
"""
serializer_class = LearnerSerializer
pagination_serializer_class = ElasticsearchDSLSearchSerializer
paginate_by_param = 'page_size'
paginate_by = learner.LEARNER_API_DEFAULT_LIST_PAGE_SIZE
pagination_class = EdxPaginationSerializer
max_paginate_by = 100 # TODO -- tweak during load testing
def _validate_query_params(self):
"""Validates various querystring parameters."""
query_params = self.request.QUERY_PARAMS
query_params = self.request.query_params
page = query_params.get('page')
if page:
try:
......@@ -222,6 +217,7 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
"""
response = super(LearnerListView, self).list(request, args, kwargs)
last_updated = self.get_last_updated()
if response.data['results'] is not None:
for result in response.data['results']:
result.update(last_updated)
return response
......@@ -232,7 +228,7 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
as a an array of dicts with fields "learner" and "last_updated".
"""
self._validate_query_params()
query_params = self.request.QUERY_PARAMS
query_params = self.request.query_params
order_by = query_params.get('order_by')
sort_order = query_params.get('sort_order')
......
analytics_data_api/v0/views/problems.py
......@@ -22,6 +22,8 @@ from analytics_data_api.v0.serializers import (
)
from analytics_data_api.utils import matching_tuple
from utils import raise_404_if_none
class ProblemResponseAnswerDistributionView(generics.ListAPIView):
"""
......@@ -98,6 +100,7 @@ class ProblemResponseAnswerDistributionView(generics.ListAPIView):
return consolidated_answers
@raise_404_if_none
def get_queryset(self):
"""Select all the answer distribution response having to do with this usage of the problem."""
problem_id = self.kwargs.get('problem_id')
......@@ -142,6 +145,7 @@ class GradeDistributionView(generics.ListAPIView):
serializer_class = GradeDistributionSerializer
allow_empty = False
@raise_404_if_none
def get_queryset(self):
"""Select all grade distributions for a particular module"""
problem_id = self.kwargs.get('problem_id')
......@@ -170,6 +174,7 @@ class SequentialOpenDistributionView(generics.ListAPIView):
serializer_class = SequentialOpenDistributionSerializer
allow_empty = False
@raise_404_if_none
def get_queryset(self):
"""Select the view count for a specific module"""
module_id = self.kwargs.get('module_id')
......
analytics_data_api/v0/views/utils.py
"""Utilities for view-level API logic."""
from django.http import Http404
def split_query_argument(argument):
......@@ -10,3 +11,15 @@ def split_query_argument(argument):
return argument.split(',')
else:
return None
def raise_404_if_none(func):
"""
Decorator for raiseing Http404 if function evaulation is falsey (e.g. empty queryset).
"""
def func_wrapper(self):
queryset = func(self)
if queryset:
return queryset
else:
raise Http404
return func_wrapper
\ No newline at end of file
analytics_data_api/v0/views/videos.py
......@@ -7,6 +7,7 @@ from rest_framework import generics
from analytics_data_api.v0.models import VideoTimeline
from analytics_data_api.v0.serializers import VideoTimelineSerializer
from utils import raise_404_if_none
class VideoTimelineView(generics.ListAPIView):
"""
......@@ -30,6 +31,7 @@ class VideoTimelineView(generics.ListAPIView):
serializer_class = VideoTimelineSerializer
allow_empty = False
@raise_404_if_none
def get_queryset(self):
"""Select the view count for a specific module"""
video_id = self.kwargs.get('video_id')
......
requirements/base.txt
boto==2.22.1 # MIT
Django==1.8.14 # BSD License
django-model-utils==2.2 # BSD
djangorestframework==2.4.4 # BSD
djangorestframework==3.4.6 # BSD
django-rest-swagger==0.2.8 # BSD
djangorestframework-csv==1.3.3 # BSD
django-countries==3.2 # MIT
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment