Skip to content

E
edx-analytics-data-api-client
Commit 94346075 by Clinton Blackburn
Options

Added course enrollment 

Change-Id: I973fcb2672b53e0aa64158bea46b8cbb9c371c17
parent 5f86255c
Showing with 209 additions and 198 deletions
analyticsclient/client.py
import logging
import requests
import requests.exceptions
from analyticsclient.course import Course
from analyticsclient.exceptions import ClientError
from analyticsclient.status import Status
log = logging.getLogger(__name__)
class Client(object):
"""A client capable of retrieving the requested resources."""
DEFAULT_TIMEOUT = 0.1 # In seconds
DEFAULT_VERSION = 'v0'
def __init__(self, version=DEFAULT_VERSION):
"""
Initialize the Client.
Arguments:
version (str): When breaking changes are made to either the resource addresses or their returned data, this
value will change. Multiple clients can be made which adhere to each version.
"""
Analytics API client
The instance has attributes `status` and `courses` that provide access to instances of
:class: `~analyticsclient.status` and :class: `~analyticsclient.course`. This is the preferred (and only supported)
way to get access to those classes and their methods.
"""
def __init__(self, base_url, auth_token=None):
"""
self.version = version
def get(self, resource, timeout=None):
"""
Retrieve the data for a resource.
Initialize the client.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: A structure consisting of simple python types (dict, list, int, str etc).
Raises: ClientError if the resource cannot be retrieved for any reason.
base_url (str): URL of the API server (e.g. http://analytics.edx.org/api/v0)
auth_token (str): Authentication token
"""
raise NotImplementedError
def has_resource(self, resource, timeout=None):
"""
Check if a resource exists.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: True iff the resource exists.
"""
raise NotImplementedError
class RestClient(Client):
"""Retrieve resources from a remote REST API."""
DEFAULT_AUTH_TOKEN = ''
DEFAULT_BASE_URL = 'http://localhost:9090'
def __init__(self, base_url=DEFAULT_BASE_URL, auth_token=DEFAULT_AUTH_TOKEN):
"""
Initialize the RestClient.
Arguments:
base_url (str): A URL containing the scheme, netloc and port of the remote service.
auth_token (str): The token that should be used to authenticate requests made to the remote service.
"""
super(RestClient, self).__init__()
self.base_url = '{0}/api/{1}'.format(base_url, self.version)
self.base_url = base_url
self.auth_token = auth_token
self.timeout = 0.1
self.status = Status(self)
self.courses = lambda course_id: Course(self, course_id)
def get(self, resource, timeout=None):
"""
Retrieve the data for a resource.
Inherited from `Client`.
Arguments:
resource (str): Path in the form of slash separated strings.
......@@ -131,7 +81,7 @@ class RestClient(Client):
def _request(self, resource, timeout=None):
if timeout is None:
timeout = self.DEFAULT_TIMEOUT
timeout = self.timeout
headers = {
'Accept': 'application/json',
......@@ -153,11 +103,3 @@ class RestClient(Client):
message = 'Unable to retrieve resource'
log.exception(message)
raise ClientError('{0} "{1}"'.format(message, resource))
# TODO: Provide more detailed errors as necessary.
class ClientError(Exception):
"""An error occurred that prevented the client from performing the requested operation."""
pass
analyticsclient/course.py
......@@ -2,34 +2,38 @@
class Course(object):
"""Course scoped analytics."""
"""
Course-related analytics.
"""
# TODO: Should we have an acceptance test that runs the hadoop job to populate the database, serves the data with
# the API server and uses the client to retrieve it and validate the various transports?
def __init__(self, client, course_key):
def __init__(self, client, course_id):
"""
Initialize the CourseUserActivity.
Initialize the Course client.
Arguments:
client (analyticsclient.client.Client): The client to use to access remote resources.
course_key (mixed): An object that when passed to unicode() returns the unique identifier for the course as
it is represented in the data pipeline results.
course_id (str): String identifying the course (e.g. edX/DemoX/Demo_Course)
"""
self.client = client
self.course_key = course_key
self.course_id = course_id
@property
def recent_active_user_count(self):
"""A count of users who have recently interacted with the course in any way."""
# TODO: should we return something more structured than a python dict?
return self.client.get('courses/{0}/recent_activity'.format(unicode(self.course_key)))
return self.client.get('courses/{0}/recent_activity'.format(self.course_id))
@property
def recent_problem_activity_count(self):
"""A count of users who have recently attempted a problem."""
# TODO: Can we avoid passing around strings like "ATTEMPTED_PROBLEM" in the data pipeline and the client?
return self.client.get(
'courses/{0}/recent_activity?label=ATTEMPTED_PROBLEM'.format(unicode(self.course_key)))
'courses/{0}/recent_activity?activity_type=ATTEMPTED_PROBLEM'.format(self.course_id))
def enrollment(self, demographic=None):
uri = 'courses/{0}/enrollment'.format(self.course_id)
if demographic:
uri += '/%s' % demographic
return self.client.get(uri)
analyticsclient/exceptions.py 0 → 100644
class ClientError(Exception):
""" Common base class for all client errors. """
pass
analyticsclient/status.py
from analyticsclient.client import ClientError
from analyticsclient.exceptions import ClientError
class Status(object):
"""
Query the status of the connection between the client and the remote service.
Arguments:
client (analyticsclient.client.Client): The client to use to access remote resources.
API server status.
"""
def __init__(self, client):
......
analyticsclient/tests/__init__.py
from unittest import TestCase
from analyticsclient.client import Client, ClientError
from analyticsclient.client import Client
class InMemoryClient(Client):
class ClientTestCase(TestCase):
"""
Base class for client-related tests.
"""
"""Serves resources that have previously been set and stored in memory."""
def setUp(self):
self.api_url = 'http://localhost:9999/api/v0'
self.client = Client(self.api_url)
def __init__(self):
"""Initialize the fake client."""
super(InMemoryClient, self).__init__()
self.resources = {}
def get_api_url(self, path):
"""
Build an API URL with the specified path.
def has_resource(self, resource, timeout=None):
"""Return True iff the resource has been previously set."""
try:
self.get(resource, timeout=timeout)
return True
except ClientError:
return False
Arguments:
path (str): Path to be appended to the URL
def get(self, resource, timeout=None):
"""Return the resource from memory."""
try:
return self.resources[resource]
except KeyError:
raise ClientError('Unable to find requested resource')
Returns:
Complete API URL and path
"""
return "{0}/{1}".format(self.client.base_url, path)
analyticsclient/tests/test_client.py
import json
from unittest import TestCase
import httpretty
from analyticsclient.client import RestClient, ClientError
from analyticsclient.client import Client
from analyticsclient.exceptions import ClientError
from analyticsclient.tests import ClientTestCase
class RestClientTest(TestCase):
BASE_URI = 'http://localhost:9091'
VERSIONED_BASE_URI = BASE_URI + '/api/v0'
TEST_ENDPOINT = 'test'
TEST_URI = VERSIONED_BASE_URI + '/' + TEST_ENDPOINT
class ClientTests(ClientTestCase):
def setUp(self):
super(ClientTests, self).setUp()
httpretty.enable()
self.client = RestClient(base_url=self.BASE_URI)
self.test_endpoint = 'test'
self.test_uri = self.get_api_url(self.test_endpoint)
def tearDown(self):
httpretty.disable()
def test_has_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='')
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), True)
httpretty.register_uri(httpretty.GET, self.test_uri, body='')
self.assertEquals(self.client.has_resource(self.test_endpoint), True)
def test_missing_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=404)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False)
httpretty.register_uri(httpretty.GET, self.test_uri, body='', status=404)
self.assertEquals(self.client.has_resource(self.test_endpoint), False)
def test_failed_authentication(self):
self.client = RestClient(base_url=self.BASE_URI, auth_token='atoken')
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=401)
client = Client(base_url=self.api_url, auth_token='atoken')
httpretty.register_uri(httpretty.GET, self.test_uri, body='', status=401)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False)
self.assertEquals(client.has_resource(self.test_endpoint), False)
self.assertEquals(httpretty.last_request().headers['Authorization'], 'Token atoken')
def test_get(self):
data = {
'foo': 'bar'
}
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data))
self.assertEquals(self.client.get(self.TEST_ENDPOINT), data)
def test_get_invalid_json(self):
data = {
'foo': 'bar'
}
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data)[:6])
data = {'foo': 'bar'}
httpretty.register_uri(httpretty.GET, self.test_uri, body=json.dumps(data))
self.assertEquals(self.client.get(self.test_endpoint), data)
# Bad JSON
httpretty.register_uri(httpretty.GET, self.test_uri, body=json.dumps(data)[:6])
with self.assertRaises(ClientError):
self.client.get(self.TEST_ENDPOINT)
self.client.get(self.test_endpoint)
analyticsclient/tests/test_course.py
from unittest import TestCase
import json
from analyticsclient.course import Course
from analyticsclient.tests import InMemoryClient
import httpretty
from analyticsclient.tests import ClientTestCase
class CourseTest(TestCase):
class CoursesTests(ClientTestCase):
def setUp(self):
self.client = InMemoryClient()
self.course = Course(self.client, 'edX/DemoX/Demo_Course')
def test_recent_activity(self):
# These tests don't feel terribly useful, since it's not really testing any substantial code... just that mock
# values are returned. The risky part of the interface (the URL and the response data) is not tested at all
# since it is mocked out.
course_id = 'edX/DemoX/Demo_Course'
expected_result = {
'course_id': 'edX/DemoX/Demo_Course',
'interval_start': '2014-05-24T00:00:00Z',
'interval_end': '2014-06-01T00:00:00Z',
'label': 'ACTIVE',
'count': 300,
super(CoursesTests, self).setUp()
self.course_id = 'edX/DemoX/Demo_Course'
self.course = self.client.courses(self.course_id)
httpretty.enable()
def tearDown(self):
super(CoursesTests, self).tearDown()
httpretty.disable()
def test_recent_active_user_count(self):
body = {
u'course_id': u'edX/DemoX/Demo_Course',
u'interval_start': u'2014-05-24T00:00:00Z',
u'interval_end': u'2014-06-01T00:00:00Z',
u'activity_type': u'any',
u'count': 300,
}
self.client.resources['courses/{0}/recent_activity'.format(course_id)] = expected_result
self.assertEquals(self.course.recent_active_user_count, expected_result)
uri = self.get_api_url('courses/{0}/recent_activity'.format(self.course_id))
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(body))
self.assertDictEqual(body, self.course.recent_active_user_count)
def assertEnrollmentResponseData(self, course, data, demographic=None):
uri = self.get_api_url('courses/{0}/enrollment'.format(course.course_id))
if demographic:
uri += '/%s' % demographic
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(data))
self.assertDictEqual(data, course.enrollment(demographic))
def test_recent_problem_activity_count(self):
body = {
u'course_id': u'edX/DemoX/Demo_Course',
u'interval_start': u'2014-05-24T00:00:00Z',
u'interval_end': u'2014-06-01T00:00:00Z',
u'activity_type': u'attempted_problem',
u'count': 200,
}
uri = self.get_api_url('courses/{0}/recent_activity?activity_type=attempted_problem'.format(self.course_id))
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(body))
self.assertDictEqual(body, self.course.recent_problem_activity_count)
def test_enrollment_birth_year(self):
data = {
u'birth_years': {
u'1894': 13,
u'1895': 19
}
}
self.assertEnrollmentResponseData(self.course, data, 'birth_year')
def test_enrollment_education(self):
data = {
u'education_levels': {
u'none': 667,
u'junior_secondary': 6051,
u'primary': 981,
u'associates': 12255,
u'bachelors': 70885,
u'masters': 53216,
u'doctorate': 9940,
u'other': 5722,
u'secondary': 51591
}
}
self.assertEnrollmentResponseData(self.course, data, 'education')
def test_enrollment_gender(self):
data = {
u'genders': {
u'm': 133240,
u'o': 423,
u'f': 77495
}
}
self.assertEnrollmentResponseData(self.course, data, 'gender')
analyticsclient/tests/test_status.py
from unittest import TestCase
import json
from analyticsclient.status import Status
from analyticsclient.tests import InMemoryClient
import httpretty
from analyticsclient.tests import ClientTestCase
class StatusTest(TestCase):
def setUp(self):
self.client = InMemoryClient()
self.status = Status(self.client)
class StatusTests(ClientTestCase):
@httpretty.activate
def test_alive(self):
self.assertEquals(self.status.alive, False)
"""
Alive status should be True if server responds with HTTP 200, otherwise False.
"""
# Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('status'))
self.assertTrue(self.client.status.alive)
self.client.resources['status'] = ''
self.assertEquals(self.status.alive, True)
# "Kill" the server (assuming there is nothing running at API_URL)
httpretty.reset()
self.assertFalse(self.client.status.alive)
@httpretty.activate
def test_authenticated(self):
self.assertEquals(self.status.authenticated, False)
"""
Authenticated status should be True if client is authenticated, otherwise False.
"""
# Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('authenticated'))
self.assertTrue(self.client.status.authenticated)
self.client.resources['authenticated'] = ''
self.assertEquals(self.status.authenticated, True)
# Non-authenticated user
httpretty.register_uri(httpretty.GET, self.get_api_url('authenticated'), status=401)
self.assertFalse(self.client.status.authenticated)
@httpretty.activate
def test_healthy(self):
self.client.resources['health'] = {
'overall_status': 'OK',
'detailed_status': {
'database_connection': 'OK'
}
}
"""
Healthy status should be True if server is alive and can respond to requests, otherwise False.
"""
self.assertEquals(self.status.healthy, True)
# Unresponsive server
self.assertFalse(self.client.status.healthy)
def test_not_healthy(self):
self.client.resources['health'] = {
'overall_status': 'UNAVAILABLE',
body = {
'overall_status': 'OK',
'detailed_status': {
'database_connection': 'UNAVAILABLE'
'database_connection': 'OK'
}
}
self.assertEquals(self.status.healthy, False)
# Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertTrue(self.client.status.healthy)
def test_invalid_health_value(self):
self.client.resources['health'] = {}
# Sick server
body['overall_status'] = 'BAD'
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertFalse(self.client.status.healthy)
self.assertEquals(self.status.healthy, False)
# Odd response
del body['overall_status']
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertFalse(self.client.status.healthy)
setup.py
......@@ -8,7 +8,7 @@ setup(
description='Client used to access edX analytics data warehouse',
long_description=open('README.rst').read(),
install_requires=[
"restnavigator==0.2.0",
"requests==2.2.0",
],
tests_require=[
"coverage==3.7.1",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment