Skip to content

E
edx-analytics-data-api-client
Commit 8e8afa54 by Clinton Blackburn
Options

Merge pull request #2 from edx/enrollment 

Added course enrollment
parents 5f86255c 94346075
Showing with 209 additions and 198 deletions
analyticsclient/client.py
import logging import logging
import requests import requests
import requests.exceptions import requests.exceptions
from analyticsclient.course import Course
from analyticsclient.exceptions import ClientError
from analyticsclient.status import Status
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Client(object): class Client(object):
"""
"""A client capable of retrieving the requested resources.""" Analytics API client
DEFAULT_TIMEOUT = 0.1 # In seconds The instance has attributes `status` and `courses` that provide access to instances of
DEFAULT_VERSION = 'v0' :class: `~analyticsclient.status` and :class: `~analyticsclient.course`. This is the preferred (and only supported)
way to get access to those classes and their methods.
def __init__(self, version=DEFAULT_VERSION): """
""" def __init__(self, base_url, auth_token=None):
Initialize the Client.
Arguments:
version (str): When breaking changes are made to either the resource addresses or their returned data, this
value will change. Multiple clients can be made which adhere to each version.
""" """
self.version = version Initialize the client.
def get(self, resource, timeout=None):
"""
Retrieve the data for a resource.
Arguments: Arguments:
base_url (str): URL of the API server (e.g. http://analytics.edx.org/api/v0)
resource (str): Path in the form of slash separated strings. auth_token (str): Authentication token
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: A structure consisting of simple python types (dict, list, int, str etc).
Raises: ClientError if the resource cannot be retrieved for any reason.
""" """
raise NotImplementedError self.base_url = base_url
def has_resource(self, resource, timeout=None):
"""
Check if a resource exists.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: True iff the resource exists.
"""
raise NotImplementedError
class RestClient(Client):
"""Retrieve resources from a remote REST API."""
DEFAULT_AUTH_TOKEN = ''
DEFAULT_BASE_URL = 'http://localhost:9090'
def __init__(self, base_url=DEFAULT_BASE_URL, auth_token=DEFAULT_AUTH_TOKEN):
"""
Initialize the RestClient.
Arguments:
base_url (str): A URL containing the scheme, netloc and port of the remote service.
auth_token (str): The token that should be used to authenticate requests made to the remote service.
"""
super(RestClient, self).__init__()
self.base_url = '{0}/api/{1}'.format(base_url, self.version)
self.auth_token = auth_token self.auth_token = auth_token
self.timeout = 0.1
self.status = Status(self)
self.courses = lambda course_id: Course(self, course_id)
def get(self, resource, timeout=None): def get(self, resource, timeout=None):
""" """
Retrieve the data for a resource. Retrieve the data for a resource.
Inherited from `Client`.
Arguments: Arguments:
resource (str): Path in the form of slash separated strings. resource (str): Path in the form of slash separated strings.
...@@ -131,7 +81,7 @@ class RestClient(Client): ...@@ -131,7 +81,7 @@ class RestClient(Client):
def _request(self, resource, timeout=None): def _request(self, resource, timeout=None):
if timeout is None: if timeout is None:
timeout = self.DEFAULT_TIMEOUT timeout = self.timeout
headers = { headers = {
'Accept': 'application/json', 'Accept': 'application/json',
...@@ -153,11 +103,3 @@ class RestClient(Client): ...@@ -153,11 +103,3 @@ class RestClient(Client):
message = 'Unable to retrieve resource' message = 'Unable to retrieve resource'
log.exception(message) log.exception(message)
raise ClientError('{0} "{1}"'.format(message, resource)) raise ClientError('{0} "{1}"'.format(message, resource))
# TODO: Provide more detailed errors as necessary.
class ClientError(Exception):
"""An error occurred that prevented the client from performing the requested operation."""
pass
analyticsclient/course.py
...@@ -2,34 +2,38 @@ ...@@ -2,34 +2,38 @@
class Course(object): class Course(object):
"""Course scoped analytics.""" """
Course-related analytics.
"""
# TODO: Should we have an acceptance test that runs the hadoop job to populate the database, serves the data with def __init__(self, client, course_id):
# the API server and uses the client to retrieve it and validate the various transports?
def __init__(self, client, course_key):
""" """
Initialize the CourseUserActivity. Initialize the Course client.
Arguments: Arguments:
client (analyticsclient.client.Client): The client to use to access remote resources. client (analyticsclient.client.Client): The client to use to access remote resources.
course_key (mixed): An object that when passed to unicode() returns the unique identifier for the course as course_id (str): String identifying the course (e.g. edX/DemoX/Demo_Course)
it is represented in the data pipeline results.
""" """
self.client = client self.client = client
self.course_key = course_key self.course_id = course_id
@property @property
def recent_active_user_count(self): def recent_active_user_count(self):
"""A count of users who have recently interacted with the course in any way.""" """A count of users who have recently interacted with the course in any way."""
# TODO: should we return something more structured than a python dict? # TODO: should we return something more structured than a python dict?
return self.client.get('courses/{0}/recent_activity'.format(unicode(self.course_key))) return self.client.get('courses/{0}/recent_activity'.format(self.course_id))
@property @property
def recent_problem_activity_count(self): def recent_problem_activity_count(self):
"""A count of users who have recently attempted a problem.""" """A count of users who have recently attempted a problem."""
# TODO: Can we avoid passing around strings like "ATTEMPTED_PROBLEM" in the data pipeline and the client? # TODO: Can we avoid passing around strings like "ATTEMPTED_PROBLEM" in the data pipeline and the client?
return self.client.get( return self.client.get(
'courses/{0}/recent_activity?label=ATTEMPTED_PROBLEM'.format(unicode(self.course_key))) 'courses/{0}/recent_activity?activity_type=ATTEMPTED_PROBLEM'.format(self.course_id))
def enrollment(self, demographic=None):
uri = 'courses/{0}/enrollment'.format(self.course_id)
if demographic:
uri += '/%s' % demographic
return self.client.get(uri)
analyticsclient/exceptions.py 0 → 100644
class ClientError(Exception):
""" Common base class for all client errors. """
pass
analyticsclient/status.py
from analyticsclient.client import ClientError from analyticsclient.exceptions import ClientError
class Status(object): class Status(object):
""" """
Query the status of the connection between the client and the remote service. API server status.
Arguments:
client (analyticsclient.client.Client): The client to use to access remote resources.
""" """
def __init__(self, client): def __init__(self, client):
......
analyticsclient/tests/__init__.py
from unittest import TestCase
from analyticsclient.client import Client, ClientError from analyticsclient.client import Client
class InMemoryClient(Client): class ClientTestCase(TestCase):
"""
Base class for client-related tests.
"""
"""Serves resources that have previously been set and stored in memory.""" def setUp(self):
self.api_url = 'http://localhost:9999/api/v0'
self.client = Client(self.api_url)
def __init__(self): def get_api_url(self, path):
"""Initialize the fake client.""" """
super(InMemoryClient, self).__init__() Build an API URL with the specified path.
self.resources = {}
def has_resource(self, resource, timeout=None): Arguments:
"""Return True iff the resource has been previously set.""" path (str): Path to be appended to the URL
try:
self.get(resource, timeout=timeout)
return True
except ClientError:
return False
def get(self, resource, timeout=None): Returns:
"""Return the resource from memory.""" Complete API URL and path
try: """
return self.resources[resource] return "{0}/{1}".format(self.client.base_url, path)
except KeyError:
raise ClientError('Unable to find requested resource')
analyticsclient/tests/test_client.py
import json import json
from unittest import TestCase
import httpretty import httpretty
from analyticsclient.client import RestClient, ClientError from analyticsclient.client import Client
from analyticsclient.exceptions import ClientError
from analyticsclient.tests import ClientTestCase
class RestClientTest(TestCase): class ClientTests(ClientTestCase):
BASE_URI = 'http://localhost:9091'
VERSIONED_BASE_URI = BASE_URI + '/api/v0'
TEST_ENDPOINT = 'test'
TEST_URI = VERSIONED_BASE_URI + '/' + TEST_ENDPOINT
def setUp(self): def setUp(self):
super(ClientTests, self).setUp()
httpretty.enable() httpretty.enable()
self.client = RestClient(base_url=self.BASE_URI) self.test_endpoint = 'test'
self.test_uri = self.get_api_url(self.test_endpoint)
def tearDown(self): def tearDown(self):
httpretty.disable() httpretty.disable()
def test_has_resource(self): def test_has_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='') httpretty.register_uri(httpretty.GET, self.test_uri, body='')
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), True) self.assertEquals(self.client.has_resource(self.test_endpoint), True)
def test_missing_resource(self): def test_missing_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=404) httpretty.register_uri(httpretty.GET, self.test_uri, body='', status=404)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False) self.assertEquals(self.client.has_resource(self.test_endpoint), False)
def test_failed_authentication(self): def test_failed_authentication(self):
self.client = RestClient(base_url=self.BASE_URI, auth_token='atoken') client = Client(base_url=self.api_url, auth_token='atoken')
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=401) httpretty.register_uri(httpretty.GET, self.test_uri, body='', status=401)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False) self.assertEquals(client.has_resource(self.test_endpoint), False)
self.assertEquals(httpretty.last_request().headers['Authorization'], 'Token atoken') self.assertEquals(httpretty.last_request().headers['Authorization'], 'Token atoken')
def test_get(self): def test_get(self):
data = { data = {'foo': 'bar'}
'foo': 'bar' httpretty.register_uri(httpretty.GET, self.test_uri, body=json.dumps(data))
} self.assertEquals(self.client.get(self.test_endpoint), data)
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data))
self.assertEquals(self.client.get(self.TEST_ENDPOINT), data) # Bad JSON
httpretty.register_uri(httpretty.GET, self.test_uri, body=json.dumps(data)[:6])
def test_get_invalid_json(self):
data = {
'foo': 'bar'
}
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data)[:6])
with self.assertRaises(ClientError): with self.assertRaises(ClientError):
self.client.get(self.TEST_ENDPOINT) self.client.get(self.test_endpoint)
analyticsclient/tests/test_course.py
from unittest import TestCase import json
from analyticsclient.course import Course import httpretty
from analyticsclient.tests import InMemoryClient
from analyticsclient.tests import ClientTestCase
class CourseTest(TestCase):
class CoursesTests(ClientTestCase):
def setUp(self): def setUp(self):
self.client = InMemoryClient() super(CoursesTests, self).setUp()
self.course = Course(self.client, 'edX/DemoX/Demo_Course') self.course_id = 'edX/DemoX/Demo_Course'
self.course = self.client.courses(self.course_id)
def test_recent_activity(self): httpretty.enable()
# These tests don't feel terribly useful, since it's not really testing any substantial code... just that mock
# values are returned. The risky part of the interface (the URL and the response data) is not tested at all def tearDown(self):
# since it is mocked out. super(CoursesTests, self).tearDown()
course_id = 'edX/DemoX/Demo_Course' httpretty.disable()
expected_result = {
'course_id': 'edX/DemoX/Demo_Course', def test_recent_active_user_count(self):
'interval_start': '2014-05-24T00:00:00Z', body = {
'interval_end': '2014-06-01T00:00:00Z', u'course_id': u'edX/DemoX/Demo_Course',
'label': 'ACTIVE', u'interval_start': u'2014-05-24T00:00:00Z',
'count': 300, u'interval_end': u'2014-06-01T00:00:00Z',
u'activity_type': u'any',
u'count': 300,
} }
self.client.resources['courses/{0}/recent_activity'.format(course_id)] = expected_result
self.assertEquals(self.course.recent_active_user_count, expected_result) uri = self.get_api_url('courses/{0}/recent_activity'.format(self.course_id))
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(body))
self.assertDictEqual(body, self.course.recent_active_user_count)
def assertEnrollmentResponseData(self, course, data, demographic=None):
uri = self.get_api_url('courses/{0}/enrollment'.format(course.course_id))
if demographic:
uri += '/%s' % demographic
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(data))
self.assertDictEqual(data, course.enrollment(demographic))
def test_recent_problem_activity_count(self):
body = {
u'course_id': u'edX/DemoX/Demo_Course',
u'interval_start': u'2014-05-24T00:00:00Z',
u'interval_end': u'2014-06-01T00:00:00Z',
u'activity_type': u'attempted_problem',
u'count': 200,
}
uri = self.get_api_url('courses/{0}/recent_activity?activity_type=attempted_problem'.format(self.course_id))
httpretty.register_uri(httpretty.GET, uri, body=json.dumps(body))
self.assertDictEqual(body, self.course.recent_problem_activity_count)
def test_enrollment_birth_year(self):
data = {
u'birth_years': {
u'1894': 13,
u'1895': 19
}
}
self.assertEnrollmentResponseData(self.course, data, 'birth_year')
def test_enrollment_education(self):
data = {
u'education_levels': {
u'none': 667,
u'junior_secondary': 6051,
u'primary': 981,
u'associates': 12255,
u'bachelors': 70885,
u'masters': 53216,
u'doctorate': 9940,
u'other': 5722,
u'secondary': 51591
}
}
self.assertEnrollmentResponseData(self.course, data, 'education')
def test_enrollment_gender(self):
data = {
u'genders': {
u'm': 133240,
u'o': 423,
u'f': 77495
}
}
self.assertEnrollmentResponseData(self.course, data, 'gender')
analyticsclient/tests/test_status.py
from unittest import TestCase import json
from analyticsclient.status import Status import httpretty
from analyticsclient.tests import InMemoryClient
from analyticsclient.tests import ClientTestCase
class StatusTest(TestCase):
def setUp(self):
self.client = InMemoryClient()
self.status = Status(self.client)
class StatusTests(ClientTestCase):
@httpretty.activate
def test_alive(self): def test_alive(self):
self.assertEquals(self.status.alive, False) """
Alive status should be True if server responds with HTTP 200, otherwise False.
"""
# Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('status'))
self.assertTrue(self.client.status.alive)
self.client.resources['status'] = '' # "Kill" the server (assuming there is nothing running at API_URL)
self.assertEquals(self.status.alive, True) httpretty.reset()
self.assertFalse(self.client.status.alive)
@httpretty.activate
def test_authenticated(self): def test_authenticated(self):
self.assertEquals(self.status.authenticated, False) """
Authenticated status should be True if client is authenticated, otherwise False.
"""
# Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('authenticated'))
self.assertTrue(self.client.status.authenticated)
self.client.resources['authenticated'] = '' # Non-authenticated user
self.assertEquals(self.status.authenticated, True) httpretty.register_uri(httpretty.GET, self.get_api_url('authenticated'), status=401)
self.assertFalse(self.client.status.authenticated)
@httpretty.activate
def test_healthy(self): def test_healthy(self):
self.client.resources['health'] = { """
'overall_status': 'OK', Healthy status should be True if server is alive and can respond to requests, otherwise False.
'detailed_status': { """
'database_connection': 'OK'
}
}
self.assertEquals(self.status.healthy, True) # Unresponsive server
self.assertFalse(self.client.status.healthy)
def test_not_healthy(self): body = {
self.client.resources['health'] = { 'overall_status': 'OK',
'overall_status': 'UNAVAILABLE',
'detailed_status': { 'detailed_status': {
'database_connection': 'UNAVAILABLE' 'database_connection': 'OK'
} }
} }
self.assertEquals(self.status.healthy, False) # Normal behavior
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertTrue(self.client.status.healthy)
def test_invalid_health_value(self): # Sick server
self.client.resources['health'] = {} body['overall_status'] = 'BAD'
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertFalse(self.client.status.healthy)
self.assertEquals(self.status.healthy, False) # Odd response
del body['overall_status']
httpretty.register_uri(httpretty.GET, self.get_api_url('health'), body=json.dumps(body))
self.assertFalse(self.client.status.healthy)
requirements.txt
restnavigator==0.2.0 requests==2.2.0
# Testing # Testing
coverage==3.7.1 coverage==3.7.1
......
setup.py
...@@ -8,7 +8,7 @@ setup( ...@@ -8,7 +8,7 @@ setup(
description='Client used to access edX analytics data warehouse', description='Client used to access edX analytics data warehouse',
long_description=open('README.rst').read(), long_description=open('README.rst').read(),
install_requires=[ install_requires=[
"restnavigator==0.2.0", "requests==2.2.0",
], ],
tests_require=[ tests_require=[
"coverage==3.7.1", "coverage==3.7.1",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment