Commit be980837 by Gabe Mulley

Initialize django project

This first pass includes:

* A django app for serving the API
* A django server to host the app
* A generic client that can be used to access the API from server side python code

Change-Id: Idd7e1a8e370e6fe216ec7413c26de66b8d51ff38
parent e78af6f3
[report]
# Regexes for lines to exclude from consideration
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
raise NotImplementedError
\ No newline at end of file
......@@ -52,3 +52,6 @@ coverage.xml
# Sphinx documentation
docs/_build/
# Sqlite Database
*.db
[pep8]
ignore=E501
max_line_length=119
exclude=settings
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=''
# Profiled execution.
profile=no
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS, migrations, settings
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
[MESSAGES CONTROL]
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once).
disable=
# Never going to use these
# I0011: Locally disabling W0232
# W0141: Used builtin function 'map'
# W0142: Used * or ** magic
# R0921: Abstract class not referenced
# R0922: Abstract class is only referenced 1 times
I0011,W0141,W0142,R0921,R0922,
# Django makes classes that trigger these
# W0232: Class has no __init__ method
W0232,
# Might use these when the code is in better shape
# C0302: Too many lines in module
# R0201: Method could be a function
# R0901: Too many ancestors
# R0902: Too many instance attributes
# R0903: Too few public methods (1/2)
# R0904: Too many public methods
# R0911: Too many return statements
# R0912: Too many branches
# R0913: Too many arguments
# R0914: Too many local variables
C0302,R0201,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,
# W0511: TODOs etc
W0511,
# E1103: maybe no member
E1103,
# C0111: missing docstring (handled by pep257)
C0111
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html
output-format=text
# Include message's id in output
include-ids=yes
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Add a comment according to your evaluation note. This is used by the global
# evaluation report (RP0004).
comment=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
ignored-classes=SQLObject
# When zope mode is activated, add a predefined set of Zope acquired attributes
# to generated-members.
zope=no
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
generated-members=
REQUEST,
acl_users,
aq_parent,
objects,
DoesNotExist,
can_read,
can_write,
get_url,
size,
content,
status_code,
# For factory_boy factories
create
[BASIC]
# Required attributes for module, separated by a comma
required-attributes=
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input
# Regular expression which should only match correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression which should only match correct module level names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$
# Regular expression which should only match correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Regular expression which should only match correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct method names
method-rgx=([a-z_][a-z0-9_]{2,60}|setUp|set[Uu]pClass|tearDown|tear[Dd]ownClass|assert[A-Z]\w*)$
# Regular expression which should only match correct instance attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct list comprehension /
# generator expression variable names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Regular expression which should only match functions or classes name which do
# not require a docstring
no-docstring-rgx=__.*__|test_.*|setUp|tearDown
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=120
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the beginning of the name of dummy variables
# (i.e. not used).
dummy-variables-rgx=_|dummy|unused|.*_unused
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branchs=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
[CLASSES]
# List of interface methods to ignore, separated by a comma. This is used for
# instance to not check methods defines in Zope's Interface base class.
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
export SECRET_KEY=test
export DATABASE_NAME=test
export DATABASE_USER=test
export DATABASE_PASSWORD=test
ROOT = $(shell echo "$$PWD")
COVERAGE = $(ROOT)/build/coverage
PACKAGES = analyticsdata analyticsdataclient
validate: test.requirements test quality
test.requirements:
pip install -q -r requirements/test.txt
clean:
find . -name '*.pyc' -delete
coverage erase
test.app: clean
. ./.test_env && ./manage.py test --settings=analyticsdataserver.settings.test \
--with-coverage --cover-inclusive --cover-branches \
--cover-html --cover-html-dir=$(COVERAGE)/html/ \
--cover-xml --cover-xml-file=$(COVERAGE)/coverage.xml \
--cover-package=analyticsdata \
analyticsdata/
test.client:
nosetests --with-coverage --cover-inclusive --cover-branches \
--cover-html --cover-html-dir=$(COVERAGE)/html/ \
--cover-xml --cover-xml-file=$(COVERAGE)/coverage.xml \
--cover-package=analyticsdataclient \
analyticsdataclient/
test: test.app test.client
diff.report:
diff-cover $(COVERAGE)/coverage.xml --html-report $(COVERAGE)/diff_cover.html
diff-quality --violations=pep8 --html-report $(COVERAGE)/diff_quality_pep8.html
diff-quality --violations=pylint --html-report $(COVERAGE)/diff_quality_pylint.html
view.diff.report:
xdg-open file:///$(COVERAGE)/diff_cover.html
xdg-open file:///$(COVERAGE)/diff_quality_pep8.html
xdg-open file:///$(COVERAGE)/diff_quality_pylint.html
quality:
pep8 --config=.pep8 $(PACKAGES)
pylint --rcfile=.pylintrc $(PACKAGES)
# Ignore module level docstrings and all test files
pep257 --ignore=D100 --match='(?!test).*py' $(PACKAGES)
from contextlib import contextmanager
from django.conf import settings
from django.contrib.auth.models import User
from django.db.utils import ConnectionHandler
from django.test import TestCase
from django.test.utils import override_settings
from mock import patch
from rest_framework.authtoken.models import Token
from analyticsdata.views import handle_internal_server_error, handle_missing_resource_error
# NOTE: Full URLs are used throughout these tests to ensure that the API contract is fulfilled. The URLs should *not*
# change for versions greater than 1.0.0. Tests target a specific version of the API, additional tests should be added
# for subsequent versions if there are breaking changes introduced in those versions.
class OperationalEndpointsTest(TestCase):
def test_status(self):
response = self.client.get('/api/v0/status')
self.assertEquals(response.status_code, 200)
def test_authentication_check_failure(self):
response = self.client.get('/api/v0/authenticated')
self.assertEquals(response.status_code, 401)
def test_authentication_check_success(self):
test_user = User.objects.create_user('tester', 'test@example.com', 'testpassword')
token = Token.objects.create(user=test_user)
response = self.client.get('/api/v0/authenticated', HTTP_AUTHORIZATION='Token ' + token.key)
self.assertEquals(response.status_code, 200)
def test_health(self):
self.assert_database_health('OK')
def assert_database_health(self, status):
response = self.client.get('/api/v0/health')
self.assertEquals(
response.data,
{
'overall_status': status,
'detailed_status': {
'database_connection': status
}
}
)
self.assertEquals(response.status_code, 200)
def test_database_down(self):
databases = {
"default": {}
}
with self.override_database_connections(databases):
self.assert_database_health('UNAVAILABLE')
@staticmethod
@contextmanager
def override_database_connections(databases):
with patch('analyticsdata.views.connections', ConnectionHandler(databases)):
yield
@override_settings(ANALYTICS_DATABASE='reporting')
def test_read_setting(self):
databases = dict(settings.DATABASES)
databases['reporting'] = {}
with self.override_database_connections(databases):
self.assert_database_health('UNAVAILABLE')
# Workaround to remove a setting from django settings. It has to be used in override_settings and then deleted.
@override_settings(ANALYTICS_DATABASE='reporting')
def test_default_setting(self):
del settings.ANALYTICS_DATABASE
databases = dict(settings.DATABASES)
databases['reporting'] = {}
with self.override_database_connections(databases):
# This would normally return UNAVAILABLE, however we have deleted the settings so it will use the default
# connection which should be OK.
self.assert_database_health('OK')
class ErrorHandlingTest(TestCase):
def test_internal_server_error_handling(self):
response = handle_internal_server_error(None)
self.validate_error_response(response, 500)
def validate_error_response(self, response, status_code):
self.assertEquals(response.content, '{{"status": {0}}}'.format(status_code))
self.assertEquals(response.status_code, status_code)
self.assertEquals(response.get('Content-Type'), 'application/json; charset=utf-8')
def test_missing_resource_handling(self):
response = handle_missing_resource_error(None)
self.validate_error_response(response, 404)
from django.conf.urls import patterns, url
from rest_framework.urlpatterns import format_suffix_patterns
from analyticsdata import views
urlpatterns = patterns(
'',
url(r'^status$', views.status),
url(r'^authenticated$', views.authenticated),
url(r'^health$', views.health),
)
urlpatterns = format_suffix_patterns(urlpatterns)
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import AllowAny
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from django.conf import settings
from django.db import connections
from django.http import HttpResponse
@api_view(['GET'])
@permission_classes((AllowAny, ))
def status(_request):
"""
A very quick check to see if the application server is alive and processing requests.
Return no data, a simple 200 OK status code is sufficient to indicate that the server is alive. This endpoint is
public and does not require an authentication token to access it.
"""
return Response()
@api_view(['GET'])
def authenticated(_request):
"""
Validate provided credentials.
Return no data, a simple 200 OK status code is sufficient to indicate that the credentials are valid.
"""
return Response()
@api_view(['GET'])
@permission_classes((AllowAny, ))
def health(_request):
"""
A more comprehensive check to see if the system is fully operational.
This endpoint is public and does not require an authentication token to access it.
The returned structure contains the following fields:
- overall_status: Can be either "OK" or "UNAVAILABLE".
- detailed_status: More detailed information about the status of the system.
- database_connection: Status of the database connection. Can be either "OK" or "UNAVAILABLE".
"""
overall_status = 'OK'
db_conn_status = 'OK'
connection_name = getattr(settings, 'ANALYTICS_DATABASE', 'default')
try:
cursor = connections[connection_name].cursor()
try:
cursor.execute("SELECT 1")
cursor.fetchone()
finally:
cursor.close()
except Exception: # pylint: disable=broad-except
overall_status = 'UNAVAILABLE'
db_conn_status = 'UNAVAILABLE'
response = {
"overall_status": overall_status,
"detailed_status": {
'database_connection': db_conn_status
}
}
return Response(response)
def handle_internal_server_error(_request):
"""Notify the client that an error occurred processing the request without providing any detail."""
return _handle_error(500)
def handle_missing_resource_error(_request):
"""Notify the client that the requested resource could not be found."""
return _handle_error(404)
def _handle_error(status_code):
info = {
'status': status_code
}
renderer = JSONRenderer()
content_type = '{media}; charset={charset}'.format(media=renderer.media_type, charset=renderer.charset)
return HttpResponse(renderer.render(info), content_type=content_type, status=status_code)
import logging
import requests
import requests.exceptions
log = logging.getLogger(__name__)
class Client(object):
"""A client capable of retrieving the requested resources."""
DEFAULT_TIMEOUT = 0.1 # In seconds
DEFAULT_VERSION = 'v0'
def __init__(self, version=DEFAULT_VERSION):
"""
Initialize the Client.
Arguments:
version (str): When breaking changes are made to either the resource addresses or their returned data, this
value will change. Multiple clients can be made which adhere to each version.
"""
self.version = version
def get(self, resource, timeout=None):
"""
Retrieve the data for a resource.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: A structure consisting of simple python types (dict, list, int, str etc).
Raises: ClientError if the resource cannot be retrieved for any reason.
"""
raise NotImplementedError
def has_resource(self, resource, timeout=None):
"""
Check if a resource exists.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: True iff the resource exists.
"""
raise NotImplementedError
class RestClient(Client):
"""Retrieve resources from a remote REST API."""
DEFAULT_AUTH_TOKEN = ''
DEFAULT_BASE_URL = 'http://localhost:9090'
def __init__(self, base_url=DEFAULT_BASE_URL, auth_token=DEFAULT_AUTH_TOKEN):
"""
Initialize the RestClient.
Arguments:
base_url (str): A URL containing the scheme, netloc and port of the remote service.
auth_token (str): The token that should be used to authenticate requests made to the remote service.
"""
super(RestClient, self).__init__()
self.base_url = '{0}/api/{1}'.format(base_url, self.version)
self.auth_token = auth_token
def get(self, resource, timeout=None):
"""
Retrieve the data for a resource.
Inherited from `Client`.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: A structure consisting of simple python types (dict, list, int, str etc).
Raises: ClientError if the resource cannot be retrieved for any reason.
"""
response = self._request(resource, timeout=timeout)
try:
return response.json()
except ValueError:
message = 'Unable to decode JSON response'
log.exception(message)
raise ClientError(message)
def has_resource(self, resource, timeout=None):
"""
Check if the server responds with a 200 OK status code when the resource is requested.
Inherited from `Client`.
Arguments:
resource (str): Path in the form of slash separated strings.
timeout (float): Continue to attempt to retrieve a resource for this many seconds before giving up and
raising an error.
Returns: True iff the resource exists.
"""
try:
self._request(resource, timeout=timeout)
return True
except ClientError:
return False
def _request(self, resource, timeout=None):
if timeout is None:
timeout = self.DEFAULT_TIMEOUT
headers = {
'Accept': 'application/json',
}
if self.auth_token:
headers['Authorization'] = 'Token ' + self.auth_token
try:
response = requests.get('{0}/{1}'.format(self.base_url, resource), headers=headers, timeout=timeout)
if response.status_code != requests.codes.ok: # pylint: disable=no-member
message = 'Resource "{0}" returned status code {1}'.format(resource, response.status_code)
log.error(message)
raise ClientError(message)
return response
except requests.exceptions.RequestException:
message = 'Unable to retrieve resource'
log.exception(message)
raise ClientError('{0} "{1}"'.format(message, resource))
# TODO: Provide more detailed errors as necessary.
class ClientError(Exception):
"""An error occurred that prevented the client from performing the requested operation."""
pass
from analyticsdataclient.client import ClientError
class Status(object):
"""
Query the status of the connection between the client and the remote service.
Arguments:
client (analyticsdataclient.client.Client): The client to use to access remote resources.
"""
def __init__(self, client):
"""
Initialize the Status.
Arguments:
client (analyticsdataclient.client.Client): The client to use to access remote resources.
"""
self.client = client
@property
def alive(self):
"""
A very fast shallow check to see if the service is functioning.
Returns: True iff the remote server responds to requests.
"""
return self.client.has_resource('status')
@property
def authenticated(self):
"""
Validate the client credentials.
Returns: True iff the client is successfully authenticated.
"""
return self.client.has_resource('authenticated')
@property
def healthy(self):
"""
A slow deep health check of the remote service.
Returns: True iff the remote service is reasonably confident that further operations will succeed.
"""
try:
health = self.client.get('health')
except ClientError:
return False
try:
return health['overall_status'] == 'OK'
except KeyError:
return False
import json
from unittest import TestCase
import httpretty
from analyticsdataclient.client import RestClient, ClientError
class RestClientTest(TestCase):
BASE_URI = 'http://localhost:9091'
VERSIONED_BASE_URI = BASE_URI + '/api/v0'
TEST_ENDPOINT = 'test'
TEST_URI = VERSIONED_BASE_URI + '/' + TEST_ENDPOINT
def setUp(self):
httpretty.enable()
self.client = RestClient(base_url=self.BASE_URI)
def tearDown(self):
httpretty.disable()
def test_has_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='')
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), True)
def test_missing_resource(self):
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=404)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False)
def test_failed_authentication(self):
self.client = RestClient(base_url=self.BASE_URI, auth_token='atoken')
httpretty.register_uri(httpretty.GET, self.TEST_URI, body='', status=401)
self.assertEquals(self.client.has_resource(self.TEST_ENDPOINT), False)
self.assertEquals(httpretty.last_request().headers['Authorization'], 'Token atoken')
def test_get(self):
data = {
'foo': 'bar'
}
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data))
self.assertEquals(self.client.get(self.TEST_ENDPOINT), data)
def test_get_invalid_json(self):
data = {
'foo': 'bar'
}
httpretty.register_uri(httpretty.GET, self.TEST_URI, body=json.dumps(data)[:6])
with self.assertRaises(ClientError):
self.client.get(self.TEST_ENDPOINT)
from unittest import TestCase
from analyticsdataclient.client import Client, ClientError
from analyticsdataclient.status import Status
class StatusTest(TestCase):
def setUp(self):
self.client = InMemoryClient()
self.status = Status(self.client)
def test_alive(self):
self.assertEquals(self.status.alive, False)
self.client.resources['status'] = ''
self.assertEquals(self.status.alive, True)
def test_authenticated(self):
self.assertEquals(self.status.authenticated, False)
self.client.resources['authenticated'] = ''
self.assertEquals(self.status.authenticated, True)
def test_healthy(self):
self.client.resources['health'] = {
'overall_status': 'OK',
'detailed_status': {
'database_connection': 'OK'
}
}
self.assertEquals(self.status.healthy, True)
def test_not_healthy(self):
self.client.resources['health'] = {
'overall_status': 'UNAVAILABLE',
'detailed_status': {
'database_connection': 'UNAVAILABLE'
}
}
self.assertEquals(self.status.healthy, False)
def test_invalid_health_value(self):
self.client.resources['health'] = {}
self.assertEquals(self.status.healthy, False)
class InMemoryClient(Client):
def __init__(self):
super(InMemoryClient, self).__init__()
self.resources = {}
def has_resource(self, resource, timeout=None):
try:
self.get(resource, timeout=timeout)
return True
except ClientError:
return False
def get(self, resource, timeout=None):
try:
return self.resources[resource]
except KeyError:
raise ClientError('Unable to find requested resource')
"""Common settings and globals."""
from os.path import abspath, basename, dirname, join, normpath
from sys import stderr
########## PATH CONFIGURATION
# Absolute filesystem path to the Django project directory:
DJANGO_ROOT = dirname(dirname(abspath(__file__)))
# Absolute filesystem path to the top-level project folder:
SITE_ROOT = dirname(DJANGO_ROOT)
# Site name:
SITE_NAME = basename(DJANGO_ROOT)
########## END PATH CONFIGURATION
########## DEBUG CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = False
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG
########## END DEBUG CONFIGURATION
########## MANAGER CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (
('Your Name', 'your_email@example.com'),
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS
########## END MANAGER CONFIGURATION
########## DATABASE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.',
'NAME': '',
'USER': '',
'PASSWORD': '',
'HOST': '',
'PORT': '',
}
}
########## END DATABASE CONFIGURATION
########## GENERAL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
TIME_ZONE = 'America/New_York'
# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'
# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1
# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True
# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True
# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True
########## END GENERAL CONFIGURATION
########## MEDIA CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = normpath(join(SITE_ROOT, 'media'))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'
########## END MEDIA CONFIGURATION
########## STATIC FILE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = normpath(join(SITE_ROOT, 'assets'))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
normpath(join(SITE_ROOT, 'static')),
)
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)
########## END STATIC FILE CONFIGURATION
########## SECRET CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Note: This key should only be used for development and testing.
SECRET_KEY = r"g)rke*$-ox1yursa_l!rjnh6tn!pd+qs^8i03xb0!#50#zhb%k"
########## END SECRET CONFIGURATION
########## SITE CONFIGURATION
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []
########## END SITE CONFIGURATION
########## FIXTURE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FIXTURE_DIRS
FIXTURE_DIRS = (
normpath(join(SITE_ROOT, 'fixtures')),
)
########## END FIXTURE CONFIGURATION
########## TEMPLATE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
TEMPLATE_CONTEXT_PROCESSORS = (
'django.contrib.auth.context_processors.auth',
'django.core.context_processors.debug',
'django.core.context_processors.i18n',
'django.core.context_processors.media',
'django.core.context_processors.static',
'django.core.context_processors.tz',
'django.contrib.messages.context_processors.messages',
'django.core.context_processors.request',
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
TEMPLATE_DIRS = (
normpath(join(SITE_ROOT, 'templates')),
)
########## END TEMPLATE CONFIGURATION
########## MIDDLEWARE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE_CLASSES = (
# Default Django middleware.
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
########## END MIDDLEWARE CONFIGURATION
########## URL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = '%s.urls' % SITE_NAME
########## END URL CONFIGURATION
########## APP CONFIGURATION
DJANGO_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.admin',
)
THIRD_PARTY_APPS = (
'south',
'rest_framework',
'rest_framework.authtoken',
)
LOCAL_APPS = (
'analyticsdata',
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
########## END APP CONFIGURATION
########## LOGGING CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'stream': stderr,
},
},
'loggers': {
'django': {
'handlers': ['console'],
'level': 'ERROR',
'propagate': False,
},
'analyticsdata': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True
},
},
}
########## END LOGGING CONFIGURATION
########## WSGI CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME
########## END WSGI CONFIGURATION
########## REST FRAMEWORK CONFIGURATION
REST_FRAMEWORK = {
'DEFAULT_MODEL_SERIALIZER_CLASS':
'rest_framework.serializers.HyperlinkedModelSerializer',
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.IsAuthenticated'
],
'DEFAULT_AUTHENTICATION_CLASSES': (
# Most clients will use token authentication
'rest_framework.authentication.TokenAuthentication',
# For the browseable API
'rest_framework.authentication.SessionAuthentication',
),
}
########## END REST FRAMEWORK CONFIGURATION
########## ANALYTICS DATA API CONFIGURATION
ANALYTICS_DATABASE = 'default'
ENABLE_ADMIN_SITE = False
########## END ANALYTICS DATA API CONFIGURATION
"""Development settings and globals."""
from os.path import join, normpath
from base import *
########## DEBUG CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = True
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG
########## END DEBUG CONFIGURATION
########## EMAIL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
########## END EMAIL CONFIGURATION
########## DATABASE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': normpath(join(DJANGO_ROOT, 'default.db')),
'USER': '',
'PASSWORD': '',
'HOST': '',
'PORT': '',
}
}
########## END DATABASE CONFIGURATION
########## CACHE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#caches
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
}
}
########## END CACHE CONFIGURATION
########## TOOLBAR CONFIGURATION
# See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation
INSTALLED_APPS += (
'debug_toolbar',
)
# See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation
INTERNAL_IPS = ('127.0.0.1',)
# See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation
MIDDLEWARE_CLASSES += (
'debug_toolbar.middleware.DebugToolbarMiddleware',
)
# See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation
DEBUG_TOOLBAR_CONFIG = {
'INTERCEPT_REDIRECTS': False,
'SHOW_TEMPLATE_CONTEXT': True,
}
########## END TOOLBAR CONFIGURATION
########## ANALYTICS DATA API CONFIGURATION
ENABLE_ADMIN_SITE = True
########## END ANALYTICS DATA API CONFIGURATION
"""Production settings and globals."""
from os import environ
from base import *
# Normally you should not import ANYTHING from Django directly
# into your settings, but ImproperlyConfigured is an exception.
from django.core.exceptions import ImproperlyConfigured
def get_env_setting(setting):
"""Get the environment setting or return exception."""
try:
return environ[setting]
except KeyError:
error_msg = "Set the %s env variable" % setting
raise ImproperlyConfigured(error_msg)
########## HOST CONFIGURATION
# See: https://docs.djangoproject.com/en/1.5/releases/1.5/#allowed-hosts-required-in-production
ALLOWED_HOSTS = ['*']
########## END HOST CONFIGURATION
########## EMAIL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-host
EMAIL_HOST = environ.get('EMAIL_HOST', 'smtp.gmail.com')
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-host-password
EMAIL_HOST_PASSWORD = environ.get('EMAIL_HOST_PASSWORD', '')
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-host-user
EMAIL_HOST_USER = environ.get('EMAIL_HOST_USER', 'your_email@example.com')
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-port
EMAIL_PORT = environ.get('EMAIL_PORT', 587)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-subject-prefix
EMAIL_SUBJECT_PREFIX = '[%s] ' % SITE_NAME
# See: https://docs.djangoproject.com/en/dev/ref/settings/#email-use-tls
EMAIL_USE_TLS = True
# See: https://docs.djangoproject.com/en/dev/ref/settings/#server-email
SERVER_EMAIL = EMAIL_HOST_USER
########## END EMAIL CONFIGURATION
########## DATABASE CONFIGURATION
DATABASES = {
'default': {
'ENGINE': environ.get('DATABASE_ENGINE', 'django.db.backends.mysql'),
'NAME': get_env_setting('DATABASE_NAME'),
'USER': get_env_setting('DATABASE_USER'),
'PASSWORD': get_env_setting('DATABASE_PASSWORD'),
'HOST': environ.get('DATABASE_HOST', 'localhost'),
'PORT': environ.get('DATABASE_PORT', '3306'),
}
}
########## END DATABASE CONFIGURATION
########## CACHE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#caches
CACHES = {}
########## END CACHE CONFIGURATION
########## SECRET CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
SECRET_KEY = get_env_setting('SECRET_KEY')
########## END SECRET CONFIGURATION
from base import *
########## IN-MEMORY TEST DATABASE
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": ":memory:",
"USER": "",
"PASSWORD": "",
"HOST": "",
"PORT": "",
},
}
INSTALLED_APPS += (
'django_nose',
)
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
from django.conf import settings
from django.conf.urls import patterns, include, url
from django.contrib import admin
urlpatterns = patterns(
'',
# Support logging in to the browseable API
url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
# Support generating tokens using a POST request
url(r'^api-token-auth/', 'rest_framework.authtoken.views.obtain_auth_token'),
# Route all reports URLs to this endpoint
# TODO: come up with a better way to handle versioning, maybe a decorator?
url(r'^api/v0/', include('analyticsdata.urls')),
)
if settings.ENABLE_ADMIN_SITE:
admin.autodiscover()
urlpatterns += patterns('', url(r'^site/admin/', include(admin.site.urls)))
handler500 = 'analyticsdata.views.handle_internal_server_error' # pylint: disable=invalid-name
handler404 = 'analyticsdata.views.handle_missing_resource_error' # pylint: disable=invalid-name
import os
# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "jajaja.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "analyticsdataserver.settings.production")
# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application() # pylint: disable=invalid-name
# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)
FORMAT: 1A
HOST: http://www.analytics.edx.org
# edX Analytics API
The edX Analytics API provides access to analytical insights related to an Open edX installation.
# Data Types
## Timestamp
All timestamps are expected to be ISO 8601 formatted: `YYYY-mm-ddTHH:MM:SS.fffZ`. All references to time are UTC. Local timezones are not supported.
# Conventions
## Versioning
All URLs are prefixed with a version number. The version number will be incremented every time a change that is made which is not backwards compatible. Version 0 is the development version that can change arbitrarily without notice.
## Authentication
All requests are expected to be made over an SSL connection and include an `Authentication` header that includes a token that can be used to authenticate the request. The format is:
Authentication: Token 987ab987987c87d97e9877ff012
# Group Operations
## Status [/api/v0/status]
A very quick check to see if the application server is alive and processing requests. Returns no data, a simple 200 OK status code is sufficient to indicate that the server is alive.
This endpoint is public and does not require an authentication token to access it.
### Get Status [GET]
+ Response 200
## Health [/api/v0/health]
A more comprehensive check to see if the system is fully operational.
This endpoint is public and does not require an authentication token to access it.
The returned structure contains the following fields:
- overall_status: Can be either "OK" or "UNAVAILABLE".
- detailed_status: More detailed information about the status of the system.
- database_connection: Status of the database connection. Can be either "OK" or "UNAVAILABLE".
### Check System Health [GET]
+ Response 200 (application/json)
+ Body
{
"overall_status": "UNAVAILABLE",
"detailed_status": {
"database_connection": "UNAVAILABLE"
}
}
## Authenticated [/api/v0/authenticated]
Validates provided credentials. Returns no data, a simple 200 OK status code is sufficient to indicate that the credentials are valid.
### Check Authentication [GET]
+ Request
+ Headers
Authentication: Token 987ab987987c87d97e9877ff012
+ Response 200
# Group Course
## User Activity [/api/v0/courses/{course_id}/user_activity{?from_date,to_date,group_by}]
Counts of unique users who performed various actions of interest. A unique user is defined as a user who performed at least one action within a time interval specified by the `group_by` parameter. This time interval used for grouping results is referred to as the `unit` and can either be a week or a day.
Each data point has the following fields:
- from_date (timestamp): All data from this timestamp up to the `to_date` was considered when computing this data point.
- to_date (timestamp): All data from `from_date` up to this timestamp was considered when computing this data point. Note that data produced at exactly this time is **not** included.
- visited (integer): The number of unique users who visited the course.
- started_video (integer): The number of unique users who started watching any video in the course.
- answered_question (integer): The number of unique users who answered any capa based question in the course.
- posted_forum (integer): The number of unique users who created a new post, responded to a post, or submitted a comment on any forum in the course.
+ Parameters
+ course_id (string) ... ID of the course.
Currently accepts url encoded slash separated course key values. In the future will also accept other course identifying strings.
+ from_date (optional, timestamp) ... A time within the first unit to include in the results.
Defaults to midnight on the first day of the most recent complete unit at the UTC time the server processes the request.
+ to_date (optional, timestamp) ... A time within the unit after the last unit to include in the results.
Defaults to midnight on the day after the last day of the most recent complete unit at the UTC time the server processes the request.
+ group_by = `week` (optional, string) ... Specifies the granularity of groups returned.
Users that appear multiple times in this interval will be counted only once.
+ Values
+ `week`
+ `day`
### User Activity over Time [GET]
+ Request
+ Headers
Authentication: Token 987ab987987c87d97e9877ff012
+ Response 200 (application/json)
+ Body
{
"count": 1,
"next": null,
"previous": null,
"results": [
{
"from_date": "2014-05-17T00:00:00.000Z",
"to_date": "2014-05-24T00:00:00.000Z",
"course_id": "edX/Demo_Course/2013_T1",
"visited": 1047,
"started_video": 931,
"answered_question": 452,
"posted_forum": 318
}
]
}
# Group Problem
Data regarding student activity on a particular problem in a course.
## Answer Distribution [/api/v0/problems/{usage_id}/answer_distribution/{attempt}]
A representation of the answer distribution associated with a problem used in a course.
The representation has the following fields:
- metadata:
- id: usage_id of the problem
- attempt: "first" or "last". The attempt for which the answer distribution is computed.
- distribution: a JSON object whose top-level keys are the ids of all the *responses* in the problem and whose top-level values
are JSON objects representing the answer distribution for that *response*.
+ Parameters
+ usage_id (String) ... ID of the problem as used in a course.
+ attempt (String) ... The attempt for which the answer distribution is computed. Valid strings are "last" and "first".
+ Model (application/json)
JSON representation of problem answer distribution
+ Body
{
"metadata": {
"id": "usageid0",
"attempt": "last"
},
"distribution": {
"response_0": {
"choice_0": {'correct': true, 'count': 15},
"choice_1": {'correct': false, 'count': 52},
"choice_2": {'correct': false, 'count': 27},
"choice_3": {'correct': false, 'count': 81}
},
"response_1": {
"": {'correct': false, 'count': 53},
"choice_0": {'correct': false, 'count': 21},
"choice_1": {'correct': true, 'count': 32},
"choice_0,choice_1": {'correct': false, 'count': 25}
},
}
}
### Problem Answer Distribution [GET]
+ Request
+ Headers
Authentication: Token 987ab987987c87d97e9877ff012
+ Response 200
[Answer Distribution][]
## Attempts Distribution [/api/v0/problems/{usage_id}/attempts_distribution]
The distribution of the number of attempts on a problem, as used in a course.
The representation has, as keys, strings of integers ascending from 0.
The values associated with each key *k* are the number of students who made
exactly *k* attempts on the problem.
+ Parameters
+ usage_id (string) ... ID of the problem as used in a course.
+ Model (application/json)
+ Body
{
"0": 235,
"1": 2923,
"2": 1098,
"3": 185
}
### Problem Attempts Distribution [GET]
+ Request
+ Headers
Authentication: Token 987ab987987c87d97e9877ff012
+ Response 200
[Attempts Distribution][]
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "analyticsdataserver.settings.local")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
Django==1.4.12
django-model-utils==1.4.0
South==0.8.1
djangorestframework==2.3.5
Markdown==2.4.1
distribute>=0.6.28, <0.7
# Local development dependencies go here
-r base.txt
-r test.txt
django-debug-toolbar==0.9.4
Sphinx==1.2b1
requests==2.3.0 # nose requires this since it loads the client during test discovery
# Pro-tip: Try not to put anything here. There should be no dependency in
# production that isn't in development.
-r base.txt
gunicorn==0.17.4
MySQL-python==1.2.4
# Test dependencies go here.
coverage==3.7
nose==1.3.3
django-nose==1.2
pep8==1.5.7
pylint==1.2.1
diff-cover >= 0.2.1
mock==1.0.1
pep257==0.3.2
httpretty==0.8.0
requests==2.3.0
from setuptools import setup
setup(
name="edx-analytics-data-client",
version="0.1.0",
packages=[
'analyticsdataclient'
],
install_requires=[
"requests==2.3.0",
],
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment