Skip to content

E
edx-notes-api
Commit cb07653c by Oleg Marshev
Options

Add model.

parent 5ed1b732
Showing with 679 additions and 225 deletions
.travis.yml
language: python
python: 2.7
services:
- elasticsearch
sudo: false
install:
- scripts/travis/install.sh
- scripts/travis/setup.sh
- pip install -r requirements/test.txt
- git fetch origin master:refs/remotes/origin/master # https://github.com/edx/diff-cover#troubleshooting
- pip install coveralls
......@@ -14,3 +14,8 @@ script:
after_success:
- coveralls
env:
- ESVER=-
- ESVER=0.90.11
- ESVER=1.4.2
Makefile
......@@ -3,9 +3,16 @@ PACKAGES = notesserver notesapi
validate: test.requirements test coverage
ifeq ($(ESVER),-)
test_settings = notesserver.settings.test_es_disabled
else
test_settings = notesserver.settings.test
endif
test: clean
./manage.py test --settings=notesserver.settings.test --with-coverage --with-ignore-docstrings \
./manage.py test --settings=$(test_settings) --with-coverage --with-ignore-docstrings \
--exclude-dir=notesserver/settings --cover-inclusive --cover-branches \
--ignore-files=search_indexes.py --ignore-files=highlight.py\
--cover-html --cover-html-dir=build/coverage/html/ \
--cover-xml --cover-xml-file=build/coverage/coverage.xml \
$(foreach package,$(PACKAGES),--cover-package=$(package)) \
......@@ -34,12 +41,15 @@ diff-quality:
coverage: diff-coverage diff-quality
create-index:
python manage.py create_index
python manage.py rebuild_index
requirements:
pip install -q -r requirements/base.txt --exists-action=w
test.requirements: requirements
pip install -q -r requirements/test.txt --exists-action=w
@# unicode QUERY_PARAMS are being improperly decoded in test client
@# remove after https://github.com/tomchristie/django-rest-framework/issues/1891 is fixed
pip install -q -e git+https://github.com/tymofij/django-rest-framework.git@bugfix/test-unicode-query-params#egg=djangorestframework
develop: test.requirements
README.rst
......@@ -11,13 +11,12 @@ Overview
--------
The edX Notes API is designed to be compatible with the
`Annotator <http://annotatorjs.org/>`__.
`Annotator <http://annotatorjs.org/>`__. Can be run with up to date ElasticSearch or legacy 0.90.x.
Getting Started
---------------
1. You'll need a recent version `ElasticSearch <http://elasticsearch.org>`__ (>=1.0.0)
installed.
1. You'll need an `ElasticSearch <http://elasticsearch.org>`__ installed.
2. Install the requirements:
......@@ -37,6 +36,19 @@ installed.
$ make run
Configuration:
--------------
``CLIENT_ID`` - OAuth2 Client ID, which is to be found in ``aud`` field of IDTokens which authorize users
``CLIENT_SECRET`` - secret with which IDTokens should be encoded
``ES_DISABLED`` - set to True when you need to run the service without ElasticSearch support.
e.g if it became corrupted and you're rebuilding the index, while still serving users
through MySQL
``HAYSTACK_CONNECTIONS['default']['url']`` - Your ElasticSearch URL
Running Tests
-------------
......
notesapi/management/commands/create_index.py deleted 100644 → 0
from django.core.management.base import BaseCommand
from annotator.annotation import Annotation
class Command(BaseCommand):
help = 'Creates the mapping in the index.'
def handle(self, *args, **options):
Annotation.create_all()
notesapi/v1/models.py 0 → 100644
import json
from django.db import models
from django.core.exceptions import ValidationError
class Note(models.Model):
"""
Annotation model.
"""
user_id = models.CharField(max_length=255, db_index=True, help_text="Anonymized user id, not course specific")
course_id = models.CharField(max_length=255, db_index=True)
usage_id = models.CharField(max_length=255, help_text="ID of XBlock where the text comes from")
quote = models.TextField(default="")
text = models.TextField(default="", help_text="Student's thoughts on the quote")
ranges = models.TextField(help_text="JSON, describes position of quote in the source text")
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
@classmethod
def create(cls, note_dict):
"""
Create the note object.
"""
if not isinstance(note_dict, dict):
raise ValidationError('Note must be a dictionary.')
if len(note_dict) == 0:
raise ValidationError('Note must have a body.')
ranges = note_dict.get('ranges', list())
if len(ranges) < 1:
raise ValidationError('Note must contain at least one range.')
note_dict['ranges'] = json.dumps(ranges)
note_dict['user_id'] = note_dict.pop('user', None)
return cls(**note_dict)
def as_dict(self):
"""
Returns the note object as a dictionary.
"""
created = self.created.isoformat() if self.created else None
updated = self.updated.isoformat() if self.updated else None
return {
'id': str(self.pk),
'user': self.user_id,
'course_id': self.course_id,
'usage_id': self.usage_id,
'text': self.text,
'quote': self.quote,
'ranges': json.loads(self.ranges),
'created': created,
'updated': updated,
}
notesapi/v1/permissions.py
......@@ -40,21 +40,26 @@ class HasAccessToken(BasePermission):
auth_user = data['sub']
if data['aud'] != settings.CLIENT_ID:
raise TokenWrongIssuer
user_found = False
for request_field in ('GET', 'POST', 'DATA'):
if 'user' in getattr(request, request_field):
req_user = getattr(request, request_field)['user']
if req_user == auth_user:
return True
user_found = True
# but we do not break or return here,
# because `user` may be present in more than one field (GET, POST)
# and we must make sure that all of them are correct
else:
logger.debug("Token user {auth_user} did not match {field} user {req_user}".format(
auth_user=auth_user, field=request_field, req_user=req_user
))
logger.debug("Token user %s did not match %s user %s", auth_user, request_field, req_user)
return False
logger.info("No user was present to compare in GET, POST or DATA")
if user_found:
return True
else:
logger.info("No user was present to compare in GET, POST or DATA")
except jwt.ExpiredSignature:
logger.debug("Token was expired: {}".format(token))
logger.debug("Token was expired: %s", token)
except jwt.DecodeError:
logger.debug("Could not decode token {}".format(token))
logger.debug("Could not decode token %s", token)
except TokenWrongIssuer:
logger.debug("Token has wrong issuer {}".format(token))
return False
\ No newline at end of file
logger.debug("Token has wrong issuer %s", token)
return False
notesapi/v1/search_indexes.py 0 → 100644
from haystack import indexes
from .models import Note
class NoteIndex(indexes.SearchIndex, indexes.Indexable):
user = indexes.CharField(model_attr='user_id', indexed=False)
course_id = indexes.CharField(model_attr='course_id', indexed=False)
usage_id = indexes.CharField(model_attr='usage_id', indexed=False)
quote = indexes.CharField(model_attr='quote')
text = indexes.CharField(document=True, model_attr='text')
ranges = indexes.CharField(model_attr='ranges', indexed=False)
created = indexes.DateTimeField(model_attr='created')
updated = indexes.DateTimeField(model_attr='updated')
def get_model(self):
return Note
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.all()
notesapi/v1/tests/test_models.py 0 → 100644
from unittest import TestCase
from notesapi.v1.models import Note
from django.core.exceptions import ValidationError
class NoteTest(TestCase):
def setUp(self):
self.note_dict = {
"user": u"test_user_id",
"usage_id": u"i4x://org/course/html/52aa9816425a4ce98a07625b8cb70811",
"course_id": u"org/course/run",
"text": u"test note text",
"quote": u"test note quote",
"ranges": [
{
"start": u"/p[1]",
"end": u"/p[1]",
"startOffset": 0,
"endOffset": 10,
}
],
}
def test_create_valid_note(self):
note = Note.create(self.note_dict.copy())
note.save()
result_note = note.as_dict()
del result_note['id']
del result_note['created']
del result_note['updated']
self.assertEqual(result_note, self.note_dict)
def test_create_invalid_note(self):
note = Note()
for empty_type in (None, '', []):
with self.assertRaises(ValidationError):
note.create(empty_type)
def test_must_have_fields_create(self):
for field in ['user', 'usage_id', 'course_id', 'ranges']:
payload = self.note_dict.copy()
payload.pop(field)
with self.assertRaises(ValidationError):
note = Note.create(payload)
note.full_clean()
notesapi/v1/urls.py
from django.conf.urls import patterns, url
from django.views.generic import RedirectView
from django.core.urlresolvers import reverse_lazy
from notesapi.v1.views import AnnotationListView, AnnotationDetailView, AnnotationSearchView
urlpatterns = patterns(
'',
url(r'^annotations/$', AnnotationListView.as_view(), name='annotations'),
url(r'^annotations/(?P<annotation_id>[a-zA-Z0-9_-]+)/?$', AnnotationDetailView.as_view(), name='annotations_detail'),
url(
r'^annotations/(?P<annotation_id>[a-zA-Z0-9_-]+)/?$',
AnnotationDetailView.as_view(),
name='annotations_detail'
),
url(r'^search/$', AnnotationSearchView.as_view(), name='annotations_search'),
)
notesapi/v1/views.py
import logging
import json
from django.conf import settings
from django.core.urlresolvers import reverse
from django.core.exceptions import ValidationError
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from annotator.annotation import Annotation
from notesapi.v1.models import Note
if not settings.ES_DISABLED:
from notesserver.highlight import SearchQuerySet
CREATE_FILTER_FIELDS = ('updated', 'created', 'consumer', 'id')
UPDATE_FILTER_FIELDS = ('updated', 'created', 'user', 'consumer')
log = logging.getLogger(__name__)
class AnnotationSearchView(APIView):
"""
Search annotations.
"""
def get(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Search annotations.
Search annotations in most appropriate storage
"""
# search in DB when ES is not available or there is no need to bother it
if settings.ES_DISABLED or 'text' not in self.request.QUERY_PARAMS.dict():
results = self.get_from_db(*args, **kwargs)
else:
results = self.get_from_es(*args, **kwargs)
return Response({'total': len(results), 'rows': results})
This method supports the limit and offset query parameters for paging
through results.
def get_from_db(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Search annotations in database
"""
params = self.request.QUERY_PARAMS.dict()
query = Note.objects.filter(
**{f: v for (f, v) in params.items() if f in ('course_id', 'usage_id')}
).order_by('-updated')
if 'offset' in params:
kwargs['offset'] = _convert_to_int(params.pop('offset'))
if 'limit' in params and _convert_to_int(params['limit']) is not None:
kwargs['limit'] = _convert_to_int(params.pop('limit'))
elif 'limit' in params and _convert_to_int(params['limit']) is None: # bad value
params.pop('limit')
kwargs['limit'] = settings.RESULTS_DEFAULT_SIZE
else:
# default
kwargs['limit'] = settings.RESULTS_DEFAULT_SIZE
if 'user' in params:
query = query.filter(user_id=params['user'])
# All remaining parameters are considered searched fields.
kwargs['query'] = params
if 'text' in params:
query = query.filter(text__icontains=params['text'])
results = Annotation.search(**kwargs)
total = Annotation.count(**kwargs)
return [note.as_dict() for note in query]
return Response({'total': total, 'rows': results})
def get_from_es(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Search annotations in ElasticSearch
"""
params = self.request.QUERY_PARAMS.dict()
query = SearchQuerySet().models(Note).filter(
**{f: v for (f, v) in params.items() if f in ('user', 'course_id', 'usage_id', 'text')}
).order_by('-updated')
if params.get('highlight'):
tag = params.get('highlight_tag', 'em')
klass = params.get('highlight_class')
opts = {
'pre_tags': ['<{tag}{klass_str}>'.format(
tag=tag,
klass_str=' class="{}"'.format(klass) if klass else ''
)],
'post_tags': ['</{tag}>'.format(tag=tag)],
}
query = query.highlight(**opts)
results = []
for item in query:
note_dict = item.get_stored_fields()
note_dict['ranges'] = json.loads(item.ranges)
note_dict['id'] = str(item.pk)
if item.highlighted:
note_dict['text'] = item.highlighted[0]
results.append(note_dict)
return results
class AnnotationListView(APIView):
......@@ -55,11 +91,14 @@ class AnnotationListView(APIView):
"""
Get a list of all annotations.
"""
self.kwargs['query'] = self.request.QUERY_PARAMS.dict()
params = self.request.QUERY_PARAMS.dict()
if 'course_id' not in params:
return Response(status=status.HTTP_400_BAD_REQUEST)
annotations = Annotation.search(**kwargs)
results = Note.objects.filter(course_id=params['course_id'], user_id=params['user']).order_by('-updated')
return Response(annotations)
return Response([result.as_dict() for result in results])
def post(self, *args, **kwargs): # pylint: disable=unused-argument
"""
......@@ -70,17 +109,18 @@ class AnnotationListView(APIView):
if 'id' in self.request.DATA:
return Response(status=status.HTTP_400_BAD_REQUEST)
filtered_payload = _filter_input(self.request.DATA, CREATE_FILTER_FIELDS)
if len(filtered_payload) == 0:
try:
note = Note.create(self.request.DATA)
note.full_clean()
except ValidationError as error:
log.debug(error, exc_info=True)
return Response(status=status.HTTP_400_BAD_REQUEST)
annotation = Annotation(filtered_payload)
annotation.save(refresh=True)
note.save()
location = reverse('api:v1:annotations_detail', kwargs={'annotation_id': annotation['id']})
location = reverse('api:v1:annotations_detail', kwargs={'annotation_id': note.id})
return Response(annotation, status=status.HTTP_201_CREATED, headers={'Location': location})
return Response(note.as_dict(), status=status.HTTP_201_CREATED, headers={'Location': location})
class AnnotationDetailView(APIView):
......@@ -94,66 +134,49 @@ class AnnotationDetailView(APIView):
"""
Get an existing annotation.
"""
annotation_id = self.kwargs.get('annotation_id')
annotation = Annotation.fetch(annotation_id)
note_id = self.kwargs.get('annotation_id')
if not annotation:
return Response(annotation, status=status.HTTP_404_NOT_FOUND)
try:
note = Note.objects.get(id=note_id)
except Note.DoesNotExist:
return Response('Annotation not found!', status=status.HTTP_404_NOT_FOUND)
return Response(annotation)
return Response(note.as_dict())
def put(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Update an existing annotation.
"""
annotation_id = self.kwargs.get('annotation_id')
annotation = Annotation.fetch(annotation_id)
note_id = self.kwargs.get('annotation_id')
if not annotation:
try:
note = Note.objects.get(id=note_id)
except Note.DoesNotExist:
return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND)
if self.request.DATA is not None:
updated = _filter_input(self.request.DATA, UPDATE_FILTER_FIELDS)
updated['id'] = annotation_id # use id from URL, regardless of what arrives in JSON payload.
annotation.update(updated)
try:
note.text = self.request.data['text']
note.full_clean()
except KeyError as error:
log.debug(error, exc_info=True)
return Response(status=status.HTTP_400_BAD_REQUEST)
refresh = self.kwargs.get('refresh') != 'false'
annotation.save(refresh=refresh)
note.save()
return Response(annotation)
return Response(note.as_dict())
def delete(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Delete an annotation.
"""
annotation_id = self.kwargs.get('annotation_id')
annotation = Annotation.fetch(annotation_id)
note_id = self.kwargs.get('annotation_id')
if not annotation:
return Response('Annotation not found! No delete performed.', status=status.HTTP_404_NOT_FOUND)
try:
note = Note.objects.get(id=note_id)
except Note.DoesNotExist:
return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND)
annotation.delete()
note.delete()
# Annotation deleted successfully.
return Response(status=status.HTTP_204_NO_CONTENT)
def _filter_input(annotation, fields):
"""
Pop given fields from annotation.
"""
for field in fields:
annotation.pop(field, None)
return annotation
def _convert_to_int(value, default=None):
"""
Convert given value to int.
"""
try:
return int(value or default)
except ValueError:
return default
notesserver/highlight.py 0 → 100644
"""
django-haystack does not support passing additional highlighting parameters
to backends, so we use our subclassed SearchQuerySet which does,
and subclassed ElasticsearchSearchBackend which passes them to ES
"""
import haystack
from haystack.backends.elasticsearch_backend import (
ElasticsearchSearchEngine as OrigElasticsearchSearchEngine,
ElasticsearchSearchQuery as OrigElasticsearchSearchQuery,
ElasticsearchSearchBackend as OrigElasticsearchSearchBackend)
from haystack.query import SearchQuerySet as OrigSearchQuerySet
class SearchQuerySet(OrigSearchQuerySet):
def highlight(self, **kwargs):
"""Adds highlighting to the results."""
clone = self._clone()
clone.query.add_highlight(**kwargs)
return clone
class ElasticsearchSearchQuery(OrigElasticsearchSearchQuery):
def add_highlight(self, **kwargs):
"""Adds highlighting to the search results."""
self.highlight = kwargs or True
class ElasticsearchSearchBackend(OrigElasticsearchSearchBackend):
"""
Subclassed backend that lets user modify highlighting options
"""
def build_search_kwargs(self, *args, **kwargs):
res = super(ElasticsearchSearchBackend, self).build_search_kwargs(*args, **kwargs)
index = haystack.connections[self.connection_alias].get_unified_index()
content_field = index.document_field
highlight = kwargs.get('highlight')
if highlight:
highlight_options = {
'fields': {
content_field: {'store': 'yes'},
}
}
if isinstance(highlight, dict):
highlight_options.update(highlight)
res['highlight'] = highlight_options
return res
class ElasticsearchSearchEngine(OrigElasticsearchSearchEngine):
backend = ElasticsearchSearchBackend
query = ElasticsearchSearchQuery
notesserver/settings/common.py
......@@ -2,6 +2,8 @@ import os
import json
import sys
from notesserver.settings.logger import get_logger_config
DEBUG = False
TEMPLATE_DEBUG = False
DISABLE_TOKEN_CHECK = False
......@@ -10,14 +12,22 @@ TIME_ZONE = 'UTC'
# This value needs to be overriden in production.
SECRET_KEY = '*^owi*4%!%9=#h@app!l^$jz8(c*q297^)4&4yn^#_m#fq=z#l'
ALLOWED_HOSTS = ['localhost', '*.edx.org']
# ID and Secret used for authenticating JWT Auth Tokens
# should match those configured for `edx-notes` Client in EdX's /admin/oauth2/client/
CLIENT_ID = 'edx-notes-id'
CLIENT_SECRET = 'edx-notes-secret'
ELASTICSEARCH_URL = 'http://127.0.0.1:9200'
ELASTICSEARCH_INDEX = 'edx-notes'
ES_DISABLED = False
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'notesserver.highlight.ElasticsearchSearchEngine',
'URL': 'http://127.0.0.1:9200/',
'INDEX_NAME': 'notes_index',
},
}
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
# Number of rows to return by default in result.
RESULTS_DEFAULT_SIZE = 25
......@@ -33,73 +43,21 @@ MIDDLEWARE_CLASSES = (
'django.middleware.csrf.CsrfViewMiddleware',
)
INSTALLED_APPS = (
INSTALLED_APPS = [
'django.contrib.staticfiles',
'rest_framework',
'rest_framework_swagger',
'corsheaders',
'haystack',
'notesapi',
'notesapi.v1',
)
]
STATIC_URL = '/static/'
WSGI_APPLICATION = 'notesserver.wsgi.application'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'stream': sys.stderr,
'formatter': 'standard',
},
},
'formatters': {
'standard': {
'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',
},
},
'loggers': {
'django': {
'handlers': ['console'],
'level': 'ERROR',
'propagate': False,
},
'notesserver': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True
},
'elasticsearch.trace': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True
},
'elasticsearch': {
'handlers': ['console'],
'level': 'WARNING',
'propagate': True
},
'annotator.elasticsearch': {
'handlers': ['console'],
'level': 'WARNING',
'propagate': True
},
'urllib3': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True
},
'notesapi.v1.permissions': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True
},
},
}
LOGGING = get_logger_config()
REST_FRAMEWORK = {
'DEFAULT_PERMISSION_CLASSES': [
......
notesserver/settings/dev.py
import annotator
from annotator import es
from .common import *
from notesserver.settings.logger import get_logger_config
DEBUG = True
ELASTICSEARCH_INDEX = 'edx-notes-dev'
ES_INDEXES = {'default': 'notes_index_dev'}
###############################################################################
# Override default annotator-store elasticsearch settings.
###############################################################################
es.host = ELASTICSEARCH_URL
es.index = ELASTICSEARCH_INDEX
annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
###############################################################################
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': 'default.db',
}
}
LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG')
notesserver/settings/logger.py 0 → 100644
"""
Logging configuration
"""
import os
import platform
import sys
from logging.handlers import SysLogHandler
def get_logger_config(log_dir='/var/tmp',
logging_env="no_env",
edx_filename="edx.log",
dev_env=False,
debug=False,
local_loglevel='INFO',
service_variant='edx-notes-api'):
"""
Return the appropriate logging config dictionary. You should assign the
result of this to the LOGGING var in your settings.
If dev_env is set to true logging will not be done via local rsyslogd,
instead, application logs will be dropped in log_dir.
"edx_filename" is ignored unless dev_env is set to true since otherwise
logging is handled by rsyslogd.
"""
# Revert to INFO if an invalid string is passed in
if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
local_loglevel = 'INFO'
hostname = platform.node().split(".")[0]
syslog_format = (
"[service_variant={service_variant}]"
"[%(name)s][env:{logging_env}] %(levelname)s "
"[{hostname} %(process)d] [%(filename)s:%(lineno)d] "
"- %(message)s"
).format(service_variant=service_variant, logging_env=logging_env, hostname=hostname)
if debug:
handlers = ['console']
else:
handlers = ['local']
logger_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '%(asctime)s %(levelname)s %(process)d '
'[%(name)s] %(filename)s:%(lineno)d - %(message)s',
},
'syslog_format': {'format': syslog_format},
'raw': {'format': '%(message)s'},
},
'handlers': {
'console': {
'level': 'DEBUG' if debug else 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'standard',
'stream': sys.stdout,
},
},
'loggers': {
'django': {
'handlers': handlers,
'propagate': True,
'level': 'INFO'
},
"elasticsearch.trace": {
'handlers': handlers,
'level': 'DEBUG',
'propagate': False,
},
'': {
'handlers': handlers,
'level': 'DEBUG',
'propagate': False
},
}
}
if dev_env:
edx_file_loc = os.path.join(log_dir, edx_filename)
logger_config['handlers'].update({
'local': {
'class': 'logging.handlers.RotatingFileHandler',
'level': local_loglevel,
'formatter': 'standard',
'filename': edx_file_loc,
'maxBytes': 1024 * 1024 * 2,
'backupCount': 5,
},
})
else:
logger_config['handlers'].update({
'local': {
'level': local_loglevel,
'class': 'logging.handlers.SysLogHandler',
# Use a different address for Mac OS X
'address': '/var/run/syslog' if sys.platform == "darwin" else '/dev/log',
'formatter': 'syslog_format',
'facility': SysLogHandler.LOG_LOCAL0,
},
})
return logger_config
notesserver/settings/test.py
from annotator import es
import annotator
from .common import *
DATABASES = {
......@@ -9,15 +7,37 @@ DATABASES = {
}
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
DISABLE_TOKEN_CHECK = False
INSTALLED_APPS += ('django_nose',)
ELASTICSEARCH_INDEX = 'edx-notes-test'
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'notesserver.highlight.ElasticsearchSearchEngine',
'URL': 'http://127.0.0.1:9200/',
'INDEX_NAME': 'notes_index_test',
},
}
###############################################################################
# Override default annotator-store elasticsearch settings.
###############################################################################
es.host = ELASTICSEARCH_URL
es.index = ELASTICSEARCH_INDEX
annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
###############################################################################
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'stream': sys.stderr,
},
},
'loggers': {
'django': {
'handlers': ['console'],
'level': 'ERROR',
'propagate': False,
},
'elasticsearch.trace': {
'handlers': ['console'],
'level': 'ERROR',
'propagate': False
}
},
}
notesserver/settings/test_es_disabled.py 0 → 100644
from .test import *
ES_DISABLED = True
HAYSTACK_CONNECTIONS = {}
INSTALLED_APPS.remove('haystack')
notesserver/settings/yaml_config.py
import yaml
import annotator
from annotator import es
from .common import * # pylint: disable=unused-wildcard-import, wildcard-import
from path import path
from django.core.exceptions import ImproperlyConfigured
###############################################################################
# Explicitly declare here in case someone changes common.py.
......@@ -12,17 +12,32 @@ TEMPLATE_DEBUG = False
DISABLE_TOKEN_CHECK = False
###############################################################################
CONFIG_ROOT = os.environ.get('EDXNOTES_CONFIG_ROOT')
EDXNOTES_CONFIG_ROOT = os.environ.get('EDXNOTES_CONFIG_ROOT')
if not EDXNOTES_CONFIG_ROOT:
raise ImproperlyConfigured("EDXNOTES_CONFIG_ROOT must be defined in the environment.")
CONFIG_ROOT = path(EDXNOTES_CONFIG_ROOT)
with open(CONFIG_ROOT / "edx-notes-api.yml") as yaml_file:
config_from_yaml = yaml.load(yaml_file)
vars().update(config_from_yaml)
###############################################################################
# Override default annotator-store elasticsearch settings.
###############################################################################
es.host = ELASTICSEARCH_URL
es.index = ELASTICSEARCH_INDEX
annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
###############################################################################
#
# Support environment overrides for migrations
DB_OVERRIDES = dict(
PASSWORD=environ.get('DB_MIGRATION_PASS', DATABASES['default']['PASSWORD']),
ENGINE=environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
USER=environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
NAME=environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
HOST=environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
PORT=environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
)
for override, value in DB_OVERRIDES.iteritems():
DATABASES['default'][override] = value
if ES_DISABLED:
HAYSTACK_CONNECTIONS = {}
INSTALLED_APPS.remove('haystack')
notesserver/test_views.py
import datetime
import base64
from unittest import skipIf
from mock import patch, Mock
from django.conf import settings
from django.core.urlresolvers import reverse
from rest_framework.test import APITestCase
from elasticsearch.exceptions import TransportError
class OperationalEndpointsTest(APITestCase):
"""
Tests for operational endpoints.
......@@ -17,16 +19,27 @@ class OperationalEndpointsTest(APITestCase):
self.assertEquals(response.status_code, 200)
self.assertEquals(response.data, {"OK": True})
@patch('annotator.elasticsearch.ElasticSearch.conn')
def test_heartbeat_failure(self, mocked_conn):
@skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
@patch('notesserver.views.get_es')
def test_heartbeat_failure_es(self, mocked_get_es):
"""
Elasticsearch is not reachable.
"""
mocked_conn.ping.return_value = False
mocked_get_es.return_value.ping.return_value = False
response = self.client.get(reverse('heartbeat'))
self.assertEquals(response.status_code, 500)
self.assertEquals(response.data, {"OK": False, "check": "es"})
@patch("django.db.backends.utils.CursorWrapper")
def test_heartbeat_failure_db(self, mocked_cursor_wrapper):
"""
Database is not reachable.
"""
mocked_cursor_wrapper.side_effect = Exception
response = self.client.get(reverse('heartbeat'))
self.assertEquals(response.status_code, 500)
self.assertEquals(response.data, {"OK": False, "check": "db"})
def test_root(self):
"""
Test root endpoint.
......@@ -41,41 +54,67 @@ class OperationalEndpointsTest(APITestCase):
}
)
class OperationalAuthEndpointsTest(APITestCase):
"""
Tests for operational authenticated endpoints.
"""
def test_selftest_status(self):
"""
Test status on authorization success.
Test status success.
"""
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 200)
@skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
@patch('notesserver.views.datetime', datetime=Mock(wraps=datetime.datetime))
@patch('annotator.elasticsearch.ElasticSearch.conn')
def test_selftest_data(self, mocked_conn, mocked_datetime):
@patch('notesserver.views.get_es')
def test_selftest_data(self, mocked_get_es, mocked_datetime):
"""
Test returned data on success.
"""
mocked_datetime.datetime.now.return_value = datetime.datetime(2014, 12, 11)
mocked_conn.info.return_value = {}
mocked_get_es.return_value.info.return_value = {}
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 200)
self.assertEquals(
response.data,
{
"es": {},
"db": "OK",
"time_elapsed": 0.0
}
)
@patch('django.conf.settings.ES_DISABLED', True)
@patch('notesserver.views.datetime', datetime=Mock(wraps=datetime.datetime))
def test_selftest_data_es_disabled(self, mocked_datetime):
"""
Test returned data on success.
"""
mocked_datetime.datetime.now.return_value = datetime.datetime(2014, 12, 11)
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 200)
self.assertEquals(
response.data,
{
"db": "OK",
"time_elapsed": 0.0
}
)
@patch('annotator.elasticsearch.ElasticSearch.conn')
def test_selftest_failure(self, mocked_conn):
@skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
@patch('notesserver.views.get_es')
def test_selftest_failure_es(self, mocked_get_es):
"""
Elasticsearch is not reachable on selftest.
"""
mocked_conn.info.side_effect = TransportError()
mocked_get_es.return_value.info.side_effect = TransportError()
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 500)
self.assertIn('es_error', response.data)
@patch("django.db.backends.utils.CursorWrapper")
def test_selftest_failure_db(self, mocked_cursor_wrapper):
"""
Database is not reachable on selftest.
"""
mocked_cursor_wrapper.side_effect = Exception
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 500)
self.assertIn('db_error', response.data)
notesserver/views.py
import traceback
import datetime
from django.db import connection
from django.conf import settings
from rest_framework import status
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.decorators import api_view, permission_classes, authentication_classes
from rest_framework.decorators import api_view, permission_classes
from elasticsearch.exceptions import TransportError
from annotator import es
if not settings.ES_DISABLED:
from haystack import connections
def get_es():
return connections['default'].get_backend().conn
@api_view(['GET'])
......@@ -26,13 +33,18 @@ def root(request): # pylint: disable=unused-argument
@permission_classes([AllowAny])
def heartbeat(request): # pylint: disable=unused-argument
"""
ElasticSearch is reachable and ready to handle requests.
ElasticSearch and database are reachable and ready to handle requests.
"""
if es.conn.ping():
return Response({"OK": True})
else:
try:
db_status()
except Exception:
return Response({"OK": False, "check": "db"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if not settings.ES_DISABLED and not get_es().ping():
return Response({"OK": False, "check": "es"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return Response({"OK": True})
@api_view(['GET'])
@permission_classes([AllowAny])
......@@ -41,18 +53,43 @@ def selftest(request): # pylint: disable=unused-argument
Manual test endpoint.
"""
start = datetime.datetime.now()
if not settings.ES_DISABLED:
try:
es_status = get_es().info()
except TransportError:
return Response(
{"es_error": traceback.format_exc()},
status=status.HTTP_500_INTERNAL_SERVER_ERROR
)
try:
es_status = es.conn.info()
except TransportError:
db_status()
database = "OK"
except Exception:
return Response(
{"es_error": traceback.format_exc()},
{"db_error": traceback.format_exc()},
status=status.HTTP_500_INTERNAL_SERVER_ERROR
)
end = datetime.datetime.now()
delta = end - start
return Response({
"es": es_status,
response = {
"db": database,
"time_elapsed": int(delta.total_seconds() * 1000) # In milliseconds.
})
}
if not settings.ES_DISABLED:
response['es'] = es_status
return Response(response)
def db_status():
"""
Return database status.
"""
with connection.cursor() as cursor:
cursor.execute("SELECT 1")
cursor.fetchone()
requirements/base.txt
Django==1.7.1
requests==2.4.3
djangorestframework==2.4.4
djangorestframework==3.0.2
django-rest-swagger==0.2.0
elasticsearch==1.2.0
annotator==0.12.0
django-haystack==2.3.1
elasticsearch==0.4.5 # only 0.4 works for ES 0.90
django-cors-headers==0.13
PyJWT==0.3.0
MySQL-python==1.2.5 # GPL License
gunicorn==19.1.1 # MIT
path.py==3.0.1
python-dateutil==2.4.0
scripts/travis/install.sh 0 → 100755
#!/bin/bash
# Installs Elasticsearch
#
# Requires ESVER environment variable to be set.
set -e
if [[ $ESVER == "-" ]];
then
exit 0
fi
echo "Installing ElasticSearch $ESVER" >&2
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ESVER.tar.gz
tar xzvf elasticsearch-$ESVER.tar.gz
scripts/travis/setup.sh 0 → 100755
#!/bin/bash
# Runs Elasticsearch
#
# Requires ESVER environment variable to be set.
# cwd is the git repository root.
set -e
if [[ $ESVER == "-" ]];
then
exit 0
fi
echo "Starting ElasticSearch $ESVER" >&2
pushd elasticsearch-$ESVER
# Elasticsearch 0.90 daemonizes automatically, but 1.0+ requires
# a -d argument.
if [[ $ESVER == 0* ]];
then
./bin/elasticsearch
else
echo "launching with -d option." >&2
./bin/elasticsearch -d
pip install elasticsearch==1.3.0
fi
popd
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment