Skip to content

E
edx-analytics-data-api
Commit 0e46ca30 by Daniel Friedman
Options

Add commands for creating/deleting es indices

parent b857f029
Showing with 124 additions and 74 deletions
analytics_data_api/management/commands/create_elasticsearch_learners_indices.py 0 → 100644
from elasticsearch import Elasticsearch
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from analytics_data_api.management.utils import elasticsearch_settings_defined
class Command(BaseCommand):
help = 'Creates Elasticsearch indices used by the Analytics Data API.'
def handle(self, *args, **options):
if not elasticsearch_settings_defined():
raise CommandError(
'You must define settings.ELASTICSEARCH_LEARNERS_HOST, '
'settings.ELASTICSEARCH_LEARNERS_INDEX, and settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX'
)
es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST])
if es.indices.exists(settings.ELASTICSEARCH_LEARNERS_INDEX):
self.stderr.write('"{}" index already exists.'.format(settings.ELASTICSEARCH_LEARNERS_INDEX))
else:
es.indices.create(
index=settings.ELASTICSEARCH_LEARNERS_INDEX,
body={
'mappings': {
'roster_entry': {
'properties': {
'name': {
'type': 'string'
},
'username': {
'type': 'string', 'index': 'not_analyzed'
},
'email': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'course_id': {
'type': 'string', 'index': 'not_analyzed'
},
'enrollment_mode': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'segments': {
'type': 'string'
},
'cohort': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'discussion_contributions': {
'type': 'integer', 'doc_values': True
},
'problems_attempted': {
'type': 'integer', 'doc_values': True
},
'problems_completed': {
'type': 'integer', 'doc_values': True
},
'problem_attempts_per_completed': {
'type': 'float', 'doc_values': True
},
'attempt_ratio_order': {
'type': 'integer', 'doc_values': True
},
'videos_viewed': {
'type': 'integer', 'doc_values': True
},
'enrollment_date': {
'type': 'date', 'doc_values': True
},
}
}
}
}
)
if es.indices.exists(settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX):
self.stderr.write('"{}" index already exists.'.format(settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX))
else:
es.indices.create(
index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
body={
'mappings': {
'marker': {
'properties': {
'date': {
'type': 'date', 'doc_values': True
},
'target_index': {
'type': 'string'
},
}
}
}
}
)
analytics_data_api/management/commands/delete_elasticsearch_learners_indices.py 0 → 100644
from elasticsearch import Elasticsearch
from django.conf import settings
from django.core.management.base import BaseCommand
class Command(BaseCommand):
help = 'Removes Elasticsearch indices used by the Analytics Data API'
def handle(self, *args, **options):
es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST])
for index in [settings.ELASTICSEARCH_LEARNERS_INDEX, settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX]:
if es.indices.exists(settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX):
es.indices.delete(index=index)
analytics_data_api/management/utils.py 0 → 100644
from django.conf import settings
def elasticsearch_settings_defined():
return all(
setting is not None for setting in (
settings.ELASTICSEARCH_LEARNERS_HOST,
settings.ELASTICSEARCH_LEARNERS_INDEX,
settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX
)
)
analytics_data_api/v0/tests/views/test_learners.py
......@@ -12,6 +12,7 @@ import pytz
from rest_framework import status
from django.conf import settings
from django.core import management
from analyticsdataserver.tests import TestCaseWithAuthentication
from analytics_data_api.constants import engagement_entity_types, engagement_events
......@@ -25,80 +26,8 @@ class LearnerAPITestMixin(object):
"""Creates the index and defines a mapping."""
super(LearnerAPITestMixin, self).setUp()
self._es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST])
for index in [settings.ELASTICSEARCH_LEARNERS_INDEX, settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX]:
# ensure the test index is deleted
def delete_index(to_delete):
if self._es.indices.exists(index=to_delete):
self._es.indices.delete(index=to_delete)
self.addCleanup(delete_index, index)
self._es.indices.create(index=index)
self._es.indices.put_mapping(
index=settings.ELASTICSEARCH_LEARNERS_INDEX,
doc_type='roster_entry',
body={
'properties': {
'name': {
'type': 'string'
},
'username': {
'type': 'string', 'index': 'not_analyzed'
},
'email': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'course_id': {
'type': 'string', 'index': 'not_analyzed'
},
'enrollment_mode': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'segments': {
'type': 'string'
},
'cohort': {
'type': 'string', 'index': 'not_analyzed', 'doc_values': True
},
'discussion_contributions': {
'type': 'integer', 'doc_values': True
},
'problems_attempted': {
'type': 'integer', 'doc_values': True
},
'problems_completed': {
'type': 'integer', 'doc_values': True
},
'problem_attempts_per_completed': {
'type': 'float', 'doc_values': True
},
'attempt_ratio_order': {
'type': 'integer', 'doc_values': True
},
'videos_viewed': {
'type': 'integer', 'doc_values': True
},
'enrollment_date': {
'type': 'date', 'doc_values': True
},
}
}
)
self._es.indices.put_mapping(
index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
doc_type='marker',
body={
'properties': {
'date': {
'type': 'date', 'doc_values': True
},
'target_index': {
'type': 'string'
},
}
}
)
management.call_command('create_elasticsearch_learners_indices')
self.addCleanup(lambda: management.call_command('delete_elasticsearch_learners_indices'))
def _create_learner(
self,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment