Merge pull request #14 from edx/clintonb/query-preview

Added page to preview queries

course_discovery/apps/courses/models.py

@@ -146,12 +146,18 @@ class Course(object):
         Returns:
             None
         """
-        client = EdxRestApiClient(settings.ECOMMERCE_API_URL, oauth_access_token=access_token)
-
-        logger.info('Refreshing course data from %s....', settings.ECOMMERCE_API_URL)
+        cls.refresh_all_ecommerce_data(access_token)
+        cls.refresh_all_course_api_data(access_token)
 
+    @classmethod
+    def refresh_all_ecommerce_data(cls, access_token):
+        ecommerce_api_url = settings.ECOMMERCE_API_URL
+        client = EdxRestApiClient(ecommerce_api_url, oauth_access_token=access_token)
         count = None
         page = 1
+
+        logger.info('Refreshing ecommerce data from %s....', ecommerce_api_url)
+
         while page:
             response = client.courses().get(include_products=True, page=page, page_size=50)
             count = response['count']
@@ -164,9 +170,36 @@ class Course(object):
                 page = None
 
             for body in results:
-                Course(body['id'], body).save()
+                Course(body['id']).update(body)
+
+        logger.info('Retrieved %d courses from %s.', count, ecommerce_api_url)
+
+    @classmethod
+    def refresh_all_course_api_data(cls, access_token):
+        course_api_url = settings.COURSES_API_URL
+        client = EdxRestApiClient(course_api_url, oauth_access_token=access_token)
+
+        count = None
+        page = 1
+
+        logger.info('Refreshing course api data from %s....', course_api_url)
+
+        while page:
+            # TODO Update API to not require username?
+            response = client.courses().get(page=page, page_size=50, username='ecommerce_worker')
+            count = response['pagination']['count']
+            results = response['results']
+            logger.info('Retrieved %d courses...', len(results))
+
+            if response['pagination']['next']:
+                page += 1
+            else:
+                page = None
+
+            for body in results:
+                Course(body['id']).update(body)
 
-        logger.info('Retrieved %d courses.', count)
+        logger.info('Retrieved %d courses from %s.', count, course_api_url)
 
     def __init__(self, id, body=None):  # pylint: disable=redefined-builtin
         if not id:
@@ -202,3 +235,20 @@ class Course(object):
         logger.info('Indexing course %s...', self.id)
         self._es_client().index(index=self._index, doc_type=self.doc_type, id=self.id, body=self.body)
         logger.info('Finished indexing course %s.', self.id)
+
+    def update(self, body):
+        """ Updates (merges) the data in the index with the provided data.
+
+        Args:
+            body (dict): Data to be merged into the index.
+
+        Returns:
+            None
+        """
+        body = {
+            'doc': body,
+            'doc_as_upsert': True,
+        }
+        logger.info('Updating course %s...', self.id)
+        self._es_client().update(index=self._index, doc_type=self.doc_type, id=self.id, body=body)
+        logger.info('Finished updating course %s.', self.id)

course_discovery/apps/courses/tests/test_models.py

@@ -10,11 +10,12 @@ from course_discovery.apps.courses.models import Course
 from course_discovery.apps.courses.tests.factories import CourseFactory
 
 ACCESS_TOKEN = 'secret'
+COURSES_API_URL = 'https://lms.example.com/api/courses/v1'
 ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
 JSON = 'application/json'
 
 
-@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL)
+@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL, COURSES_API_URL=COURSES_API_URL)
 class CourseTests(ElasticsearchTestMixin, TestCase):
     def assert_course_attrs(self, course, attrs):
         """
@@ -30,14 +31,12 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
     @responses.activate  # pylint: disable=no-member
     def mock_refresh_all(self):
         """
-        Mock the E-Commerce API and refresh all course data.
+        Mock the external APIs and refresh all course data.
 
         Returns:
             [dict]: List of dictionaries representing course content bodies.
         """
 
-        # Mock the call to the E-Commerce API, simulating multiple pages of data
-        url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
         course_bodies = [
             {
                 'id': 'a/b/c',
@@ -57,30 +56,64 @@ class CourseTests(ElasticsearchTestMixin, TestCase):
             }
         ]
 
-        def request_callback(request):
-            # pylint: disable=redefined-builtin
-            next = None
-            count = len(course_bodies)
+        def ecommerce_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(course_bodies)
+
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'count': count,
+                    'next': next,
+                    'previous': None,
+                    'results': [data[page - 1]]
+                }
 
-            # Use the querystring to determine which page should be returned. Default to page 1.
-            # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-            qs = parse_qs(urlparse(request.path_url).query)
-            page = int(qs.get('page', [1])[0])
+                return 200, {}, json.dumps(body)
 
-            if page < count:
-                next = '{}?page={}'.format(url, page)
+            return request_callback
 
-            body = {
-                'count': count,
-                'next': next,
-                'previous': None,
-                'results': [course_bodies[page - 1]]
-            }
+        def courses_api_callback(url, data):
+            def request_callback(request):
+                # pylint: disable=redefined-builtin
+                next = None
+                count = len(course_bodies)
 
-            return 200, {}, json.dumps(body)
+                # Use the querystring to determine which page should be returned. Default to page 1.
+                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+                qs = parse_qs(urlparse(request.path_url).query)
+                page = int(qs.get('page', [1])[0])
+
+                if page < count:
+                    next = '{}?page={}'.format(url, page)
+
+                body = {
+                    'pagination': {
+                        'count': count,
+                        'next': next,
+                        'previous': None,
+                    },
+                    'results': [data[page - 1]]
+                }
+
+                return 200, {}, json.dumps(body)
+
+            return request_callback
 
         # pylint: disable=no-member
-        responses.add_callback(responses.GET, url, callback=request_callback, content_type=JSON)
+        url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
+        responses.add_callback(responses.GET, url, callback=ecommerce_api_callback(url, course_bodies),
+                               content_type=JSON)
+        url = '{host}/courses/'.format(host=COURSES_API_URL)
+        responses.add_callback(responses.GET, url, callback=courses_api_callback(url, course_bodies), content_type=JSON)
 
         # Refresh all course data
         Course.refresh_all(ACCESS_TOKEN)

course_discovery/apps/courses/views.py

+from django.views.generic import TemplateView
+
+
+class QueryPreviewView(TemplateView):
+    template_name = 'catalogs/preview.html'

course_discovery/settings/base.py

@@ -258,9 +258,10 @@ SWAGGER_SETTINGS = {
 }
 
 ELASTICSEARCH = {
-    'host': '',
+    'host': 'localhost:9200',
     'index': 'course_discovery',
 }
 
 # TODO Replace with None and document.
 ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
+COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'

course_discovery/settings/local.py

@@ -57,6 +57,9 @@ ENABLE_AUTO_AUTH = True
 
 JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
 
+ECOMMERCE_API_URL = 'http://localhost:8002/api/v2/'
+COURSES_API_URL = 'http://localhost:8000/api/courses/v1/'
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):

course_discovery/static/css/catalog-preview.css

+#queryForm {
+    margin: 25px 0;
+}
+
+.intro,
+.examples,
+.fields,
+.preview {
+    margin-bottom: 30px;
+}

course_discovery/static/js/catalog-preview.js

+var $alertNoResults, $alertQueryInvalid, $query, $table;
+
+function processApiResponse(response) {
+    $table.rows.add(response.results).draw();
+
+    if (response.next) {
+        getApiResponse(response.next);
+    }
+    else if (response.previous == null && response.results.length == 0) {
+        $alertNoResults.removeClass('hidden');
+    }
+}
+
+function getApiResponse(url) {
+    $.get(url)
+        .done(processApiResponse)
+        .fail(function () {
+            $alertQueryInvalid.removeClass('hidden');
+        });
+}
+
+/**
+ * Form submission handler. Sends the query to the server and displays the list of courses.\
+ */
+function onSubmit(e) {
+    var query = {
+            "query": {
+                "query_string": {
+                    "query": $query.val(),
+                    "analyze_wildcard": true
+                }
+            }
+        },
+        url = '/api/v1/courses/?q=' + encodeURIComponent(JSON.stringify(query));
+
+    e.preventDefault();
+
+    $table.clear();
+    $alertNoResults.addClass('hidden');
+    $alertQueryInvalid.addClass('hidden');
+
+    getApiResponse(url);
+}
+
+/**
+ * Click handler. Populates the query input with the content of the
+ * clicked example query.
+ */
+function populateQueryWithExample(e) {
+    $query.val($(e.target).text());
+    $query.focus();
+}
+
+/**
+ * Populate the list of Elasticsearch fields
+ */
+function populateFieldsTable() {
+    var data = [
+        ['end', 'Course end date'],
+        ['enrollment_start', 'Enrollment start date'],
+        ['enrollment_end', 'Enrollment end date'],
+        ['id', 'Course ID'],
+        ['name', 'Course name'],
+        ['number', 'Course number (e.g. 6.002x)'],
+        ['org', 'Organization (e.g. MITx)'],
+        ['start', 'Course start date'],
+        ['type', 'Type of course (audit, credit, professional, verified)'],
+        ['verification_deadline', 'Final date to submit identity verification'],
+    ];
+    $("#fields").DataTable({
+        info: false,
+        paging: false,
+        columns: [
+            {title: 'Name'},
+            {title: 'Description'}
+        ],
+        oLanguage: {
+            sSearch: "Filter: "
+        },
+        data: data
+    });
+}
+
+$(document).ready(function () {
+    $alertNoResults = $('#alertNoResults');
+    $alertQueryInvalid = $('#alertQueryInvalid');
+    $query = $('#query');
+    $table = $('#courses').DataTable({
+        info: true,
+        paging: true,
+        columns: [
+            {
+                title: 'Course ID',
+                data: 'id'
+            },
+            {
+                title: 'Name',
+                data: 'name'
+            }
+        ],
+        oLanguage: {
+            sSearch: "Filter: "
+        }
+    });
+
+    $('#queryForm').submit(onSubmit);
+
+    $('.example').click(populateQueryWithExample);
+
+    populateFieldsTable();
+});

course_discovery/templates/catalogs/preview.html

+{% load staticfiles %}
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+
+  <title>Query Preview | Course Discovery</title>
+  <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"
+        integrity="sha256-7s5uDGW3AHqw6xtJmNNtr+OBRJUlgkNJEo78P4b0yRw= sha512-nNo+yCHEyn0smMxSswnf/OnX6/KwJuZTlNZBjauKhTK0c+zT+q5JOCx0UFhXQ6rJR9jg6Es8gPuD2uZcYDLqSw=="
+        crossorigin="anonymous">
+  <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.10/css/dataTables.bootstrap.min.css">
+  <link rel="stylesheet" type="text/css" href="{% static 'css/catalog-preview.css' %}">
+</head>
+<body>
+<div class="container">
+  <h1 class="page-header">Query Preview</h1>
+
+  <div class="intro">
+    <div class="alert alert-warning" role="alert">
+      Please take a moment to review the <a
+        href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax"
+        target="_blank" class="alert-link">Elasticsearch query string syntax</a>.
+    </div>
+    <p>This page is a demonstration of the query language that will power dynamic course catalogs use for affiliates
+      and coupons.</p>
+  </div>
+
+  <div class="fields">
+    <h3>Fields</h3>
+    <p>A number of fields can be used to search for courses. A complete list is below.</p>
+    <p></p>
+
+    <table class="table table-striped table-bordered" id="fields">
+      <thead>
+      <tr>
+        <th>Name</th>
+        <th>Description</th>
+      </tr>
+      </thead>
+    </table>
+  </div>
+
+  <div class="examples">
+    <h3>Example Queries</h3>
+    <div class="alert alert-info" role="alert">
+      Click an example to populate the query field with the example query.
+    </div>
+
+    <table class="table table-striped table-bordered">
+      <thead>
+      <tr>
+        <th>Description</th>
+        <th>Query</th>
+      </tr>
+      </thead>
+      <tbody>
+      <tr>
+        <td>Courses belonging to a specific organization</td>
+        <td><a class="example">org:(MITx OR HarvardX)</a></td>
+      </tr>
+      <tr>
+        <td>Courses NOT belonging to a specific organization</td>
+        <td><a class="example">org:(-MITx OR -HarvardX)</a></td>
+      </tr>
+      <tr>
+        <td>Courses of a particular type. Options include audit, credit, honor, professional, verified.</td>
+        <td><a class="example">type:credit</a></td>
+      </tr>
+      <tr>
+        <td>Courses starting in a specific time period</td>
+        <td><a class="example">start:[2016-01-01 TO 2016-12-31]</a></td>
+      </tr>
+      <tr>
+        <td>All runs of a particular course</td>
+        <td><a class="example">number:6.002x*</a></td>
+      </tr>
+      </tbody>
+    </table>
+  </div>
+
+  <div class="preview">
+    <h3>Preview</h3>
+    <form id="queryForm" class="form-horizontal">
+      <div class="input-group">
+        <input id="query" type="text" class="form-control" placeholder="Query">
+            <span class="input-group-btn">
+              <button class="btn btn-primary" type="submit">Search</button>
+            </span>
+      </div>
+    </form>
+
+    <div class="alert alert-warning alert-dismissible hidden" role="alert" id="alertNoResults">
+      <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span
+          aria-hidden="true">&times;</span></button>
+      The query returned no results.
+    </div>
+
+    <div class="alert alert-danger alert-dismissible hidden" role="alert" id="alertQueryInvalid">
+      <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span
+          aria-hidden="true">&times;</span></button>
+      The query is invalid.
+    </div>
+
+    <hr>
+    <div class="results">
+      <table id="courses" class="table table-striped table-bordered" cellspacing="0">
+        <thead>
+        <tr>
+          <th>Course ID</th>
+          <th>Name</th>
+        </tr>
+        </thead>
+      </table>
+    </div>
+  </div>
+</div>
+
+<script type="text/javascript" language="javascript" src="//code.jquery.com/jquery-1.11.3.min.js"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"
+        integrity="sha256-KXn5puMvxCw+dAYznun+drMdG1IFl3agK0p/pqT9KAo= sha512-2e8qq0ETcfWRI4HJBzQiA3UoyFk6tbNyG+qSaIBZLyW9Xf3sWZHN/lxe9fTh1U45DpPf07yj94KsUHHWe4Yk1A=="
+        crossorigin="anonymous"></script>
+<script type="text/javascript" language="javascript"
+        src="https://cdn.datatables.net/1.10.10/js/jquery.dataTables.min.js"></script>
+<script type="text/javascript" language="javascript"
+        src="https://cdn.datatables.net/1.10.10/js/dataTables.bootstrap.min.js"></script>
+
+<script src="{% static 'js/catalog-preview.js' %}"></script>
+</body>
+</html>

course_discovery/urls.py

@@ -23,6 +23,7 @@ from django.core.urlresolvers import reverse_lazy
 from django.views.generic import RedirectView
 
 from course_discovery.apps.core import views as core_views
+from course_discovery.apps.courses.views import QueryPreviewView
 
 admin.autodiscover()
 
@@ -40,6 +41,7 @@ urlpatterns = [
     url(r'^login/$', login, name='login'),
     url(r'^logout/$', logout, name='logout'),
     url('', include('social.apps.django_app.urls', namespace='social')),
+    url('^$', QueryPreviewView.as_view()),
 ]
 
 if settings.DEBUG and os.environ.get('ENABLE_DJANGO_TOOLBAR', False):  # pragma: no cover