Merge pull request #6506 from edx/feature/courseware_search

Courseware search

cms/djangoapps/contentstore/views/course.py

@@ -10,7 +10,7 @@ from django.utils.translation import ugettext as _
 import django.utils
 from django.contrib.auth.decorators import login_required
 from django.conf import settings
-from django.views.decorators.http import require_http_methods
+from django.views.decorators.http import require_http_methods, require_GET
 from django.core.exceptions import PermissionDenied
 from django.core.urlresolvers import reverse
 from django.http import HttpResponseBadRequest, HttpResponseNotFound, HttpResponse, Http404
@@ -22,6 +22,7 @@ from edxmako.shortcuts import render_to_response
 from xmodule.course_module import DEFAULT_START_DATE
 from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.courseware_index import CoursewareSearchIndexer, SearchIndexingError
 from xmodule.contentstore.content import StaticContent
 from xmodule.tabs import PDFTextbookTabs
 from xmodule.partitions.partitions import UserPartition
@@ -75,6 +76,7 @@ from course_action_state.managers import CourseActionStateItemNotFoundError
 from microsite_configuration import microsite
 from xmodule.course_module import CourseFields
 from xmodule.split_test_module import get_split_user_partitions
+from student.auth import has_course_author_access
 
 from util.milestones_helpers import (
     set_prerequisite_courses,
@@ -90,7 +92,7 @@ CONTENT_GROUP_CONFIGURATION_DESCRIPTION = 'The groups in this configuration can
 CONTENT_GROUP_CONFIGURATION_NAME = 'Content Group Configuration'
 
 __all__ = ['course_info_handler', 'course_handler', 'course_listing',
-           'course_info_update_handler',
+           'course_info_update_handler', 'course_search_index_handler',
            'course_rerun_handler',
            'settings_handler',
            'grading_handler',
@@ -121,6 +123,15 @@ def get_course_and_check_access(course_key, user, depth=0):
     return course_module
 
 
+def reindex_course_and_check_access(course_key, user):
+    """
+    Internal method used to restart indexing on a course.
+    """
+    if not has_course_author_access(user, course_key):
+        raise PermissionDenied()
+    return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key)
+
+
 @login_required
 def course_notifications_handler(request, course_key_string=None, action_state_id=None):
     """
@@ -283,6 +294,28 @@ def course_rerun_handler(request, course_key_string):
             })
 
 
+@login_required
+@ensure_csrf_cookie
+@require_GET
+def course_search_index_handler(request, course_key_string):
+    """
+    The restful handler for course indexing.
+    GET
+        html: return status of indexing task
+    """
+    # Only global staff (PMs) are able to index courses
+    if not GlobalStaff().has_user(request.user):
+        raise PermissionDenied()
+    course_key = CourseKey.from_string(course_key_string)
+    with modulestore().bulk_operations(course_key):
+        try:
+            reindex_course_and_check_access(course_key, request.user)
+        except SearchIndexingError as search_err:
+            return HttpResponse(search_err.error_list, status=500)
+
+        return HttpResponse({}, status=200)
+
+
 def _course_outline_json(request, course_module):
     """
     Returns a JSON representation of the course module and recursively all of its children.

cms/envs/aws.py

@@ -316,3 +316,7 @@ API_DATE_FORMAT = ENV_TOKENS.get('API_DATE_FORMAT', API_DATE_FORMAT)
 # Video Caching. Pairing country codes with CDN URLs.
 # Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='}
 VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {})
+
+if FEATURES['ENABLE_COURSEWARE_INDEX']:
+    # Use ElasticSearch for the search engine
+    SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"

cms/envs/bok_choy.py

@@ -77,6 +77,13 @@ YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
 YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
 YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)
 
+FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+# Path at which to store the mock index
+MOCK_SEARCH_BACKING_FILE = (
+    TEST_ROOT / "index_file.dat"  # pylint: disable=no-value-for-parameter
+).abspath()
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

cms/envs/common.py

@@ -146,6 +146,9 @@ FEATURES = {
 
     # Toggle course entrance exams feature
     'ENTRANCE_EXAMS': False,
+
+    # Enable the courseware search functionality
+    'ENABLE_COURSEWARE_INDEX': False,
 }
 
 ENABLE_JASMINE = False
@@ -868,3 +871,11 @@ FILES_AND_UPLOAD_TYPE_FILTERS = {
         'application/vnd.ms-powerpoint',
     ],
 }
+
+# Default to no Search Engine
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+ELASTIC_FIELD_MAPPINGS = {
+    "start_date": {
+        "type": "date"
+    }
+}

cms/envs/devstack.py

@@ -83,6 +83,9 @@ FEATURES['MILESTONES_APP'] = True
 ################################ ENTRANCE EXAMS ################################
 FEATURES['ENTRANCE_EXAMS'] = True
 
+################################ SEARCH INDEX ################################
+FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"
 
 ###############################################################################
 # See if the developer has any local overrides.

cms/envs/test.py

@@ -254,3 +254,11 @@ ENTRANCE_EXAM_MIN_SCORE_PCT = 50
 VIDEO_CDN_URL = {
     'CN': 'http://api.xuetangx.com/edx/video?s3_url='
 }
+
+# Courseware Search Index
+FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+# Path at which to store the mock index
+MOCK_SEARCH_BACKING_FILE = (
+    TEST_ROOT / "index_file.dat"  # pylint: disable=no-value-for-parameter
+).abspath()

cms/static/js/spec/views/pages/course_outline_spec.js

@@ -8,7 +8,7 @@ define(["jquery", "js/common_helpers/ajax_helpers", "js/views/utils/view_utils",
                 getItemsOfType, getItemHeaders, verifyItemsExpanded, expandItemsAndVerifyState,
                 collapseItemsAndVerifyState, createMockCourseJSON, createMockSectionJSON, createMockSubsectionJSON,
                 verifyTypePublishable, mockCourseJSON, mockEmptyCourseJSON, mockSingleSectionCourseJSON,
-                createMockVerticalJSON,
+                createMockVerticalJSON, createMockIndexJSON,
                 mockOutlinePage = readFixtures('mock/mock-course-outline-page.underscore'),
                 mockRerunNotification = readFixtures('mock/mock-course-rerun-notification.underscore');
 
@@ -88,6 +88,21 @@ define(["jquery", "js/common_helpers/ajax_helpers", "js/views/utils/view_utils",
                 }, options);
             };
 
+            createMockIndexJSON = function(option) {
+                if(option){
+                    return {
+                        status: 200,
+                        responseText: ''
+                    };
+                }
+                else {
+                    return {
+                        status: 500,
+                        responseText: JSON.stringify('Could not index item: course/slashes:mock+item')
+                    };
+                }
+            };
+
             getItemsOfType = function(type) {
                 return outlinePage.$('.outline-' + type);
             };
@@ -308,6 +323,28 @@ define(["jquery", "js/common_helpers/ajax_helpers", "js/views/utils/view_utils",
                     outlinePage.$('.nav-actions .button-toggle-expand-collapse .expand-all').click();
                     verifyItemsExpanded('section', true);
                 });
+
+                it('can start reindex of a course - respond success', function() {
+                    createCourseOutlinePage(this, mockSingleSectionCourseJSON);
+                    var reindexSpy = spyOn(outlinePage, 'startReIndex').andCallThrough();
+                    var successSpy = spyOn(outlinePage, 'onIndexSuccess').andCallThrough();
+                    var reindexButton = outlinePage.$('.button.button-reindex');
+                    reindexButton.trigger('click');
+                    AjaxHelpers.expectJsonRequest(requests, 'GET', '/course_search_index/5');
+                    AjaxHelpers.respondWithJson(requests, createMockIndexJSON(true));
+                    expect(reindexSpy).toHaveBeenCalled();
+                    expect(successSpy).toHaveBeenCalled();
+                });
+
+                it('can start reindex of a course - respond fail', function() {
+                    createCourseOutlinePage(this, mockSingleSectionCourseJSON);
+                    var reindexSpy = spyOn(outlinePage, 'startReIndex').andCallThrough();
+                    var reindexButton = outlinePage.$('.button.button-reindex');
+                    reindexButton.trigger('click');
+                    AjaxHelpers.expectJsonRequest(requests, 'GET', '/course_search_index/5');
+                    AjaxHelpers.respondWithJson(requests, createMockIndexJSON(false));
+                    expect(reindexSpy).toHaveBeenCalled();
+                });
             });
 
             describe("Empty course", function() {

cms/static/js/views/pages/course_outline.js

@@ -2,8 +2,8 @@
  * This page is used to show the user an outline of the course.
  */
 define(["jquery", "underscore", "gettext", "js/views/pages/base_page", "js/views/utils/xblock_utils",
-        "js/views/course_outline", "js/views/utils/view_utils"],
-    function ($, _, gettext, BasePage, XBlockViewUtils, CourseOutlineView, ViewUtils) {
+        "js/views/course_outline", "js/views/utils/view_utils", "js/views/feedback_alert"],
+    function ($, _, gettext, BasePage, XBlockViewUtils, CourseOutlineView, ViewUtils, AlertView) {
         var expandedLocators, CourseOutlinePage;
 
         CourseOutlinePage = BasePage.extend({
@@ -24,6 +24,9 @@ define(["jquery", "underscore", "gettext", "js/views/pages/base_page", "js/views
                 this.$('.button-new').click(function(event) {
                     self.outlineView.handleAddEvent(event);
                 });
+                this.$('.button.button-reindex').click(function(event) {
+                    self.handleReIndexEvent(event);
+                });
                 this.model.on('change', this.setCollapseExpandVisibility, this);
                 $('.dismiss-button').bind('click', ViewUtils.deleteNotificationHandler(function () {
                     $('.wrapper-alert-announcement').removeClass('is-shown').addClass('is-hidden');
@@ -100,6 +103,32 @@ define(["jquery", "underscore", "gettext", "js/views/pages/base_page", "js/views
                         }
                     }, this);
                 }
+            },
+
+            handleReIndexEvent: function(event) {
+                var self = this;
+                event.preventDefault();
+                var target = $(event.currentTarget);
+                target.css('cursor', 'wait');
+                this.startReIndex()
+                    .done(function() {self.onIndexSuccess();})
+                    .always(function() {target.css('cursor', 'pointer');});
+            },
+
+            startReIndex: function() {
+                var locator =  window.course.id;
+                return $.ajax({
+                    url: '/course_search_index/' + locator,
+                    method: 'GET'
+                    });
+            },
+
+            onIndexSuccess: function() {
+                var msg = new AlertView.Announcement({
+                        title: gettext('Course Index'),
+                        message: gettext('Course has been successfully reindexed.')
+                    });
+                msg.show();
             }
         });
 

cms/templates/course_outline.html

@@ -69,6 +69,11 @@ from contentstore.utils import reverse_usage_url
                     </a>
                 </li>
                 <li class="nav-item">
+                    <a href="#" class="button button-reindex" data-category="reindex" title="${_('Reindex current course')}">
+                        <i class="icon-arrow-right"></i>${_('Reindex')}
+                    </a>
+                </li>
+                <li class="nav-item">
                     <a href="#" class="button button-toggle button-toggle-expand-collapse collapse-all is-hidden">
                         <span class="collapse-all"><i class="icon fa fa-arrow-up"></i> <span class="label">${_("Collapse All Sections")}</span></span>
                         <span class="expand-all"><i class="icon fa fa-arrow-down"></i> <span class="label">${_("Expand All Sections")}</span></span>

cms/templates/js/mock/mock-course-outline-page.underscore

@@ -22,6 +22,11 @@
                         </a>
                     </li>
                     <li class="nav-item">
+                        <a title="Reindex current course" data-category="reindex" class="button button-reindex" href="#">
+                            <i class="icon-arrow-right"></i>Reindex
+                        </a>
+                    </li>
+                    <li class="nav-item">
                         <a href="#" rel="external" class="button view-button view-live-button" title="Click to open the courseware in the LMS in a new tab">View Live</a>
                     </li>
                 </ul>

cms/urls.py

@@ -80,6 +80,11 @@ urlpatterns += patterns(
         'course_info_update_handler'
     ),
     url(r'^home/$', 'course_listing', name='home'),
+    url(
+        r'^course_search_index/{}?$'.format(settings.COURSE_KEY_PATTERN),
+        'course_search_index_handler',
+        name='course_search_index_handler'
+    ),
     url(r'^course/{}?$'.format(settings.COURSE_KEY_PATTERN), 'course_handler', name='course_handler'),
     url(r'^course_notifications/{}/(?P<action_state_id>\d+)?$'.format(settings.COURSE_KEY_PATTERN), 'course_notifications_handler'),
     url(r'^course_rerun/{}$'.format(settings.COURSE_KEY_PATTERN), 'course_rerun_handler', name='course_rerun_handler'),

common/lib/xmodule/xmodule/html_module.py

@@ -17,7 +17,8 @@ import textwrap
 from xmodule.contentstore.content import StaticContent
 from xblock.core import XBlock
 from xmodule.edxnotes_utils import edxnotes
-
+from xmodule.annotator_mixin import html_to_text
+import re
 
 log = logging.getLogger("edx.courseware")
 
@@ -253,6 +254,25 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
         non_editable_fields.append(HtmlDescriptor.use_latex_compiler)
         return non_editable_fields
 
+    def index_dictionary(self):
+        xblock_body = super(HtmlDescriptor, self).index_dictionary()
+        # Removing HTML-encoded non-breaking space characters
+        html_content = re.sub(r"(\s| |//)+", " ", html_to_text(self.data))
+        # Removing HTML CDATA
+        html_content = re.sub(r"<!\[CDATA\[.*\]\]>", "", html_content)
+        # Removing HTML comments
+        html_content = re.sub(r"<!--.*-->", "", html_content)
+        html_body = {
+            "html_content": html_content,
+            "display_name": self.display_name,
+        }
+        if "content" in xblock_body:
+            xblock_body["content"].update(html_body)
+        else:
+            xblock_body["content"] = html_body
+        xblock_body["content_type"] = "HTML Content"
+        return xblock_body
+
 
 class AboutFields(object):
     display_name = String(

common/lib/xmodule/xmodule/modulestore/courseware_index.py

+""" Code to allow module store to interface with courseware index """
+from __future__ import absolute_import
+
+import logging
+
+from django.utils.translation import ugettext as _
+from opaque_keys.edx.locator import CourseLocator
+from search.search_engine_base import SearchEngine
+
+from . import ModuleStoreEnum
+from .exceptions import ItemNotFoundError
+
+# Use default index and document names for now
+INDEX_NAME = "courseware_index"
+DOCUMENT_TYPE = "courseware_content"
+
+log = logging.getLogger('edx.modulestore')
+
+
+class SearchIndexingError(Exception):
+    """ Indicates some error(s) occured during indexing """
+
+    def __init__(self, message, error_list):
+        super(SearchIndexingError, self).__init__(message)
+        self.error_list = error_list
+
+
+class CoursewareSearchIndexer(object):
+    """
+    Class to perform indexing for courseware search from different modulestores
+    """
+
+    @staticmethod
+    def add_to_search_index(modulestore, location, delete=False, raise_on_error=False):
+        """
+        Add to courseware search index from given location and its children
+        """
+        error_list = []
+        # TODO - inline for now, need to move this out to a celery task
+        searcher = SearchEngine.get_search_engine(INDEX_NAME)
+        if not searcher:
+            return
+
+        if isinstance(location, CourseLocator):
+            course_key = location
+        else:
+            course_key = location.course_key
+
+        location_info = {
+            "course": unicode(course_key),
+        }
+
+        def _fetch_item(item_location):
+            """ Fetch the item from the modulestore location, log if not found, but continue """
+            try:
+                if isinstance(item_location, CourseLocator):
+                    item = modulestore.get_course(item_location)
+                else:
+                    item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only)
+            except ItemNotFoundError:
+                log.warning('Cannot find: %s', item_location)
+                return None
+
+            return item
+
+        def index_item_location(item_location, current_start_date):
+            """ add this item to the search index """
+            item = _fetch_item(item_location)
+            if not item:
+                return
+
+            is_indexable = hasattr(item, "index_dictionary")
+            # if it's not indexable and it does not have children, then ignore
+            if not is_indexable and not item.has_children:
+                return
+
+            # if it has a defined start, then apply it and to it's children
+            if item.start and (not current_start_date or item.start > current_start_date):
+                current_start_date = item.start
+
+            if item.has_children:
+                for child_loc in item.children:
+                    index_item_location(child_loc, current_start_date)
+
+            item_index = {}
+            item_index_dictionary = item.index_dictionary() if is_indexable else None
+
+            # if it has something to add to the index, then add it
+            if item_index_dictionary:
+                try:
+                    item_index.update(location_info)
+                    item_index.update(item_index_dictionary)
+                    item_index['id'] = unicode(item.scope_ids.usage_id)
+                    if current_start_date:
+                        item_index['start_date'] = current_start_date
+
+                    searcher.index(DOCUMENT_TYPE, item_index)
+                except Exception as err:  # pylint: disable=broad-except
+                    # broad exception so that index operation does not fail on one item of many
+                    log.warning('Could not index item: %s - %s', item_location, unicode(err))
+                    error_list.append(_('Could not index item: {}').format(item_location))
+
+        def remove_index_item_location(item_location):
+            """ remove this item from the search index """
+            item = _fetch_item(item_location)
+            if item:
+                if item.has_children:
+                    for child_loc in item.children:
+                        remove_index_item_location(child_loc)
+
+                searcher.remove(DOCUMENT_TYPE, unicode(item.scope_ids.usage_id))
+
+        try:
+            if delete:
+                remove_index_item_location(location)
+            else:
+                index_item_location(location, None)
+        except Exception as err:  # pylint: disable=broad-except
+            # broad exception so that index operation does not prevent the rest of the application from working
+            log.exception(
+                "Indexing error encountered, courseware index may be out of date %s - %s",
+                course_key,
+                unicode(err)
+            )
+            error_list.append(_('General indexing error occurred'))
+
+        if raise_on_error and error_list:
+            raise SearchIndexingError(_('Error(s) present during indexing'), error_list)
+
+    @classmethod
+    def do_course_reindex(cls, modulestore, course_key):
+        """
+        (Re)index all content within the given course
+        """
+        return cls.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True)

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -278,7 +278,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
                     raw_metadata = json_data.get('metadata', {})
                     # published_on was previously stored as a list of time components instead of a datetime
                     if raw_metadata.get('published_date'):
-                        module._edit_info['published_date'] = datetime(*raw_metadata.get('published_date')[0:6]).replace(tzinfo=UTC)
+                        module._edit_info['published_date'] = datetime(
+                            *raw_metadata.get('published_date')[0:6]
+                        ).replace(tzinfo=UTC)
                     module._edit_info['published_by'] = raw_metadata.get('published_by')
 
                 # decache any computed pending field settings
@@ -1804,3 +1806,25 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
 
         # To allow prioritizing draft vs published material
         self.collection.create_index('_id.revision')
+
+    # Some overrides that still need to be implemented by subclasses
+    def convert_to_draft(self, location, user_id):
+        raise NotImplementedError()
+
+    def delete_item(self, location, user_id, **kwargs):
+        raise NotImplementedError()
+
+    def has_changes(self, xblock):
+        raise NotImplementedError()
+
+    def has_published_version(self, xblock):
+        raise NotImplementedError()
+
+    def publish(self, location, user_id):
+        raise NotImplementedError()
+
+    def revert_to_published(self, location, user_id):
+        raise NotImplementedError()
+
+    def unpublish(self, location, user_id):
+        raise NotImplementedError()

common/lib/xmodule/xmodule/modulestore/mongo/draft.py

@@ -12,6 +12,7 @@ import logging
 from opaque_keys.edx.locations import Location
 from xmodule.exceptions import InvalidVersionError
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
 from xmodule.modulestore.exceptions import (
     ItemNotFoundError, DuplicateItemError, DuplicateCourseError, InvalidBranchSetting
 )
@@ -509,7 +510,8 @@ class DraftModuleStore(MongoModuleStore):
                 parent_locations = [draft_parent.location]
         # there could be 2 parents if
         #   Case 1: the draft item moved from one parent to another
-        #   Case 2: revision==ModuleStoreEnum.RevisionOption.all and the single parent has 2 versions: draft and published
+        # Case 2: revision==ModuleStoreEnum.RevisionOption.all and the single
+        # parent has 2 versions: draft and published
         for parent_location in parent_locations:
             # don't remove from direct_only parent if other versions of this still exists (this code
             # assumes that there's only one parent_location in this case)
@@ -541,6 +543,10 @@ class DraftModuleStore(MongoModuleStore):
             )
         self._delete_subtree(location, as_functions)
 
+        # Remove this location from the courseware search index so that searches
+        # will refrain from showing it as a result
+        CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
+
     def _delete_subtree(self, location, as_functions, draft_only=False):
         """
         Internal method for deleting all of the subtree whose revisions match the as_functions
@@ -713,6 +719,10 @@ class DraftModuleStore(MongoModuleStore):
             bulk_record = self._get_bulk_ops_record(location.course_key)
             bulk_record.dirty = True
             self.collection.remove({'_id': {'$in': to_be_deleted}})
+
+        # Now it's been published, add the object to the courseware search index so that it appears in search results
+        CoursewareSearchIndexer.add_to_search_index(self, location)
+
         return self.get_item(as_published(location))
 
     def unpublish(self, location, user_id, **kwargs):

common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py

@@ -5,6 +5,7 @@ Module for the dual-branch fall-back Draft->Published Versioning ModuleStore
 from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL
 from xmodule.exceptions import InvalidVersionError
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
 from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError
 from xmodule.modulestore.draft_and_published import (
     ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError
@@ -203,6 +204,10 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
                 if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES:
                     self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
 
+        # Remove this location from the courseware search index so that searches
+        # will refrain from showing it as a result
+        CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
+
     def _map_revision_to_branch(self, key, revision=None):
         """
         Maps RevisionOptions to BranchNames, inserting them into the key
@@ -345,6 +350,10 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
             [location],
             blacklist=blacklist
         )
+
+        # Now it's been published, add the object to the courseware search index so that it appears in search results
+        CoursewareSearchIndexer.add_to_search_index(self, location)
+
         return self.get_item(location.for_branch(ModuleStoreEnum.BranchName.published), **kwargs)
 
     def unpublish(self, location, user_id, **kwargs):

common/lib/xmodule/xmodule/seq_module.py

@@ -188,3 +188,22 @@ class SequenceDescriptor(SequenceFields, MakoModuleDescriptor, XmlDescriptor):
         for child in self.get_children():
             self.runtime.add_block_as_child_node(child, xml_object)
         return xml_object
+
+    def index_dictionary(self):
+        """
+        Return dictionary prepared with module content and type for indexing.
+        """
+        # return key/value fields in a Python dict object
+        # values may be numeric / string or dict
+        # default implementation is an empty dict
+        xblock_body = super(SequenceDescriptor, self).index_dictionary()
+        html_body = {
+            "display_name": self.display_name,
+        }
+        if "content" in xblock_body:
+            xblock_body["content"].update(html_body)
+        else:
+            xblock_body["content"] = html_body
+        xblock_body["content_type"] = self.category.title()
+
+        return xblock_body

common/lib/xmodule/xmodule/tests/test_html_module.py

@@ -3,9 +3,25 @@ import unittest
 from mock import Mock
 
 from xblock.field_data import DictFieldData
-from xmodule.html_module import HtmlModule
+from xmodule.html_module import HtmlModule, HtmlDescriptor
 
-from . import get_test_system
+from . import get_test_system, get_test_descriptor_system
+from opaque_keys.edx.locations import SlashSeparatedCourseKey
+from xblock.fields import ScopeIds
+
+
+def instantiate_descriptor(**field_data):
+    """
+    Instantiate descriptor with most properties.
+    """
+    system = get_test_descriptor_system()
+    course_key = SlashSeparatedCourseKey('org', 'course', 'run')
+    usage_key = course_key.make_usage_key('html', 'SampleHtml')
+    return system.construct_xblock_from_class(
+        HtmlDescriptor,
+        scope_ids=ScopeIds(None, None, usage_key, usage_key),
+        field_data=DictFieldData(field_data),
+    )
 
 
 class HtmlModuleSubstitutionTestCase(unittest.TestCase):
@@ -36,3 +52,71 @@ class HtmlModuleSubstitutionTestCase(unittest.TestCase):
         module_system.anonymous_student_id = None
         module = HtmlModule(self.descriptor, module_system, field_data, Mock())
         self.assertEqual(module.get_html(), sample_xml)
+
+
+class HtmlDescriptorIndexingTestCase(unittest.TestCase):
+    """
+    Make sure that HtmlDescriptor can format data for indexing as expected.
+    """
+
+    def test_index_dictionary(self):
+        sample_xml = '''
+            <html>
+                <p>Hello World!</p>
+            </html>
+        '''
+        descriptor = instantiate_descriptor(data=sample_xml)
+        self.assertEqual(descriptor.index_dictionary(), {
+            "content": {"html_content": " Hello World! ", "display_name": "Text"},
+            "content_type": "HTML Content"
+        })
+
+        sample_xml_cdata = '''
+            <html>
+                <p>This has CDATA in it.</p>
+                <![CDATA[This is just a CDATA!]]>
+            </html>
+        '''
+        descriptor = instantiate_descriptor(data=sample_xml_cdata)
+        self.assertEqual(descriptor.index_dictionary(), {
+            "content": {"html_content": " This has CDATA in it. ", "display_name": "Text"},
+            "content_type": "HTML Content"
+        })
+
+        sample_xml_tab_spaces = '''
+            <html>
+                <p>     Text has spaces :)  </p>
+            </html>
+        '''
+        descriptor = instantiate_descriptor(data=sample_xml_tab_spaces)
+        self.assertEqual(descriptor.index_dictionary(), {
+            "content": {"html_content": " Text has spaces :) ", "display_name": "Text"},
+            "content_type": "HTML Content"
+        })
+
+        sample_xml_comment = '''
+            <html>
+                <p>This has HTML comment in it.</p>
+                <!-- Html Comment -->
+            </html>
+        '''
+        descriptor = instantiate_descriptor(data=sample_xml_comment)
+        self.assertEqual(descriptor.index_dictionary(), {
+            "content": {"html_content": " This has HTML comment in it. ", "display_name": "Text"},
+            "content_type": "HTML Content"
+        })
+
+        sample_xml_mix_comment_cdata = '''
+            <html>
+                <!-- Beginning of the html -->
+                <p>This has HTML comment in it.<!-- Commenting Content --></p>
+                <!-- Here comes CDATA -->
+                <![CDATA[This is just a CDATA!]]>
+                <p>HTML end.</p>
+            </html>
+        '''
+        descriptor = instantiate_descriptor(data=sample_xml_mix_comment_cdata)
+        self.assertEqual(descriptor.index_dictionary(), {
+            "content": {"html_content": " This has HTML comment in it. HTML end. ", "display_name": "Text"},
+            "content_type": "HTML Content"
+        })

common/lib/xmodule/xmodule/video_module/video_module.py

@@ -32,6 +32,7 @@ from xmodule.x_module import XModule, module_attr
 from xmodule.editing_module import TabsEditingDescriptor
 from xmodule.raw_module import EmptyDataRawDescriptor
 from xmodule.xml_module import is_pointer_tag, name_to_pathname, deserialize_field
+from xmodule.exceptions import NotFoundError
 
 from .transcripts_utils import VideoTranscriptsMixin
 from .video_utils import create_youtube_string, get_video_from_cdn
@@ -607,3 +608,34 @@ class VideoDescriptor(VideoFields, VideoTranscriptsMixin, VideoStudioViewHandler
             field_data['download_track'] = True
 
         return field_data
+
+    def index_dictionary(self):
+        xblock_body = super(VideoDescriptor, self).index_dictionary()
+        video_body = {
+            "display_name": self.display_name,
+        }
+
+        def _update_transcript_for_index(language=None):
+            """ Find video transcript - if not found, don't update index """
+            try:
+                transcript = self.get_transcript(transcript_format='txt', lang=language)[0].replace("\n", " ")
+                transcript_index_name = "transcript_{}".format(language if language else self.transcript_language)
+                video_body.update({transcript_index_name: transcript})
+            except NotFoundError:
+                pass
+
+        if self.sub:
+            _update_transcript_for_index()
+
+        # check to see if there are transcripts in other languages besides default transcript
+        if self.transcripts:
+            for language in self.transcripts.keys():
+                _update_transcript_for_index(language)
+
+        if "content" in xblock_body:
+            xblock_body["content"].update(video_body)
+        else:
+            xblock_body["content"] = video_body
+        xblock_body["content_type"] = "Video"
+
+        return xblock_body

common/test/acceptance/pages/lms/courseware_search.py

+"""
+Courseware search
+"""
+
+from .course_page import CoursePage
+
+
+class CoursewareSearchPage(CoursePage):
+    """
+    Coursware page featuring a search form
+    """
+
+    url_path = "courseware/"
+    search_bar_selector = '#courseware-search-bar'
+
+    @property
+    def search_results(self):
+        """ search results list showing """
+        return self.q(css='#courseware-search-results')
+
+    def is_browser_on_page(self):
+        """ did we find the search bar in the UI """
+        return self.q(css=self.search_bar_selector).present
+
+    def enter_search_term(self, text):
+        """ enter the search term into the box """
+        self.q(css=self.search_bar_selector + ' input[type="text"]').fill(text)
+
+    def search(self):
+        """ execute the search """
+        self.q(css=self.search_bar_selector + ' [type="submit"]').click()
+        self.wait_for_element_visibility('.search-info', 'Search results are shown')
+
+    def search_for_term(self, text):
+        """
+        Search and return results
+        """
+        self.enter_search_term(text)
+        self.search()

common/test/acceptance/pages/studio/overview.py

@@ -505,6 +505,12 @@ class CourseOutlinePage(CoursePage, CourseOutlineContainer):
         """
         self.q(css=self.EXPAND_COLLAPSE_CSS).click()
 
+    def start_reindex(self):
+        """
+        Starts course reindex by clicking reindex button
+        """
+        self.reindex_button.click()
+
     @property
     def bottom_add_section_button(self):
         """
@@ -545,6 +551,13 @@ class CourseOutlinePage(CoursePage, CourseOutlineContainer):
         else:
             return ExpandCollapseLinkState.EXPAND
 
+    @property
+    def reindex_button(self):
+        """
+        Returns reindex button.
+        """
+        return self.q(css=".button.button-reindex")[0]
+
     def expand_all_subsections(self):
         """
         Expands all the subsections in this course.

common/test/acceptance/pages/studio/utils.py

@@ -130,6 +130,34 @@ def add_component(page, item_type, specific_type):
     page.wait_for_ajax()
 
 
+def add_html_component(page, menu_index, boilerplate=None):
+    """
+    Adds an instance of the HTML component with the specified name.
+
+    menu_index specifies which instance of the menus should be used (based on vertical
+    placement within the page).
+    """
+    # Click on the HTML icon.
+    page.wait_for_component_menu()
+    click_css(page, 'a>span.large-html-icon', menu_index, require_notification=False)
+
+    # Make sure that the menu of HTML components is visible before clicking
+    page.wait_for_element_visibility('.new-component-html', 'HTML component menu is visible')
+
+    # Now click on the component to add it.
+    component_css = 'a[data-category=html]'
+    if boilerplate:
+        component_css += '[data-boilerplate={}]'.format(boilerplate)
+    else:
+        component_css += ':not([data-boilerplate])'
+
+    page.wait_for_element_visibility(component_css, 'HTML component {} is visible'.format(boilerplate))
+
+    # Adding some components will make an ajax call but we should be OK because
+    # the click_css method is written to handle that.
+    click_css(page, component_css, 0)
+
+
 @js_defined('window.jQuery')
 def type_in_codemirror(page, index, text, find_prefix="$"):
     script = """

common/test/acceptance/tests/lms/test_lms_courseware_search.py

+"""
+Test courseware search
+"""
+import os
+import json
+
+from ..helpers import UniqueCourseTest
+from ...pages.common.logout import LogoutPage
+from ...pages.studio.utils import add_html_component, click_css, type_in_codemirror
+from ...pages.studio.auto_auth import AutoAuthPage
+from ...pages.studio.overview import CourseOutlinePage
+from ...pages.studio.container import ContainerPage
+from ...pages.lms.courseware_search import CoursewareSearchPage
+from ...fixtures.course import CourseFixture, XBlockFixtureDesc
+
+
+class CoursewareSearchTest(UniqueCourseTest):
+    """
+    Test courseware search.
+    """
+    USERNAME = 'STUDENT_TESTER'
+    EMAIL = 'student101@example.com'
+
+    STAFF_USERNAME = "STAFF_TESTER"
+    STAFF_EMAIL = "staff101@example.com"
+
+    HTML_CONTENT = """
+            Someday I'll wish upon a star
+            And wake up where the clouds are far
+            Behind me.
+            Where troubles melt like lemon drops
+            Away above the chimney tops
+            That's where you'll find me.
+        """
+    SEARCH_STRING = "chimney"
+    EDITED_CHAPTER_NAME = "Section 2 - edited"
+    EDITED_SEARCH_STRING = "edited"
+
+    TEST_INDEX_FILENAME = "test_root/index_file.dat"
+
+    def setUp(self):
+        """
+        Create search page and course content to search
+        """
+        # create test file in which index for this test will live
+        with open(self.TEST_INDEX_FILENAME, "w+") as index_file:
+            json.dump({}, index_file)
+
+        super(CoursewareSearchTest, self).setUp()
+        self.courseware_search_page = CoursewareSearchPage(self.browser, self.course_id)
+
+        self.course_outline = CourseOutlinePage(
+            self.browser,
+            self.course_info['org'],
+            self.course_info['number'],
+            self.course_info['run']
+        )
+
+        course_fix = CourseFixture(
+            self.course_info['org'],
+            self.course_info['number'],
+            self.course_info['run'],
+            self.course_info['display_name']
+        )
+
+        course_fix.add_children(
+            XBlockFixtureDesc('chapter', 'Section 1').add_children(
+                XBlockFixtureDesc('sequential', 'Subsection 1')
+            )
+        ).add_children(
+            XBlockFixtureDesc('chapter', 'Section 2').add_children(
+                XBlockFixtureDesc('sequential', 'Subsection 2')
+            )
+        ).install()
+
+    def tearDown(self):
+        os.remove(self.TEST_INDEX_FILENAME)
+
+    def _auto_auth(self, username, email, staff):
+        """
+        Logout and login with given credentials.
+        """
+        LogoutPage(self.browser).visit()
+        AutoAuthPage(self.browser, username=username, email=email,
+                     course_id=self.course_id, staff=staff).visit()
+
+    def test_page_existence(self):
+        """
+        Make sure that the page is accessible.
+        """
+        self._auto_auth(self.USERNAME, self.EMAIL, False)
+        self.courseware_search_page.visit()
+
+    def _studio_publish_content(self, section_index):
+        """
+        Publish content on studio course page under specified section
+        """
+        self.course_outline.visit()
+        subsection = self.course_outline.section_at(section_index).subsection_at(0)
+        subsection.toggle_expand()
+        unit = subsection.unit_at(0)
+        unit.publish()
+
+    def _studio_edit_chapter_name(self, section_index):
+        """
+        Edit chapter name on studio course page under specified section
+        """
+        self.course_outline.visit()
+        section = self.course_outline.section_at(section_index)
+        section.change_name(self.EDITED_CHAPTER_NAME)
+
+    def _studio_add_content(self, section_index):
+        """
+        Add content on studio course page under specified section
+        """
+
+        # create a unit in course outline
+        self.course_outline.visit()
+        subsection = self.course_outline.section_at(section_index).subsection_at(0)
+        subsection.toggle_expand()
+        subsection.add_unit()
+
+        # got to unit and create an HTML component and save (not publish)
+        unit_page = ContainerPage(self.browser, None)
+        unit_page.wait_for_page()
+        add_html_component(unit_page, 0)
+        unit_page.wait_for_element_presence('.edit-button', 'Edit button is visible')
+        click_css(unit_page, '.edit-button', 0, require_notification=False)
+        unit_page.wait_for_element_visibility('.modal-editor', 'Modal editor is visible')
+        type_in_codemirror(unit_page, 0, self.HTML_CONTENT)
+        click_css(unit_page, '.action-save', 0)
+
+    def _studio_reindex(self):
+        """
+        Reindex course content on studio course page
+        """
+
+        self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
+        self.course_outline.visit()
+        self.course_outline.start_reindex()
+        self.course_outline.wait_for_ajax()
+
+    def test_search(self):
+        """
+        Make sure that you can search for something.
+        """
+
+        # Create content in studio without publishing.
+        self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
+        self._studio_add_content(0)
+
+        # Do a search, there should be no results shown.
+        self._auto_auth(self.USERNAME, self.EMAIL, False)
+        self.courseware_search_page.visit()
+        self.courseware_search_page.search_for_term(self.SEARCH_STRING)
+        assert self.SEARCH_STRING not in self.courseware_search_page.search_results.html[0]
+
+        # Publish in studio to trigger indexing.
+        self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
+        self._studio_publish_content(0)
+
+        # Do the search again, this time we expect results.
+        self._auto_auth(self.USERNAME, self.EMAIL, False)
+        self.courseware_search_page.visit()
+        self.courseware_search_page.search_for_term(self.SEARCH_STRING)
+        assert self.SEARCH_STRING in self.courseware_search_page.search_results.html[0]
+
+    def test_reindex(self):
+        """
+        Make sure new content gets reindexed on button press.
+        """
+
+        # Create content in studio without publishing.
+        self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
+        self._studio_add_content(1)
+
+        # Publish in studio to trigger indexing, and edit chapter name afterwards.
+        self._studio_publish_content(1)
+        self._studio_edit_chapter_name(1)
+
+        # Do a search, there should be no results shown.
+        self._auto_auth(self.USERNAME, self.EMAIL, False)
+        self.courseware_search_page.visit()
+        self.courseware_search_page.search_for_term(self.EDITED_SEARCH_STRING)
+        assert self.EDITED_SEARCH_STRING not in self.courseware_search_page.search_results.html[0]
+
+        # Do a ReIndex from studio, to add edited chapter name
+        self._studio_reindex()
+
+        # Do the search again, this time we expect results.
+        self._auto_auth(self.USERNAME, self.EMAIL, False)
+        self.courseware_search_page.visit()
+        self.courseware_search_page.search_for_term(self.EDITED_SEARCH_STRING)
+        assert self.EDITED_SEARCH_STRING in self.courseware_search_page.search_results.html[0]

lms/envs/acceptance.py

@@ -179,3 +179,7 @@ XQUEUE_INTERFACE = {
 YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
 YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
 YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)
+
+if FEATURES.get('ENABLE_COURSEWARE_SEARCH'):
+    # Use MockSearchEngine as the search engine for test scenario
+    SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"

lms/envs/aws.py

@@ -500,3 +500,7 @@ PDF_RECEIPT_LOGO_HEIGHT_MM = ENV_TOKENS.get('PDF_RECEIPT_LOGO_HEIGHT_MM', PDF_RE
 PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM = ENV_TOKENS.get(
     'PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM', PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM
 )
+
+if FEATURES.get('ENABLE_COURSEWARE_SEARCH'):
+    # Use ElasticSearch as the search engine herein
+    SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"

lms/envs/bok_choy.py

@@ -118,6 +118,15 @@ FEATURES['ADVANCED_SECURITY'] = False
 PASSWORD_MIN_LENGTH = None
 PASSWORD_COMPLEXITY = {}
 
+# Enable courseware search for tests
+FEATURES['ENABLE_COURSEWARE_SEARCH'] = True
+# Use MockSearchEngine as the search engine for test scenario
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+# Path at which to store the mock index
+MOCK_SEARCH_BACKING_FILE = (
+    TEST_ROOT / "index_file.dat"  # pylint: disable=no-value-for-parameter
+).abspath()
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

lms/envs/common.py

@@ -328,6 +328,9 @@ FEATURES = {
 
     # For easily adding modes to courses during acceptance testing
     'MODE_CREATION_FOR_TESTING': False,
+
+    # Courseware search feature
+    'ENABLE_COURSEWARE_SEARCH': False,
 }
 
 # Ignore static asset files on import which match this pattern
@@ -1039,6 +1042,7 @@ courseware_js = (
         for pth in ['courseware', 'histogram', 'navigation', 'time']
     ] +
     ['js/' + pth + '.js' for pth in ['ajax-error']] +
+    ['js/search/main.js'] +
     sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/modules/**/*.js'))
 )
 
@@ -2011,3 +2015,8 @@ PDF_RECEIPT_LOGO_HEIGHT_MM = 12
 PDF_RECEIPT_COBRAND_LOGO_PATH = PROJECT_ROOT + '/static/images/default-theme/logo.png'
 # Height of the Co-brand Logo in mm
 PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM = 12
+
+# Use None for the default search engine
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+# Use the LMS specific result processor
+SEARCH_RESULT_PROCESSOR = "lms.lib.courseware_search.lms_result_processor.LmsSearchResultProcessor"

lms/envs/devstack.py

@@ -113,6 +113,11 @@ FEATURES['MILESTONES_APP'] = True
 FEATURES['ENTRANCE_EXAMS'] = True
 
 
+########################## Courseware Search #######################
+FEATURES['ENABLE_COURSEWARE_SEARCH'] = True
+SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"
+
+
 #####################################################################
 # See if the developer has any local overrides.
 try:

lms/envs/test.py

@@ -455,3 +455,8 @@ FEATURES['MILESTONES_APP'] = True
 
 # ENTRANCE EXAMS
 FEATURES['ENTRANCE_EXAMS'] = True
+
+# Enable courseware search for tests
+FEATURES['ENABLE_COURSEWARE_SEARCH'] = True
+# Use MockSearchEngine as the search engine for test scenario
+SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"

lms/lib/courseware_search/__init__.py

+"""
+Search overrides for courseware search
+Implement overrides for:
+* SearchResultProcessor
+    - to mix in path to result
+    - to provide last-ditch access check
+* SearchFilterGenerator
+    - to provide additional filter fields (for cohorted values etc.)
+    - to inject specific field restrictions if/when desired
+"""

lms/lib/courseware_search/lms_result_processor.py

+"""
+This file contains implementation override of SearchResultProcessor which will allow
+    * Blends in "location" property
+    * Confirms user access to object
+"""
+from django.core.urlresolvers import reverse
+
+from opaque_keys.edx.locations import SlashSeparatedCourseKey
+from search.result_processor import SearchResultProcessor
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.search import path_to_location
+
+from courseware.access import has_access
+
+
+class LmsSearchResultProcessor(SearchResultProcessor):
+
+    """ SearchResultProcessor for LMS Search """
+    _course_key = None
+    _usage_key = None
+    _module_store = None
+    _module_temp_dictionary = {}
+
+    def get_course_key(self):
+        """ fetch course key object from string representation - retain result for subsequent uses """
+        if self._course_key is None:
+            self._course_key = SlashSeparatedCourseKey.from_deprecated_string(self._results_fields["course"])
+        return self._course_key
+
+    def get_usage_key(self):
+        """ fetch usage key for component from string representation - retain result for subsequent uses """
+        if self._usage_key is None:
+            self._usage_key = self.get_course_key().make_usage_key_from_deprecated_string(self._results_fields["id"])
+        return self._usage_key
+
+    def get_module_store(self):
+        """ module store accessor - retain result for subsequent uses """
+        if self._module_store is None:
+            self._module_store = modulestore()
+        return self._module_store
+
+    def get_item(self, usage_key):
+        """ fetch item from the modulestore - don't refetch if we've already retrieved it beforehand """
+        if usage_key not in self._module_temp_dictionary:
+            self._module_temp_dictionary[usage_key] = self.get_module_store().get_item(usage_key)
+        return self._module_temp_dictionary[usage_key]
+
+    @property
+    def url(self):
+        """
+        Property to display the url for the given location, useful for allowing navigation
+        """
+        if "course" not in self._results_fields or "id" not in self._results_fields:
+            raise ValueError("Must have course and id in order to build url")
+
+        return reverse(
+            "jump_to",
+            kwargs={"course_id": self._results_fields["course"], "location": self._results_fields["id"]}
+        )
+
+    @property
+    def location(self):
+        """
+        Blend "location" property into the resultset, so that the path to the found component can be shown within the UI
+        """
+        # TODO: update whern changes to "cohorted-courseware" branch are merged in
+        (course_key, chapter, section, position) = path_to_location(self.get_module_store(), self.get_usage_key())
+
+        def get_display_name(category, item_id):
+            """ helper to get display name from object """
+            item = self.get_item(course_key.make_usage_key(category, item_id))
+            return getattr(item, "display_name", None)
+
+        def get_position_name(section, position):
+            """ helper to fetch name corresponding to the position therein """
+            pos = int(position)
+            section_item = self.get_item(course_key.make_usage_key("sequential", section))
+            if section_item.has_children and len(section_item.children) >= pos:
+                item = self.get_item(section_item.children[pos - 1])
+                return getattr(item, "display_name", None)
+            return None
+
+        location_description = []
+        if chapter:
+            location_description.append(get_display_name("chapter", chapter))
+        if section:
+            location_description.append(get_display_name("sequential", section))
+        if position:
+            location_description.append(get_position_name(section, position))
+
+        return location_description
+
+    def should_remove(self, user):
+        """ Test to see if this result should be removed due to access restriction """
+        return not has_access(
+            user,
+            "load",
+            self.get_item(self.get_usage_key()),
+            self.get_course_key()
+        )

lms/lib/courseware_search/test/test_lms_result_processor.py

+"""
+Tests for the lms_result_processor
+"""
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+
+from courseware.tests.factories import UserFactory
+
+from lms.lib.courseware_search.lms_result_processor import LmsSearchResultProcessor
+
+
+class LmsSearchResultProcessorTestCase(ModuleStoreTestCase):
+    """ Test case class to test search result processor """
+
+    def build_course(self):
+        """
+        Build up a course tree with an html control
+        """
+        self.global_staff = UserFactory(is_staff=True)
+
+        self.course = CourseFactory.create(
+            org='Elasticsearch',
+            course='ES101',
+            run='test_run',
+            display_name='Elasticsearch test course',
+        )
+        self.section = ItemFactory.create(
+            parent=self.course,
+            category='chapter',
+            display_name='Test Section',
+        )
+        self.subsection = ItemFactory.create(
+            parent=self.section,
+            category='sequential',
+            display_name='Test Subsection',
+        )
+        self.vertical = ItemFactory.create(
+            parent=self.subsection,
+            category='vertical',
+            display_name='Test Unit',
+        )
+        self.html = ItemFactory.create(
+            parent=self.vertical, category='html',
+            display_name='Test Html control',
+        )
+
+    def setUp(self):
+        # from nose.tools import set_trace
+        # set_trace()
+        self.build_course()
+
+    def test_url_parameter(self):
+        fake_url = ""
+        srp = LmsSearchResultProcessor({}, "test")
+        with self.assertRaises(ValueError):
+            fake_url = srp.url
+        self.assertEqual(fake_url, "")
+
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.html.scope_ids.usage_id),
+                "content": {"text": "This is the html text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(
+            srp.url, "/courses/{}/jump_to/{}".format(unicode(self.course.id), unicode(self.html.scope_ids.usage_id)))
+
+    def test_location_parameter(self):
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.html.scope_ids.usage_id),
+                "content": {"text": "This is html test text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(len(srp.location), 3)
+        self.assertEqual(srp.location[0], 'Test Section')
+        self.assertEqual(srp.location[1], 'Test Subsection')
+        self.assertEqual(srp.location[2], 'Test Unit')
+
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.vertical.scope_ids.usage_id),
+                "content": {"text": "This is html test text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(len(srp.location), 3)
+        self.assertEqual(srp.location[0], 'Test Section')
+        self.assertEqual(srp.location[1], 'Test Subsection')
+        self.assertEqual(srp.location[2], 'Test Unit')
+
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.subsection.scope_ids.usage_id),
+                "content": {"text": "This is html test text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(len(srp.location), 2)
+        self.assertEqual(srp.location[0], 'Test Section')
+        self.assertEqual(srp.location[1], 'Test Subsection')
+
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.section.scope_ids.usage_id),
+                "content": {"text": "This is html test text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(len(srp.location), 1)
+        self.assertEqual(srp.location[0], 'Test Section')
+
+    def test_should_remove(self):
+        """
+        Tests that "visible_to_staff_only" overrides start date.
+        """
+        srp = LmsSearchResultProcessor(
+            {
+                "course": unicode(self.course.id),
+                "id": unicode(self.html.scope_ids.usage_id),
+                "content": {"text": "This is html test text"}
+            },
+            "test"
+        )
+
+        self.assertEqual(srp.should_remove(self.global_staff), False)

lms/static/js/fixtures/search_form.html

+<div id="courseware-search-bar" class="search-container">
+    <form role="search-form">
+        <input type="text" class="search-field"/>
+        <button type="submit" class="search-button" aria-label="Search">search <i class="icon-search"></i></button>
+        <button type="button" class="cancel-button" aria-label="Cancel"><i class="icon-remove"></i></button>
+    </form>
+</div>

lms/static/js/search/collections/search_collection.js

+;(function (define) {
+
+define([
+    'backbone',
+    'js/search/models/search_result'
+], function (Backbone, SearchResult) {
+    'use strict';
+
+    return Backbone.Collection.extend({
+
+        model: SearchResult,
+        pageSize: 20,
+        totalCount: 0,
+        accessDeniedCount: 0,
+        searchTerm: '',
+        page: 0,
+        url: '/search/',
+        fetchXhr: null,
+
+        initialize: function (models, options) {
+            // call super constructor
+            Backbone.Collection.prototype.initialize.apply(this, arguments);
+            if (options && options.course_id) {
+                this.url += options.course_id;
+            }
+        },
+
+        performSearch: function (searchTerm) {
+            this.fetchXhr && this.fetchXhr.abort();
+            this.searchTerm = searchTerm || '';
+            this.totalCount = 0;
+            this.accessDeniedCount = 0;
+            this.page = 0;
+            this.fetchXhr = this.fetch({
+                data: {
+                    search_string: searchTerm,
+                    page_size: this.pageSize,
+                    page_index: 0
+                },
+                type: 'POST',
+                success: function (self, xhr) {
+                    self.trigger('search');
+                },
+                error: function (self, xhr) {
+                    self.trigger('error');
+                }
+            });
+        },
+
+        loadNextPage: function () {
+            this.fetchXhr && this.fetchXhr.abort();
+            this.fetchXhr = this.fetch({
+                data: {
+                    search_string: this.searchTerm,
+                    page_size: this.pageSize,
+                    page_index: this.page + 1
+                },
+                type: 'POST',
+                success: function (self, xhr) {
+                    self.page += 1;
+                    self.trigger('next');
+                },
+                error: function (self, xhr) {
+                    self.trigger('error');
+                }
+            });
+        },
+
+        cancelSearch: function () {
+            this.fetchXhr && this.fetchXhr.abort();
+            this.page = 0;
+            this.totalCount = 0;
+            this.accessDeniedCount = 0;
+        },
+
+        parse: function(response) {
+            this.totalCount = response.total;
+            this.accessDeniedCount += response.access_denied_count;
+            this.totalCount -= this.accessDeniedCount;
+            return _.map(response.results, function(result){ return result.data; });
+        },
+
+        hasNextPage: function () {
+            return this.totalCount - ((this.page + 1) * this.pageSize) > 0;
+        }
+
+    });
+
+});
+
+
+})(define || RequireJS.define);

lms/static/js/search/main.js

+RequireJS.require([
+    'jquery',
+    'backbone',
+    'js/search/search_app'
+], function ($, Backbone, SearchApp) {
+    'use strict';
+
+    var course_id = $('#courseware-search-results').attr('data-course-id');
+    var app = new SearchApp(course_id);
+    Backbone.history.start();
+
+});

lms/static/js/search/models/search_result.js

+;(function (define) {
+
+define(['backbone'], function (Backbone) {
+    'use strict';
+
+    return Backbone.Model.extend({
+        defaults: {
+            location: [],
+            content_type: '',
+            excerpt: '',
+            url: ''
+        }
+    });
+
+});
+
+})(define || RequireJS.define);

lms/static/js/search/search_app.js

+;(function (define) {
+
+define([
+    'backbone',
+    'js/search/search_router',
+    'js/search/views/search_form',
+    'js/search/views/search_list_view',
+    'js/search/collections/search_collection'
+], function(Backbone, SearchRouter, SearchForm, SearchListView, SearchCollection) {
+    'use strict';
+
+    return function (course_id) {
+
+        var self = this;
+
+        this.router = new SearchRouter();
+        this.form = new SearchForm();
+        this.collection = new SearchCollection([], { course_id: course_id });
+        this.results = new SearchListView({ collection: this.collection });
+
+        this.form.on('search', this.results.showLoadingMessage, this.results);
+        this.form.on('search', this.collection.performSearch, this.collection);
+        this.form.on('search', function (term) {
+            self.router.navigate('search/' + term, { replace: true });
+        });
+        this.form.on('clear', this.collection.cancelSearch, this.collection);
+        this.form.on('clear', this.results.clear, this.results);
+        this.form.on('clear', this.router.navigate, this.router);
+
+        this.results.on('next', this.collection.loadNextPage, this.collection);
+        this.router.on('route:search', this.form.doSearch, this.form);
+
+    };
+
+});
+
+})(define || RequireJS.define);

lms/static/js/search/search_router.js

+;(function (define) {
+
+define(['backbone'], function (Backbone) {
+    'use strict';
+
+    return Backbone.Router.extend({
+      routes: {
+        'search/:query': 'search'
+      }
+    });
+
+});
+
+})(define || RequireJS.define);

lms/static/js/search/views/search_form.js

+;(function (define) {
+
+define(['jquery', 'backbone'], function ($, Backbone) {
+   'use strict';
+
+    return Backbone.View.extend({
+
+        el: '#courseware-search-bar',
+        events: {
+            'submit form': 'submitForm',
+            'click .cancel-button': 'clearSearch',
+        },
+
+        initialize: function () {
+            this.$searchField = this.$el.find('.search-field');
+            this.$searchButton = this.$el.find('.search-button');
+            this.$cancelButton = this.$el.find('.cancel-button');
+        },
+
+        submitForm: function (event) {
+            event.preventDefault();
+            this.doSearch();
+        },
+
+        doSearch: function (term) {
+            if (term) {
+                this.$searchField.val(term);
+            }
+            else {
+                term = this.$searchField.val();
+            }
+
+            var trimmed = $.trim(term);
+            if (trimmed) {
+                this.setActiveStyle();
+                this.trigger('search', trimmed);
+            }
+            else {
+                this.clearSearch();
+            }
+        },
+
+        clearSearch: function () {
+            this.$searchField.val('');
+            this.setInitialStyle();
+            this.trigger('clear');
+        },
+
+        setActiveStyle: function () {
+            this.$searchField.addClass('is-active');
+            this.$searchButton.hide();
+            this.$cancelButton.show();
+        },
+
+        setInitialStyle: function () {
+            this.$searchField.removeClass('is-active');
+            this.$searchButton.show();
+            this.$cancelButton.hide();
+        }
+
+    });
+
+});
+
+})(define || RequireJS.define);

lms/static/js/search/views/search_item_view.js

+;(function (define) {
+
+define([
+    'jquery',
+    'underscore',
+    'backbone',
+    'gettext'
+], function ($, _, Backbone, gettext) {
+   'use strict';
+
+    return Backbone.View.extend({
+
+        tagName: 'li',
+        className: 'search-results-item',
+        attributes: {
+            'role': 'region',
+            'aria-label': 'search result'
+        },
+
+        initialize: function () {
+            this.tpl = _.template($('#search_item-tpl').html());
+        },
+
+        render: function () {
+            this.$el.html(this.tpl(this.model.attributes));
+            return this;
+        }
+    });
+
+});
+
+})(define || RequireJS.define);

lms/static/js/search/views/search_list_view.js

+;(function (define) {
+
+define([
+    'jquery',
+    'underscore',
+    'backbone',
+    'gettext',
+    'js/search/views/search_item_view'
+], function ($, _, Backbone, gettext, SearchItemView) {
+
+   'use strict';
+
+    return Backbone.View.extend({
+
+        el: '#courseware-search-results',
+        events: {
+            'click .search-load-next': 'loadNext'
+        },
+        spinner: '.icon',
+
+        initialize: function () {
+            this.courseName = this.$el.attr('data-course-name');
+            this.$courseContent = $('#course-content');
+            this.listTemplate = _.template($('#search_list-tpl').html());
+            this.loadingTemplate = _.template($('#search_loading-tpl').html());
+            this.errorTemplate = _.template($('#search_error-tpl').html());
+            this.collection.on('search', this.render, this);
+            this.collection.on('next', this.renderNext, this);
+            this.collection.on('error', this.showErrorMessage, this);
+        },
+
+        render: function () {
+            this.$el.html(this.listTemplate({
+                courseName: this.courseName,
+                totalCount: this.collection.totalCount,
+                totalCountMsg: this.totalCountMsg(),
+                pageSize: this.collection.pageSize,
+                hasMoreResults: this.collection.hasNextPage()
+            }));
+            this.renderItems();
+            this.$courseContent.hide();
+            this.$el.show();
+            return this;
+        },
+
+        renderNext: function () {
+            // total count may have changed
+            this.$el.find('.search-count').text(this.totalCountMsg());
+            this.renderItems();
+            if (! this.collection.hasNextPage()) {
+                this.$el.find('.search-load-next').remove();
+            }
+            this.$el.find(this.spinner).hide();
+        },
+
+        renderItems: function () {
+            var items = this.collection.map(function (result) {
+                var item = new SearchItemView({ model: result });
+                return item.render().el;
+            });
+            this.$el.find('.search-results').append(items);
+        },
+
+        totalCountMsg: function () {
+            var fmt = ngettext('%s result', '%s results', this.collection.totalCount);
+            return interpolate(fmt, [this.collection.totalCount]);
+        },
+
+        clear: function () {
+            this.$el.hide().empty();
+            this.$courseContent.show();
+        },
+
+        showLoadingMessage: function () {
+            this.$el.html(this.loadingTemplate());
+            this.$el.show();
+            this.$courseContent.hide();
+        },
+
+        showErrorMessage: function () {
+            this.$el.html(this.errorTemplate());
+            this.$el.show();
+            this.$courseContent.hide();
+        },
+
+        loadNext: function (event) {
+            event && event.preventDefault();
+            this.$el.find(this.spinner).show();
+            this.trigger('next');
+        }
+
+    });
+
+});
+
+
+})(define || RequireJS.define);

lms/static/js/spec/main.js

@@ -560,7 +560,8 @@
         'lms/include/js/spec/edxnotes/models/note_spec.js',
         'lms/include/js/spec/edxnotes/plugins/events_spec.js',
         'lms/include/js/spec/edxnotes/plugins/scroller_spec.js',
-        'lms/include/js/spec/edxnotes/collections/notes_spec.js'
+        'lms/include/js/spec/edxnotes/collections/notes_spec.js',
+        'lms/include/js/spec/search/search_spec.js'
     ]);
 
 }).call(this, requirejs, define);

lms/static/js_test.yml

@@ -78,6 +78,7 @@ spec_paths:
 #   loadFixtures('path/to/fixture/fixture.html');
 #
 fixture_paths:
+    - js/fixtures
     - templates/instructor/instructor_dashboard_2
     - templates/dashboard
     - templates/edxnotes
@@ -86,6 +87,7 @@ fixture_paths:
     - templates/verify_student
     - templates/file-upload.underscore
     - js/fixtures/edxnotes
+    - templates/courseware_search
 
 requirejs:
   paths:

lms/static/sass/course.scss.mako

@@ -41,6 +41,11 @@
 @import 'course/courseware/sidebar';
 @import 'course/courseware/amplifier';
 
+## Import styles for courseware search
+% if env["FEATURES"].get("ENABLE_COURSEWARE_SEARCH"):
+    @import 'course/courseware/courseware_search';
+% endif
+
 // course - modules
 @import 'course/modules/student-notes'; // student notes
 @import 'course/modules/calculator'; // calculator utility

lms/static/sass/course/courseware/_courseware_search.scss

+.course-index .courseware-search-bar {
+
+  @include box-sizing(border-box);
+  position: relative;
+  padding: 5px;
+  box-shadow: 0 1px 0 #fff inset, 0 -1px 0 rgba(0, 0, 0, .1) inset;
+  font-family: $sans-serif;
+
+  .search-field {
+    @include box-sizing(border-box);
+    top: 5px;
+    width: 100%;
+    @include border-radius(4px);
+    background: $white-t1;
+    &.is-active {
+      background: $white;
+    }
+  }
+
+  .search-button, .cancel-button {
+    @include box-sizing(border-box);
+    color: #888;
+    font-size: 14px;
+    font-weight: normal;
+    display: block;
+    position: absolute;
+    right: 12px;
+    top: 5px;
+    height: 35px;
+    line-height: 35px;
+    padding: 0;
+    border: none;
+    box-shadow: none;
+    background: transparent;
+  }
+
+  .cancel-button {
+    display: none;
+  }
+
+}
+
+
+.courseware-search-results {
+
+  display: none;
+  padding: 40px;
+
+  .search-info {
+    padding-bottom: lh(.75);
+    border-bottom: 1px solid lighten($border-color, 10%);
+    .search-count {
+      float: right;
+      color: $gray;
+    }
+  }
+
+  .search-results {
+    padding: 0;
+  }
+
+  .search-results-item {
+    position: relative;
+    border-bottom: 1px solid lighten($border-color, 10%);
+    list-style-type: none;
+    margin-bottom: lh(.75);
+    padding-bottom: lh(.75);
+    padding-right: 140px;
+
+    .sri-excerpt {
+      color: $gray;
+      margin-bottom: lh(1);
+    }
+    .sri-type {
+      position: absolute;
+      right: 0;
+      top: 0;
+      color: $gray;
+    }
+    .sri-link {
+      position: absolute;
+      right: 0;
+      line-height: 1.6em;
+      bottom: lh(.75);
+      text-transform: uppercase;
+    }
+  }
+
+  .search-load-next {
+    display: block;
+    text-transform: uppercase;
+    color: $base-font-color;
+    border: 2px solid $link-color;
+    @include border-radius(3px);
+    padding: 1rem;
+    .icon-spin {
+      display: none;
+    }
+  }
+
+}

lms/templates/courseware/courseware.html

@@ -23,8 +23,14 @@ ${page_title_breadcrumbs(course_name())}
     <%static:include path="js/${template_name}.underscore" />
 </script>
 % endfor
-</%block>
 
+% for template_name in ["search_item", "search_list", "search_loading", "search_error"]:
+<script type="text/template" id="${template_name}-tpl">
+    <%static:include path="courseware_search/${template_name}.underscore" />
+</script>
+% endfor
+
+</%block>
 
 <%block name="headextra">
 <%static:css group='style-course-vendor'/>
@@ -201,6 +207,16 @@ ${fragment.foot_html()}
         <a href="#">${_("close")}</a>
       </header>
 
+      % if settings.FEATURES.get('ENABLE_COURSEWARE_SEARCH'):
+        <div id="courseware-search-bar" class="courseware-search-bar">
+          <form role="search-form">
+            <input type="text" class="search-field"/>
+            <button type="submit" class="search-button" aria-label="${_('Search')}">${_('search')} <i class="icon fa fa-search"></i></button>
+            <button type="button" class="cancel-button" aria-label="${_('Cancel')}"><i class="icon fa fa-remove"></i></button>
+          </form>
+        </div>
+      % endif
+
       <div id="accordion" style="display: none">
         <nav aria-label="${_('Course Navigation')}">
           % if accordion.strip():
@@ -212,10 +228,13 @@ ${fragment.foot_html()}
       </div>
     </div>
 % endif
-
     <section class="course-content" id="course-content">
       ${fragment.body_html()}
     </section>
+    % if settings.FEATURES.get('ENABLE_COURSEWARE_SEARCH'):
+      <section class="courseware-search-results" id="courseware-search-results" data-course-id="${course.id}" data-course-name="${course.display_name_with_default}">
+      </section>
+    % endif
   </div>
 </div>
 

lms/templates/courseware_search/search_error.underscore

+<%= gettext("There was an error, try searching again.") %>

lms/templates/courseware_search/search_item.underscore

+<div class='sri-excerpt'><%= excerpt %></div>
+<span class='sri-type'><%- content_type %></span>
+<span class='sri-location'><%- location.join(' ▸ ') %></span>
+<a class="sri-link" href="<%- url %>"><%= gettext("View") %> <i class="icon-arrow-right"></i></a>

lms/templates/courseware_search/search_list.underscore

+<div class="search-info">
+    <%- interpolate(gettext("Searching %s"), [courseName]) %>
+    <div class="search-count"><%- totalCountMsg %></div>
+</div>
+
+<% if (totalCount > 0 ) { %>
+
+    <ol class='search-results'></ol>
+
+    <% if (hasMoreResults) { %>
+        <a class="search-load-next" href="javascript:void(0);">
+            <%- interpolate(gettext("Load next %s results"), [pageSize]) %>
+            <i class="icon fa-spinner fa-spin"></i>
+        </a>
+    <% } %>
+
+<% } else { %>
+
+    <p><%- gettext("Sorry, no results were found.") %></p>
+
+<% } %>

lms/templates/courseware_search/search_loading.underscore

+<i class="icon fa fa-spinner fa-spin"></i> <%= gettext("Loading") %>
+

lms/urls.py

@@ -10,6 +10,8 @@ from microsite_configuration import microsite
 if settings.DEBUG or settings.FEATURES.get('ENABLE_DJANGO_ADMIN_SITE'):
     admin.autodiscover()
 
+# Use urlpatterns formatted as within the Django docs with first parameter "stuck" to the open parenthesis
+# pylint: disable=bad-continuation
 urlpatterns = ('',  # nopep8
     # certificate view
     url(r'^update_certificate$', 'certificates.views.update_certificate'),
@@ -79,6 +81,9 @@ urlpatterns = ('',  # nopep8
     # CourseInfo API RESTful endpoints
     url(r'^api/course/details/v0/', include('course_about.urls')),
 
+    # Courseware search endpoints
+    url(r'^search/', include('search.urls')),
+
 )
 
 if settings.FEATURES["ENABLE_MOBILE_REST_API"]:

requirements/edx/base.txt

@@ -35,6 +35,7 @@ django-threaded-multihost==1.4-1
 django-method-override==0.1.0
 djangorestframework==2.3.14
 django==1.4.18
+elasticsearch==0.4.5
 feedparser==5.1.3
 firebase-token-generator==1.3.2
 # Master pyfs has a bug working with VPC auth. This is a fix. We should switch

requirements/edx/github.txt

@@ -21,7 +21,7 @@
 git+https://github.com/mitocw/django-cas.git@60a5b8e5a62e63e0d5d224a87f0b489201a0c695#egg=django-cas
 
 # Our libraries:
--e git+https://github.com/edx/XBlock.git@9c634481dfc85a17dcb3351ca232d7098a38e10e#egg=XBlock
+-e git+https://github.com/edx/XBlock.git@3682847a91acac6640a330fbe797ef56ce988517#egg=XBlock
 -e git+https://github.com/edx/codejail.git@2b095e820ff752a108653bb39d518b122f7154db#egg=codejail
 -e git+https://github.com/edx/js-test-tool.git@v0.1.6#egg=js_test_tool
 -e git+https://github.com/edx/event-tracking.git@0.1.0#egg=event-tracking
@@ -36,3 +36,4 @@ git+https://github.com/mitocw/django-cas.git@60a5b8e5a62e63e0d5d224a87f0b489201a
 -e git+https://github.com/edx/edx-val.git@ba00a5f2e0571e9a3f37d293a98efe4cbca850d5#egg=edx-val
 -e git+https://github.com/pmitros/RecommenderXBlock.git@9b07e807c89ba5761827d0387177f71aa57ef056#egg=recommender-xblock
 -e git+https://github.com/edx/edx-milestones.git@547f2250ee49e73ce8d7ff4e78ecf1b049892510#egg=edx-milestones
+-e git+https://github.com/edx/edx-search.git@264bb3317f98e9cb22b932aa11b89d0651fd741c#egg=edx-search