SOL-495 Cohort-Aware content search

cms/djangoapps/contentstore/courseware_index.py

@@ -8,8 +8,10 @@ from six import add_metaclass
 
 from django.conf import settings
 from django.utils.translation import ugettext as _
+from django.core.urlresolvers import resolve
 
 from contentstore.utils import course_image_url
+from contentstore.course_group_config import GroupConfiguration
 from course_modes.models import CourseMode
 from eventtracking import tracker
 from search.search_engine_base import SearchEngine
@@ -151,7 +153,7 @@ class SearchIndexerBase(object):
         # list - those are ready to be destroyed
         indexed_items = set()
 
-        def index_item(item, skip_index=False):
+        def index_item(item, skip_index=False, groups_usage_info=None):
             """
             Add this item to the search index and indexed_items list
 
@@ -162,6 +164,9 @@ class SearchIndexerBase(object):
                 older than the REINDEX_AGE window and would have been already indexed.
                 This should really only be passed from the recursive child calls when
                 this method has determined that it is safe to do so
+
+            Returns:
+            item_content_groups - content groups assigned to indexed item
             """
             is_indexable = hasattr(item, "index_dictionary")
             item_index_dictionary = item.index_dictionary() if is_indexable else None
@@ -169,15 +174,29 @@ class SearchIndexerBase(object):
             if not item_index_dictionary and not item.has_children:
                 return
 
+            item_content_groups = None
+            if groups_usage_info:
+                item_location = item.location.version_agnostic().replace(branch=None)
+                item_content_groups = groups_usage_info.get(unicode(item_location), None)
+
             item_id = unicode(cls._id_modifier(item.scope_ids.usage_id))
             indexed_items.add(item_id)
             if item.has_children:
                 # determine if it's okay to skip adding the children herein based upon how recently any may have changed
                 skip_child_index = skip_index or \
                     (triggered_at is not None and (triggered_at - item.subtree_edited_on) > reindex_age)
+                children_groups_usage = []
                 for child_item in item.get_children():
                     if modulestore.has_published_version(child_item):
-                        index_item(child_item, skip_index=skip_child_index)
+                        children_groups_usage.append(
+                            index_item(
+                                child_item,
+                                skip_index=skip_child_index,
+                                groups_usage_info=groups_usage_info
+                            )
+                        )
+                if None in children_groups_usage:
+                    item_content_groups = None
 
             if skip_index or not item_index_dictionary:
                 return
@@ -190,10 +209,11 @@ class SearchIndexerBase(object):
                 item_index['id'] = item_id
                 if item.start:
                     item_index['start_date'] = item.start
+                item_index['content_groups'] = item_content_groups if item_content_groups else None
                 item_index.update(cls.supplemental_fields(item))
-
                 searcher.index(cls.DOCUMENT_TYPE, item_index)
                 indexed_count["count"] += 1
+                return item_content_groups
             except Exception as err:  # pylint: disable=broad-except
                 # broad exception so that index operation does not fail on one item of many
                 log.warning('Could not index item: %s - %r', item.location, err)
@@ -202,13 +222,14 @@ class SearchIndexerBase(object):
         try:
             with modulestore.branch_setting(ModuleStoreEnum.RevisionOption.published_only):
                 structure = cls._fetch_top_level(modulestore, structure_key)
+                groups_usage_info = cls.fetch_group_usage(modulestore, structure)
 
                 # First perform any additional indexing from the structure object
                 cls.supplemental_index_information(modulestore, structure)
 
                 # Now index the content
                 for item in structure.get_children():
-                    index_item(item)
+                    index_item(item, groups_usage_info=groups_usage_info)
                 cls.remove_deleted_items(searcher, structure_key, indexed_items)
         except Exception as err:  # pylint: disable=broad-except
             # broad exception so that index operation does not prevent the rest of the application from working
@@ -258,6 +279,13 @@ class SearchIndexerBase(object):
         )
 
     @classmethod
+    def fetch_group_usage(cls, modulestore, structure):  # pylint: disable=unused-argument
+        """
+        Base implementation of fetch group usage on course/library.
+        """
+        return None
+
+    @classmethod
     def supplemental_index_information(cls, modulestore, structure):
         """
         Perform any supplemental indexing given that the structure object has
@@ -319,6 +347,27 @@ class CoursewareSearchIndexer(SearchIndexerBase):
         return cls._do_reindex(modulestore, course_key)
 
     @classmethod
+    def fetch_group_usage(cls, modulestore, structure):
+        groups_usage_dict = {}
+        groups_usage_info = GroupConfiguration.get_content_groups_usage_info(modulestore, structure).items()
+        groups_usage_info.extend(
+            GroupConfiguration.get_content_groups_items_usage_info(
+                modulestore,
+                structure
+            ).items()
+        )
+        if groups_usage_info:
+            for name, group in groups_usage_info:
+                for module in group:
+                    view, args, kwargs = resolve(module['url'])  # pylint: disable=unused-variable
+                    usage_key_string = unicode(kwargs['usage_key_string'])
+                    if groups_usage_dict.get(usage_key_string, None):
+                        groups_usage_dict[usage_key_string].append(name)
+                    else:
+                        groups_usage_dict[usage_key_string] = [name]
+        return groups_usage_dict
+
+    @classmethod
     def supplemental_index_information(cls, modulestore, structure):
         """
         Perform additional indexing from loaded structure object

cms/djangoapps/contentstore/views/tests/test_group_configurations.py

@@ -7,7 +7,7 @@ import json
 from mock import patch
 from contentstore.utils import reverse_course_url, reverse_usage_url
 from contentstore.views.component import SPLIT_TEST_COMPONENT_TYPE
-from contentstore.views.course import GroupConfiguration
+from contentstore.course_group_config import GroupConfiguration
 from contentstore.tests.utils import CourseTestCase
 from xmodule.partitions.partitions import Group, UserPartition
 from xmodule.modulestore.tests.factories import ItemFactory

common/test/acceptance/pages/lms/courseware_search.py

@@ -37,3 +37,10 @@ class CoursewareSearchPage(CoursePage):
         """
         self.enter_search_term(text)
         self.search()
+
+    def clear_search(self):
+        """
+        Clear search bar after search.
+        """
+        self.q(css=self.search_bar_selector + ' .cancel-button').click()
+        self.wait_for_element_visibility('#course-content', 'Search bar is cleared')

lms/envs/common.py

@@ -2225,6 +2225,8 @@ PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM = 12
 
 # Use None for the default search engine
 SEARCH_ENGINE = None
+# Use LMS specific search initializer
+SEARCH_INITIALIZER = "lms.lib.courseware_search.lms_search_initializer.LmsSearchInitializer"
 # Use the LMS specific result processor
 SEARCH_RESULT_PROCESSOR = "lms.lib.courseware_search.lms_result_processor.LmsSearchResultProcessor"
 # Use the LMS specific filter generator

lms/lib/courseware_search/lms_filter_generator.py

@@ -3,14 +3,44 @@ This file contains implementation override of SearchFilterGenerator which will a
     * Filter by all courses in which the user is enrolled in
 """
 from microsite_configuration import microsite
+
 from student.models import CourseEnrollment
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 from search.filter_generator import SearchFilterGenerator
+from openedx.core.djangoapps.course_groups.partition_scheme import get_cohorted_user_partition
+from courseware.access import get_user_role
 
 
 class LmsSearchFilterGenerator(SearchFilterGenerator):
     """ SearchFilterGenerator for LMS Search """
 
+    def filter_dictionary(self, **kwargs):
+        """ base implementation which filters via start_date """
+        filter_dictionary = super(LmsSearchFilterGenerator, self).filter_dictionary(**kwargs)
+        if 'user' in kwargs and 'course_id' in kwargs and kwargs['course_id']:
+            user = kwargs['user']
+            try:
+                course_key = CourseKey.from_string(kwargs['course_id'])
+            except InvalidKeyError:
+                course_key = SlashSeparatedCourseKey.from_deprecated_string(kwargs['course_id'])
+
+            # Staff user looking at course as staff user
+            if get_user_role(user, course_key) == 'staff':
+                return filter_dictionary
+
+            cohorted_user_partition = get_cohorted_user_partition(course_key)
+            if cohorted_user_partition:
+                partition_group = cohorted_user_partition.scheme.get_group_for_user(
+                    course_key,
+                    user,
+                    cohorted_user_partition,
+                )
+                filter_dictionary['content_groups'] = unicode(partition_group.id) if partition_group else None
+        return filter_dictionary
+
     def field_dictionary(self, **kwargs):
         """ add course if provided otherwise add courses in which the user is enrolled in """
         field_dictionary = super(LmsSearchFilterGenerator, self).field_dictionary(**kwargs)

lms/lib/courseware_search/lms_search_initializer.py

+"""
+This file contains implementation override of SearchInitializer which will allow
+    * To set initial set of masquerades and other parameters
+"""
+
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+from opaque_keys.edx.locations import SlashSeparatedCourseKey
+
+from search.initializer import SearchInitializer
+from courseware.masquerade import setup_masquerade
+from courseware.access import has_access
+
+
+class LmsSearchInitializer(SearchInitializer):
+    """ SearchInitializer for LMS Search """
+    def initialize(self, **kwargs):
+        if 'request' in kwargs and kwargs['request'] and kwargs['course_id']:
+            request = kwargs['request']
+            try:
+                course_key = CourseKey.from_string(kwargs['course_id'])
+            except InvalidKeyError:
+                course_key = SlashSeparatedCourseKey.from_deprecated_string(kwargs['course_id'])
+            staff_access = has_access(request.user, 'staff', course_key)
+            setup_masquerade(request, course_key, staff_access)

lms/lib/courseware_search/test/test_lms_filter_generator.py

@@ -3,11 +3,18 @@ Tests for the lms_filter_generator
 """
 from mock import patch, Mock
 
-from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from student.tests.factories import UserFactory
 from student.models import CourseEnrollment
 
+from xmodule.partitions.partitions import Group, UserPartition
+from openedx.core.djangoapps.course_groups.partition_scheme import CohortPartitionScheme
+from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory, config_course_cohorts
+from openedx.core.djangoapps.course_groups.cohorts import add_user_to_cohort
+from openedx.core.djangoapps.course_groups.views import link_cohort_to_partition_group
+from opaque_keys import InvalidKeyError
 from lms.lib.courseware_search.lms_filter_generator import LmsSearchFilterGenerator
 
 
@@ -35,13 +42,78 @@ class LmsSearchFilterGeneratorTestCase(ModuleStoreTestCase):
             )
         ]
 
+        self.chapter = ItemFactory.create(
+            parent_location=self.courses[0].location,
+            category='chapter',
+            display_name="Week 1",
+            publish_item=True,
+        )
+
+        self.groups = [Group(1, 'Group 1'), Group(2, 'Group 2')]
+
+        self.content_groups = [1, 2]
+
     def setUp(self):
         super(LmsSearchFilterGeneratorTestCase, self).setUp()
         self.build_courses()
+        self.user_partition = None
+        self.first_cohort = None
+        self.second_cohort = None
         self.user = UserFactory.create(username="jack", email="jack@fake.edx.org", password='test')
+
         for course in self.courses:
             CourseEnrollment.enroll(self.user, course.location.course_key)
 
+    def add_seq_with_content_groups(self, groups=None):
+        """
+        Adds sequential and two content groups to first course in courses list.
+        """
+        config_course_cohorts(self.courses[0], is_cohorted=True)
+
+        if groups is None:
+            groups = self.groups
+
+        self.user_partition = UserPartition(
+            id=0,
+            name='Partition 1',
+            description='This is partition 1',
+            groups=groups,
+            scheme=CohortPartitionScheme
+        )
+
+        self.user_partition.scheme.name = "cohort"
+
+        ItemFactory.create(
+            parent_location=self.chapter.location,
+            category='sequential',
+            display_name="Lesson 1",
+            publish_item=True,
+            metadata={u"user_partitions": [self.user_partition.to_json()]}
+        )
+
+        self.first_cohort, self.second_cohort = [
+            CohortFactory(course_id=self.courses[0].id) for _ in range(2)
+        ]
+
+        self.courses[0].user_partitions = [self.user_partition]
+        self.courses[0].save()
+        modulestore().update_item(self.courses[0], self.user.id)
+
+    def add_user_to_cohort_group(self):
+        """
+        adds user to cohort and links cohort to content group
+        """
+        add_user_to_cohort(self.first_cohort, self.user.username)
+
+        link_cohort_to_partition_group(
+            self.first_cohort,
+            self.user_partition.id,
+            self.groups[0].id,
+        )
+
+        self.courses[0].save()
+        modulestore().update_item(self.courses[0], self.user.id)
+
     def test_course_id_not_provided(self):
         """
         Tests that we get the list of IDs of courses the user is enrolled in when the course ID is null or not provided
@@ -118,3 +190,90 @@ class LmsSearchFilterGeneratorTestCase(ModuleStoreTestCase):
         self.assertNotIn('org', exclude_dictionary)
         self.assertIn('org', field_dictionary)
         self.assertEqual('TestMicrosite3', field_dictionary['org'])
+
+    def test_content_group_id_provided(self):
+        """
+        Tests that we get the content group ID when course is assigned to cohort
+        which is assigned content group.
+        """
+        self.add_seq_with_content_groups()
+        self.add_user_to_cohort_group()
+        field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
+            user=self.user,
+            course_id=unicode(self.courses[0].id)
+        )
+
+        self.assertTrue('start_date' in filter_dictionary)
+        self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
+        self.assertEqual(unicode(self.content_groups[0]), filter_dictionary['content_groups'])
+
+    def test_content_multiple_groups_id_provided(self):
+        """
+        Tests that we get content groups IDs when course is assigned to cohort
+        which is assigned to multiple content groups.
+        """
+        self.add_seq_with_content_groups()
+        self.add_user_to_cohort_group()
+
+        # Second cohort link
+        link_cohort_to_partition_group(
+            self.second_cohort,
+            self.user_partition.id,
+            self.groups[0].id,
+        )
+
+        self.courses[0].save()
+        modulestore().update_item(self.courses[0], self.user.id)
+
+        field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
+            user=self.user,
+            course_id=unicode(self.courses[0].id)
+        )
+
+        self.assertTrue('start_date' in filter_dictionary)
+        self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
+        # returns only first group, relevant to current user
+        self.assertEqual(unicode(self.content_groups[0]), filter_dictionary['content_groups'])
+
+    def test_content_group_id_not_provided(self):
+        """
+        Tests that we don't get content group ID when course is assigned a cohort
+        but cohort is not assigned to content group.
+        """
+        self.add_seq_with_content_groups(groups=[])
+
+        field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
+            user=self.user,
+            course_id=unicode(self.courses[0].id)
+        )
+
+        self.assertTrue('start_date' in filter_dictionary)
+        self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
+        self.assertEqual(None, filter_dictionary['content_groups'])
+
+    def test_content_group_with_cohort_not_provided(self):
+        """
+        Tests that we don't get content group ID when course has no cohorts
+        """
+        self.add_seq_with_content_groups()
+
+        field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
+            user=self.user,
+            course_id=unicode(self.courses[0].id)
+        )
+
+        self.assertTrue('start_date' in filter_dictionary)
+        self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
+        self.assertEqual(None, filter_dictionary['content_groups'])
+
+    def test_invalid_course_key(self):
+        """
+        Test system raises an error if no course found.
+        """
+
+        self.add_seq_with_content_groups()
+        with self.assertRaises(InvalidKeyError):
+            LmsSearchFilterGenerator.generate_field_filters(
+                user=self.user,
+                course_id='this_is_false_course_id'
+            )

lms/lib/courseware_search/test/test_lms_search_initializer.py

+"""
+Tests for the lms_search_initializer
+"""
+
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.partitions.partitions import Group, UserPartition
+from xmodule.modulestore.django import modulestore
+
+from courseware.tests.factories import UserFactory
+from courseware.tests.test_masquerade import StaffMasqueradeTestCase
+from courseware.masquerade import handle_ajax
+
+from lms.lib.courseware_search.lms_search_initializer import LmsSearchInitializer
+from lms.lib.courseware_search.lms_filter_generator import LmsSearchFilterGenerator
+
+
+class LmsSearchInitializerTestCase(StaffMasqueradeTestCase):
+    """ Test case class to test search initializer """
+
+    def build_course(self):
+        """
+        Build up a course tree with an html control
+        """
+        self.global_staff = UserFactory(is_staff=True)
+
+        self.course = CourseFactory.create(
+            org='Elasticsearch',
+            course='ES101',
+            run='test_run',
+            display_name='Elasticsearch test course',
+        )
+        self.section = ItemFactory.create(
+            parent=self.course,
+            category='chapter',
+            display_name='Test Section',
+        )
+        self.subsection = ItemFactory.create(
+            parent=self.section,
+            category='sequential',
+            display_name='Test Subsection',
+        )
+        self.vertical = ItemFactory.create(
+            parent=self.subsection,
+            category='vertical',
+            display_name='Test Unit',
+        )
+        self.html = ItemFactory.create(
+            parent=self.vertical,
+            category='html',
+            display_name='Test Html control 1',
+        )
+        self.html = ItemFactory.create(
+            parent=self.vertical,
+            category='html',
+            display_name='Test Html control 2',
+        )
+
+    def setUp(self):
+        super(LmsSearchInitializerTestCase, self).setUp()
+        self.build_course()
+        self.user_partition = UserPartition(
+            id=0,
+            name='Test User Partition',
+            description='',
+            groups=[Group(0, 'Group 1'), Group(1, 'Group 2')],
+            scheme_id='cohort'
+        )
+        self.course.user_partitions.append(self.user_partition)
+        modulestore().update_item(self.course, self.global_staff.id)  # pylint: disable=no-member
+
+    def test_staff_masquerading_added_to_group(self):
+        """
+        Tests that initializer sets masquerading for a staff user in a group.
+        """
+        # Verify that there is no masquerading group initially
+        _, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters(  # pylint: disable=unused-variable
+            user=self.global_staff,
+            course_id=unicode(self.course.id)
+        )
+        self.assertIsNone(filter_directory['content_groups'])
+
+        # Install a masquerading group
+        request = self._create_mock_json_request(
+            self.global_staff,
+            body='{"role": "student", "user_partition_id": 0, "group_id": 1}'
+        )
+        handle_ajax(request, unicode(self.course.id))
+
+        # Call initializer
+        LmsSearchInitializer.set_search_enviroment(
+            request=request,
+            course_id=unicode(self.course.id)
+        )
+
+        # Verify that there is masquerading group after masquerade
+        _, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters(  # pylint: disable=unused-variable
+            user=self.global_staff,
+            course_id=unicode(self.course.id)
+        )
+        self.assertEqual(filter_directory['content_groups'], unicode(1))
+
+    def test_staff_masquerading_as_a_staff_user(self):
+        """
+        Tests that initializer sets masquerading for a staff user as staff.
+        """
+
+        # Install a masquerading group
+        request = self._create_mock_json_request(
+            self.global_staff,
+            body='{"role": "staff"}'
+        )
+        handle_ajax(request, unicode(self.course.id))
+
+        # Call initializer
+        LmsSearchInitializer.set_search_enviroment(
+            request=request,
+            course_id=unicode(self.course.id)
+        )
+
+        # Verify that there is masquerading group after masquerade
+        _, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters(  # pylint: disable=unused-variable
+            user=self.global_staff,
+            course_id=unicode(self.course.id)
+        )
+        self.assertNotIn('content_groups', filter_directory)
+
+    def test_staff_masquerading_as_a_student_user(self):
+        """
+        Tests that initializer sets masquerading for a staff user as student.
+        """
+
+        # Install a masquerading group
+        request = self._create_mock_json_request(
+            self.global_staff,
+            body='{"role": "student"}'
+        )
+        handle_ajax(request, unicode(self.course.id))
+
+        # Call initializer
+        LmsSearchInitializer.set_search_enviroment(
+            request=request,
+            course_id=unicode(self.course.id)
+        )
+
+        # Verify that there is masquerading group after masquerade
+        _, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters(  # pylint: disable=unused-variable
+            user=self.global_staff,
+            course_id=unicode(self.course.id)
+        )
+        self.assertEqual(filter_directory['content_groups'], None)

requirements/edx/github.txt

@@ -45,7 +45,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c
 -e git+https://github.com/edx/edx-val.git@b1e11c9af3233bc06a17acbb33179f46d43c3b87#egg=edx-val
 -e git+https://github.com/pmitros/RecommenderXBlock.git@9b07e807c89ba5761827d0387177f71aa57ef056#egg=recommender-xblock
 -e git+https://github.com/edx/edx-milestones.git@547f2250ee49e73ce8d7ff4e78ecf1b049892510#egg=edx-milestones
--e git+https://github.com/edx/edx-search.git@59c7b4a8b61e8f7c4607669ea48e070555cca2fe#egg=edx-search
+-e git+https://github.com/edx/edx-search.git@ae459ead41962c656ce794619f58cdae46eb7896#egg=edx-search
 git+https://github.com/edx/edx-lint.git@8bf82a32ecb8598c415413df66f5232ab8d974e9#egg=edx_lint==0.2.1
 -e git+https://github.com/edx/xblock-utils.git@581ed636c862b286002bb9a3724cc883570eb54c#egg=xblock-utils
 -e git+https://github.com/edx-solutions/xblock-google-drive.git@138e6fa0bf3a2013e904a085b9fed77dab7f3f21#egg=xblock-google-drive