optimize libraries iter for studio dashboard

cms/djangoapps/contentstore/views/course.py

@@ -484,7 +484,7 @@ def _accessible_libraries_iter(user, org=None):
     if org is not None:
         libraries = [] if org == '' else modulestore().get_libraries(org=org)
     else:
-        libraries = modulestore().get_libraries()
+        libraries = modulestore().get_library_summaries()
     # No need to worry about ErrorDescriptors - split's get_libraries() never returns them.
     return (lib for lib in libraries if has_studio_read_access(user, lib.location.library_key))
 
@@ -493,7 +493,7 @@ def _accessible_libraries_iter(user, org=None):
 @ensure_csrf_cookie
 def course_listing(request):
     """
-    List all courses available to the logged in user
+    List all courses and libraries available to the logged in user
     """
 
     optimization_enabled = GlobalStaff().has_user(request.user) and \
@@ -502,7 +502,10 @@ def course_listing(request):
     org = request.GET.get('org', '') if optimization_enabled else None
     courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org)
     user = request.user
+    # remove this temporary log after getting results
+    start_time = time.time()
     libraries = _accessible_libraries_iter(request.user, org) if LIBRARIES_ENABLED else []
+    log.info("_accessible_libraries_iter completed in [%f]", (time.time() - start_time))
 
     def format_in_process_course_view(uca):
         """
@@ -529,6 +532,7 @@ def course_listing(request):
         """
         Return a dict of the data which the view requires for each library
         """
+
         return {
             u'display_name': library.display_name,
             u'library_key': unicode(library.location.library_key),

common/lib/xmodule/xmodule/library_content_module.py

@@ -640,3 +640,39 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe
             if field.is_set_on(self):
                 xml_object.set(field_name, unicode(field.read_from(self)))
         return xml_object
+
+
+class LibrarySummary(object):
+    """
+    A library summary object which contains the fields required for library listing on studio.
+    """
+
+    def __init__(self, library_locator, display_name):
+        """
+        Initialize LibrarySummary
+
+        Arguments:
+        library_locator (LibraryLocator):  LibraryLocator object of the library.
+
+        display_name (unicode): display name of the library.
+        """
+        self.display_name = display_name if display_name else _(u"Empty")
+
+        self.id = library_locator  # pylint: disable=invalid-name
+        self.location = library_locator.make_usage_key('library', 'library')
+
+    @property
+    def display_org_with_default(self):
+        """
+        Org display names are not implemented. This just provides API compatibility with CourseDescriptor.
+        Always returns the raw 'org' field from the key.
+        """
+        return self.location.library_key.org
+
+    @property
+    def display_number_with_default(self):
+        """
+        Display numbers are not implemented. This just provides API compatibility with CourseDescriptor.
+        Always returns the raw 'library' field from the key.
+        """
+        return self.location.library_key.library

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -18,6 +18,13 @@ class MultipleCourseBlocksFound(Exception):
     pass
 
 
+class MultipleLibraryBlocksFound(Exception):
+    """
+    Raise this exception when Iterating over the library blocks return multiple library blocks.
+    """
+    pass
+
+
 class InsufficientSpecificationError(Exception):
     pass
 

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -322,6 +322,23 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         return courses.values()
 
     @strip_key
+    def get_library_summaries(self, **kwargs):
+        """
+        Returns a list of LibrarySummary objects.
+        Information contains `location`, `display_name`, `locator` of the libraries in this modulestore.
+        """
+        library_summaries = {}
+        for store in self.modulestores:
+            if not hasattr(store, 'get_libraries'):
+                continue
+            # fetch library summaries and filter out any duplicated entry across/within stores
+            for library_summary in store.get_library_summaries(**kwargs):
+                library_id = self._clean_locator_for_mapping(library_summary.location)
+                if library_id not in library_summaries:
+                    library_summaries[library_id] = library_summary
+        return library_summaries.values()
+
+    @strip_key
     def get_libraries(self, **kwargs):
         """
         Returns a list containing the top level XBlock of the libraries (LibraryRoot) in this modulestore.

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -362,20 +362,21 @@ class MongoConnection(object):
             return docs
 
     @autoretry_read()
-    def find_course_blocks_by_id(self, ids, course_context=None):
+    def find_courselike_blocks_by_id(self, ids, block_type, course_context=None):
         """
-        Find all structures that specified in `ids`. Among the blocks only return block whose type is `course`.
+        Find all structures that specified in `ids`. Among the blocks only return block whose type is `block_type`.
 
         Arguments:
             ids (list): A list of structure ids
+            block_type: type of block to return
         """
-        with TIMER.timer("find_course_blocks_by_id", course_context) as tagger:
+        with TIMER.timer("find_courselike_blocks_by_id", course_context) as tagger:
             tagger.measure("requested_ids", len(ids))
             docs = [
                 structure_from_mongo(structure, course_context)
                 for structure in self.structures.find(
                     {'_id': {'$in': ids}},
-                    {'blocks': {'$elemMatch': {'block_type': 'course'}}, 'root': 1}
+                    {'blocks': {'$elemMatch': {'block_type': block_type}}, 'root': 1}
                 )
             ]
             tagger.measure("structures", len(docs))

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -70,6 +70,7 @@ from bson.objectid import ObjectId
 from xblock.core import XBlock
 from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
 from xmodule.course_module import CourseSummary
+from xmodule.library_content_module import LibrarySummary
 from xmodule.errortracker import null_error_tracker
 from opaque_keys.edx.keys import CourseKey
 from opaque_keys.edx.locator import (
@@ -582,16 +583,15 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
 
         return course_indexes
 
-    def find_course_blocks_by_id(self, ids):
+    def find_courselike_blocks_by_id(self, ids, block_type):
         """
-        Find all structures that specified in `ids`. Filter the course blocks to only return whose
-        `block_type` is `course`
-
+        Find all structures that specified in `ids`. Return blocks matching with block_type.
         Arguments:
             ids (list): A list of structure ids
+            block_type: type of block to return
         """
         ids = set(ids)
-        return self.db_connection.find_course_blocks_by_id(list(ids))
+        return self.db_connection.find_courselike_blocks_by_id(list(ids), block_type)
 
     def find_structures_by_id(self, ids):
         """
@@ -691,6 +691,9 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
     # version) but those functions will have an optional arg for setting these.
     SEARCH_TARGET_DICT = ['wiki_slug']
 
+    DEFAULT_ROOT_LIBRARY_BLOCK_TYPE = 'library'
+    DEFAULT_ROOT_COURSE_BLOCK_TYPE = 'course'
+
     def __init__(self, contentstore, doc_store_config, fs_root, render_template,
                  default_class=None,
                  error_tracker=null_error_tracker,
@@ -913,16 +916,19 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         # add it in the envelope for the structure.
         return CourseEnvelope(course_key.replace(version_guid=version_guid), entry)
 
-    def _get_course_blocks_for_branch(self, branch, **kwargs):
+    def _get_courselike_blocks_for_branch(self, branch, **kwargs):
         """
-        Internal generator for fetching lists of courses without loading them.
+        Internal generator for fetching lists of courselike without loading them.
         """
         version_guids, id_version_map = self.collect_ids_from_matching_indexes(branch, **kwargs)
 
         if not version_guids:
             return
 
-        for entry in self.find_course_blocks_by_id(version_guids):
+        block_type = SplitMongoModuleStore.DEFAULT_ROOT_LIBRARY_BLOCK_TYPE \
+            if branch == 'library' else SplitMongoModuleStore.DEFAULT_ROOT_COURSE_BLOCK_TYPE
+
+        for entry in self.find_courselike_blocks_by_id(version_guids, block_type):
             for course_index in id_version_map[entry['_id']]:
                 yield entry, course_index
 
@@ -1039,7 +1045,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             }
 
         courses_summaries = []
-        for entry, structure_info in self._get_course_blocks_for_branch(branch, **kwargs):
+        for entry, structure_info in self._get_courselike_blocks_for_branch(branch, **kwargs):
             course_locator = self._create_course_locator(structure_info, branch=None)
             course_block = [
                 block_data
@@ -1059,6 +1065,42 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             )
         return courses_summaries
 
+    @autoretry_read()
+    def get_library_summaries(self, **kwargs):
+        """
+        Returns a list of `LibrarySummary` objects.
+        kwargs can be valid db fields to match against active_versions
+        collection e.g org='example_org'.
+        """
+        branch = 'library'
+        libraries_summaries = []
+        for entry, structure_info in self._get_courselike_blocks_for_branch(branch, **kwargs):
+            library_locator = self._create_library_locator(structure_info, branch=None)
+            library_block = [
+                block_data
+                for block_key, block_data in entry['blocks'].items()
+                if block_key.type == "library"
+            ]
+            if not library_block:
+                raise ItemNotFoundError
+
+            if len(library_block) > 1:
+                raise MultipleLibraryBlocksFound(
+                    "Expected 1 library block, but found {0}".format(len(library_block))
+                )
+
+            library_block_fields = library_block[0].fields
+            display_name = ''
+
+            if 'display_name' in library_block_fields:
+                display_name = library_block_fields['display_name']
+
+            libraries_summaries.append(
+                LibrarySummary(library_locator, display_name)
+            )
+
+        return libraries_summaries
+
     def get_libraries(self, branch="library", **kwargs):
         """
         Returns a list of "library" root blocks matching any given qualifiers.