add a discussion module write through cache to speed up Forum rendering

cms/djangoapps/contentstore/utils.py

@@ -6,6 +6,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_tab', 'course_info']
 
 
+
 def get_modulestore(location):
     """
     Returns the correct modulestore to use for modifying the specified location

cms/one_time_startup.py

 from dogapi import dog_http_api, dog_stats_api
 from django.conf import settings
 from xmodule.modulestore.django import modulestore
+from cache_toolbox.discussion_cache import discussion_cache_register_for_updates
+from django.dispatch import Signal
 
 from django.core.cache import get_cache, InvalidCacheBackendError
 
@@ -9,6 +11,12 @@ for store_name in settings.MODULESTORE:
     store = modulestore(store_name)
     store.metadata_inheritance_cache = cache
 
+    modulestore_update_signal = Signal(
+                            providing_args=['modulestore', 'course_id', 'location']
+                        )
+    store.modulestore_update_signal = modulestore_update_signal
+    discussion_cache_register_for_updates(store)
+
 if hasattr(settings, 'DATADOG_API'):
     dog_http_api.api_key = settings.DATADOG_API
     dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)

common/djangoapps/cache_toolbox/discussion_cache.py

+import logging
+from django.core.cache import cache, get_cache
+from datetime import datetime, timedelta
+
+def _get_discussion_cache():
+    return cache
+
+
+def get_discussion_cache_key(course_id):
+    return 'discussion_{0}'.format(course_id)
+
+
+def get_discussion_cache_entry(modulestore, course_id):
+    cache_entry = None
+    cache = _get_discussion_cache()
+    
+    if cache is not None:
+        cache_entry = cache.get(get_discussion_cache_key(course_id), None)
+        if cache_entry is not None:
+            delta = datetime.now() - cache_entry.get('timestamp', datetime.min)
+            if delta > Timedelta(0,300):
+                cache_entry = None
+    
+    if cache_entry is None:
+        cache_entry = generate_discussion_cache_entry(modulestore, course_id)
+
+    return cache_entry.get('modules',[])
+
+
+def generate_discussion_cache_entry(modulestore, course_id):
+    components = course_id.split('/')
+    all_discussion_modules = modulestore.get_items(['i4x', components[0], components[1], 'discussion', None], 
+        course_id=course_id)
+
+    cache = _get_discussion_cache()
+    if cache is not None:
+        cache.set(get_discussion_cache_key(course_id), {'modules': all_discussion_modules, 'timestamp': datetime.now()})
+    return all_discussion_modules
+
+
+def modulestore_update_signal_handler(modulestore = None, course_id = None, location = None, **kwargs):
+    """called when there is an write event in our modulestore
+    """
+    if location.category == 'discussion':
+        logging.debug('******* got modulestore update signal. Regenerating discussion cache for {0}'.format(course_id))
+        # refresh the cache entry if we've changed a discussion module
+        generate_discussion_cache_entry(modulestore, course_id)
+
+
+def discussion_cache_register_for_updates(modulestore):
+    if modulestore.modulestore_update_signal is not None:
+        modulestore.modulestore_update_signal.connect(modulestore_update_signal_handler)
\ No newline at end of file

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -251,7 +251,6 @@ class Location(_LocationBase):
     def __repr__(self):
         return "Location%s" % repr(tuple(self))
 
-
     @property
     def course_id(self):
         """Return the ID of the Course that this item belongs to by looking
@@ -413,7 +412,6 @@ class ModuleStore(object):
         return courses
 
 
-
 class ModuleStoreBase(ModuleStore):
     '''
     Implement interface functionality that can be shared.
@@ -424,6 +422,7 @@ class ModuleStoreBase(ModuleStore):
         '''
         self._location_errors = {}    # location -> ErrorLog
         self.metadata_inheritance_cache = None
+        self.modulestore_update_signal = None  # can be set by runtime to route notifications of datastore changes
 
     def _get_errorlog(self, location):
         """

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -31,6 +31,10 @@ log = logging.getLogger(__name__)
 # there is only one revision for each item. Once we start versioning inside the CMS,
 # that assumption will have to change
 
+def get_course_id_no_run(location):
+    '''
+    '''
+    return "/".join([location.org, location.course])
 
 class MongoKeyValueStore(KeyValueStore):
     """
@@ -527,6 +531,15 @@ class MongoModuleStore(ModuleStoreBase):
 
         # recompute (and update) the metadata inheritance tree which is cached
         self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True)
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
+
+    def fire_updated_modulestore_signal(self, course_id, location):
+        if self.modulestore_update_signal is not None:
+            self.modulestore_update_signal.send(None,
+                modulestore = self,
+                course_id = course_id,
+                location = location
+            )
 
     def get_course_for_item(self, location, depth=0):
         '''
@@ -594,6 +607,8 @@ class MongoModuleStore(ModuleStoreBase):
         self._update_single_item(location, {'definition.children': children})
         # recompute (and update) the metadata inheritance tree which is cached
         self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True)
+        # fire signal that we've written to DB
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
 
     def update_metadata(self, location, metadata):
         """
@@ -620,6 +635,7 @@ class MongoModuleStore(ModuleStoreBase):
         self._update_single_item(location, {'metadata': metadata})
         # recompute (and update) the metadata inheritance tree which is cached
         self.get_cached_metadata_inheritance_tree(loc, force_refresh = True)  
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))   
 
     def delete_item(self, location):
         """
@@ -640,7 +656,7 @@ class MongoModuleStore(ModuleStoreBase):
         self.collection.remove({'_id': Location(location).dict()})
         # recompute (and update) the metadata inheritance tree which is cached
         self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True)  
-
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))   
 
     def get_parent_locations(self, location, course_id):
         '''Find all locations that are the parents of this location in this

lms/djangoapps/django_comment_client/utils.py

@@ -16,6 +16,7 @@ from django.utils import simplejson
 from django_comment_client.models import Role
 from django_comment_client.permissions import check_permissions_by_view
 from xmodule.modulestore.exceptions import NoPathToItem
+from cache_toolbox.discussion_cache import get_discussion_cache_entry
 
 from mitxmako import middleware
 import pystache_custom as pystache
@@ -146,28 +147,15 @@ def sort_map_entries(category_map):
 
 
 def initialize_discussion_info(course):
-
     global _DISCUSSIONINFO
 
-    # only cache in-memory discussion information for 10 minutes
-    # this is because we need a short-term hack fix for
-    # mongo-backed courseware whereby new discussion modules can be added
-    # without LMS service restart
-
-    if _DISCUSSIONINFO[course.id]:
-        timestamp = _DISCUSSIONINFO[course.id].get('timestamp', datetime.now())
-        age = datetime.now() - timestamp
-        # expire every 5 minutes
-        if age.seconds < 300:
-            return
-
     course_id = course.id
 
     discussion_id_map = {}
     unexpanded_category_map = defaultdict(list)
 
     # get all discussion models within this course_id
-    all_modules = modulestore().get_items(['i4x', course.location.org, course.location.course, 'discussion', None], course_id=course_id)
+    all_modules = get_discussion_cache_entry(modulestore(), course_id)
 
     for module in all_modules:
         skip_module = False