Merge pull request #690 from cpennington/one-time-startup

One time startup centralization

CHANGELOG.rst

@@ -11,6 +11,9 @@ It is hidden behind a false defaulted course level flag.
 Studio: Allow course authors to set their course image on the schedule
 and details page, with support for JPEG and PNG images.
 
+LMS, Studio: Centralized startup code to manage.py and wsgi.py files.
+Made studio runnable using wsgi.
+
 Blades: Took videoalpha out of alpha, replacing the old video player
 
 Common: Allow instructors to input complicated expressions as answers to

cms/djangoapps/contentstore/management/commands/check_course.py

@@ -3,11 +3,6 @@ from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml_importer import check_module_metadata_editability
 from xmodule.course_module import CourseDescriptor
 
-from request_cache.middleware import RequestCache
-
-from django.core.cache import get_cache
-
-CACHE = get_cache('mongo_metadata_inheritance')
 
 class Command(BaseCommand):
     help = '''Enumerates through the course and find common errors'''
@@ -21,12 +16,6 @@ class Command(BaseCommand):
         loc = CourseDescriptor.id_to_location(loc_str)
         store = modulestore()
 
-        # setup a request cache so we don't throttle the DB with all the metadata inheritance requests
-        store.set_modulestore_configuration({
-            'metadata_inheritance_cache_subsystem': CACHE,
-            'request_cache': RequestCache.get_request_cache()
-        })
-
         course = store.get_item(loc, depth=3)
 
         err_cnt = 0

cms/djangoapps/contentstore/management/commands/clone_course.py

@@ -9,14 +9,10 @@ from xmodule.course_module import CourseDescriptor
 
 from auth.authz import _copy_course_group
 
+
 #
 # To run from command line: rake cms:clone SOURCE_LOC=MITx/111/Foo1 DEST_LOC=MITx/135/Foo3
 #
-from request_cache.middleware import RequestCache
-from django.core.cache import get_cache
-
-CACHE = get_cache('mongo_metadata_inheritance')
-
 class Command(BaseCommand):
     """Clone a MongoDB-backed course to another location"""
     help = 'Clone a MongoDB backed course to another location'
@@ -32,11 +28,6 @@ class Command(BaseCommand):
         mstore = modulestore('direct')
         cstore = contentstore()
 
-        mstore.set_modulestore_configuration({
-            'metadata_inheritance_cache_subsystem': CACHE,
-            'request_cache': RequestCache.get_request_cache()
-        })
-
         org, course_num, run = dest_course_id.split("/")
         mstore.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
 

cms/djangoapps/contentstore/management/commands/delete_course.py

@@ -9,14 +9,11 @@ from xmodule.course_module import CourseDescriptor
 from .prompt import query_yes_no
 
 from auth.authz import _delete_course_group
-from request_cache.middleware import RequestCache
-from django.core.cache import get_cache
+
 
 #
 # To run from command line: rake cms:delete_course LOC=MITx/111/Foo1
 #
-
-CACHE = get_cache('mongo_metadata_inheritance')
 class Command(BaseCommand):
     help = '''Delete a MongoDB backed course'''
 
@@ -36,11 +33,6 @@ class Command(BaseCommand):
         ms = modulestore('direct')
         cs = contentstore()
 
-        ms.set_modulestore_configuration({
-            'metadata_inheritance_cache_subsystem': CACHE,
-            'request_cache': RequestCache.get_request_cache()
-        })
-
         org, course_num, run = course_id.split("/")
         ms.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
 

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -1140,12 +1140,15 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         wrapper = MongoCollectionFindWrapper(module_store.collection.find)
         module_store.collection.find = wrapper.find
+        print module_store.metadata_inheritance_cache_subsystem
+        print module_store.request_cache
         course = module_store.get_item(location, depth=2)
 
         # make sure we haven't done too many round trips to DB
-        # note we say 4 round trips here for 1) the course, 2 & 3) for the chapters and sequentials, and
-        # 4) because of the RT due to calculating the inherited metadata
-        self.assertEqual(wrapper.counter, 4)
+        # note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
+        # Because we're querying from the top of the tree, we cache information needed for inheritance,
+        # so we don't need to make an extra query to compute it.
+        self.assertEqual(wrapper.counter, 3)
 
         # make sure we pre-fetched a known sequential which should be at depth=2
         self.assertTrue(Location(['i4x', 'edX', 'toy', 'sequential',

cms/one_time_startup.py

-from dogapi import dog_http_api, dog_stats_api
-from django.conf import settings
-from xmodule.modulestore.django import modulestore
-from django.dispatch import Signal
-from request_cache.middleware import RequestCache
-
-from django.core.cache import get_cache
-
-CACHE = get_cache('mongo_metadata_inheritance')
-for store_name in settings.MODULESTORE:
-    store = modulestore(store_name)
-
-    store.set_modulestore_configuration({
-        'metadata_inheritance_cache_subsystem': CACHE,
-        'request_cache': RequestCache.get_request_cache()
-    })
-
-    modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location'])
-    store.modulestore_update_signal = modulestore_update_signal
-if hasattr(settings, 'DATADOG_API'):
-    dog_http_api.api_key = settings.DATADOG_API
-    dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)

cms/startup.py

+"""
+Module with code executed during Studio startup
+"""
+from django.conf import settings
+
+# Force settings to run so that the python path is modified
+settings.INSTALLED_APPS  # pylint: disable=W0104
+
+from django_startup import autostartup
+
+# TODO: Remove this code once Studio/CMS runs via wsgi in all environments
+INITIALIZED = False
+
+
+def run():
+    """
+    Executed during django startup
+    """
+    global INITIALIZED
+    if INITIALIZED:
+        return
+
+    INITIALIZED = True
+    autostartup()
+

cms/urls.py

 from django.conf import settings
 from django.conf.urls import patterns, include, url
 
-# Import this file so it can do its work, even though we don't use the name.
-# pylint: disable=W0611
-from . import one_time_startup
+# TODO: This should be removed once the CMS is running via wsgi on all production servers
+import cms.startup as startup
+startup.run()
 
 # There is a course creators admin table.
 from ratelimitbackend import admin

cms/wsgi.py

+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws")
+
+import cms.startup as startup
+startup.run()
+
+# This application object is used by the development server
+# as well as any WSGI server configured to use this file.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+

common/djangoapps/datadog/startup.py

+from django.conf import settings
+from dogapi import dog_http_api, dog_stats_api
+
+def run():
+    """
+    Initialize connection to datadog during django startup.
+
+    Expects the datadog api key in the DATADOG_API settings key
+    """
+    if hasattr(settings, 'DATADOG_API'):
+        dog_http_api.api_key = settings.DATADOG_API
+        dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)

common/djangoapps/terrain/browser.py

@@ -16,11 +16,6 @@ from requests import put
 from base64 import encodestring
 from json import dumps
 
-# Let the LMS and CMS do their one-time setup
-# For example, setting up mongo caches
-# These names aren't used, but do important work on import.
-from lms import one_time_startup        # pylint: disable=W0611
-from cms import one_time_startup        # pylint: disable=W0611
 from pymongo import MongoClient
 import xmodule.modulestore.django
 from xmodule.contentstore.django import _CONTENTSTORE

common/lib/django_startup.py

+from importlib import import_module
+from django.conf import settings
+
+def autostartup():
+    """
+    Execute app.startup:run() for all installed django apps
+    """
+    for app in settings.INSTALLED_APPS:
+        try:
+            mod = import_module('{}.startup')
+            if hasattr(mod, 'run'):
+                mod.run()
+        except ImportError:
+            continue

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -386,13 +386,6 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
-    def set_modulestore_configuration(self, config_dict):
-        '''
-        Allows for runtime configuration of the modulestore. In particular this is how the
-        application (LMS/CMS) can pass down Django related configuration information, e.g. caches, etc.
-        '''
-        raise NotImplementedError
-
     def get_modulestore_type(self, course_id):
         """
         Returns a type which identifies which modulestore is servicing the given
@@ -405,13 +398,14 @@ class ModuleStoreBase(ModuleStore):
     '''
     Implement interface functionality that can be shared.
     '''
-    def __init__(self):
+    def __init__(self, metadata_inheritance_cache_subsystem=None, request_cache=None, modulestore_update_signal=None):
         '''
         Set up the error-tracking logic.
         '''
         self._location_errors = {}  # location -> ErrorLog
-        self.modulestore_configuration = {}
-        self.modulestore_update_signal = None  # can be set by runtime to route notifications of datastore changes
+        self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
+        self.modulestore_update_signal = modulestore_update_signal
+        self.request_cache = request_cache
 
     def _get_errorlog(self, location):
         """
@@ -455,27 +449,6 @@ class ModuleStoreBase(ModuleStore):
                 return c
         return None
 
-    @property
-    def metadata_inheritance_cache_subsystem(self):
-        """
-        Exposes an accessor to the runtime configuration for the metadata inheritance cache
-        """
-        return self.modulestore_configuration.get('metadata_inheritance_cache_subsystem', None)
-
-    @property
-    def request_cache(self):
-        """
-        Exposes an accessor to the runtime configuration for the request cache
-        """
-        return self.modulestore_configuration.get('request_cache', None)
-
-    def set_modulestore_configuration(self, config_dict):
-        """
-        This is the base implementation of the interface, all we need to do is store
-        two possible configurations as attributes on the class
-        """
-        self.modulestore_configuration = config_dict
-
 
 def namedtuple_to_son(namedtuple, prefix=''):
     """

common/lib/xmodule/xmodule/modulestore/django.py

@@ -8,8 +8,17 @@ from __future__ import absolute_import
 from importlib import import_module
 
 from django.conf import settings
+from django.core.cache import get_cache, InvalidCacheBackendError
+from django.dispatch import Signal
 from xmodule.modulestore.loc_mapper_store import LocMapperStore
 
+# We may not always have the request_cache module available
+try:
+    from request_cache.middleware import RequestCache
+    HAS_REQUEST_CACHE = True
+except ImportError:
+    HAS_REQUEST_CACHE = False
+
 _MODULESTORES = {}
 
 FUNCTION_KEYS = ['render_template']
@@ -39,7 +48,20 @@ def create_modulestore_instance(engine, options):
         if key in _options and isinstance(_options[key], basestring):
             _options[key] = load_function(_options[key])
 
+    if HAS_REQUEST_CACHE:
+        request_cache = RequestCache.get_request_cache()
+    else:
+        request_cache = None
+
+    try:
+        metadata_inheritance_cache = get_cache('mongo_metadata_inheritance')
+    except InvalidCacheBackendError:
+        metadata_inheritance_cache = get_cache('default')
+
     return class_(
+        metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
+        request_cache=request_cache,
+        modulestore_update_signal=Signal(providing_args=['modulestore', 'course_id', 'location']),
         **_options
     )
 

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -17,12 +17,12 @@ class MixedModuleStore(ModuleStoreBase):
     """
     ModuleStore that can be backed by either XML or Mongo
     """
-    def __init__(self, mappings, stores):
+    def __init__(self, mappings, stores, **kwargs):
         """
         Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
         collection of other modulestore configuration informations
         """
-        super(MixedModuleStore, self).__init__()
+        super(MixedModuleStore, self).__init__(**kwargs)
 
         self.modulestores = {}
         self.mappings = mappings
@@ -132,14 +132,6 @@ class MixedModuleStore(ModuleStoreBase):
         """
         return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id)
 
-    def set_modulestore_configuration(self, config_dict):
-        """
-        This implementation of the interface method will pass along the configuration to all ModuleStore
-        instances
-        """
-        for store in self.modulestores.values():
-            store.set_modulestore_configuration(config_dict)
-
     def get_modulestore_type(self, course_id):
         """
         Returns a type which identifies which modulestore is servicing the given

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -270,15 +270,18 @@ class MongoModuleStore(ModuleStoreBase):
     def __init__(self, host, db, collection, fs_root, render_template,
                  port=27017, default_class=None,
                  error_tracker=null_error_tracker,
-                 user=None, password=None, **kwargs):
+                 user=None, password=None, mongo_options=None, **kwargs):
 
-        super(MongoModuleStore, self).__init__()
+        super(MongoModuleStore, self).__init__(**kwargs)
+
+        if mongo_options is None:
+            mongo_options = {}
 
         self.collection = pymongo.connection.Connection(
             host=host,
             port=port,
             tz_aware=True,
-            **kwargs
+            **mongo_options
         )[db][collection]
 
         if user is not None and password is not None:

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -50,15 +50,18 @@ class SplitMongoModuleStore(ModuleStoreBase):
                  port=27017, default_class=None,
                  error_tracker=null_error_tracker,
                  user=None, password=None,
+                 mongo_options=None,
                  **kwargs):
 
         ModuleStoreBase.__init__(self)
+        if mongo_options is None:
+            mongo_options = {}
 
         self.db = pymongo.database.Database(pymongo.MongoClient(
             host=host,
             port=port,
             tz_aware=True,
-            **kwargs
+            **mongo_options
         ), db)
 
         self.course_index = self.db[collection + '.active_versions']

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -5,9 +5,16 @@ from uuid import uuid4
 from xmodule.tests import DATA_DIR
 from xmodule.modulestore import Location, MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
 from xmodule.modulestore.exceptions import ItemNotFoundError
-from xmodule.modulestore.mixed import MixedModuleStore
 from xmodule.modulestore.xml_importer import import_from_xml
 
+# Mixed modulestore depends on django, so we'll manually configure some django settings
+# before importing the module
+from django.conf import settings
+if not settings.configured:
+    settings.configure()
+
+from xmodule.modulestore.mixed import MixedModuleStore
+
 
 HOST = 'localhost'
 PORT = 27017
@@ -234,22 +241,3 @@ class TestMixedModuleStore(object):
         assert_equals(Location(parents[0]).org, 'edX')
         assert_equals(Location(parents[0]).course, 'toy')
         assert_equals(Location(parents[0]).name, '2012_Fall')
-
-    # pylint: disable=W0212
-    def test_set_modulestore_configuration(self):
-        config = {'foo': 'bar'}
-        self.store.set_modulestore_configuration(config)
-        assert_equals(
-            config,
-            self.store._get_modulestore_for_courseid(IMPORT_COURSEID).modulestore_configuration
-        )
-
-        assert_equals(
-            config,
-            self.store._get_modulestore_for_courseid(XML_COURSEID1).modulestore_configuration
-        )
-
-        assert_equals(
-            config,
-            self.store._get_modulestore_for_courseid(XML_COURSEID2).modulestore_configuration
-        )

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -257,7 +257,7 @@ class XMLModuleStore(ModuleStoreBase):
     """
     An XML backed ModuleStore
     """
-    def __init__(self, data_dir, default_class=None, course_dirs=None, load_error_modules=True):
+    def __init__(self, data_dir, default_class=None, course_dirs=None, load_error_modules=True, **kwargs):
         """
         Initialize an XMLModuleStore from data_dir
 
@@ -269,7 +269,7 @@ class XMLModuleStore(ModuleStoreBase):
         course_dirs: If specified, the list of course_dirs to load. Otherwise,
             load all course dirs
         """
-        super(XMLModuleStore, self).__init__()
+        super(XMLModuleStore, self).__init__(**kwargs)
 
         self.data_dir = path(data_dir)
         self.modules = defaultdict(dict)  # course_id -> dict(location -> XModuleDescriptor)

common/lib/xmodule/xmodule/peer_grading_module.py

@@ -8,10 +8,9 @@ from pkg_resources import resource_string
 from .capa_module import ComplexEncoder
 from .x_module import XModule
 from xmodule.raw_module import RawDescriptor
-from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from .timeinfo import TimeInfo
-from xblock.core import Dict, String, Scope, Boolean, Integer, Float
+from xblock.core import Dict, String, Scope, Boolean, Float
 from xmodule.fields import Date, Timedelta
 
 from xmodule.open_ended_grading_classes.peer_grading_service import PeerGradingService, GradingServiceError, MockPeerGradingService
@@ -104,7 +103,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
 
         if self.use_for_single_location:
             try:
-                self.linked_problem = modulestore().get_instance(self.system.course_id, self.link_to_location)
+                self.linked_problem = self.system.get_module(self.link_to_location)
             except ItemNotFoundError:
                 log.error("Linked location {0} for peer grading module {1} does not exist".format(
                     self.link_to_location, self.location))
@@ -632,3 +631,11 @@ class PeerGradingDescriptor(PeerGradingFields, RawDescriptor):
         non_editable_fields = super(PeerGradingDescriptor, self).non_editable_metadata_fields
         non_editable_fields.extend([PeerGradingFields.due, PeerGradingFields.graceperiod])
         return non_editable_fields
+
+    def get_required_module_descriptors(self):
+        """Returns a list of XModuleDescritpor instances upon which this module depends, but are
+        not children of this module"""
+        if self.use_for_single_location:
+            return [self.system.load_item(self.link_to_location)]
+        else:
+            return []

common/lib/xmodule/xmodule/template_module.py

@@ -2,7 +2,6 @@ from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
 from lxml import etree
 from mako.template import Template
-from xmodule.modulestore.django import modulestore
 
 
 class CustomTagModule(XModule):
@@ -56,7 +55,7 @@ class CustomTagDescriptor(RawDescriptor):
         # cdodge: look up the template as a module
         template_loc = self.location.replace(category='custom_tag_template', name=template_name)
 
-        template_module = modulestore().get_instance(system.course_id, template_loc)
+        template_module = system.load_item(template_loc)
         template_module_data = template_module.data
         template = Template(template_module_data)
         return template.render(**params)

common/lib/xmodule/xmodule/tests/test_capa_module.py

@@ -322,7 +322,8 @@ class CapaModuleTest(unittest.TestCase):
 
         # We have to set up Django settings in order to use QueryDict
         from django.conf import settings
-        settings.configure()
+        if not settings.configured:
+            settings.configure()
 
         # Valid GET param dict
         valid_get_dict = self._querydict_from_dict({'input_1': 'test',

common/lib/xmodule/xmodule/video_module.py

@@ -24,9 +24,6 @@ from xmodule.editing_module import TabsEditingDescriptor
 from xmodule.raw_module import EmptyDataRawDescriptor
 from xmodule.xml_module import is_pointer_tag, name_to_pathname
 from xmodule.modulestore import Location
-from xmodule.modulestore.mongo import MongoModuleStore
-from xmodule.modulestore.django import modulestore
-from xmodule.contentstore.content import StaticContent
 from xblock.core import Scope, String, Boolean, Float, List, Integer
 
 import datetime

common/test/data/uploads/test

-This is an arbitrary file for testing uploads
+This is an arbitrary file for testing uploads
\ No newline at end of file

lms/djangoapps/courseware/tests/__init__.py

@@ -50,7 +50,8 @@ class BaseTestXmodule(ModuleStoreTestCase):
         self.course = CourseFactory.create(data=self.COURSE_DATA)
 
         # Turn off cache.
-        modulestore().set_modulestore_configuration({})
+        modulestore().request_cache = None
+        modulestore().metadata_inheritance_cache_subsystem = None
 
         chapter = ItemFactory.create(
             parent_location=self.course.location,

lms/djangoapps/courseware/tests/test_model_data.py

@@ -318,8 +318,6 @@ class StorageTestBase(object):
         self.assertEquals(exception.saved_field_names[0], 'existing_field')
 
 
-
-
 class TestSettingsStorage(StorageTestBase, TestCase):
     factory = SettingsFactory
     scope = Scope.settings

lms/djangoapps/instructor/tests/test_legacy_gradebook.py

@@ -26,7 +26,8 @@ class TestGradebook(ModuleStoreTestCase):
         self.client.login(username=instructor.username, password='test')
 
         # remove the caches
-        modulestore().set_modulestore_configuration({})
+        modulestore().request_cache = None
+        modulestore().metadata_inheritance_cache_subsystem = None
 
         kwargs = {}
         if self.grading_policy is not None:

lms/one_time_startup.py

-from dogapi import dog_http_api, dog_stats_api
-from django.conf import settings
-from xmodule.modulestore.django import modulestore
-from request_cache.middleware import RequestCache
-
-from django.core.cache import get_cache
-
-cache = get_cache('mongo_metadata_inheritance')
-for store_name in settings.MODULESTORE:
-    store = modulestore(store_name)
-    store.set_modulestore_configuration({
-        'metadata_inheritance_cache_subsystem': cache,
-        'request_cache': RequestCache.get_request_cache()
-    })
-
-if hasattr(settings, 'DATADOG_API'):
-    dog_http_api.api_key = settings.DATADOG_API
-    dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)

lms/startup.py

+"""
+Module for code that should run during LMS startup
+"""
+from django.conf import settings
+
+# Force settings to run so that the python path is modified
+settings.INSTALLED_APPS  # pylint: disable=W0104
+
+from django_startup import autostartup
+
+
+def run():
+    """
+    Executed during django startup
+    """
+    autostartup()

lms/urls.py

@@ -3,9 +3,6 @@ from django.conf.urls import patterns, include, url
 from ratelimitbackend import admin
 from django.conf.urls.static import static
 
-# Not used, the work is done in the imported module.
-from . import one_time_startup      # pylint: disable=W0611
-
 import django.contrib.auth.views
 
 # Uncomment the next two lines to enable the admin:

lms/wsgi.py

@@ -2,13 +2,11 @@ import os
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
 
+import lms.startup as startup
+startup.run()
+
 # This application object is used by the development server
 # as well as any WSGI server configured to use this file.
 from django.core.wsgi import get_wsgi_application
 application = get_wsgi_application()
 
-from django.conf import settings
-from xmodule.modulestore.django import modulestore
-
-for store_name in settings.MODULESTORE:
-    modulestore(store_name)

manage.py

@@ -13,8 +13,7 @@ Any arguments not understood by this manage.py will be passed to django-admin.py
 
 import os
 import sys
-import glob2
-import imp
+import importlib
 from argparse import ArgumentParser
 
 def parse_args():
@@ -41,7 +40,8 @@ def parse_args():
     lms.set_defaults(
         help_string=lms.format_help(),
         settings_base='lms/envs',
-        default_settings='lms.envs.dev'
+        default_settings='lms.envs.dev',
+        startup='lms.startup',
     )
 
     cms = subparsers.add_parser(
@@ -59,7 +59,8 @@ def parse_args():
         help_string=cms.format_help(),
         settings_base='cms/envs',
         default_settings='cms.envs.dev',
-        service_variant='cms'
+        service_variant='cms',
+        startup='cms.startup',
     )
 
     edx_args, django_args = parser.parse_known_args()
@@ -86,6 +87,9 @@ if __name__ == "__main__":
         # This will trigger django-admin.py to print out its help
         django_args.append('--help')
 
+    startup = importlib.import_module(edx_args.startup)
+    startup.run()
+
     from django.core.management import execute_from_command_line
 
     execute_from_command_line([sys.argv[0]] + django_args)