Commit 407b02b3 by Calen Pennington

Centralize startup code, and execute in all contexts

Inspired by: http://eldarion.com/blog/2013/02/14/entry-point-hook-django-projects/
Moves startup code to lms.startup and cms.startup, and calls the startup
methods in wsgi.py and manage.py for both projects.
parent 0e4b1142
......@@ -11,6 +11,9 @@ It is hidden behind a false defaulted course level flag.
Studio: Allow course authors to set their course image on the schedule
and details page, with support for JPEG and PNG images.
LMS, Studio: Centralized startup code to manage.py and wsgi.py files.
Made studio runnable using wsgi.
Blades: Took videoalpha out of alpha, replacing the old video player
Common: Allow instructors to input complicated expressions as answers to
......
......@@ -3,11 +3,6 @@ from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.course_module import CourseDescriptor
from request_cache.middleware import RequestCache
from django.core.cache import get_cache
CACHE = get_cache('mongo_metadata_inheritance')
class Command(BaseCommand):
help = '''Enumerates through the course and find common errors'''
......@@ -21,12 +16,6 @@ class Command(BaseCommand):
loc = CourseDescriptor.id_to_location(loc_str)
store = modulestore()
# setup a request cache so we don't throttle the DB with all the metadata inheritance requests
store.set_modulestore_configuration({
'metadata_inheritance_cache_subsystem': CACHE,
'request_cache': RequestCache.get_request_cache()
})
course = store.get_item(loc, depth=3)
err_cnt = 0
......
......@@ -9,14 +9,10 @@ from xmodule.course_module import CourseDescriptor
from auth.authz import _copy_course_group
#
# To run from command line: rake cms:clone SOURCE_LOC=MITx/111/Foo1 DEST_LOC=MITx/135/Foo3
#
from request_cache.middleware import RequestCache
from django.core.cache import get_cache
CACHE = get_cache('mongo_metadata_inheritance')
class Command(BaseCommand):
"""Clone a MongoDB-backed course to another location"""
help = 'Clone a MongoDB backed course to another location'
......@@ -32,11 +28,6 @@ class Command(BaseCommand):
mstore = modulestore('direct')
cstore = contentstore()
mstore.set_modulestore_configuration({
'metadata_inheritance_cache_subsystem': CACHE,
'request_cache': RequestCache.get_request_cache()
})
org, course_num, run = dest_course_id.split("/")
mstore.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
......
......@@ -9,14 +9,11 @@ from xmodule.course_module import CourseDescriptor
from .prompt import query_yes_no
from auth.authz import _delete_course_group
from request_cache.middleware import RequestCache
from django.core.cache import get_cache
#
# To run from command line: rake cms:delete_course LOC=MITx/111/Foo1
#
CACHE = get_cache('mongo_metadata_inheritance')
class Command(BaseCommand):
help = '''Delete a MongoDB backed course'''
......@@ -36,11 +33,6 @@ class Command(BaseCommand):
ms = modulestore('direct')
cs = contentstore()
ms.set_modulestore_configuration({
'metadata_inheritance_cache_subsystem': CACHE,
'request_cache': RequestCache.get_request_cache()
})
org, course_num, run = course_id.split("/")
ms.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num))
......
......@@ -1139,12 +1139,15 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
wrapper = MongoCollectionFindWrapper(module_store.collection.find)
module_store.collection.find = wrapper.find
print module_store.metadata_inheritance_cache_subsystem
print module_store.request_cache
course = module_store.get_item(location, depth=2)
# make sure we haven't done too many round trips to DB
# note we say 4 round trips here for 1) the course, 2 & 3) for the chapters and sequentials, and
# 4) because of the RT due to calculating the inherited metadata
self.assertEqual(wrapper.counter, 4)
# note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
# Because we're querying from the top of the tree, we cache information needed for inheritance,
# so we don't need to make an extra query to compute it.
self.assertEqual(wrapper.counter, 3)
# make sure we pre-fetched a known sequential which should be at depth=2
self.assertTrue(Location(['i4x', 'edX', 'toy', 'sequential',
......
from dogapi import dog_http_api, dog_stats_api
from django.conf import settings
from xmodule.modulestore.django import modulestore
from django.dispatch import Signal
from request_cache.middleware import RequestCache
from django.core.cache import get_cache
CACHE = get_cache('mongo_metadata_inheritance')
for store_name in settings.MODULESTORE:
store = modulestore(store_name)
store.set_modulestore_configuration({
'metadata_inheritance_cache_subsystem': CACHE,
'request_cache': RequestCache.get_request_cache()
})
modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location'])
store.modulestore_update_signal = modulestore_update_signal
if hasattr(settings, 'DATADOG_API'):
dog_http_api.api_key = settings.DATADOG_API
dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
"""
Module with code executed during Studio startup
"""
from django.conf import settings
# Force settings to run so that the python path is modified
settings.INSTALLED_APPS # pylint: disable=W0104
from django_startup import autostartup
# TODO: Remove this code once Studio/CMS runs via wsgi in all environments
INITIALIZED = False
def run():
"""
Executed during django startup
"""
global INITIALIZED
if INITIALIZED:
return
INITIALIZED = True
autostartup()
from django.conf import settings
from django.conf.urls import patterns, include, url
# Import this file so it can do its work, even though we don't use the name.
# pylint: disable=W0611
from . import one_time_startup
# TODO: This should be removed once the CMS is running via wsgi on all production servers
import cms.startup as startup
startup.run()
# There is a course creators admin table.
from ratelimitbackend import admin
......
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws")
import cms.startup as startup
startup.run()
# This application object is used by the development server
# as well as any WSGI server configured to use this file.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from django.conf import settings
from dogapi import dog_http_api, dog_stats_api
def run():
"""
Initialize connection to datadog during django startup.
Expects the datadog api key in the DATADOG_API settings key
"""
if hasattr(settings, 'DATADOG_API'):
dog_http_api.api_key = settings.DATADOG_API
dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
......@@ -16,11 +16,6 @@ from requests import put
from base64 import encodestring
from json import dumps
# Let the LMS and CMS do their one-time setup
# For example, setting up mongo caches
# These names aren't used, but do important work on import.
from lms import one_time_startup # pylint: disable=W0611
from cms import one_time_startup # pylint: disable=W0611
from pymongo import MongoClient
import xmodule.modulestore.django
from xmodule.contentstore.django import _CONTENTSTORE
......
from importlib import import_module
from django.conf import settings
def autostartup():
"""
Execute app.startup:run() for all installed django apps
"""
for app in settings.INSTALLED_APPS:
try:
mod = import_module('{}.startup')
if hasattr(mod, 'run'):
mod.run()
except ImportError:
continue
......@@ -386,13 +386,6 @@ class ModuleStore(object):
"""
raise NotImplementedError
def set_modulestore_configuration(self, config_dict):
'''
Allows for runtime configuration of the modulestore. In particular this is how the
application (LMS/CMS) can pass down Django related configuration information, e.g. caches, etc.
'''
raise NotImplementedError
def get_modulestore_type(self, course_id):
"""
Returns a type which identifies which modulestore is servicing the given
......@@ -405,13 +398,14 @@ class ModuleStoreBase(ModuleStore):
'''
Implement interface functionality that can be shared.
'''
def __init__(self):
def __init__(self, metadata_inheritance_cache_subsystem=None, request_cache=None, modulestore_update_signal=None):
'''
Set up the error-tracking logic.
'''
self._location_errors = {} # location -> ErrorLog
self.modulestore_configuration = {}
self.modulestore_update_signal = None # can be set by runtime to route notifications of datastore changes
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
self.modulestore_update_signal = modulestore_update_signal
self.request_cache = request_cache
def _get_errorlog(self, location):
"""
......@@ -455,27 +449,6 @@ class ModuleStoreBase(ModuleStore):
return c
return None
@property
def metadata_inheritance_cache_subsystem(self):
"""
Exposes an accessor to the runtime configuration for the metadata inheritance cache
"""
return self.modulestore_configuration.get('metadata_inheritance_cache_subsystem', None)
@property
def request_cache(self):
"""
Exposes an accessor to the runtime configuration for the request cache
"""
return self.modulestore_configuration.get('request_cache', None)
def set_modulestore_configuration(self, config_dict):
"""
This is the base implementation of the interface, all we need to do is store
two possible configurations as attributes on the class
"""
self.modulestore_configuration = config_dict
def namedtuple_to_son(namedtuple, prefix=''):
"""
......
......@@ -8,8 +8,17 @@ from __future__ import absolute_import
from importlib import import_module
from django.conf import settings
from django.core.cache import get_cache, InvalidCacheBackendError
from django.dispatch import Signal
from xmodule.modulestore.loc_mapper_store import LocMapperStore
# We may not always have the request_cache module available
try:
from request_cache.middleware import RequestCache
HAS_REQUEST_CACHE = True
except ImportError:
HAS_REQUEST_CACHE = False
_MODULESTORES = {}
FUNCTION_KEYS = ['render_template']
......@@ -39,7 +48,20 @@ def create_modulestore_instance(engine, options):
if key in _options and isinstance(_options[key], basestring):
_options[key] = load_function(_options[key])
if HAS_REQUEST_CACHE:
request_cache = RequestCache.get_request_cache()
else:
request_cache = None
try:
metadata_inheritance_cache = get_cache('mongo_metadata_inheritance')
except InvalidCacheBackendError:
metadata_inheritance_cache = get_cache('default')
return class_(
metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
request_cache=request_cache,
modulestore_update_signal=Signal(providing_args=['modulestore', 'course_id', 'location']),
**_options
)
......
......@@ -17,12 +17,12 @@ class MixedModuleStore(ModuleStoreBase):
"""
ModuleStore that can be backed by either XML or Mongo
"""
def __init__(self, mappings, stores):
def __init__(self, mappings, stores, **kwargs):
"""
Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
collection of other modulestore configuration informations
"""
super(MixedModuleStore, self).__init__()
super(MixedModuleStore, self).__init__(**kwargs)
self.modulestores = {}
self.mappings = mappings
......@@ -132,14 +132,6 @@ class MixedModuleStore(ModuleStoreBase):
"""
return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id)
def set_modulestore_configuration(self, config_dict):
"""
This implementation of the interface method will pass along the configuration to all ModuleStore
instances
"""
for store in self.modulestores.values():
store.set_modulestore_configuration(config_dict)
def get_modulestore_type(self, course_id):
"""
Returns a type which identifies which modulestore is servicing the given
......
......@@ -270,15 +270,18 @@ class MongoModuleStore(ModuleStoreBase):
def __init__(self, host, db, collection, fs_root, render_template,
port=27017, default_class=None,
error_tracker=null_error_tracker,
user=None, password=None, **kwargs):
user=None, password=None, mongo_options=None, **kwargs):
super(MongoModuleStore, self).__init__()
super(MongoModuleStore, self).__init__(**kwargs)
if mongo_options is None:
mongo_options = {}
self.collection = pymongo.connection.Connection(
host=host,
port=port,
tz_aware=True,
**kwargs
**mongo_options
)[db][collection]
if user is not None and password is not None:
......
......@@ -50,15 +50,18 @@ class SplitMongoModuleStore(ModuleStoreBase):
port=27017, default_class=None,
error_tracker=null_error_tracker,
user=None, password=None,
mongo_options=None,
**kwargs):
ModuleStoreBase.__init__(self)
if mongo_options is None:
mongo_options = {}
self.db = pymongo.database.Database(pymongo.MongoClient(
host=host,
port=port,
tz_aware=True,
**kwargs
**mongo_options
), db)
self.course_index = self.db[collection + '.active_versions']
......
......@@ -5,9 +5,16 @@ from uuid import uuid4
from xmodule.tests import DATA_DIR
from xmodule.modulestore import Location, MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.mixed import MixedModuleStore
from xmodule.modulestore.xml_importer import import_from_xml
# Mixed modulestore depends on django, so we'll manually configure some django settings
# before importing the module
from django.conf import settings
if not settings.configured:
settings.configure()
from xmodule.modulestore.mixed import MixedModuleStore
HOST = 'localhost'
PORT = 27017
......@@ -234,22 +241,3 @@ class TestMixedModuleStore(object):
assert_equals(Location(parents[0]).org, 'edX')
assert_equals(Location(parents[0]).course, 'toy')
assert_equals(Location(parents[0]).name, '2012_Fall')
# pylint: disable=W0212
def test_set_modulestore_configuration(self):
config = {'foo': 'bar'}
self.store.set_modulestore_configuration(config)
assert_equals(
config,
self.store._get_modulestore_for_courseid(IMPORT_COURSEID).modulestore_configuration
)
assert_equals(
config,
self.store._get_modulestore_for_courseid(XML_COURSEID1).modulestore_configuration
)
assert_equals(
config,
self.store._get_modulestore_for_courseid(XML_COURSEID2).modulestore_configuration
)
......@@ -257,7 +257,7 @@ class XMLModuleStore(ModuleStoreBase):
"""
An XML backed ModuleStore
"""
def __init__(self, data_dir, default_class=None, course_dirs=None, load_error_modules=True):
def __init__(self, data_dir, default_class=None, course_dirs=None, load_error_modules=True, **kwargs):
"""
Initialize an XMLModuleStore from data_dir
......@@ -269,7 +269,7 @@ class XMLModuleStore(ModuleStoreBase):
course_dirs: If specified, the list of course_dirs to load. Otherwise,
load all course dirs
"""
super(XMLModuleStore, self).__init__()
super(XMLModuleStore, self).__init__(**kwargs)
self.data_dir = path(data_dir)
self.modules = defaultdict(dict) # course_id -> dict(location -> XModuleDescriptor)
......
......@@ -8,10 +8,9 @@ from pkg_resources import resource_string
from .capa_module import ComplexEncoder
from .x_module import XModule
from xmodule.raw_module import RawDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from .timeinfo import TimeInfo
from xblock.core import Dict, String, Scope, Boolean, Integer, Float
from xblock.core import Dict, String, Scope, Boolean, Float
from xmodule.fields import Date, Timedelta
from xmodule.open_ended_grading_classes.peer_grading_service import PeerGradingService, GradingServiceError, MockPeerGradingService
......@@ -104,7 +103,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
if self.use_for_single_location:
try:
self.linked_problem = modulestore().get_instance(self.system.course_id, self.link_to_location)
self.linked_problem = self.system.get_module(self.link_to_location)
except ItemNotFoundError:
log.error("Linked location {0} for peer grading module {1} does not exist".format(
self.link_to_location, self.location))
......@@ -632,3 +631,11 @@ class PeerGradingDescriptor(PeerGradingFields, RawDescriptor):
non_editable_fields = super(PeerGradingDescriptor, self).non_editable_metadata_fields
non_editable_fields.extend([PeerGradingFields.due, PeerGradingFields.graceperiod])
return non_editable_fields
def get_required_module_descriptors(self):
"""Returns a list of XModuleDescritpor instances upon which this module depends, but are
not children of this module"""
if self.use_for_single_location:
return [self.system.load_item(self.link_to_location)]
else:
return []
......@@ -2,7 +2,6 @@ from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor
from lxml import etree
from mako.template import Template
from xmodule.modulestore.django import modulestore
class CustomTagModule(XModule):
......@@ -56,7 +55,7 @@ class CustomTagDescriptor(RawDescriptor):
# cdodge: look up the template as a module
template_loc = self.location.replace(category='custom_tag_template', name=template_name)
template_module = modulestore().get_instance(system.course_id, template_loc)
template_module = system.load_item(template_loc)
template_module_data = template_module.data
template = Template(template_module_data)
return template.render(**params)
......
......@@ -322,7 +322,8 @@ class CapaModuleTest(unittest.TestCase):
# We have to set up Django settings in order to use QueryDict
from django.conf import settings
settings.configure()
if not settings.configured:
settings.configure()
# Valid GET param dict
valid_get_dict = self._querydict_from_dict({'input_1': 'test',
......
......@@ -24,9 +24,6 @@ from xmodule.editing_module import TabsEditingDescriptor
from xmodule.raw_module import EmptyDataRawDescriptor
from xmodule.xml_module import is_pointer_tag, name_to_pathname
from xmodule.modulestore import Location
from xmodule.modulestore.mongo import MongoModuleStore
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.content import StaticContent
from xblock.core import Scope, String, Boolean, Float, List, Integer
import datetime
......
This is an arbitrary file for testing uploads
This is an arbitrary file for testing uploads
\ No newline at end of file
......@@ -50,7 +50,8 @@ class BaseTestXmodule(ModuleStoreTestCase):
self.course = CourseFactory.create(data=self.COURSE_DATA)
# Turn off cache.
modulestore().set_modulestore_configuration({})
modulestore().request_cache = None
modulestore().metadata_inheritance_cache_subsystem = None
chapter = ItemFactory.create(
parent_location=self.course.location,
......
......@@ -318,8 +318,6 @@ class StorageTestBase(object):
self.assertEquals(exception.saved_field_names[0], 'existing_field')
class TestSettingsStorage(StorageTestBase, TestCase):
factory = SettingsFactory
scope = Scope.settings
......
......@@ -26,7 +26,8 @@ class TestGradebook(ModuleStoreTestCase):
self.client.login(username=instructor.username, password='test')
# remove the caches
modulestore().set_modulestore_configuration({})
modulestore().request_cache = None
modulestore().metadata_inheritance_cache_subsystem = None
kwargs = {}
if self.grading_policy is not None:
......
from dogapi import dog_http_api, dog_stats_api
from django.conf import settings
from xmodule.modulestore.django import modulestore
from request_cache.middleware import RequestCache
from django.core.cache import get_cache
cache = get_cache('mongo_metadata_inheritance')
for store_name in settings.MODULESTORE:
store = modulestore(store_name)
store.set_modulestore_configuration({
'metadata_inheritance_cache_subsystem': cache,
'request_cache': RequestCache.get_request_cache()
})
if hasattr(settings, 'DATADOG_API'):
dog_http_api.api_key = settings.DATADOG_API
dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
"""
Module for code that should run during LMS startup
"""
from django.conf import settings
# Force settings to run so that the python path is modified
settings.INSTALLED_APPS # pylint: disable=W0104
from django_startup import autostartup
def run():
"""
Executed during django startup
"""
autostartup()
......@@ -3,9 +3,6 @@ from django.conf.urls import patterns, include, url
from ratelimitbackend import admin
from django.conf.urls.static import static
# Not used, the work is done in the imported module.
from . import one_time_startup # pylint: disable=W0611
import django.contrib.auth.views
# Uncomment the next two lines to enable the admin:
......
......@@ -2,13 +2,11 @@ import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
import lms.startup as startup
startup.run()
# This application object is used by the development server
# as well as any WSGI server configured to use this file.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from django.conf import settings
from xmodule.modulestore.django import modulestore
for store_name in settings.MODULESTORE:
modulestore(store_name)
......@@ -13,8 +13,7 @@ Any arguments not understood by this manage.py will be passed to django-admin.py
import os
import sys
import glob2
import imp
import importlib
from argparse import ArgumentParser
def parse_args():
......@@ -41,7 +40,8 @@ def parse_args():
lms.set_defaults(
help_string=lms.format_help(),
settings_base='lms/envs',
default_settings='lms.envs.dev'
default_settings='lms.envs.dev',
startup='lms.startup',
)
cms = subparsers.add_parser(
......@@ -59,7 +59,8 @@ def parse_args():
help_string=cms.format_help(),
settings_base='cms/envs',
default_settings='cms.envs.dev',
service_variant='cms'
service_variant='cms',
startup='cms.startup',
)
edx_args, django_args = parser.parse_known_args()
......@@ -86,6 +87,9 @@ if __name__ == "__main__":
# This will trigger django-admin.py to print out its help
django_args.append('--help')
startup = importlib.import_module(edx_args.startup)
startup.run()
from django.core.management import execute_from_command_line
execute_from_command_line([sys.argv[0]] + django_args)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment