Merge pull request #856 from MITx/feature/cale/edge-configs

Configuration changes to get edge running on AWS

cms/envs/aws.py

@@ -3,8 +3,8 @@ This is the default template for our main set of AWS servers.
 """
 import json
 
-from .logsettings import get_logger_config
 from .common import *
+from logsettings import get_logger_config
 
 ############################### ALWAYS THE SAME ################################
 DEBUG = False
@@ -27,6 +27,8 @@ LOG_DIR = ENV_TOKENS['LOG_DIR']
 
 CACHES = ENV_TOKENS['CACHES']
 
+SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
+
 for feature, value in ENV_TOKENS.get('MITX_FEATURES', {}).items():
     MITX_FEATURES[feature] = value
 
@@ -48,3 +50,4 @@ AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"]
 AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"]
 DATABASES = AUTH_TOKENS['DATABASES']
 MODULESTORE = AUTH_TOKENS['MODULESTORE']
+CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']

cms/envs/dev.py

@@ -2,7 +2,7 @@
 This config file runs the simplest dev environment"""
 
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 
 import logging
 import sys

cms/envs/logsettings.py

-import os
-import os.path
-import platform
-import sys
-
-def get_logger_config(log_dir,
-                      logging_env="no_env",
-                      tracking_filename=None,
-                      syslog_addr=None,
-                      debug=False):
-    """Return the appropriate logging config dictionary. You should assign the
-    result of this to the LOGGING var in your settings. The reason it's done
-    this way instead of registering directly is because I didn't want to worry
-    about resetting the logging state if this is called multiple times when
-    settings are extended."""
-
-    # If we're given an explicit place to put tracking logs, we do that (say for
-    # debugging). However, logging is not safe for multiple processes hitting
-    # the same file. So if it's left blank, we dynamically create the filename
-    # based on the PID of this worker process.
-    if tracking_filename:
-        tracking_file_loc = os.path.join(log_dir, tracking_filename)
-    else:
-        pid = os.getpid() # So we can log which process is creating the log
-        tracking_file_loc = os.path.join(log_dir, "tracking_{0}.log".format(pid))
-
-    hostname = platform.node().split(".")[0]
-    syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s [{hostname} " +
-                     " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format(
-                        logging_env=logging_env, hostname=hostname)
-
-    handlers = ['console'] if debug else ['console', 'syslogger', 'newrelic']
-
-    return {
-        'version': 1,
-        'disable_existing_loggers': False,
-        'formatters' : {
-            'standard' : {
-                'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',
-            },
-            'syslog_format' : { 'format' : syslog_format },
-            'raw' : { 'format' : '%(message)s' },
-        },
-        'handlers' : {
-            'console' : {
-                'level' : 'DEBUG' if debug else 'INFO',
-                'class' : 'logging.StreamHandler',
-                'formatter' : 'standard',
-                'stream' : sys.stdout,
-            },
-            'syslogger' : {
-                'level' : 'INFO',
-                'class' : 'logging.handlers.SysLogHandler',
-                'address' : syslog_addr,
-                'formatter' : 'syslog_format',
-            },
-            'tracking' : {
-                'level' : 'DEBUG',
-                'class' : 'logging.handlers.WatchedFileHandler',
-                'filename' : tracking_file_loc,
-                'formatter' : 'raw',
-            },
-            'newrelic' : {
-                'level': 'ERROR',
-                'class': 'newrelic_logging.NewRelicHandler',
-                'formatter': 'raw',
-            }
-        },
-        'loggers' : {
-            'django' : {
-                'handlers' : handlers,
-                'propagate' : True,
-                'level' : 'INFO'
-            },
-            'tracking' : {
-                'handlers' : ['tracking'],
-                'level' : 'DEBUG',
-                'propagate' : False,
-            },
-            '' : {
-                'handlers' : handlers,
-                'level' : 'DEBUG',
-                'propagate' : False
-            },
-            'mitx' : {
-                'handlers' : handlers,
-                'level' : 'DEBUG',
-                'propagate' : False
-            },
-            'keyedcache' : {
-                'handlers' : handlers,
-                'level' : 'DEBUG',
-                'propagate' : False
-            },
-        }
-    }

cms/urls.py

@@ -44,6 +44,8 @@ urlpatterns = ('',
 
     # temporary landing page for edge
     url(r'^edge$', 'contentstore.views.edge', name='edge'),
+
+    url(r'^heartbeat$', include('heartbeat.urls')),
 )
 
 # User creation and updating views

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -14,9 +14,13 @@ from xmodule.exceptions import NotFoundError
 
 
 class MongoContentStore(ContentStore):
-    def __init__(self, host, db, port=27017):
+    def __init__(self, host, db, port=27017, user=None, password=None, **kwargs):
         logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db))
-        _db = Connection(host=host, port=port)[db]
+        _db = Connection(host=host, port=port, **kwargs)[db]
+
+        if self.user is not None and self.password is not None:
+            _db.authenticate(user, password)
+
         self.fs = gridfs.GridFS(_db)
         self.fs_files = _db["fs.files"] # the underlying collection GridFS uses
 

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -97,15 +97,21 @@ class MongoModuleStore(ModuleStoreBase):
     # TODO (cpennington): Enable non-filesystem filestores
     def __init__(self, host, db, collection, fs_root, render_template,
                  port=27017, default_class=None,
-                 error_tracker=null_error_tracker):
+                 error_tracker=null_error_tracker,
+                 user=None, password=None, **kwargs):
 
         ModuleStoreBase.__init__(self)
 
         self.collection = pymongo.connection.Connection(
             host=host,
-            port=port
+            port=port,
+            **kwargs
         )[db][collection]
 
+        if user is not None and password is not None:
+            self.collection.database.authenticate(user, password)
+
+
         # Force mongo to report errors, at the expense of performance
         self.collection.safe = True
 

common/lib/xmodule/xmodule/x_module.py

@@ -409,7 +409,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
 
     # cdodge: this is a list of metadata names which are 'system' metadata
     # and should not be edited by an end-user
-    system_metadata_fields = ['data_dir', 'published_date', 'published_by']
+    system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft']
     
     # A list of descriptor attributes that must be equal for the descriptors to
     # be equal

lms/djangoapps/courseware/tabs.py

@@ -220,9 +220,6 @@ def get_default_tabs(user, course, active_page):
         link = reverse('django_comment_client.forum.views.forum_form_discussion',
                               args=[course.id])
         tabs.append(CourseTab('Discussion', link, active_page == 'discussion'))
-    elif settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
-        ## This is Askbot, which we should be retiring soon...
-        tabs.append(CourseTab('Discussion', reverse('questions'), active_page == 'discussion'))
 
     tabs.extend(_wiki({'name': 'Wiki', 'type': 'wiki'}, user, course, active_page))
 

lms/envs/aws.py

@@ -8,8 +8,8 @@ Common traits:
 """
 import json
 
-from .logsettings import get_logger_config
 from .common import *
+from logsettings import get_logger_config
 
 ############################### ALWAYS THE SAME ################################
 DEBUG = False
@@ -20,7 +20,6 @@ SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
 DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
 
 # Disable askbot, enable Berkeley forums
-MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = True
 
 # IMPORTANT: With this enabled, the server must always be behind a proxy that 
@@ -37,6 +36,7 @@ with open(ENV_ROOT / "env.json") as env_file:
     ENV_TOKENS = json.load(env_file)
 
 SITE_NAME = ENV_TOKENS['SITE_NAME']
+SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
 
 BOOK_URL = ENV_TOKENS['BOOK_URL']
 MEDIA_URL = ENV_TOKENS['MEDIA_URL']
@@ -76,6 +76,11 @@ DATABASES = AUTH_TOKENS['DATABASES']
 
 XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE']
 
+# Get the MODULESTORE from auth.json, but if it doesn't exist,
+# use the one from common.py
+MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE)
+CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)
+
 if 'COURSE_ID' in ENV_TOKENS:
     ASKBOT_URL = "courses/{0}/discussions/".format(ENV_TOKENS['COURSE_ID'])
 

lms/envs/cms/aws.py

@@ -2,7 +2,7 @@
 Settings for the LMS that runs alongside the CMS on AWS
 """
 
-from .aws import *
+from ..aws import *
 
 with open(ENV_ROOT / "cms.auth.json") as auth_file:
     CMS_AUTH_TOKENS = json.load(auth_file)

lms/envs/cms/dev.py

@@ -19,3 +19,11 @@ MODULESTORE = {
         'OPTIONS': modulestore_options
     },
 }
+
+CONTENTSTORE = {
+    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
+    'OPTIONS': {
+        'host': 'localhost',
+        'db' : 'xcontent',
+    }
+}

lms/envs/common.py

@@ -26,12 +26,10 @@ from xmodule.static_content import write_module_styles, write_module_js
 import djcelery
 from path import path
 
-from .askbotsettings import * # this is where LIVESETTINGS_OPTIONS comes from
 from .discussionsettings import *
 
 ################################### FEATURES ###################################
 COURSEWARE_ENABLED = True
-ASKBOT_ENABLED = False
 GENERATE_RANDOM_USER_CREDENTIALS = False
 PERFSTATS = False
 
@@ -65,7 +63,6 @@ MITX_FEATURES = {
                                         # set to None to do no university selection
 
     'ENABLE_TEXTBOOK' : True,
-    'ENABLE_DISCUSSION' : False,
     'ENABLE_DISCUSSION_SERVICE': True,
 
     'ENABLE_PSYCHOMETRICS': False,	# real-time psychometrics (eg item response theory analysis in instructor dashboard)
@@ -97,15 +94,12 @@ PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()
 COMMON_ROOT = REPO_ROOT / "common"
 ENV_ROOT = REPO_ROOT.dirname()  # virtualenv dir /mitx is in
-ASKBOT_ROOT = REPO_ROOT / "askbot"
 COURSES_ROOT = ENV_ROOT / "data"
 
 # FIXME: To support multiple courses, we should walk the courses dir at startup
 DATA_DIR = COURSES_ROOT
 
 sys.path.append(REPO_ROOT)
-sys.path.append(ASKBOT_ROOT)
-sys.path.append(ASKBOT_ROOT / "askbot" / "deps")
 sys.path.append(PROJECT_ROOT / 'djangoapps')
 sys.path.append(PROJECT_ROOT / 'lib')
 sys.path.append(COMMON_ROOT / 'djangoapps')
@@ -149,10 +143,8 @@ TEMPLATE_DIRS = (
 TEMPLATE_CONTEXT_PROCESSORS = (
     'django.core.context_processors.request',
     'django.core.context_processors.static',
-    'askbot.context.application_settings',
     'django.contrib.messages.context_processors.messages',
     #'django.core.context_processors.i18n',
-    'askbot.user_messages.context_processors.user_messages',#must be before auth
     'django.contrib.auth.context_processors.auth', #this is required for admin
     'django.core.context_processors.csrf', #necessary for csrf protection
     
@@ -229,6 +221,7 @@ MODULESTORE = {
         }
     }
 }
+CONTENTSTORE = None
 
 ############################ SIGNAL HANDLERS ################################
 # This is imported to register the exception signal handling that logs exceptions
@@ -263,7 +256,6 @@ STATIC_ROOT = ENV_ROOT / "staticfiles"
 STATICFILES_DIRS = [
     COMMON_ROOT / "static",
     PROJECT_ROOT / "static",
-    PROJECT_ROOT / "askbot" / "skins",
 ]
 if os.path.isdir(DATA_DIR):
     # Add the full course repo if there is no static directory
@@ -309,35 +301,6 @@ ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
 # in the global settings.py
 AWS_QUERYSTRING_EXPIRE = 10 * 365 * 24 * 60 * 60 # 10 years
 
-################################### ASKBOT #####################################
-LIVESETTINGS_OPTIONS['MITX_ROOT_URL'] = MITX_ROOT_URL
-skin_settings = LIVESETTINGS_OPTIONS[1]['SETTINGS']['GENERAL_SKIN_SETTINGS']
-skin_settings['SITE_FAVICON'] = unicode(MITX_ROOT_URL) + skin_settings['SITE_FAVICON']
-skin_settings['SITE_LOGO_URL'] = unicode(MITX_ROOT_URL) +  skin_settings['SITE_LOGO_URL']
-skin_settings['LOCAL_LOGIN_ICON'] = unicode(MITX_ROOT_URL) + skin_settings['LOCAL_LOGIN_ICON']
-LIVESETTINGS_OPTIONS[1]['SETTINGS']['LOGIN_PROVIDERS']['WORDPRESS_SITE_ICON'] = unicode(MITX_ROOT_URL) + LIVESETTINGS_OPTIONS[1]['SETTINGS']['LOGIN_PROVIDERS']['WORDPRESS_SITE_ICON']
-LIVESETTINGS_OPTIONS[1]['SETTINGS']['LICENSE_SETTINGS']['LICENSE_LOGO_URL'] = unicode(MITX_ROOT_URL) + LIVESETTINGS_OPTIONS[1]['SETTINGS']['LICENSE_SETTINGS']['LICENSE_LOGO_URL']
-
-# ASKBOT_EXTRA_SKINS_DIR = ASKBOT_ROOT / "askbot" / "skins"
-ASKBOT_EXTRA_SKINS_DIR =  PROJECT_ROOT / "askbot" / "skins"
-ASKBOT_ALLOWED_UPLOAD_FILE_TYPES = ('.jpg', '.jpeg', '.gif', '.bmp', '.png', '.tiff')
-ASKBOT_MAX_UPLOAD_FILE_SIZE = 1024 * 1024 # result in bytes
-
-CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True
-CACHE_PREFIX = SITE_ID
-ASKBOT_URL = 'discussion/'
-LOGIN_REDIRECT_URL = MITX_ROOT_URL + '/'
-LOGIN_URL = MITX_ROOT_URL + '/'
-
-ALLOW_UNICODE_SLUGS = False
-ASKBOT_USE_STACKEXCHANGE_URLS = False # mimic url scheme of stackexchange
-ASKBOT_CSS_DEVEL = True
-
-# Celery Settings
-BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
-CELERY_ALWAYS_EAGER = True
-djcelery.setup_loader()
-
 ################################# SIMPLEWIKI ###################################
 SIMPLE_WIKI_REQUIRE_LOGIN_EDIT = True
 SIMPLE_WIKI_REQUIRE_LOGIN_VIEW = False
@@ -373,11 +336,11 @@ TEMPLATE_LOADERS = (
     # 'django.template.loaders.filesystem.Loader',
     # 'django.template.loaders.app_directories.Loader',
     
-    #'askbot.skins.loaders.filesystem_load_template_source',
     # 'django.template.loaders.eggs.Loader',
 )
 
 MIDDLEWARE_CLASSES = (
+    'contentserver.middleware.StaticContentServer',
     'django_comment_client.middleware.AjaxExceptionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
@@ -393,13 +356,7 @@ MIDDLEWARE_CLASSES = (
     
     'course_wiki.course_nav.Middleware',
 
-    'askbot.middleware.anon_user.ConnectToSessionMessagesMiddleware',
-    'askbot.middleware.forum_mode.ForumModeMiddleware',
-    'askbot.middleware.cancel.CancelActionMiddleware',
     'django.middleware.transaction.TransactionMiddleware',
-    'askbot.middleware.view_log.ViewLogMiddleware',
-    'askbot.middleware.spaceless.SpacelessMiddleware',
-    # 'askbot.middleware.pagesize.QuestionsPageSizeMiddleware',
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',
 
     'django_comment_client.utils.ViewNameMiddleware',
@@ -586,16 +543,4 @@ INSTALLED_APPS = (
 
     # Discussion
     'django_comment_client',
-
-    # For Askbot
-    'django.contrib.sitemaps',
-    'django.contrib.admin',
-    'django_countries',
-    'djcelery',
-    'djkombu',
-    'askbot',
-    'askbot.deps.livesettings',
-    'followit',
-    'keyedcache',
-    'robots'
 )

lms/envs/dev.py

@@ -8,7 +8,7 @@ sessions. Assumes structure:
         /log  # Where we're going to write log files
 """
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 
 DEBUG = True
 TEMPLATE_DEBUG = True

lms/envs/dev_edx4edx.py

@@ -14,7 +14,7 @@ if 'eecs1' in socket.gethostname():
     MITX_ROOT_URL = '/mitx2'
 
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 from .dev import *
 
 if 'eecs1' in socket.gethostname():

lms/envs/dev_ike.py

@@ -8,13 +8,12 @@ sessions. Assumes structure:
         /log  # Where we're going to write log files
 """
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 from .dev import *
 import socket
 
 WIKI_ENABLED = False
 MITX_FEATURES['ENABLE_TEXTBOOK'] = False
-MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = False

lms/envs/static.py

@@ -8,7 +8,7 @@ sessions. Assumes structure:
         /log  # Where we're going to write log files
 """
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 
 STATIC_GRAB = True
 

lms/envs/test.py

@@ -8,7 +8,7 @@ sessions. Assumes structure:
         /log  # Where we're going to write log files
 """
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 import os
 from path import path
 

lms/envs/test_ike.py

@@ -8,7 +8,7 @@ sessions. Assumes structure:
         /log  # Where we're going to write log files
 """
 from .common import *
-from .logsettings import get_logger_config
+from logsettings import get_logger_config
 import os
 
 DEBUG = True

lms/urls.py

@@ -219,14 +219,6 @@ if settings.QUICKEDIT:
     urlpatterns += (url(r'^quickedit/(?P<id>[^/]*)$', 'dogfood.views.quickedit'),)
     urlpatterns += (url(r'^dogfood/(?P<id>[^/]*)$', 'dogfood.views.df_capa_problem'),)
 
-if settings.ASKBOT_ENABLED:
-    urlpatterns += (url(r'^%s' % settings.ASKBOT_URL, include('askbot.urls')), \
-                    url(r'^settings/', include('askbot.deps.livesettings.urls')), \
-                    url(r'^followit/', include('followit.urls')), \
-#                       url(r'^robots.txt$', include('robots.urls')),
-                              )
-
-
 
 if settings.DEBUG:
     ## Jasmine and admin