Add sysadmin dashboard

For seeing overview of system status, for deleting and loading
courses, for seeing log of git imports of courseware.  Includes command
for importing course XML from git repositories.

Added a lot of tests for additional coverage with some minor fixes
those tests discovered

cms/djangoapps/contentstore/management/commands/import.py

@@ -31,9 +31,16 @@ class Command(BaseCommand):
             course_dirs = args[1:]
         else:
             course_dirs = None
-        print("Importing.  Data_dir={data}, course_dirs={courses}".format(
+        self.stdout.write("Importing.  Data_dir={data}, course_dirs={courses}\n".format(
             data=data_dir,
             courses=course_dirs,
             dis=do_import_static))
-        import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,
+        try:
+            mstore = modulestore('direct')
+        except KeyError:
+            self.stdout.write('Unable to load direct modulestore, trying '
+                              'default\n')
+            mstore = modulestore('default')
+
+        import_from_xml(mstore, data_dir, course_dirs, load_error_modules=False,
                         static_content_store=contentstore(), verbose=True, do_import_static=do_import_static)

lms/djangoapps/courseware/management/commands/import.py

+../../../../../cms/djangoapps/contentstore/management/commands/import.py
\ No newline at end of file

lms/djangoapps/dashboard/management/commands/git_add_course.py

+"""
+Script for importing courseware from git/xml into a mongo modulestore
+"""
+
+import os
+import re
+import datetime
+import StringIO
+import subprocess
+import logging
+
+from django.conf import settings
+from django.core import management
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext as _
+import mongoengine
+
+from dashboard.models import CourseImportLog
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
+
+log = logging.getLogger(__name__)
+
+GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
+GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
+
+GIT_IMPORT_NO_DIR = -1
+GIT_IMPORT_URL_BAD = -2
+GIT_IMPORT_CANNOT_PULL = -3
+GIT_IMPORT_XML_IMPORT_FAILED = -4
+GIT_IMPORT_UNSUPPORTED_STORE = -5
+GIT_IMPORT_MONGODB_FAIL = -6
+GIT_IMPORT_BAD_REPO = -7
+
+
+def add_repo(repo, rdir_in):
+    """This will add a git repo into the mongo modulestore"""
+    # pylint: disable=R0915
+
+    # Set defaults even if it isn't defined in settings
+    mongo_db = {
+        'host': 'localhost',
+        'user': '',
+        'password': '',
+        'db': 'xlog',
+    }
+
+    # Allow overrides
+    if hasattr(settings, 'MONGODB_LOG'):
+        for config_item in ['host', 'user', 'password', 'db', ]:
+            mongo_db[config_item] = settings.MONGODB_LOG.get(
+                config_item, mongo_db[config_item])
+
+    if not os.path.isdir(GIT_REPO_DIR):
+        log.critical(_("Path {0} doesn't exist, please create it, "
+                       "or configure a different path with "
+                       "GIT_REPO_DIR").format(GIT_REPO_DIR))
+        return GIT_IMPORT_NO_DIR
+
+    # pull from git
+    if not repo.endswith('.git') or not (
+            repo.startswith('http:') or
+            repo.startswith('https:') or
+            repo.startswith('git:') or
+            repo.startswith('file:')):
+
+        log.error(_('Oops, not a git ssh url?'))
+        log.error(_('Expecting something like '
+                    'git@github.com:mitocw/edx4edx_lite.git'))
+        return GIT_IMPORT_URL_BAD
+
+    if rdir_in:
+        rdir = rdir_in
+        rdir = os.path.basename(rdir)
+    else:
+        rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
+
+    log.debug('rdir = {0}'.format(rdir))
+
+    rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
+    if os.path.exists(rdirp):
+        log.info(_('directory already exists, doing a git pull instead '
+                   'of git clone'))
+        cmd = ['git', 'pull', ]
+        cwd = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
+    else:
+        cmd = ['git', 'clone', repo, ]
+        cwd = GIT_REPO_DIR
+
+    log.debug(cmd)
+    cwd = os.path.abspath(cwd)
+    try:
+        ret_git = subprocess.check_output(cmd, cwd=cwd)
+    except subprocess.CalledProcessError:
+        log.exception(_('git clone or pull failed!'))
+        return GIT_IMPORT_CANNOT_PULL
+    log.debug(ret_git)
+
+    # get commit id
+    cmd = ['git', 'log', '-1', '--format=%H', ]
+    try:
+        commit_id = subprocess.check_output(cmd, cwd=rdirp)
+    except subprocess.CalledProcessError:
+        log.exception(_('Unable to get git log'))
+        return GIT_IMPORT_BAD_REPO
+
+    ret_git += _('\nCommit ID: {0}').format(commit_id)
+
+    # get branch
+    cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ]
+    try:
+        branch = subprocess.check_output(cmd, cwd=rdirp)
+    except subprocess.CalledProcessError:
+        log.exception(_('Unable to get branch info'))
+        return GIT_IMPORT_BAD_REPO
+
+    ret_git += '   \nBranch: {0}'.format(branch)
+
+    # Get XML logging logger and capture debug to parse results
+    output = StringIO.StringIO()
+    import_log_handler = logging.StreamHandler(output)
+    import_log_handler.setLevel(logging.DEBUG)
+
+    logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course',
+                    'xmodule.modulestore.xml', 'xmodule.seq_module', ]
+    loggers = []
+
+    for logger_name in logger_names:
+        logger = logging.getLogger(logger_name)
+        logger.old_level = logger.level
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(import_log_handler)
+        loggers.append(logger)
+
+    try:
+        management.call_command('import', GIT_REPO_DIR, rdir,
+                                nostatic=not GIT_IMPORT_STATIC)
+    except CommandError:
+        log.exception(_('Unable to run import command.'))
+        return GIT_IMPORT_XML_IMPORT_FAILED
+    except NotImplementedError:
+        log.exception(_('The underlying module store does not support import.'))
+        return GIT_IMPORT_UNSUPPORTED_STORE
+
+    ret_import = output.getvalue()
+
+    # Remove handler hijacks
+    for logger in loggers:
+        logger.setLevel(logger.old_level)
+        logger.removeHandler(import_log_handler)
+
+    course_id = 'unknown'
+    location = 'unknown'
+
+    # extract course ID from output of import-command-run and make symlink
+    # this is needed in order for custom course scripts to work
+    match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)',
+                      ret_import)
+    if match:
+        location = match.group(1).strip()
+        log.debug('location = {0}'.format(location))
+        course_id = location.replace('i4x://', '').replace(
+            '/course/', '/').split('\n')[0].strip()
+
+        cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1])
+        log.debug(_('Studio course dir = {0}').format(cdir))
+
+        if os.path.exists(cdir) and not os.path.islink(cdir):
+            log.debug(_('   -> exists, but is not symlink'))
+            log.debug(subprocess.check_output(['ls', '-l', ],
+                                              cwd=os.path.abspath(cdir)))
+            try:
+                os.rmdir(os.path.abspath(cdir))
+            except OSError:
+                log.exception(_('Failed to remove course directory'))
+
+        if not os.path.exists(cdir):
+            log.debug(_('   -> creating symlink between {0} and {1}').format(rdirp, cdir))
+            try:
+                os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
+            except OSError:
+                log.exception(_('Unable to create course symlink'))
+            log.debug(subprocess.check_output(['ls', '-l', ],
+                                              cwd=os.path.abspath(cdir)))
+
+    # store import-command-run output in mongo
+    mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
+
+    try:
+        if mongo_db['user'] and mongo_db['password']:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
+        else:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
+    except mongoengine.connection.ConnectionError:
+        log.exception(_('Unable to connect to mongodb to save log, please '
+                        'check MONGODB_LOG settings'))
+        return GIT_IMPORT_MONGODB_FAIL
+    cil = CourseImportLog(
+        course_id=course_id,
+        location=location,
+        repo_dir=rdir,
+        created=datetime.datetime.now(),
+        import_log=ret_import,
+        git_log=ret_git,
+    )
+    cil.save()
+
+    log.debug(_('saved CourseImportLog for {0}').format(cil.course_id))
+    mdb.disconnect()
+    return 0
+
+
+class Command(BaseCommand):
+    """
+    Pull a git repo and import into the mongo based content database.
+    """
+
+    help = _('Import the specified git repository into the '
+             'modulestore and directory')
+
+    def handle(self, *args, **options):
+        """Check inputs and run the command"""
+
+        if isinstance(modulestore, XMLModuleStore):
+            raise CommandError(_('This script requires a mongo module store'))
+
+        if len(args) < 1:
+            raise CommandError(_('This script requires at least one argument, '
+                                 'the git URL'))
+
+        if len(args) > 2:
+            raise CommandError(_('This script requires no more than two '
+                                 'arguments'))
+
+        rdir_arg = None
+
+        if len(args) > 1:
+            rdir_arg = args[1]
+
+        if add_repo(args[0], rdir_arg) != 0:
+            raise CommandError(_('Repo was not added, check log output '
+                                 'for details'))

lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py

+"""
+Provide tests for git_add_course management command.
+"""
+
+import unittest
+import os
+import shutil
+import subprocess
+
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.test.utils import override_settings
+
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+import dashboard.management.commands.git_add_course as git_add_course
+
+TEST_MONGODB_LOG = {
+    'host': 'localhost',
+    'user': '',
+    'password': '',
+    'db': 'test_xlog',
+}
+
+FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy()
+FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
+@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
+                     "ENABLE_SYSADMIN_DASHBOARD not set")
+class TestGitAddCourse(ModuleStoreTestCase):
+    """
+    Tests the git_add_course management command for proper functions.
+    """
+
+    TEST_REPO = 'https://github.com/mitocw/edx4edx_lite.git'
+
+    def assertCommandFailureRegexp(self, regex, *args):
+        """
+        Convenience function for testing command failures
+        """
+        with self.assertRaises(SystemExit):
+            self.assertRaisesRegexp(CommandError, regex,
+                                    call_command('git_add_course', *args))
+
+    def test_command_args(self):
+        """
+        Validate argument checking
+        """
+        self.assertCommandFailureRegexp(
+            'This script requires at least one argument, the git URL')
+        self.assertCommandFailureRegexp(
+            'This script requires no more than two arguments',
+            'blah', 'blah', 'blah')
+        self.assertCommandFailureRegexp(
+            'Repo was not added, check log output for details',
+            'blah')
+        # Test successful import from command
+        try:
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+
+        # Make a course dir that will be replaced with a symlink
+        # while we are at it.
+        if not os.path.isdir(getattr(settings, 'GIT_REPO_DIR') / 'edx4edx'):
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR') / 'edx4edx')
+
+        call_command('git_add_course', self.TEST_REPO,
+                     getattr(settings, 'GIT_REPO_DIR') / 'edx4edx_lite')
+        if os.path.isdir(getattr(settings, 'GIT_REPO_DIR')):
+            shutil.rmtree(getattr(settings, 'GIT_REPO_DIR'))
+
+    def test_add_repo(self):
+        """
+        Various exit path tests for test_add_repo
+        """
+        self.assertEqual(git_add_course.GIT_IMPORT_NO_DIR,
+                         git_add_course.add_repo(self.TEST_REPO, None))
+        try:
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+        self.assertEqual(git_add_course.GIT_IMPORT_URL_BAD,
+                         git_add_course.add_repo('foo', None))
+
+        self.assertEqual(
+            git_add_course.GIT_IMPORT_CANNOT_PULL,
+            git_add_course.add_repo('file:///foobar.git', None)
+        )
+
+        # Test git repo that exists, but is "broken"
+        bare_repo = os.path.abspath('{0}/{1}'.format(settings.TEST_ROOT, 'bare.git'))
+        os.mkdir(os.path.abspath(bare_repo))
+        subprocess.call(['git', '--bare', 'init', ], cwd=bare_repo)
+
+        self.assertEqual(
+            git_add_course.GIT_IMPORT_BAD_REPO,
+            git_add_course.add_repo('file://{0}'.format(bare_repo), None)
+        )
+        shutil.rmtree(bare_repo)

lms/djangoapps/dashboard/models.py

-# Create your models here.
+"""Models for dashboard application"""
+
+import mongoengine
+
+
+class CourseImportLog(mongoengine.Document):
+    """Mongoengine model for git log"""
+    # pylint: disable=R0924
+
+    course_id = mongoengine.StringField(max_length=128)
+    location = mongoengine.StringField(max_length=168)
+    import_log = mongoengine.StringField(max_length=20 * 65535)
+    git_log = mongoengine.StringField(max_length=65535)
+    repo_dir = mongoengine.StringField(max_length=128)
+    created = mongoengine.DateTimeField()
+    meta = {'indexes': ['course_id', 'created'],
+            'allow_inheritance': False}

lms/djangoapps/dashboard/sysadmin_urls.py

+"""
+Urls for sysadmin dashboard feature
+"""
+# pylint: disable=E1120
+
+from django.conf.urls import patterns, url
+
+from dashboard import sysadmin
+
+urlpatterns = patterns(
+    '',
+    url(r'^$', sysadmin.Users.as_view(), name="sysadmin"),
+    url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"),
+    url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),
+    url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"),
+    url(r'^gitlogs/(?P<course_id>.+)$', sysadmin.GitLogs.as_view(),
+        name="gitlogs_detail"),
+)

lms/envs/aws.py

@@ -237,6 +237,10 @@ ZENDESK_URL = ENV_TOKENS.get("ZENDESK_URL")
 FEEDBACK_SUBMISSION_EMAIL = ENV_TOKENS.get("FEEDBACK_SUBMISSION_EMAIL")
 MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
 
+# git repo loading  environment
+GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', '/edx/var/edxapp/course_repos')
+GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True)
+
 for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():
     oldvalue = CODE_JAIL.get(name)
     if isinstance(oldvalue, dict):
@@ -251,6 +255,11 @@ COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", [])
 if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
     TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")
 
+# SSL external authentication settings
+SSL_AUTH_EMAIL_DOMAIN = ENV_TOKENS.get("SSL_AUTH_EMAIL_DOMAIN", "MIT.EDU")
+SSL_AUTH_DN_FORMAT_STRING = ENV_TOKENS.get("SSL_AUTH_DN_FORMAT_STRING",
+                                           "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}")
+
 ############################## SECURE AUTH ITEMS ###############
 # Secret things: passwords, access keys, etc.
 
@@ -286,6 +295,7 @@ XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE']
 MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE)
 CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)
 DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG',DOC_STORE_CONFIG)
+MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {})
 
 OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE',
                                                OPEN_ENDED_GRADING_INTERFACE)

lms/envs/common.py

@@ -89,6 +89,8 @@ FEATURES = {
 
     'ENABLE_MASQUERADE': True,  # allow course staff to change to student view of courseware
 
+    'ENABLE_SYSADMIN_DASHBOARD': False,  # sysadmin dashboard, to see what courses are loaded, to delete & load courses
+
     'DISABLE_LOGIN_BUTTON': False,  # used in systems where login is automatic, eg MIT SSL
 
     # extrernal access methods
@@ -944,6 +946,7 @@ INSTALLED_APPS = (
     'eventtracking.django',
     'util',
     'certificates',
+    'dashboard',
     'instructor',
     'instructor_task',
     'open_ended_grading',

lms/envs/test.py

@@ -207,6 +207,10 @@ CC_PROCESSOR['CyberSource']['MERCHANT_ID'] = "edx"
 CC_PROCESSOR['CyberSource']['SERIAL_NUMBER'] = "0123456789012345678901"
 CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = "/shoppingcart/payment_fake"
 
+########################### SYSADMIN DASHBOARD ################################
+FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
+GIT_REPO_DIR = TEST_ROOT / "course_repos"
+
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True

lms/templates/sysadmin_dashboard.html

+<%inherit file="/main.html" />
+<%! from django.core.urlresolvers import reverse %>
+<%! from django.utils.translation import ugettext as _ %>
+<%namespace name='static' file='/static_content.html'/>
+
+<%block name="headextra">
+  <%static:css group='style-course'/>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
+</%block>
+
+<style type="text/css">
+a.active-section {
+	color: #551A8B;
+}
+.sysadmin-dashboard-content h2 a {
+	margin-right: 1.2em;
+}
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+
+a.selectedmode { background-color: yellow; }
+
+textarea {
+  height: 200px;
+}
+</style>
+
+<section class="container">
+<div class="sysadmin-dashboard-wrapper">
+
+  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
+    <h1>${_('Sysadmin Dashboard')}</h1>
+	<hr />
+    <h2 class="instructor-nav">
+	  <a href="${reverse('sysadmin')}" class="${modeflag.get('users')}">${_('Users')}</a>
+      <a href="${reverse('sysadmin_courses')}" class="${modeflag.get('courses')}">${_('Courses')}</a>
+      <a href="${reverse('sysadmin_staffing')}" class="${modeflag.get('staffing')}">${_('Staffing and Enrollment')}</a> 
+      <a href="${reverse('gitlogs')}">${_('Git Logs')}</a>
+    </h2>
+	<hr />
+%if modeflag.get('users'):
+	<h3>${_('User Management')}</h3>
+
+    <form name="action" method="POST">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
+
+	<ul class="list-input">
+	  <li class="field text" style="padding-bottom: 1.2em">
+		<label for="student_uname">${_('Email or username')}</label>
+		<input type="text" name="student_uname" size=40 />
+	  </li>
+
+	  <li class="field text">
+		<label for="student_fullname">${_('Full Name')}</label>
+		<input type="text" name="student_fullname" size=40 />
+	  </li>
+	  <li class="field text">
+		<label for="student_password">${_('Password')}</label>
+		<input type="password" name="student_password" size=40 />
+	  </li>
+	</ul>
+
+	<div class="form-actions">
+	  <p>
+		<button type="submit" name="action" value="del_user">${_('Delete user')}</button>
+		<button type="submit" name="action" value="create_user">${_('Create user')}</button>
+	  </p>
+    </div>
+
+	<hr />
+    <p>
+      <button type="submit" name="action" value="download_users" style="width: 350px;">
+		${_('Download list of all users (csv file)')}
+	  </button>
+    </p>
+
+    <p>
+    <button type="submit" name="action" value="repair_eamap" style="width: 350px;">
+	  ${_('Check and repair external authentication map')}
+	</button>
+    </p>
+
+    <hr width="40%" style="align:left">
+	</form>
+ %endif
+
+%if modeflag.get('staffing'):
+
+<p>${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}</p>
+<hr />
+
+<h3>${_('Manage course staff and instructors')}</h3><br/>
+<form name="action" method="POST">
+  <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
+  <button type="submit" name="action" value="get_staff_csv">${_('Download staff and instructor list (csv file)')}</button>
+</form>
+
+%endif
+
+%if modeflag.get('courses'):
+<h3>${_('Administer Courses')}</h3><br/>
+
+<form name="action" method="POST">
+  <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
+  <ul class="list-input">
+	<li class="field text">
+	  <label for="repo_location">
+		${_('Repo location')}:
+	  </label>
+	  <input type="text" name="repo_location" style="width:60%" />
+	</li>
+  </ul>
+  <div class="form-actions">
+	<button type="submit" name="action" value="add_course">${_('Load new course from github')}</button>
+  </div>
+  <hr />
+  <ul class="list-input">
+	<li class="field text">
+	  <label for="course_id">
+		${_('Course ID or dir')}:
+	  </label>
+	  <input type="text" name="course_id" style="width:60%" />
+	</li>
+  </ul>
+  <div class="form-actions">
+	<button type="submit" name="action" value="del_course">${_('Delete course from site')}</button>
+  </div>
+</form>
+<hr style="width:40%" />
+%endif
+
+%if msg:
+    <p>${msg}</p>
+%endif
+
+%if datatable:
+
+    <br/>
+    <br/>
+    <p>
+      <hr width="100%">
+      <h2>${datatable['title']}</h2>
+      <table class="stat_table">
+        <tr>
+	%for hname in datatable['header']:
+	  <th>${hname}</th>
+	%endfor
+	</tr>
+        %for row in datatable['data']:
+          <tr>
+	    %for value in row:
+	      <td>${value}</td>
+	    %endfor
+	  </tr>
+        %endfor
+      </table>
+    </p>
+%endif
+
+%if plots:
+
+    %for plot in plots:
+      <br/>
+      <h3>${plot['title']}</h3>
+      <br/>
+      <p>${plot['info']}</p>
+      <br/>
+      <div id="plot_${plot['id']}" style="width:600px;height:300px;"></div>
+      <script type="text/javascript">
+          $(function () {
+              ${plot['data']}
+              $.plot($("#plot_${plot['id']}"), ${plot['cmd']}  );
+          });
+      </script>
+      <br/>
+      <br/>
+    %endfor
+
+%endif
+
+  </section>
+  <div style="text-align:right; float: right"><span id="djangopid">${_('Django PID')}: ${djangopid}</span>
+  | <span id="mitxver">${_('Platform Version')}: ${mitx_version}</span></div>
+</div>
+</section>

lms/templates/sysadmin_dashboard_gitlogs.html

+<%inherit file="/main.html" />
+<%! from django.core.urlresolvers import reverse %>
+<%! from django.utils.translation import ugettext as _ %>
+<%namespace name='static' file='/static_content.html'/>
+
+<%block name="headextra">
+  <%static:css group='style-course'/>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
+</%block>
+<style type="text/css">
+a.active-section {
+	color: #551A8B;
+}
+.sysadmin-dashboard-content h2 a {
+	margin-right: 1.2em;
+}
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+
+a.selectedmode { background-color: yellow; }
+
+textarea {
+  height: 200px;
+}
+</style>
+
+<section class="container">
+<div class="sysadmin-dashboard-wrapper">
+
+  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
+    <h1>${_('Sysadmin Dashboard')}</h1>
+	<hr />
+    <h2 class="instructor-nav">
+	  <a href="${reverse('sysadmin')}">${_('Users')}</a>
+      <a href="${reverse('sysadmin_courses')}">${_('Courses')}</a>
+      <a href="${reverse('sysadmin_staffing')}">${_('Staffing and Enrollment')}</a> 
+      <a href="${reverse('gitlogs')}" class="active-section">${_('Git Logs')}</a>
+    </h2>
+	<hr />
+
+    <form name="dashform" method="POST" action="${reverse('sysadmin')}">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+    <input type="hidden" name="dash_mode" value="">
+
+    <h3>${_('Git Logs')}</h3>
+
+%if course_id is None:
+    <table class="stat_table">
+	  <thead>
+		<tr>
+		  <th>${_('Date')}</th>
+		  <th>${_('Course ID')}</th>
+		  <th>${_('Git Action')}</th>
+		</tr>
+	  </thead>
+	  <tbody>
+		%for cil in cilset[:10]:
+        <tr>
+	      <td>${cil.created}</td>
+	      <td><a href="${reverse('gitlogs')}/${cil.course_id}">${cil.course_id}</a></td>
+	      <td>${cil.git_log}</td>
+		</tr>
+		%endfor
+	  </tbody>
+    </table>
+
+%else:
+
+    <h2>${_('Recent git load activity for')} ${course_id}</h2>
+	%if error_msg:
+	<h3>${_('Error')}:</h3>
+	<p>${error_msg}</p>
+	%endif
+
+    <table class="stat_table">
+	  <thead>
+		<tr>
+		  <th>${_('Date')}</th>
+		  <th>${_('Course ID')}</th>
+		  <th>${_('git action')}</th>
+		</tr>
+	  </thead>
+	  <tbody>
+		% for cil in cilset[:2]:
+        <tr>
+	      <td>${cil.created}</td>
+	      <td><a href="${reverse('gitlogs')}/${cil.course_id}">${cil.course_id}</a></td>
+	      <td>${cil.git_log}</td>
+		</tr>
+	
+		<tr>
+		  <td colspan="3">
+			<pre>${cil.import_log | h}</pre>
+		  </td>
+		</tr>
+		% endfor
+      </tbody>
+    </table>
+
+% endif
+
+  </section>
+</div>
+</section>

lms/urls.py

@@ -80,6 +80,12 @@ urlpatterns += (
     url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
 )
 
+# sysadmin dashboard, to see what courses are loaded, to delete & load courses
+if settings.FEATURES["ENABLE_SYSADMIN_DASHBOARD"]:
+    urlpatterns += (
+        url(r'^sysadmin/', include('dashboard.sysadmin_urls')),
+    )
+
 #Semi-static views (these need to be rendered and have the login bar, but don't change)
 urlpatterns += (
     url(r'^404$', 'static_template_view.views.render',

requirements/edx/base.txt

@@ -42,6 +42,7 @@ lazy==1.1
 lxml==3.0.1
 mako==0.7.3
 Markdown==2.2.1
+mongoengine==0.7.10
 networkx==1.7
 nltk==2.0.4
 oauthlib==0.5.1