Skip to content

E
edx-platform
Commit dca434ad by Calen Pennington
Options

Merge pull request #448 from MITx/feature/ichuang/add-gitreload-to-migrate 

add gitreload to lms-migration code
parents 61192e52 65ea54f7
Showing with 285 additions and 147 deletions
common/djangoapps/track/views.py
import json
import logging
import os
import pytz
import datetime
import dateutil.parser
......@@ -84,15 +85,33 @@ def server_track(request, event_type, event, page=None):
"time": datetime.datetime.utcnow().isoformat(),
}
if event_type=="/event_logs" and request.user.is_staff: # don't log
if event_type.startswith("/event_logs") and request.user.is_staff: # don't log
return
log_event(event)
@login_required
@ensure_csrf_cookie
def view_tracking_log(request):
def view_tracking_log(request,args=''):
if not request.user.is_staff:
return redirect('/')
record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
nlen = 100
username = ''
if args:
for arg in args.split('/'):
if arg.isdigit():
nlen = int(arg)
if arg.startswith('username='):
username = arg[9:]
record_instances = TrackingLog.objects.all().order_by('-time')
if username:
record_instances = record_instances.filter(username=username)
record_instances = record_instances[0:nlen]
# fix dtstamp
fmt = '%a %d-%b-%y %H:%M:%S' # "%Y-%m-%d %H:%M:%S %Z%z"
for rinst in record_instances:
rinst.dtstr = rinst.time.replace(tzinfo=pytz.utc).astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
return render_to_response('tracking_log.html',{'records':record_instances})
common/lib/capa/capa/inputtypes.py
......@@ -150,6 +150,7 @@ def optioninput(element, value, status, render_template, msg=''):
'state': status,
'msg': msg,
'options': osetdict,
'inline': element.get('inline',''),
}
html = render_template("optioninput.html", context)
......@@ -294,7 +295,9 @@ def textline(element, value, status, render_template, msg=""):
hidden = element.get('hidden', '') # if specified, then textline is hidden and id is stored in div of name given by hidden
escapedict = {'"': '"'}
value = saxutils.escape(value, escapedict) # otherwise, answers with quotes in them crashes the system!
context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden}
context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden,
'inline': element.get('inline',''),
}
html = render_template("textinput.html", context)
try:
xhtml = etree.XML(html)
......
common/lib/capa/capa/templates/textinput.html
<section id="textinput_${id}" class="textinput">
<% doinline = "inline" if inline else "" %>
<section id="textinput_${id}" class="textinput ${doinline}" >
% if state == 'unsubmitted':
<div class="unanswered" id="status_${id}">
<div class="unanswered ${doinline}" id="status_${id}">
% elif state == 'correct':
<div class="correct" id="status_${id}">
<div class="correct ${doinline}" id="status_${id}">
% elif state == 'incorrect':
<div class="incorrect" id="status_${id}">
<div class="incorrect ${doinline}" id="status_${id}">
% elif state == 'incomplete':
<div class="incorrect" id="status_${id}">
<div class="incorrect ${doinline}" id="status_${id}">
% endif
% if hidden:
<div style="display:none;" name="${hidden}" inputid="input_${id}" />
......
common/lib/xmodule/xmodule/css/capa/display.scss
......@@ -27,6 +27,10 @@ section.problem {
}
}
.inline {
display: inline;
}
div {
p {
&.answer {
......
utility-scripts/create_groups.py → lms/djangoapps/lms_migration/management/commands/create_groups.py
......@@ -6,41 +6,53 @@
import os, sys, string, re
sys.path.append(os.path.abspath('.'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
try:
from lms.envs.dev import *
except Exception as err:
print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
sys.exit(-1)
from django.core.management.base import BaseCommand
from django.conf import settings
from django.contrib.auth.models import User, Group
from path import path
from lxml import etree
data_dir = settings.DATA_DIR
print "data_dir = %s" % data_dir
def create_groups():
'''
Create staff and instructor groups for all classes in the data_dir
'''
data_dir = settings.DATA_DIR
print "data_dir = %s" % data_dir
for course_dir in os.listdir(data_dir):
for course_dir in os.listdir(data_dir):
# print course_dir
if not os.path.isdir(path(data_dir) / course_dir):
continue
if course_dir.startswith('.'):
continue
if not os.path.isdir(path(data_dir) / course_dir):
continue
cxfn = path(data_dir) / course_dir / 'course.xml'
try:
coursexml = etree.parse(cxfn)
except Exception as err:
print "Oops, cannot read %s, skipping" % cxfn
continue
cxmlroot = coursexml.getroot()
course = cxmlroot.get('course') # TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
if course is None:
print "oops, can't get course id for %s" % course_dir
continue
print "course=%s for course_dir=%s" % (course,course_dir)
cxfn = path(data_dir) / course_dir / 'course.xml'
coursexml = etree.parse(cxfn)
cxmlroot = coursexml.getroot()
course = cxmlroot.get('course')
if course is None:
print "oops, can't get course id for %s" % course_dir
continue
print "course=%s for course_dir=%s" % (course,course_dir)
gname = 'staff_%s' % course
create_group('staff_%s' % course) # staff group
create_group('instructor_%s' % course) # instructor group (can manage staff group list)
def create_group(gname):
if Group.objects.filter(name=gname):
print "group exists for %s" % gname
continue
print " group exists for %s" % gname
return
g = Group(name=gname)
g.save()
print "created group %s" % gname
print " created group %s" % gname
class Command(BaseCommand):
help = "Create groups associated with all courses in data_dir."
def handle(self, *args, **options):
create_groups()
utility-scripts/create_user.py → lms/djangoapps/lms_migration/management/commands/create_user.py
......@@ -8,21 +8,13 @@ import os, sys, string, re
import datetime
from getpass import getpass
import json
from random import choice
import readline
sys.path.append(os.path.abspath('.'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
try:
from lms.envs.dev import *
except Exception as err:
print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
sys.exit(-1)
from django.core.management.base import BaseCommand
from student.models import UserProfile, Registration
from external_auth.models import ExternalAuthMap
from django.contrib.auth.models import User, Group
from random import choice
class MyCompleter(object): # Custom completer
......@@ -47,103 +39,108 @@ def GenPasswd(length=8, chars=string.letters + string.digits):
return ''.join([choice(chars) for i in range(length)])
#-----------------------------------------------------------------------------
# main
# main command
while True:
uname = raw_input('username: ')
if User.objects.filter(username=uname):
print "username %s already taken" % uname
else:
break
class Command(BaseCommand):
help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
make_eamap = False
if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
email = '%s@MIT.EDU' % uname
if not email.endswith('@MIT.EDU'):
print "Failed - email must be @MIT.EDU"
sys.exit(-1)
mit_domain = 'ssl:MIT'
if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
print "Failed - email %s already exists as external_id" % email
sys.exit(-1)
make_eamap = True
password = GenPasswd(12)
# get name from kerberos
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
name = raw_input('Full name: [%s] ' % kname).strip()
if name=='':
name = kname
print "name = %s" % name
else:
while True:
password = getpass()
password2 = getpass()
if password == password2:
break
print "Oops, passwords do not match, please retry"
def handle(self, *args, **options):
while True:
email = raw_input('email: ')
if User.objects.filter(email=email):
print "email %s already taken" % email
while True:
uname = raw_input('username: ')
if User.objects.filter(username=uname):
print "username %s already taken" % uname
else:
break
make_eamap = False
if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
email = '%s@MIT.EDU' % uname
if not email.endswith('@MIT.EDU'):
print "Failed - email must be @MIT.EDU"
sys.exit(-1)
mit_domain = 'ssl:MIT'
if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
print "Failed - email %s already exists as external_id" % email
sys.exit(-1)
make_eamap = True
password = GenPasswd(12)
# get name from kerberos
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
name = raw_input('Full name: [%s] ' % kname).strip()
if name=='':
name = kname
print "name = %s" % name
else:
break
while True:
password = getpass()
password2 = getpass()
if password == password2:
break
print "Oops, passwords do not match, please retry"
name = raw_input('Full name: ')
while True:
email = raw_input('email: ')
if User.objects.filter(email=email):
print "email %s already taken" % email
else:
break
name = raw_input('Full name: ')
user = User(username=uname, email=email, is_active=True)
user.set_password(password)
try:
user.save()
except IntegrityError:
print "Oops, failed to create user %s, IntegrityError" % user
raise
r = Registration()
r.register(user)
up = UserProfile(user=user)
up.name = name
up.save()
if make_eamap:
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
eamap = ExternalAuthMap(external_id = email,
external_email = email,
external_domain = mit_domain,
external_name = name,
internal_password = password,
external_credentials = json.dumps(credentials),
)
eamap.user = user
eamap.dtsignup = datetime.datetime.now()
eamap.save()
print "User %s created successfully!" % user
if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
sys.exit(0)
print "Here are the groups available:"
groups = [str(g.name) for g in Group.objects.all()]
print groups
completer = MyCompleter(groups)
readline.set_completer(completer.complete)
readline.parse_and_bind('tab: complete')
while True:
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
if not gname:
break
if not gname in groups:
print "Unknown group %s" % gname
continue
g = Group.objects.get(name=gname)
user.groups.add(g)
print "Added %s to group %s" % (user,g)
print "Done!"
user = User(username=uname, email=email, is_active=True)
user.set_password(password)
try:
user.save()
except IntegrityError:
print "Oops, failed to create user %s, IntegrityError" % user
raise
r = Registration()
r.register(user)
up = UserProfile(user=user)
up.name = name
up.save()
if make_eamap:
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
eamap = ExternalAuthMap(external_id = email,
external_email = email,
external_domain = mit_domain,
external_name = name,
internal_password = password,
external_credentials = json.dumps(credentials),
)
eamap.user = user
eamap.dtsignup = datetime.datetime.now()
eamap.save()
print "User %s created successfully!" % user
if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
sys.exit(0)
print "Here are the groups available:"
groups = [str(g.name) for g in Group.objects.all()]
print groups
completer = MyCompleter(groups)
readline.set_completer(completer.complete)
readline.parse_and_bind('tab: complete')
while True:
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
if not gname:
break
if not gname in groups:
print "Unknown group %s" % gname
continue
g = Group.objects.get(name=gname)
user.groups.add(g)
print "Added %s to group %s" % (user,g)
print "Done!"
lms/djangoapps/lms_migration/migrate.py
......@@ -2,13 +2,21 @@
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#
import json
import logging
import os
from pprint import pprint
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore
from django.http import HttpResponse
from django.conf import settings
import track.views
try:
from django.views.decorators.csrf import csrf_exempt
except ImportError:
from django.contrib.csrf.middleware import csrf_exempt
log = logging.getLogger("mitx.lms_migrate")
LOCAL_DEBUG = True
......@@ -18,6 +26,15 @@ def escape(s):
"""escape HTML special characters in string"""
return str(s).replace('<','&lt;').replace('>','&gt;')
def getip(request):
'''
Extract IP address of requester from header, even if behind proxy
'''
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
if not ip:
ip = request.META.get('REMOTE_ADDR','None')
return ip
def manage_modulestores(request,reload_dir=None):
'''
Manage the static in-memory modulestores.
......@@ -32,9 +49,7 @@ def manage_modulestores(request,reload_dir=None):
#----------------------------------------
# check on IP address of requester
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
if not ip:
ip = request.META.get('REMOTE_ADDR','None')
ip = getip(request)
if LOCAL_DEBUG:
html += '<h3>IP address: %s ' % ip
......@@ -48,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
html += 'Permission denied'
html += "</body></html>"
log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
return HttpResponse(html)
return HttpResponse(html, status=403)
#----------------------------------------
# reload course if specified
......@@ -108,3 +123,66 @@ def manage_modulestores(request,reload_dir=None):
html += "</body></html>"
return HttpResponse(html)
@csrf_exempt
def gitreload(request, reload_dir=None):
'''
This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
If reload_dir is not None, then instruct the xml loader to reload that course directory.
'''
html = "<html><body>"
ip = getip(request)
html += '<h3>IP address: %s ' % ip
html += '<h3>User: %s ' % request.user
ALLOWED_IPS = [] # allow none by default
if hasattr(settings,'ALLOWED_GITRELOAD_IPS'): # allow override in settings
ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
if request.user and request.user.is_staff:
log.debug('request allowed because user=%s is staff' % request.user)
else:
html += 'Permission denied'
html += "</body></html>"
log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
return HttpResponse(html)
#----------------------------------------
# see if request is from github (POST with JSON)
if reload_dir is None and 'payload' in request.POST:
payload = request.POST['payload']
log.debug("payload=%s" % payload)
gitargs = json.loads(payload)
log.debug("gitargs=%s" % gitargs)
reload_dir = gitargs['repository']['name']
log.debug("github reload_dir=%s" % reload_dir)
gdir = settings.DATA_DIR / reload_dir
if not os.path.exists(gdir):
log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
return HttpResponse('Error')
cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
log.debug(os.popen(cmd).read())
if hasattr(settings,'GITRELOAD_HOOK'): # hit this hook after reload, if set
gh = settings.GITRELOAD_HOOK
if gh:
ghurl = '%s/%s' % (gh,reload_dir)
r = requests.get(ghurl)
log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
#----------------------------------------
# reload course if specified
if reload_dir is not None:
def_ms = modulestore()
if reload_dir not in def_ms.courses:
html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
else:
html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
def_ms.try_load_course(reload_dir)
track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
return HttpResponse(html)
lms/envs/common.py
......@@ -260,6 +260,14 @@ USE_L10N = True
# Messages
MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
#################################### GITHUB #######################################
# gitreload is used in LMS-workflow to pull content from github
# gitreload requests are only allowed from these IP addresses, which are
# the advertised public IPs of the github WebHook servers.
# These are listed, eg at https://github.com/MITx/mitx/admin/hooks
ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
#################################### AWS #######################################
# S3BotoStorage insists on a timeout for uploaded assets. We should make it
# permanent instead, but rather than trying to figure out exactly where that
......
lms/envs/dev.py
......@@ -73,6 +73,8 @@ MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll
MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
INSTALLED_APPS += ('lms_migration',)
LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
################################ OpenID Auth #################################
......
lms/envs/dev_ike.py
......@@ -17,14 +17,19 @@ MITX_FEATURES['ENABLE_TEXTBOOK'] = False
MITX_FEATURES['ENABLE_DISCUSSION'] = False
MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll
MITX_FEATURES['DISABLE_START_DATES'] = True
# MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
myhost = socket.gethostname()
if ('edxvm' in myhost) or ('ocw' in myhost):
MITX_FEATURES['DISABLE_LOGIN_BUTTON'] = True # auto-login with MIT certificate
MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa' # needs to be ssl or browser blocks it
MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
if ('domU' in myhost):
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu' # nonempty string = address for all activation emails
MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') # django 1.4 for nginx ssl proxy
......@@ -33,4 +38,9 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') # django 1.4 fo
INSTALLED_APPS = tuple([ app for app in INSTALLED_APPS if not app.startswith('debug_toolbar') ])
MIDDLEWARE_CLASSES = tuple([ mcl for mcl in MIDDLEWARE_CLASSES if not mcl.startswith('debug_toolbar') ])
TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('mitxmako') ])
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
)
lms/templates/tracking_log.html
......@@ -3,7 +3,7 @@
<table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
% for rec in records:
<tr>
<td>${rec.time}</td>
<td>${rec.dtstr}</td>
<td>${rec.username}</td>
<td>${rec.ip}</td>
<td>${rec.event_source}</td>
......
lms/urls.py
......@@ -217,11 +217,14 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
urlpatterns += (
url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
url(r'^gitreload$', 'lms_migration.migrate.gitreload'),
url(r'^gitreload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.gitreload'),
)
if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
urlpatterns += (
url(r'^event_logs$', 'track.views.view_tracking_log'),
url(r'^event_logs/(?P<args>.+)$', 'track.views.view_tracking_log'),
)
urlpatterns = patterns(*urlpatterns)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment