Merge pull request #448 from MITx/feature/ichuang/add-gitreload-to-migrate

add gitreload to lms-migration code

common/djangoapps/track/views.py

 import json
 import logging
 import os
+import pytz
 import datetime
 import dateutil.parser
 
@@ -84,15 +85,33 @@ def server_track(request, event_type, event, page=None):
         "time": datetime.datetime.utcnow().isoformat(),
         }
 
-    if event_type=="/event_logs" and request.user.is_staff:	# don't log
+    if event_type.startswith("/event_logs") and request.user.is_staff:	# don't log
         return
     log_event(event)
 
 @login_required
 @ensure_csrf_cookie
-def view_tracking_log(request):
+def view_tracking_log(request,args=''):
     if not request.user.is_staff:
         return redirect('/')
-    record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
+    nlen = 100
+    username = ''
+    if args:
+        for arg in args.split('/'):
+            if arg.isdigit():
+                nlen = int(arg)
+            if arg.startswith('username='):
+                username = arg[9:]
+            
+    record_instances = TrackingLog.objects.all().order_by('-time')
+    if username:
+        record_instances = record_instances.filter(username=username)
+    record_instances = record_instances[0:nlen]
+    
+    # fix dtstamp
+    fmt = '%a %d-%b-%y %H:%M:%S'  # "%Y-%m-%d %H:%M:%S %Z%z"
+    for rinst in record_instances:
+        rinst.dtstr = rinst.time.replace(tzinfo=pytz.utc).astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
+
     return render_to_response('tracking_log.html',{'records':record_instances})
 

common/lib/capa/capa/inputtypes.py

@@ -150,6 +150,7 @@ def optioninput(element, value, status, render_template, msg=''):
              'state': status,
              'msg': msg,
              'options': osetdict,
+             'inline': element.get('inline',''),
              }
 
     html = render_template("optioninput.html", context)
@@ -294,7 +295,9 @@ def textline(element, value, status, render_template, msg=""):
     hidden = element.get('hidden', '')	 # if specified, then textline is hidden and id is stored in div of name given by hidden
     escapedict = {'"': '"'}
     value = saxutils.escape(value, escapedict)	 # otherwise, answers with quotes in them crashes the system!
-    context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden}
+    context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden,
+               'inline': element.get('inline',''),
+               }
     html = render_template("textinput.html", context)
     try:
         xhtml = etree.XML(html)

common/lib/capa/capa/templates/textinput.html

-<section id="textinput_${id}" class="textinput">
+<% doinline = "inline" if inline else "" %>
+
+<section id="textinput_${id}" class="textinput ${doinline}" >
     % if state == 'unsubmitted':
-    <div class="unanswered" id="status_${id}">
+    <div class="unanswered ${doinline}" id="status_${id}">
     % elif state == 'correct':
-    <div class="correct" id="status_${id}">
+    <div class="correct ${doinline}" id="status_${id}">
     % elif state == 'incorrect':
-    <div class="incorrect" id="status_${id}">
+    <div class="incorrect ${doinline}" id="status_${id}">
     % elif state == 'incomplete':
-    <div class="incorrect" id="status_${id}">
+    <div class="incorrect ${doinline}" id="status_${id}">
     % endif
     % if hidden:
       <div style="display:none;" name="${hidden}" inputid="input_${id}" />

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -27,6 +27,10 @@ section.problem {
     }
   }
 
+  .inline {
+      display: inline;
+  }
+
   div {
     p {
       &.answer {

utility-scripts/create_groups.py → lms/djangoapps/lms_migration/management/commands/create_groups.py

@@ -6,41 +6,53 @@
 
 import os, sys, string, re
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-try:
-    from lms.envs.dev import *
-except Exception as err:
-    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-    sys.exit(-1)
-
+from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.contrib.auth.models import User, Group
 from path import path
 from lxml import etree
 
-data_dir = settings.DATA_DIR
-print "data_dir = %s" % data_dir
+def create_groups():
+    '''
+    Create staff and instructor groups for all classes in the data_dir
+    '''
+
+    data_dir = settings.DATA_DIR
+    print "data_dir = %s" % data_dir
 
-for course_dir in os.listdir(data_dir):
-    # print course_dir
+    for course_dir in os.listdir(data_dir):
+
+        if course_dir.startswith('.'):
+            continue
         if not os.path.isdir(path(data_dir) / course_dir):
             continue
         
         cxfn = path(data_dir) / course_dir / 'course.xml'
+        try:
             coursexml = etree.parse(cxfn)
+        except Exception as err:
+            print "Oops, cannot read %s, skipping" % cxfn
+            continue
         cxmlroot = coursexml.getroot()
-    course = cxmlroot.get('course')
+        course = cxmlroot.get('course')		# TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
         if course is None:
             print "oops, can't get course id for %s" % course_dir
             continue
         print "course=%s for course_dir=%s" % (course,course_dir)
     
-    gname = 'staff_%s' % course
+        create_group('staff_%s' % course)		# staff group
+        create_group('instructor_%s' % course)		# instructor group (can manage staff group list)
+
+def create_group(gname):
     if Group.objects.filter(name=gname):
-        print "group exists for %s" % gname
-        continue
+        print "    group exists for %s" % gname
+        return
     g = Group(name=gname)
     g.save()
-    print "created group %s" % gname
+    print "    created group %s" % gname
+
+class Command(BaseCommand):
+    help = "Create groups associated with all courses in data_dir."
+
+    def handle(self, *args, **options):
+        create_groups()

utility-scripts/create_user.py → lms/djangoapps/lms_migration/management/commands/create_user.py

@@ -8,21 +8,13 @@ import os, sys, string, re
 import datetime
 from getpass import getpass
 import json
+from random import choice
 import readline
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-try:
-    from lms.envs.dev import *
-except Exception as err:
-    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-    sys.exit(-1)
-
+from django.core.management.base import BaseCommand
 from student.models import UserProfile, Registration
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.models import User, Group
-from random import choice
 
 class MyCompleter(object):  # Custom completer
 
@@ -47,17 +39,22 @@ def GenPasswd(length=8, chars=string.letters + string.digits):
     return ''.join([choice(chars) for i in range(length)])
 
 #-----------------------------------------------------------------------------
-# main
+# main command
+
+class Command(BaseCommand):
+    help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
 
-while True:
+    def handle(self, *args, **options):
+
+        while True:
             uname = raw_input('username: ')
             if User.objects.filter(username=uname):
                 print "username %s already taken" % uname
             else:
                 break
         
-make_eamap = False
-if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+        make_eamap = False
+        if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
             email = '%s@MIT.EDU' % uname
             if not email.endswith('@MIT.EDU'):
                 print "Failed - email must be @MIT.EDU"
@@ -75,7 +72,7 @@ if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
             if name=='':
                 name = kname
             print "name = %s" % name
-else:
+        else:
             while True:
                 password = getpass()
                 password2 = getpass()
@@ -93,22 +90,22 @@ else:
             name = raw_input('Full name: ')
         
                 
-user = User(username=uname, email=email, is_active=True)
-user.set_password(password)
-try:
+        user = User(username=uname, email=email, is_active=True)
+        user.set_password(password)
+        try:
             user.save()
-except IntegrityError:
+        except IntegrityError:
             print "Oops, failed to create user %s, IntegrityError" % user
             raise
             
-r = Registration()
-r.register(user)
+        r = Registration()
+        r.register(user)
         
-up = UserProfile(user=user)
-up.name = name
-up.save()
+        up = UserProfile(user=user)
+        up.name = name
+        up.save()
         
-if make_eamap:
+        if make_eamap:
             credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
             eamap = ExternalAuthMap(external_id = email,
                                     external_email = email,
@@ -121,21 +118,21 @@ if make_eamap:
             eamap.dtsignup = datetime.datetime.now()
             eamap.save()
         
-print "User %s created successfully!" % user
+        print "User %s created successfully!" % user
         
-if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+        if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
             sys.exit(0)
         
-print "Here are the groups available:"
+        print "Here are the groups available:"
         
-groups = [str(g.name) for g in Group.objects.all()]
-print groups
+        groups = [str(g.name) for g in Group.objects.all()]
+        print groups
         
-completer = MyCompleter(groups)
-readline.set_completer(completer.complete)
-readline.parse_and_bind('tab: complete')
+        completer = MyCompleter(groups)
+        readline.set_completer(completer.complete)
+        readline.parse_and_bind('tab: complete')
         
-while True:
+        while True:
             gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
             if not gname:
                 break
@@ -146,4 +143,4 @@ while True:
             user.groups.add(g)
             print "Added %s to group %s" % (user,g)
         
-print "Done!"
+        print "Done!"

lms/djangoapps/lms_migration/migrate.py

@@ -2,13 +2,21 @@
 # migration tools for content team to go from stable-edx4edx to LMS+CMS
 #
 
+import json
 import logging
+import os
 from pprint import pprint
 import xmodule.modulestore.django as xmodule_django
 from xmodule.modulestore.django import modulestore
 
 from django.http import HttpResponse
 from django.conf import settings
+import track.views
+
+try:
+    from django.views.decorators.csrf import csrf_exempt
+except ImportError:
+    from django.contrib.csrf.middleware import csrf_exempt
 
 log = logging.getLogger("mitx.lms_migrate")
 LOCAL_DEBUG = True
@@ -18,6 +26,15 @@ def escape(s):
     """escape HTML special characters in string"""
     return str(s).replace('<','&lt;').replace('>','&gt;')
 
+def getip(request):
+    '''
+    Extract IP address of requester from header, even if behind proxy
+    '''
+    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
+    if not ip:
+        ip = request.META.get('REMOTE_ADDR','None')
+    return ip
+
 def manage_modulestores(request,reload_dir=None):
     '''
     Manage the static in-memory modulestores.
@@ -32,9 +49,7 @@ def manage_modulestores(request,reload_dir=None):
     #----------------------------------------
     # check on IP address of requester
 
-    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
-    if not ip:
-        ip = request.META.get('REMOTE_ADDR','None')
+    ip = getip(request)
 
     if LOCAL_DEBUG:
         html += '<h3>IP address: %s ' % ip
@@ -48,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
             html += 'Permission denied'
             html += "</body></html>"
             log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
-            return HttpResponse(html)
+            return HttpResponse(html, status=403)
 
     #----------------------------------------
     # reload course if specified
@@ -108,3 +123,66 @@ def manage_modulestores(request,reload_dir=None):
 
     html += "</body></html>"
     return HttpResponse(html)
+
+@csrf_exempt
+def gitreload(request, reload_dir=None):
+    '''
+    This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
+
+    If reload_dir is not None, then instruct the xml loader to reload that course directory.
+    '''
+    html = "<html><body>"
+    ip = getip(request)
+
+    html += '<h3>IP address: %s ' % ip
+    html += '<h3>User: %s ' % request.user
+
+    ALLOWED_IPS = []	# allow none by default
+    if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
+        ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
+
+    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
+        if request.user and request.user.is_staff:
+            log.debug('request allowed because user=%s is staff' % request.user)
+        else:
+            html += 'Permission denied'
+            html += "</body></html>"
+            log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
+            return HttpResponse(html)    
+        
+    #----------------------------------------
+    # see if request is from github (POST with JSON)
+
+    if reload_dir is None and 'payload' in request.POST:
+        payload = request.POST['payload']
+        log.debug("payload=%s" % payload)
+        gitargs = json.loads(payload)
+        log.debug("gitargs=%s" % gitargs)
+        reload_dir = gitargs['repository']['name']
+        log.debug("github reload_dir=%s" % reload_dir)
+        gdir = settings.DATA_DIR / reload_dir 
+        if not os.path.exists(gdir):
+            log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
+            return HttpResponse('Error')
+        cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
+        log.debug(os.popen(cmd).read())
+        if hasattr(settings,'GITRELOAD_HOOK'):	# hit this hook after reload, if set
+            gh = settings.GITRELOAD_HOOK
+            if gh:
+                ghurl = '%s/%s' % (gh,reload_dir)
+                r = requests.get(ghurl)
+                log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
+        
+    #----------------------------------------
+    # reload course if specified
+
+    if reload_dir is not None:
+        def_ms = modulestore()
+        if reload_dir not in def_ms.courses:
+            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
+        else:
+            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
+            def_ms.try_load_course(reload_dir)
+            track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
+
+    return HttpResponse(html)    

lms/envs/common.py

@@ -260,6 +260,14 @@ USE_L10N = True
 # Messages
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
+#################################### GITHUB #######################################
+# gitreload is used in LMS-workflow to pull content from github
+# gitreload requests are only allowed from these IP addresses, which are
+# the advertised public IPs of the github WebHook servers.
+# These are listed, eg at https://github.com/MITx/mitx/admin/hooks
+
+ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
+
 #################################### AWS #######################################
 # S3BotoStorage insists on a timeout for uploaded assets. We should make it
 # permanent instead, but rather than trying to figure out exactly where that

lms/envs/dev.py

@@ -73,6 +73,8 @@ MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
 
+INSTALLED_APPS += ('lms_migration',)
+
 LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 
 ################################ OpenID Auth #################################

lms/envs/dev_ike.py

@@ -17,14 +17,19 @@ MITX_FEATURES['ENABLE_TEXTBOOK'] = False
 MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 
+MITX_FEATURES['DISABLE_START_DATES'] = True
+# MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
+
 myhost = socket.gethostname()
 if ('edxvm' in myhost) or ('ocw' in myhost):
     MITX_FEATURES['DISABLE_LOGIN_BUTTON'] = True	# auto-login with MIT certificate
     MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa'	# needs to be ssl or browser blocks it
+    MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
 if ('domU' in myhost):
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
     MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu'	# nonempty string = address for all activation emails
+    MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')	# django 1.4 for nginx ssl proxy
 
@@ -33,4 +38,9 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')	# django 1.4 fo
 
 INSTALLED_APPS = tuple([ app for app in INSTALLED_APPS if not app.startswith('debug_toolbar') ])
 MIDDLEWARE_CLASSES = tuple([ mcl for mcl in MIDDLEWARE_CLASSES if not mcl.startswith('debug_toolbar') ])
-TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
+#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
+#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('mitxmako') ])
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+    )

lms/templates/tracking_log.html

@@ -3,7 +3,7 @@
 <table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
 % for rec in records:
     <tr>
-    <td>${rec.time}</td> 
+    <td>${rec.dtstr}</td> 
     <td>${rec.username}</td>  
     <td>${rec.ip}</td>  
     <td>${rec.event_source}</td>   

lms/urls.py

@@ -217,11 +217,14 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
     urlpatterns += (
         url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
         url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
+        url(r'^gitreload$', 'lms_migration.migrate.gitreload'),
+        url(r'^gitreload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.gitreload'),
         )
 
 if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
     urlpatterns += (
         url(r'^event_logs$', 'track.views.view_tracking_log'),
+        url(r'^event_logs/(?P<args>.+)$', 'track.views.view_tracking_log'),
         )
 
 urlpatterns = patterns(*urlpatterns)