github IPs in lms.envs.common; made create_user, create_groups,

manage_course_groups management commands (in lms_migration)

utility-scripts/create_groups.py → lms/djangoapps/lms_migration/management/commands/create_groups.py

@@ -6,44 +6,53 @@
 
 import os, sys, string, re
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-#try:
-#    from lms.envs.dev import *
-#except Exception as err:
-#    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-#    sys.exit(-1)
-
+from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.contrib.auth.models import User, Group
 from path import path
 from lxml import etree
 
-print "configured=",settings.configured
-print settings.SETTINGS_MODULE
-
-data_dir = settings.DATA_DIR
-print "data_dir = %s" % data_dir
-
-for course_dir in os.listdir(data_dir):
-    # print course_dir
-    if not os.path.isdir(path(data_dir) / course_dir):
-        continue
+def create_groups():
+    '''
+    Create staff and instructor groups for all classes in the data_dir
+    '''
+
+    data_dir = settings.DATA_DIR
+    print "data_dir = %s" % data_dir
+
+    for course_dir in os.listdir(data_dir):
+
+        if course_dir.startswith('.'):
+            continue
+        if not os.path.isdir(path(data_dir) / course_dir):
+            continue
+        
+        cxfn = path(data_dir) / course_dir / 'course.xml'
+        try:
+            coursexml = etree.parse(cxfn)
+        except Exception as err:
+            print "Oops, cannot read %s, skipping" % cxfn
+            continue
+        cxmlroot = coursexml.getroot()
+        course = cxmlroot.get('course')		# TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
+        if course is None:
+            print "oops, can't get course id for %s" % course_dir
+            continue
+        print "course=%s for course_dir=%s" % (course,course_dir)
     
-    cxfn = path(data_dir) / course_dir / 'course.xml'
-    coursexml = etree.parse(cxfn)
-    cxmlroot = coursexml.getroot()
-    course = cxmlroot.get('course')
-    if course is None:
-        print "oops, can't get course id for %s" % course_dir
-        continue
-    print "course=%s for course_dir=%s" % (course,course_dir)
-
-    gname = 'staff_%s' % course
+        create_group('staff_%s' % course)		# staff group
+        create_group('instructor_%s' % course)		# instructor group (can manage staff group list)
+
+def create_group(gname):
     if Group.objects.filter(name=gname):
-        print "group exists for %s" % gname
-        continue
+        print "    group exists for %s" % gname
+        return
     g = Group(name=gname)
     g.save()
-    print "created group %s" % gname
+    print "    created group %s" % gname
+
+class Command(BaseCommand):
+    help = "Create groups associated with all courses in data_dir."
+
+    def handle(self, *args, **options):
+        create_groups()

utility-scripts/create_user.py → lms/djangoapps/lms_migration/management/commands/create_user.py

@@ -8,23 +8,13 @@ import os, sys, string, re
 import datetime
 from getpass import getpass
 import json
+from random import choice
 import readline
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-#try:
-#    from lms.envs.dev import *
-#except Exception as err:
-#    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-#    sys.exit(-1)
-
-sys.path.append(os.path.abspath('common/djangoapps'))
-
+from django.core.management.base import BaseCommand
 from student.models import UserProfile, Registration
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.models import User, Group
-from random import choice
 
 class MyCompleter(object):  # Custom completer
 
@@ -49,103 +39,108 @@ def GenPasswd(length=8, chars=string.letters + string.digits):
     return ''.join([choice(chars) for i in range(length)])
 
 #-----------------------------------------------------------------------------
-# main
+# main command
 
-while True:
-    uname = raw_input('username: ')
-    if User.objects.filter(username=uname):
-        print "username %s already taken" % uname
-    else:
-        break
+class Command(BaseCommand):
+    help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
 
-make_eamap = False
-if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
-    email = '%s@MIT.EDU' % uname
-    if not email.endswith('@MIT.EDU'):
-        print "Failed - email must be @MIT.EDU"
-        sys.exit(-1)
-    mit_domain = 'ssl:MIT'
-    if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
-        print "Failed - email %s already exists as external_id" % email
-        sys.exit(-1)
-    make_eamap = True
-    password = GenPasswd(12)
-    
-    # get name from kerberos
-    kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
-    name = raw_input('Full name: [%s] ' % kname).strip()
-    if name=='':
-        name = kname
-    print "name = %s" % name
-else:
-    while True:
-        password = getpass()
-        password2 = getpass()
-        if password == password2:
-            break
-        print "Oops, passwords do not match, please retry"
+    def handle(self, *args, **options):
 
-    while True:
-        email = raw_input('email: ')
-        if User.objects.filter(email=email):
-            print "email %s already taken" % email
+        while True:
+            uname = raw_input('username: ')
+            if User.objects.filter(username=uname):
+                print "username %s already taken" % uname
+            else:
+                break
+        
+        make_eamap = False
+        if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+            email = '%s@MIT.EDU' % uname
+            if not email.endswith('@MIT.EDU'):
+                print "Failed - email must be @MIT.EDU"
+                sys.exit(-1)
+            mit_domain = 'ssl:MIT'
+            if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+                print "Failed - email %s already exists as external_id" % email
+                sys.exit(-1)
+            make_eamap = True
+            password = GenPasswd(12)
+            
+            # get name from kerberos
+            kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+            name = raw_input('Full name: [%s] ' % kname).strip()
+            if name=='':
+                name = kname
+            print "name = %s" % name
         else:
-            break
+            while True:
+                password = getpass()
+                password2 = getpass()
+                if password == password2:
+                    break
+                print "Oops, passwords do not match, please retry"
         
-    name = raw_input('Full name: ')
-
+            while True:
+                email = raw_input('email: ')
+                if User.objects.filter(email=email):
+                    print "email %s already taken" % email
+                else:
+                    break
+                
+            name = raw_input('Full name: ')
         
-user = User(username=uname, email=email, is_active=True)
-user.set_password(password)
-try:
-    user.save()
-except IntegrityError:
-    print "Oops, failed to create user %s, IntegrityError" % user
-    raise
-    
-r = Registration()
-r.register(user)
-
-up = UserProfile(user=user)
-up.name = name
-up.save()
-
-if make_eamap:
-    credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
-    eamap = ExternalAuthMap(external_id = email,
-                            external_email = email,
-                            external_domain = mit_domain,
-                            external_name = name,
-                            internal_password = password,
-                            external_credentials = json.dumps(credentials),
-        )
-    eamap.user = user
-    eamap.dtsignup = datetime.datetime.now()
-    eamap.save()
-
-print "User %s created successfully!" % user
-
-if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
-    sys.exit(0)
-
-print "Here are the groups available:"
-
-groups = [str(g.name) for g in Group.objects.all()]
-print groups
-
-completer = MyCompleter(groups)
-readline.set_completer(completer.complete)
-readline.parse_and_bind('tab: complete')
-
-while True:
-    gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
-    if not gname:
-        break
-    if not gname in groups:
-        print "Unknown group %s" % gname
-        continue
-    g = Group.objects.get(name=gname)
-    user.groups.add(g)
-    print "Added %s to group %s" % (user,g)
-
-print "Done!"
+                
+        user = User(username=uname, email=email, is_active=True)
+        user.set_password(password)
+        try:
+            user.save()
+        except IntegrityError:
+            print "Oops, failed to create user %s, IntegrityError" % user
+            raise
+            
+        r = Registration()
+        r.register(user)
+        
+        up = UserProfile(user=user)
+        up.name = name
+        up.save()
+        
+        if make_eamap:
+            credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+            eamap = ExternalAuthMap(external_id = email,
+                                    external_email = email,
+                                    external_domain = mit_domain,
+                                    external_name = name,
+                                    internal_password = password,
+                                    external_credentials = json.dumps(credentials),
+                )
+            eamap.user = user
+            eamap.dtsignup = datetime.datetime.now()
+            eamap.save()
+        
+        print "User %s created successfully!" % user
+        
+        if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+            sys.exit(0)
+        
+        print "Here are the groups available:"
+        
+        groups = [str(g.name) for g in Group.objects.all()]
+        print groups
+        
+        completer = MyCompleter(groups)
+        readline.set_completer(completer.complete)
+        readline.parse_and_bind('tab: complete')
+        
+        while True:
+            gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
+            if not gname:
+                break
+            if not gname in groups:
+                print "Unknown group %s" % gname
+                continue
+            g = Group.objects.get(name=gname)
+            user.groups.add(g)
+            print "Added %s to group %s" % (user,g)
+        
+        print "Done!"

lms/djangoapps/lms_migration/migrate.py

@@ -63,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
             html += 'Permission denied'
             html += "</body></html>"
             log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
-            return HttpResponse(html)
+            return HttpResponse(html, status=403)
 
     #----------------------------------------
     # reload course if specified
@@ -137,7 +137,7 @@ def gitreload(request, reload_dir=None):
     html += '<h3>IP address: %s ' % ip
     html += '<h3>User: %s ' % request.user
 
-    ALLOWED_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']	# hardcoded to github
+    ALLOWED_IPS = []	# allow none by default
     if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
         ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
 

lms/envs/common.py

@@ -254,6 +254,14 @@ USE_L10N = True
 # Messages
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
+#################################### GITHUB #######################################
+# gitreload is used in LMS-workflow to pull content from github
+# gitreload requests are only allowed from these IP addresses, which are
+# the advertised public IPs of the github WebHook servers.
+# These are listed, eg at https://github.com/MITx/mitx/admin/hooks
+
+ALLOWED_GITRELOAD = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
+
 #################################### AWS #######################################
 # S3BotoStorage insists on a timeout for uploaded assets. We should make it
 # permanent instead, but rather than trying to figure out exactly where that

lms/envs/dev.py

@@ -73,6 +73,8 @@ MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
 
+INSTALLED_APPS += ('lms_migration',)
+
 LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 
 ################################ OpenID Auth #################################

utility-scripts/manage_class_groups.py

-#!/usr/bin/python
-#
-# File:   manage_class_groups
-#
-# list and edit membership in class staff group
-
-import os, sys, string, re
-import datetime
-from getpass import getpass
-import json
-import readline
-
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-#try:
-#    from lms.envs.dev import *
-#except Exception as err:
-#    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-#    sys.exit(-1)
-
-from django.conf import settings
-from django.contrib.auth.models import User, Group
-
-#-----------------------------------------------------------------------------
-# get all staff groups
-
-gset = Group.objects.all()
-
-print "Groups:"
-for cnt,g in zip(range(len(gset)), gset):
-    print "%d. %s" % (cnt,g)
-
-gnum = int(raw_input('Choose group to manage (enter #): '))
-
-group = gset[gnum]
-
-#-----------------------------------------------------------------------------
-# users in group
-
-uall = User.objects.all()
-print "----"
-print "List of All Users: %s" % [str(x.username) for x in uall]
-print "----"
-
-while True:
-
-    print "Users in the group:"
-    
-    uset = group.user_set.all()
-    for cnt, u in zip(range(len(uset)), uset):
-        print "%d. %s" % (cnt, u)
-
-    action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
-
-    m = re.match('^[0-9]+$',action)
-    if m:
-        unum = int(action)
-        u = uset[unum]
-        print "Deleting user %s" % u
-        u.groups.remove(group)
-    
-    else:
-        for uname in action.split(','):
-            try:
-                user = User.objects.get(username=action)
-            except Exception as err:
-                print "Error %s" % err
-                continue
-            print "adding %s to group %s" % (user, group)
-            user.groups.add(group)
-        
-    
-