Merge pull request #250 from MITx/pmitros/release-fixes

Tracking issues, import/export scripts

common/djangoapps/student/management/commands/6002exportusers.py

+##
+## One-off script to export 6.002x users into the edX framework
+##
+## Could be modified to be general by:
+## * Changing user_keys and up_keys to handle dates more cleanly
+## * Providing a generic set of tables, rather than just users and user profiles
+## * Handling certificates and grades
+## * Handling merge/forks of UserProfile.meta
+
+
+import datetime
+import json
+
+import os.path
+
+from lxml import etree
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from student.models import UserProfile
+
+import mitxmako.middleware as middleware
+
+middleware.MakoMiddleware()
+
+class Command(BaseCommand):
+    help = \
+'''Exports all users and user profiles. 
+Caveat: Should be looked over before any run
+for schema changes. 
+
+Current version grabs user_keys from 
+django.contrib.auth.models.User and up_keys
+from student.userprofile. '''
+    def handle(self, *args, **options):
+        users = list(User.objects.all())
+        user_profiles = list(UserProfile.objects.all())
+        user_profile_dict = dict([(up.user_id, up) for up in user_profiles])
+    
+        user_tuples = [(user_profile_dict[u.id], u) for u in users if u.id in user_profile_dict]
+    
+        user_keys = ['id', 'username', 'email', 'password', 'is_staff', 
+                     'is_active', 'is_superuser', 'last_login', 'date_joined', 
+                     'password']
+        up_keys = ['language', 'location','meta','name', 'id','user_id']
+        
+        def extract_dict(keys, object):
+            d = {}
+            for key in keys:
+                item = object.__getattribute__(key)
+                if type(item) == datetime.datetime:
+                    item = item.isoformat()
+                d[key] = item
+            return d
+
+        extracted = [{'up':extract_dict(up_keys, t[0]), 'u':extract_dict(user_keys, t[1])} for t in user_tuples]
+        fp = open('transfer_users.txt', 'w')
+        json.dump(extracted, fp)
+        fp.close()

common/djangoapps/student/management/commands/6002importusers.py

+##
+## One-off script to import 6.002x users into the edX framework
+## See export for more info
+
+
+import datetime
+import json
+
+import dateutil.parser
+
+import os.path
+
+from lxml import etree
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from student.models import UserProfile
+
+import mitxmako.middleware as middleware
+
+middleware.MakoMiddleware()
+
+def import_user(u):
+    user_info = u['u']
+    up_info = u['up']
+
+    # HACK to handle dates
+    user_info['last_login'] = dateutil.parser.parse(user_info['last_login'])
+    user_info['date_joined'] = dateutil.parser.parse(user_info['date_joined'])
+
+    user_keys = ['id', 'username', 'email', 'password', 'is_staff', 
+                 'is_active', 'is_superuser', 'last_login', 'date_joined', 
+                 'password']
+    up_keys = ['language', 'location','meta','name', 'id','user_id']
+
+
+    u = User()
+    for key in user_keys:
+        u.__setattr__(key, user_info[key])
+    u.save()
+
+    up = UserProfile()
+    up.user = u
+    for key in up_keys:
+        up.__setattr__(key, up_info[key])
+    up.save()
+
+class Command(BaseCommand):
+    help = \
+'''Exports all users and user profiles. 
+Caveat: Should be looked over before any run
+for schema changes. 
+
+Current version grabs user_keys from 
+django.contrib.auth.models.User and up_keys
+from student.userprofile. '''
+    def handle(self, *args, **options):
+        extracted = json.load(open('transfer_users.txt'))
+        n=0
+        for u in extracted:
+            import_user(u)
+            if n%100 == 0: 
+                print n
+            n = n+1

common/djangoapps/track/middleware.py

@@ -12,8 +12,32 @@ class TrackMiddleware:
             if request.META['PATH_INFO'] in ['/event', '/login']:
                 return
             
-            event = { 'GET'  : dict(request.GET),
-                      'POST' : dict(request.POST)}
+            # Removes passwords from the tracking logs
+            # WARNING: This list needs to be changed whenever we change
+            # password handling functionality. 
+            #
+            # As of the time of this comment, only 'password' is used
+            # The rest are there for future extension. 
+            #
+            # Passwords should never be sent as GET requests, but 
+            # this can happen due to older browser bugs. We censor
+            # this too. 
+            # 
+            # We should manually confirm no passwords make it into log
+            # files when we change this. 
+
+            censored_strings = ['password', 'newpassword', 'new_password', 
+                                'oldpassword', 'old_password']
+            post_dict = dict(request.POST)
+            get_dict = dict(request.GET)
+            for string in censored_strings: 
+                if string in post_dict: 
+                    post_dict[string] = '*'*8                    
+                if string in get_dict: 
+                    get_dict[string] = '*'*8                    
+
+            event = { 'GET'  : dict(get_dict),
+                      'POST' : dict(post_dict)}
             
             # TODO: Confirm no large file uploads
             event = json.dumps(event)