Skip to content

E
edx-platform
Commit ca28796a by David Ormsbee
Options

Merge pull request #1345 from MITx/feature/apenney/pearson-merge 

Pearson export/import code. (No merging before 25th Jan)
parents 72c9d9f7 a5477855
Showing with 421 additions and 95 deletions
common/djangoapps/student/management/commands/pearson_export_cdd.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir
from optparse import make_option
from django.core.management.base import BaseCommand
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from student.models import TestCenterUser
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
# Skipping optional field CandidateID
("ClientCandidateID", "client_candidate_id"),
......@@ -34,43 +36,52 @@ class Command(BaseCommand):
("FAXCountryCode", "fax_country_code"),
("CompanyName", "company_name"),
# Skipping optional field CustomQuestion
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
# define defaults, even thought 'store_true' shouldn't need them.
# (call_command will set None as default value for all options that don't have one,
# so one cannot rely on presence/absence of flags in that world.)
option_list = BaseCommand.option_list + (
make_option(
'--dump_all',
action='store_true',
dest='dump_all',
),
make_option('--dest-from-settings',
action='store_true',
dest='dest-from-settings',
default=False,
help='Retrieve the destination to export to from django.'),
make_option('--destination',
action='store',
dest='destination',
default=None,
help='Where to store the exported files')
)
args = '<output_file_or_dir>'
help = """
Export user demographic information from TestCenterUser model into a tab delimited
text file with a format that Pearson expects.
"""
def handle(self, *args, **kwargs):
if len(args) < 1:
print Command.help
return
def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at
# field
uploaded_at = datetime.utcnow()
# if specified destination is an existing directory, then
# if specified destination is an existing directory, then
# create a filename for it automatically. If it doesn't exist,
# or exists as a file, then we will just write to it.
# then we will create the directory.
# Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps
# but it should at least be consistent with the other timestamps
# used in the system.
dest = args[0]
if isdir(dest):
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
if 'dest-from-settings' in options and options['dest-from-settings']:
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif 'destination' in options and options['destination']:
dest = options['destination']
else:
destfile = dest
raise CommandError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
# strings must be in latin-1 format. CSV parser will
# otherwise convert unicode objects to ascii.
def ensure_encoding(value):
......@@ -78,8 +89,8 @@ class Command(BaseCommand):
return value.encode('iso-8859-1')
else:
return value
dump_all = kwargs['dump_all']
# dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -89,7 +100,7 @@ class Command(BaseCommand):
extrasaction='ignore')
writer.writeheader()
for tcu in TestCenterUser.objects.order_by('id'):
if dump_all or tcu.needs_uploading:
if tcu.needs_uploading: # or dump_all
record = dict((csv_field, ensure_encoding(getattr(tcu, model_field)))
for csv_field, model_field
in Command.CSV_TO_MODEL_FIELDS.items())
......@@ -97,6 +108,3 @@ class Command(BaseCommand):
writer.writerow(record)
tcu.uploaded_at = uploaded_at
tcu.save()
common/djangoapps/student/management/commands/pearson_export_ead.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir, join
from optparse import make_option
from django.core.management.base import BaseCommand
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from student.models import TestCenterRegistration, ACCOMMODATION_REJECTED_CODE
from student.models import TestCenterRegistration
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
('AuthorizationTransactionType', 'authorization_transaction_type'),
('AuthorizationID', 'authorization_id'),
......@@ -20,51 +22,60 @@ class Command(BaseCommand):
('Accommodations', 'accommodation_code'),
('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
args = '<output_file_or_dir>'
help = """
Export user registration information from TestCenterRegistration model into a tab delimited
text file with a format that Pearson expects.
"""
option_list = BaseCommand.option_list + (
make_option(
'--dump_all',
action='store_true',
dest='dump_all',
),
make_option(
'--force_add',
action='store_true',
dest='force_add',
),
make_option('--dest-from-settings',
action='store_true',
dest='dest-from-settings',
default=False,
help='Retrieve the destination to export to from django.'),
make_option('--destination',
action='store',
dest='destination',
default=None,
help='Where to store the exported files'),
make_option('--dump_all',
action='store_true',
dest='dump_all',
default=False,
),
make_option('--force_add',
action='store_true',
dest='force_add',
default=False,
),
)
def handle(self, *args, **kwargs):
if len(args) < 1:
print Command.help
return
# update time should use UTC in order to be comparable to the user_updated_at
def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at
# field
uploaded_at = datetime.utcnow()
# if specified destination is an existing directory, then
# if specified destination is an existing directory, then
# create a filename for it automatically. If it doesn't exist,
# or exists as a file, then we will just write to it.
# then we will create the directory.
# Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps
# but it should at least be consistent with the other timestamps
# used in the system.
dest = args[0]
if isdir(dest):
destfile = join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
if 'dest-from-settings' in options and options['dest-from-settings']:
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif 'destination' in options and options['destination']:
dest = options['destination']
else:
destfile = dest
raise CommandError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
dump_all = kwargs['dump_all']
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -81,13 +92,11 @@ class Command(BaseCommand):
record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
if kwargs['force_add']:
if record["Accommodations"] == ACCOMMODATION_REJECTED_CODE:
record["Accommodations"] = ""
if options['force_add']:
record['AuthorizationTransactionType'] = 'Add'
writer.writerow(record)
tcr.uploaded_at = uploaded_at
tcr.save()
common/djangoapps/student/management/commands/pearson_import_conf_zip.py 0 → 100644
import csv
from zipfile import ZipFile, is_zipfile
from time import strptime, strftime
from collections import OrderedDict
from datetime import datetime
from os.path import isdir
from optparse import make_option
from dogapi import dog_http_api, dog_stats_api
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from student.models import TestCenterUser, TestCenterRegistration
class Command(BaseCommand):
dog_http_api.api_key = settings.DATADOG_API
args = '<input zip file>'
help = """
Import Pearson confirmation files and update TestCenterUser
and TestCenterRegistration tables with status.
"""
@staticmethod
def datadog_error(string, tags):
dog_http_api.event("Pearson Import", string, alert_type='error', tags=[tags])
def handle(self, *args, **kwargs):
if len(args) < 1:
print Command.help
return
source_zip = args[0]
if not is_zipfile(source_zip):
error = "Input file is not a zipfile: \"{}\"".format(source_zip)
Command.datadog_error(error, source_zip)
raise CommandError(error)
# loop through all files in zip, and process them based on filename prefix:
with ZipFile(source_zip, 'r') as zipfile:
for fileinfo in zipfile.infolist():
with zipfile.open(fileinfo) as zipentry:
if fileinfo.filename.startswith("eac-"):
self.process_eac(zipentry)
elif fileinfo.filename.startswith("vcdc-"):
self.process_vcdc(zipentry)
else:
error = "Unrecognized confirmation file type\"{}\" in confirmation zip file \"{}\"".format(fileinfo.filename, zipfile)
Command.datadog_error(error, source_zip)
raise CommandError(error)
def process_eac(self, eacfile):
print "processing eac"
reader = csv.DictReader(eacfile, delimiter="\t")
for row in reader:
client_authorization_id = row['ClientAuthorizationID']
if not client_authorization_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
else:
try:
registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
Command.datadog_error("Found authorization record for user {}".format(registration.testcenter_user.user.username), eacfile)
# now update the record:
registration.upload_status = row['Status']
registration.upload_error_message = row['Message']
try:
registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
# store the authorization Id if one is provided. (For debugging)
if row['AuthorizationID']:
try:
registration.authorization_id = int(row['AuthorizationID'])
except ValueError as ve:
Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
registration.confirmed_at = datetime.utcnow()
registration.save()
except TestCenterRegistration.DoesNotExist:
Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
def process_vcdc(self, vcdcfile):
print "processing vcdc"
reader = csv.DictReader(vcdcfile, delimiter="\t")
for row in reader:
client_candidate_id = row['ClientCandidateID']
if not client_candidate_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
else:
try:
tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
# now update the record:
tcuser.upload_status = row['Status']
tcuser.upload_error_message = row['Message']
try:
tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
# store the candidate Id if one is provided. (For debugging)
if row['CandidateID']:
try:
tcuser.candidate_id = int(row['CandidateID'])
except ValueError as ve:
Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
tcuser.confirmed_at = datetime.utcnow()
tcuser.save()
except TestCenterUser.DoesNotExist:
Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)
common/djangoapps/student/management/commands/pearson_make_tc_registration.py
......@@ -71,6 +71,12 @@ class Command(BaseCommand):
dest='ignore_registration_dates',
help='find exam info for course based on exam_series_code, even if the exam is not active.'
),
make_option(
'--create_dummy_exam',
action='store_true',
dest='create_dummy_exam',
help='create dummy exam info for course, even if course exists'
),
)
args = "<student_username course_id>"
help = "Create or modify a TestCenterRegistration entry for a given Student"
......@@ -98,15 +104,20 @@ class Command(BaseCommand):
except TestCenterUser.DoesNotExist:
raise CommandError("User \"{}\" does not have an existing demographics record".format(username))
# check to see if a course_id was specified, and use information from that:
try:
course = course_from_id(course_id)
if 'ignore_registration_dates' in our_options:
examlist = [exam for exam in course.test_center_exams if exam.exam_series_code == our_options.get('exam_series_code')]
exam = examlist[0] if len(examlist) > 0 else None
else:
exam = course.current_test_center_exam
except ItemNotFoundError:
# get an "exam" object. Check to see if a course_id was specified, and use information from that:
exam = None
create_dummy_exam = 'create_dummy_exam' in our_options and our_options['create_dummy_exam']
if not create_dummy_exam:
try:
course = course_from_id(course_id)
if 'ignore_registration_dates' in our_options:
examlist = [exam for exam in course.test_center_exams if exam.exam_series_code == our_options.get('exam_series_code')]
exam = examlist[0] if len(examlist) > 0 else None
else:
exam = course.current_test_center_exam
except ItemNotFoundError:
pass
else:
# otherwise use explicit values (so we don't have to define a course):
exam_name = "Dummy Placeholder Name"
exam_info = { 'Exam_Series_Code': our_options['exam_series_code'],
......@@ -120,7 +131,7 @@ class Command(BaseCommand):
our_options['eligibility_appointment_date_last'] = strftime("%Y-%m-%d", exam.last_eligible_appointment_date)
if exam is None:
raise CommandError("Exam for course_id {%s} does not exist".format(course_id))
raise CommandError("Exam for course_id {} does not exist".format(course_id))
exam_code = exam.exam_series_code
......
common/djangoapps/student/management/commands/pearson_make_tc_user.py
from optparse import make_option
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from django.core.management.base import BaseCommand, CommandError
from student.models import TestCenterUser, TestCenterUserForm
......@@ -161,15 +161,16 @@ class Command(BaseCommand):
if form.is_valid():
form.update_and_save()
else:
errorlist = []
if (len(form.errors) > 0):
print "Field Form errors encountered:"
for fielderror in form.errors:
print "Field Form Error: %s" % fielderror
if (len(form.non_field_errors()) > 0):
print "Non-field Form errors encountered:"
for nonfielderror in form.non_field_errors:
print "Non-field Form Error: %s" % nonfielderror
errorlist.append("Field Form errors encountered:")
for fielderror in form.errors:
errorlist.append("Field Form Error: {}".format(fielderror))
if (len(form.non_field_errors()) > 0):
errorlist.append("Non-field Form errors encountered:")
for nonfielderror in form.non_field_errors:
errorlist.append("Non-field Form Error: {}".format(nonfielderror))
raise CommandError("\n".join(errorlist))
else:
print "No changes necessary to make to existing user's demographics."
......
common/djangoapps/student/management/commands/pearson_transfer.py 0 → 100644
import os
from optparse import make_option
from stat import S_ISDIR
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from dogapi import dog_http_api, dog_stats_api
import paramiko
import boto
dog_http_api.api_key = settings.DATADOG_API
class Command(BaseCommand):
help = """
This command handles the importing and exporting of student records for
Pearson. It uses some other Django commands to export and import the
files and then uploads over SFTP to Pearson and stuffs the entry in an
S3 bucket for archive purposes.
Usage: django-admin.py pearson-transfer --mode [import|export|both]
"""
option_list = BaseCommand.option_list + (
make_option('--mode',
action='store',
dest='mode',
default='both',
choices=('import', 'export', 'both'),
help='mode is import, export, or both'),
)
def handle(self, **options):
if not hasattr(settings, 'PEARSON'):
raise CommandError('No PEARSON entries in auth/env.json.')
# check settings needed for either import or export:
for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']:
if not hasattr(settings, value):
raise CommandError('No entry in the AWS settings'
'(env/auth.json) for {0}'.format(value))
# check additional required settings for import and export:
if options['mode'] in ('export', 'both'):
for value in ['LOCAL_EXPORT','SFTP_EXPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
# make sure that the import directory exists or can be created:
source_dir = settings.PEARSON['LOCAL_EXPORT']
if not os.path.isdir(source_dir):
os.makedirs(source_dir)
if options['mode'] in ('import', 'both'):
for value in ['LOCAL_IMPORT','SFTP_IMPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
# make sure that the import directory exists or can be created:
dest_dir = settings.PEARSON['LOCAL_IMPORT']
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
def sftp(files_from, files_to, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try:
t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t)
if mode == 'export':
try:
sftp.chdir(files_to)
except IOError:
raise CommandError('SFTP destination path does not exist: {}'.format(files_to))
for filename in os.listdir(files_from):
sftp.put(files_from + '/' + filename, filename)
if deleteAfterCopy:
os.remove(os.path.join(files_from, filename))
else:
try:
sftp.chdir(files_from)
except IOError:
raise CommandError('SFTP source path does not exist: {}'.format(files_from))
for filename in sftp.listdir('.'):
# skip subdirectories
if not S_ISDIR(sftp.stat(filename).st_mode):
sftp.get(filename, files_to + '/' + filename)
# delete files from sftp server once they are successfully pulled off:
if deleteAfterCopy:
sftp.remove(filename)
except:
dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed',
alert_type='error')
raise
finally:
sftp.close()
t.close()
def s3(files_from, bucket, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try:
for filename in os.listdir(files_from):
source_file = os.path.join(files_from, filename)
# use mode as name of directory into which to write files
dest_file = os.path.join(mode, filename)
upload_file_to_s3(bucket, source_file, dest_file)
if deleteAfterCopy:
os.remove(files_from + '/' + filename)
except:
dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed')
raise
def upload_file_to_s3(bucket, source_file, dest_file):
"""
Upload file to S3
"""
s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
from boto.s3.key import Key
b = s3.get_bucket(bucket)
k = Key(b)
k.key = "{filename}".format(filename=dest_file)
k.set_contents_from_filename(source_file)
def export_pearson():
options = { 'dest-from-settings' : True }
call_command('pearson_export_cdd', **options)
call_command('pearson_export_ead', **options)
mode = 'export'
sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy = False)
s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True)
def import_pearson():
mode = 'import'
try:
sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy = True)
s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
except Exception as e:
dog_http_api.event('Pearson Import failure', str(e))
raise e
else:
for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
call_command('pearson_import_conf_zip', filepath)
os.remove(filepath)
# actually do the work!
if options['mode'] in ('export', 'both'):
export_pearson()
if options['mode'] in ('import', 'both'):
import_pearson()
common/djangoapps/student/models.py
......@@ -428,6 +428,10 @@ class TestCenterRegistration(models.Model):
# TODO: figure out if this should really go in the database (with a default value).
return 1
@property
def needs_uploading(self):
return self.uploaded_at is None or self.uploaded_at < self.user_updated_at
@classmethod
def create(cls, testcenter_user, exam, accommodation_request):
registration = cls(testcenter_user = testcenter_user)
......@@ -549,7 +553,11 @@ def get_testcenter_registration(user, course_id, exam_series_code):
except TestCenterUser.DoesNotExist:
return []
return TestCenterRegistration.objects.filter(testcenter_user=tcu, course_id=course_id, exam_series_code=exam_series_code)
# nosetests thinks that anything with _test_ in the name is a test.
# Correct this (https://nose.readthedocs.org/en/latest/finding_tests.html)
get_testcenter_registration.__test__ = False
def unique_id_for_user(user):
"""
Return a unique id for a user, suitable for inserting into
......
github-requirements.txt
......@@ -3,3 +3,4 @@
-e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
-e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
-e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
-e git://github.com/MITx/dogapi.git@003a4fc9#egg=dogapi
lms/envs/aws.py
......@@ -88,3 +88,9 @@ PEER_GRADING_INTERFACE = AUTH_TOKENS.get('PEER_GRADING_INTERFACE', PEER_GRADING_
PEARSON_TEST_USER = "pearsontest"
PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")
# Pearson hash for import/export
PEARSON = AUTH_TOKENS.get("PEARSON")
# Datadog for events!
DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
requirements.txt
......@@ -58,4 +58,4 @@ factory_boy
Shapely==1.2.16
ipython==0.13.1
xmltodict==0.4.1
paramiko==1.9.0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment