Commit 1199b1ec by Brian Wilson Committed by Ashley Penney

debug pearson import and update unit tests

parent d395c444
...@@ -10,6 +10,7 @@ from optparse import make_option ...@@ -10,6 +10,7 @@ from optparse import make_option
from dogapi import dog_http_api, dog_stats_api from dogapi import dog_http_api, dog_stats_api
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from student.models import TestCenterUser, TestCenterRegistration from student.models import TestCenterUser, TestCenterRegistration
...@@ -23,6 +24,7 @@ class Command(BaseCommand): ...@@ -23,6 +24,7 @@ class Command(BaseCommand):
and TestCenterRegistration tables with status. and TestCenterRegistration tables with status.
""" """
@staticmethod
def datadog_error(string, tags): def datadog_error(string, tags):
dog_http_api.event("Pearson Import", string, alert_type='error', tags=tags) dog_http_api.event("Pearson Import", string, alert_type='error', tags=tags)
......
from optparse import make_option from optparse import make_option
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand, CommandError
from student.models import TestCenterUser, TestCenterUserForm from student.models import TestCenterUser, TestCenterUserForm
...@@ -161,15 +161,16 @@ class Command(BaseCommand): ...@@ -161,15 +161,16 @@ class Command(BaseCommand):
if form.is_valid(): if form.is_valid():
form.update_and_save() form.update_and_save()
else: else:
errorlist = []
if (len(form.errors) > 0): if (len(form.errors) > 0):
print "Field Form errors encountered:" errorlist.append("Field Form errors encountered:")
for fielderror in form.errors: for fielderror in form.errors:
print "Field Form Error: %s" % fielderror errorlist.append("Field Form Error: {}".format(fielderror))
if (len(form.non_field_errors()) > 0): if (len(form.non_field_errors()) > 0):
print "Non-field Form errors encountered:" errorlist.append("Non-field Form errors encountered:")
for nonfielderror in form.non_field_errors: for nonfielderror in form.non_field_errors:
print "Non-field Form Error: %s" % nonfielderror errorlist.append("Non-field Form Error: {}".format(nonfielderror))
raise CommandError("\n".join(errorlist))
else: else:
print "No changes necessary to make to existing user's demographics." print "No changes necessary to make to existing user's demographics."
......
import os import os
from optparse import make_option from optparse import make_option
from stat import S_ISDIR
from django.conf import settings from django.conf import settings
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
...@@ -26,85 +27,99 @@ class Command(BaseCommand): ...@@ -26,85 +27,99 @@ class Command(BaseCommand):
action='store', action='store',
dest='mode', dest='mode',
default='both', default='both',
choices=('import', 'export', 'both'),
help='mode is import, export, or both'), help='mode is import, export, or both'),
) )
def handle(self, **options): def handle(self, **options):
# TODO: this doesn't work. Need to check if it's a property. if not hasattr(settings, 'PEARSON'):
if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.') raise CommandError('No PEARSON entries in auth/env.json.')
for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'LOCAL_EXPORT', # check settings needed for either import or export:
'SFTP_EXPORT', 'SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD']: for value in ['SFTP_HOSTNAME', 'SFTP_USERNAME', 'SFTP_PASSWORD', 'S3_BUCKET']:
if value not in settings.PEARSON: if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings' raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value)) '(env/auth.json) for {0}'.format(value))
def import_pearson(): for value in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']:
try: if not hasattr(settings, value):
sftp(settings.PEARSON['SFTP_IMPORT'], raise CommandError('No entry in the AWS settings'
settings.PEARSON['LOCAL_IMPORT'], options['mode']) '(env/auth.json) for {0}'.format(value))
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode']) # check additional required settings for import and export:
except Exception as e: if options['mode'] in ('export', 'both'):
dog_http_api.event('Pearson Import failure', str(e)) for value in ['LOCAL_EXPORT','SFTP_EXPORT']:
else: if value not in settings.PEARSON:
for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']): raise CommandError('No entry in the PEARSON settings'
call_command('pearson_import_conf_zip', '(env/auth.json) for {0}'.format(value))
settings.PEARSON['LOCAL_IMPORT'] + '/' + file) # make sure that the import directory exists or can be created:
os.remove(file) source_dir = settings.PEARSON['LOCAL_EXPORT']
if not os.path.isdir(source_dir):
os.makedirs(source_dir)
if options['mode'] in ('import', 'both'):
for value in ['LOCAL_IMPORT','SFTP_IMPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
# make sure that the import directory exists or can be created:
dest_dir = settings.PEARSON['LOCAL_IMPORT']
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
def export_pearson():
call_command('pearson_export_cdd', 'dest_from_settings')
call_command('pearson_export_ead', 'dest_from_settings')
sftp(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['SFTP_EXPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['BUCKET'], options['mode'])
if options['mode'] == 'export':
export_pearson()
elif options['mode'] == 'import':
import_pearson()
else:
export_pearson()
import_pearson()
def sftp(files_from, files_to, mode): def sftp(files_from, files_to, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try: try:
t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22)) t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'], t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD']) password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t) sftp = paramiko.SFTPClient.from_transport(t)
if os.path.isdir(files_from):
if mode == 'export':
try:
sftp.chdir(files_to)
except IOError:
raise CommandError('SFTP destination path does not exist: {}'.format(files_to))
for filename in os.listdir(files_from): for filename in os.listdir(files_from):
sftp.put(files_from + '/' + filename, sftp.put(files_from + '/' + filename, filename)
files_to + '/' + filename) if deleteAfterCopy:
os.remove(os.path.join(files_from, filename))
else: else:
for filename in sftp.listdir(files_from): try:
sftp.get(files_from + '/' + filename, sftp.chdir(files_from)
files_to + '/' + filename) except IOError:
sftp.remove(files_from + '/' + filename) raise CommandError('SFTP source path does not exist: {}'.format(files_from))
t.close() for filename in sftp.listdir('.'):
# skip subdirectories
if not S_ISDIR(sftp.stat(filename).st_mode):
sftp.get(filename, files_to + '/' + filename)
# delete files from sftp server once they are successfully pulled off:
if deleteAfterCopy:
sftp.remove(filename)
except: except:
dog_http_api.event('pearson {0}'.format(mode), dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed', 'sftp uploading failed',
alert_type='error') alert_type='error')
raise raise
finally:
sftp.close()
t.close()
def s3(files_from, bucket, mode): def s3(files_from, bucket, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try: try:
for filename in os.listdir(files_from): for filename in os.listdir(files_from):
upload_file_to_s3(bucket, files_from + '/' + filename) upload_file_to_s3(bucket, files_from, filename)
if deleteAfterCopy:
os.remove(files_from + '/' + filename)
except: except:
dog_http_api.event('pearson {0}'.format(mode), dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed') 's3 archiving failed')
raise raise
def upload_file_to_s3(bucket, filename): def upload_file_to_s3(bucket, source_dir, filename):
""" """
Upload file to S3 Upload file to S3
""" """
...@@ -114,4 +129,32 @@ class Command(BaseCommand): ...@@ -114,4 +129,32 @@ class Command(BaseCommand):
b = s3.get_bucket(bucket) b = s3.get_bucket(bucket)
k = Key(b) k = Key(b)
k.key = "{filename}".format(filename=filename) k.key = "{filename}".format(filename=filename)
k.set_contents_from_filename(filename) k.set_contents_from_filename(os.path.join(source_dir, filename))
def export_pearson():
options = { 'dest-from-settings' : True }
call_command('pearson_export_cdd', **options)
call_command('pearson_export_ead', **options)
mode = 'export'
sftp(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['SFTP_EXPORT'], mode, deleteAfterCopy = False)
s3(settings.PEARSON['LOCAL_EXPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=True)
def import_pearson():
mode = 'import'
try:
sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy = True)
s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
except Exception as e:
dog_http_api.event('Pearson Import failure', str(e))
raise e
else:
for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
call_command('pearson_import_conf_zip', filepath)
os.remove(filepath)
# actually do the work!
if options['mode'] in ('export', 'both'):
export_pearson()
if options['mode'] in ('import', 'both'):
import_pearson()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment