Commit 482cefd2 by Ashley Penney

Bunch of fixes to pep8 formatting, missing imports, change of kwargs to

options, quoting variables and a few other fixups.
parent 5431332c
import csv import csv
import os
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime from datetime import datetime
from os.path import isdir
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from student.models import TestCenterUser from student.models import TestCenterUser
class Command(BaseCommand): class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([ CSV_TO_MODEL_FIELDS = OrderedDict([
...@@ -62,8 +63,8 @@ class Command(BaseCommand): ...@@ -62,8 +63,8 @@ class Command(BaseCommand):
# but it should at least be consistent with the other timestamps # but it should at least be consistent with the other timestamps
# used in the system. # used in the system.
if 'dest-from-settings' in options: if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON: if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT] dest = settings.PEARSON['LOCAL_EXPORT']
else: else:
raise CommandError('--dest-from-settings was enabled but the' raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.') 'PEARSON[LOCAL_EXPORT] setting was not set.')
...@@ -72,7 +73,6 @@ class Command(BaseCommand): ...@@ -72,7 +73,6 @@ class Command(BaseCommand):
else: else:
raise CommandError('--destination or --dest-from-settings must be used') raise CommandError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest): if not os.path.isdir(dest):
os.makedirs(dest) os.makedirs(dest)
...@@ -86,7 +86,7 @@ class Command(BaseCommand): ...@@ -86,7 +86,7 @@ class Command(BaseCommand):
else: else:
return value return value
dump_all = kwargs['dump_all'] dump_all = options['dump_all']
with open(destfile, "wb") as outfile: with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile, writer = csv.DictWriter(outfile,
...@@ -104,6 +104,3 @@ class Command(BaseCommand): ...@@ -104,6 +104,3 @@ class Command(BaseCommand):
writer.writerow(record) writer.writerow(record)
tcu.uploaded_at = uploaded_at tcu.uploaded_at = uploaded_at
tcu.save() tcu.save()
import csv import csv
import os
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime from datetime import datetime
from os.path import isdir, join
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from student.models import TestCenterRegistration from student.models import TestCenterRegistration
class Command(BaseCommand): class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([ CSV_TO_MODEL_FIELDS = OrderedDict([
...@@ -56,12 +57,12 @@ class Command(BaseCommand): ...@@ -56,12 +57,12 @@ class Command(BaseCommand):
# but it should at least be consistent with the other timestamps # but it should at least be consistent with the other timestamps
# used in the system. # used in the system.
if 'dest-from-settings' in options: if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON: if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT] dest = settings.PEARSON['LOCAL_EXPORT']
else: else:
raise CommandError('--dest-from-settings was enabled but the' raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.') 'PEARSON[LOCAL_EXPORT] setting was not set.')
elif destinations in options: elif 'destinations' in options:
dest = options['destination'] dest = options['destination']
else: else:
raise CommandError('--destination or --dest-from-settings must be used') raise CommandError('--destination or --dest-from-settings must be used')
...@@ -71,7 +72,7 @@ class Command(BaseCommand): ...@@ -71,7 +72,7 @@ class Command(BaseCommand):
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = kwargs['dump_all'] dump_all = options['dump_all']
with open(destfile, "wb") as outfile: with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile, writer = csv.DictWriter(outfile,
...@@ -88,13 +89,9 @@ class Command(BaseCommand): ...@@ -88,13 +89,9 @@ class Command(BaseCommand):
record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S") record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d") record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d") record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
if kwargs['force_add']: if options['force_add']:
record['AuthorizationTransactionType'] = 'Add' record['AuthorizationTransactionType'] = 'Add'
writer.writerow(record) writer.writerow(record)
tcr.uploaded_at = uploaded_at tcr.uploaded_at = uploaded_at
tcr.save() tcr.save()
from optparse import make_option from optparse import make_option
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
import re
from dogapi import dog_http_api, dog_stats_api from dogapi import dog_http_api, dog_stats_api
import paramiko import paramiko
import boto import boto
import os
dog_http_api.api_key = settings.DATADOG_API dog_http_api.api_key = settings.DATADOG_API
...@@ -25,16 +24,26 @@ class Command(BaseCommand): ...@@ -25,16 +24,26 @@ class Command(BaseCommand):
if not settings.PEARSON: if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.') raise CommandError('No PEARSON entries in auth/env.json.')
for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
'SFTP_EXPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
def import_pearson(): def import_pearson():
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT]) sftp(settings.PEARSON['SFTP_IMPORT'],
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET]) settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
call_command('pearson_import', 'dest_from_settings') call_command('pearson_import', 'dest_from_settings')
def export_pearson(): def export_pearson():
call_command('pearson_export_ccd', 'dest_from_settings') call_command('pearson_export_ccd', 'dest_from_settings')
call_command('pearson_export_ead', 'dest_from_settings') call_command('pearson_export_ead', 'dest_from_settings')
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT]) sftp(settings.PEARSON['LOCAL_EXPORT'],
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET]) settings.PEARSON['SFTP_EXPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['BUCKET'], options['mode'])
if options['mode'] == 'export': if options['mode'] == 'export':
export_pearson() export_pearson()
...@@ -44,38 +53,38 @@ class Command(BaseCommand): ...@@ -44,38 +53,38 @@ class Command(BaseCommand):
export_pearson() export_pearson()
import_pearson() import_pearson()
def sftp(files_from, files_to, mode):
def sftp(files_from, files_to):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try: try:
t = paramiko.Transport((hostname, 22)) t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON[SFTP_USERNAME], t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON[SFTP_PASSWORD]) password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t) sftp = paramiko.SFTPClient.from_transport(t)
if os.path.isdir(files_from): if os.path.isdir(files_from):
for filename in os.listdir(files_from): for filename in os.listdir(files_from):
sftp.put(files_from+'/'+filename, sftp.put(files_from + '/' + filename,
files_to+'/'+filename) files_to + '/' + filename)
else: else:
for filename in sftp.listdir(files_from): for filename in sftp.listdir(files_from):
sftp.get(files_from+'/'+filename, sftp.get(files_from + '/' + filename,
files_to+'/'+filename) files_to + '/' + filename)
t.close() t.close()
except: except:
dog_http_api.event('pearson {0}'.format(mode), dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed', alert_type='error') 'sftp uploading failed',
alert_type='error')
raise raise
def s3(files_from, bucket): def s3(files_from, bucket, mode):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try: try:
for filename in os.listdir(files): for filename in os.listdir(files_from):
upload_file_to_s3(bucket, files_from+'/'+filename) upload_file_to_s3(bucket, files_from + '/' + filename)
except: except:
dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed') dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed')
raise raise
def upload_file_to_s3(bucket, filename): def upload_file_to_s3(bucket, filename):
""" """
Upload file to S3 Upload file to S3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment