Skip to content

E
edx-platform
Commit 482cefd2 by Ashley Penney
Options

Bunch of fixes to pep8 formatting, missing imports, change of kwargs to 

options, quoting variables and a few other fixups.
parent 5431332c
Showing with 55 additions and 52 deletions
common/djangoapps/student/management/commands/pearson_export_cdd.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir
from optparse import make_option
from django.core.management.base import BaseCommand
from student.models import TestCenterUser
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
# Skipping optional field CandidateID
("ClientCandidateID", "client_candidate_id"),
......@@ -34,7 +35,7 @@ class Command(BaseCommand):
("FAXCountryCode", "fax_country_code"),
("CompanyName", "company_name"),
# Skipping optional field CustomQuestion
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
option_list = BaseCommand.option_list + (
......@@ -55,29 +56,28 @@ class Command(BaseCommand):
# field
uploaded_at = datetime.utcnow()
# if specified destination is an existing directory, then
# if specified destination is an existing directory, then
# create a filename for it automatically. If it doesn't exist,
# then we will create the directory.
# Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps
# but it should at least be consistent with the other timestamps
# used in the system.
if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT]
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif 'destination' in options:
dest = options['destination']
else:
raise CommandError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
# strings must be in latin-1 format. CSV parser will
# otherwise convert unicode objects to ascii.
def ensure_encoding(value):
......@@ -85,8 +85,8 @@ class Command(BaseCommand):
return value.encode('iso-8859-1')
else:
return value
dump_all = kwargs['dump_all']
dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -104,6 +104,3 @@ class Command(BaseCommand):
writer.writerow(record)
tcu.uploaded_at = uploaded_at
tcu.save()
common/djangoapps/student/management/commands/pearson_export_ead.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir, join
from optparse import make_option
from django.core.management.base import BaseCommand
from student.models import TestCenterRegistration
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
('AuthorizationTransactionType', 'authorization_transaction_type'),
('AuthorizationID', 'authorization_id'),
......@@ -20,7 +21,7 @@ class Command(BaseCommand):
('Accommodations', 'accommodation_code'),
('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
])
option_list = BaseCommand.option_list + (
......@@ -37,11 +38,11 @@ class Command(BaseCommand):
make_option('--dump_all',
action='store_true',
dest='dump_all',
),
),
make_option('--force_add',
action='store_true',
dest='force_add',
),
),
)
def handle(self, **options):
......@@ -56,12 +57,12 @@ class Command(BaseCommand):
# but it should at least be consistent with the other timestamps
# used in the system.
if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT]
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif destinations in options:
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif 'destinations' in options:
dest = options['destination']
else:
raise CommandError('--destination or --dest-from-settings must be used')
......@@ -71,7 +72,7 @@ class Command(BaseCommand):
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = kwargs['dump_all']
dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -88,13 +89,9 @@ class Command(BaseCommand):
record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
if kwargs['force_add']:
if options['force_add']:
record['AuthorizationTransactionType'] = 'Add'
writer.writerow(record)
tcr.uploaded_at = uploaded_at
tcr.save()
common/djangoapps/student/management/commands/pearson_transfer.py
from optparse import make_option
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand, CommandError
import re
from dogapi import dog_http_api, dog_stats_api
import paramiko
import boto
import os
dog_http_api.api_key = settings.DATADOG_API
......@@ -25,16 +24,26 @@ class Command(BaseCommand):
if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.')
for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
'SFTP_EXPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
def import_pearson():
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
call_command('pearson_import', 'dest_from_settings')
def export_pearson():
call_command('pearson_export_ccd', 'dest_from_settings')
call_command('pearson_export_ead', 'dest_from_settings')
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
sftp(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['SFTP_EXPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['BUCKET'], options['mode'])
if options['mode'] == 'export':
export_pearson()
......@@ -44,44 +53,44 @@ class Command(BaseCommand):
export_pearson()
import_pearson()
def sftp(files_from, files_to):
def sftp(files_from, files_to, mode):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try:
t = paramiko.Transport((hostname, 22))
t.connect(username=settings.PEARSON[SFTP_USERNAME],
password=settings.PEARSON[SFTP_PASSWORD])
t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t)
if os.path.isdir(files_from):
for filename in os.listdir(files_from):
sftp.put(files_from+'/'+filename,
files_to+'/'+filename)
sftp.put(files_from + '/' + filename,
files_to + '/' + filename)
else:
for filename in sftp.listdir(files_from):
sftp.get(files_from+'/'+filename,
files_to+'/'+filename)
sftp.get(files_from + '/' + filename,
files_to + '/' + filename)
t.close()
except:
dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed', alert_type='error')
'sftp uploading failed',
alert_type='error')
raise
def s3(files_from, bucket):
def s3(files_from, bucket, mode):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try:
for filename in os.listdir(files):
upload_file_to_s3(bucket, files_from+'/'+filename)
for filename in os.listdir(files_from):
upload_file_to_s3(bucket, files_from + '/' + filename)
except:
dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed')
dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed')
raise
def upload_file_to_s3(bucket, filename):
"""
Upload file to S3
"""
s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
settings.AWS_SECRET_ACCESS_KEY)
from boto.s3.key import Key
b = s3.get_bucket(bucket)
k = Key(b)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment