Skip to content

E
edx-platform
Commit 482cefd2 by Ashley Penney
Options

Bunch of fixes to pep8 formatting, missing imports, change of kwargs to 

options, quoting variables and a few other fixups.
parent 5431332c
Showing with 42 additions and 39 deletions
common/djangoapps/student/management/commands/pearson_export_cdd.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir
from optparse import make_option
from django.core.management.base import BaseCommand
from student.models import TestCenterUser
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
......@@ -62,8 +63,8 @@ class Command(BaseCommand):
# but it should at least be consistent with the other timestamps
# used in the system.
if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT]
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
......@@ -72,7 +73,6 @@ class Command(BaseCommand):
else:
raise CommandError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
......@@ -86,7 +86,7 @@ class Command(BaseCommand):
else:
return value
dump_all = kwargs['dump_all']
dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -104,6 +104,3 @@ class Command(BaseCommand):
writer.writerow(record)
tcu.uploaded_at = uploaded_at
tcu.save()
common/djangoapps/student/management/commands/pearson_export_ead.py
import csv
import os
from collections import OrderedDict
from datetime import datetime
from os.path import isdir, join
from optparse import make_option
from django.core.management.base import BaseCommand
from student.models import TestCenterRegistration
class Command(BaseCommand):
CSV_TO_MODEL_FIELDS = OrderedDict([
......@@ -56,12 +57,12 @@ class Command(BaseCommand):
# but it should at least be consistent with the other timestamps
# used in the system.
if 'dest-from-settings' in options:
if LOCAL_EXPORT in settings.PEARSON:
dest = settings.PEARSON[LOCAL_EXPORT]
if 'LOCAL_EXPORT' in settings.PEARSON:
dest = settings.PEARSON['LOCAL_EXPORT']
else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif destinations in options:
elif 'destinations' in options:
dest = options['destination']
else:
raise CommandError('--destination or --dest-from-settings must be used')
......@@ -71,7 +72,7 @@ class Command(BaseCommand):
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = kwargs['dump_all']
dump_all = options['dump_all']
with open(destfile, "wb") as outfile:
writer = csv.DictWriter(outfile,
......@@ -88,13 +89,9 @@ class Command(BaseCommand):
record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
if kwargs['force_add']:
if options['force_add']:
record['AuthorizationTransactionType'] = 'Add'
writer.writerow(record)
tcr.uploaded_at = uploaded_at
tcr.save()
common/djangoapps/student/management/commands/pearson_transfer.py
from optparse import make_option
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand, CommandError
import re
from dogapi import dog_http_api, dog_stats_api
import paramiko
import boto
import os
dog_http_api.api_key = settings.DATADOG_API
......@@ -25,16 +24,26 @@ class Command(BaseCommand):
if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.')
for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
'SFTP_EXPORT']:
if value not in settings.PEARSON:
raise CommandError('No entry in the PEARSON settings'
'(env/auth.json) for {0}'.format(value))
def import_pearson():
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
call_command('pearson_import', 'dest_from_settings')
def export_pearson():
call_command('pearson_export_ccd', 'dest_from_settings')
call_command('pearson_export_ead', 'dest_from_settings')
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
sftp(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['SFTP_EXPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_EXPORT'],
settings.PEARSON['BUCKET'], options['mode'])
if options['mode'] == 'export':
export_pearson()
......@@ -44,38 +53,38 @@ class Command(BaseCommand):
export_pearson()
import_pearson()
def sftp(files_from, files_to):
def sftp(files_from, files_to, mode):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try:
t = paramiko.Transport((hostname, 22))
t.connect(username=settings.PEARSON[SFTP_USERNAME],
password=settings.PEARSON[SFTP_PASSWORD])
t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t)
if os.path.isdir(files_from):
for filename in os.listdir(files_from):
sftp.put(files_from+'/'+filename,
files_to+'/'+filename)
sftp.put(files_from + '/' + filename,
files_to + '/' + filename)
else:
for filename in sftp.listdir(files_from):
sftp.get(files_from+'/'+filename,
files_to+'/'+filename)
sftp.get(files_from + '/' + filename,
files_to + '/' + filename)
t.close()
except:
dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed', alert_type='error')
'sftp uploading failed',
alert_type='error')
raise
def s3(files_from, bucket):
def s3(files_from, bucket, mode):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try:
for filename in os.listdir(files):
upload_file_to_s3(bucket, files_from+'/'+filename)
for filename in os.listdir(files_from):
upload_file_to_s3(bucket, files_from + '/' + filename)
except:
dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed')
dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed')
raise
def upload_file_to_s3(bucket, filename):
"""
Upload file to S3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment