Bunch of fixes to pep8 formatting, missing imports, change of kwargs to

options, quoting variables and a few other fixups.

common/djangoapps/student/management/commands/pearson_export_cdd.py

 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterUser
 
+
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         # Skipping optional field CandidateID
         ("ClientCandidateID", "client_candidate_id"),
@@ -34,7 +35,7 @@ class Command(BaseCommand):
         ("FAXCountryCode", "fax_country_code"),
         ("CompanyName", "company_name"),
         # Skipping optional field CustomQuestion
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
     option_list = BaseCommand.option_list + (
@@ -55,29 +56,28 @@ class Command(BaseCommand):
         # field
         uploaded_at = datetime.utcnow()
 
-        # if specified destination is an existing directory, then 
+        # if specified destination is an existing directory, then
         # create a filename for it automatically.  If it doesn't exist,
         # then we will create the directory.
         # Name will use timestamp -- this is UTC, so it will look funny,
-        # but it should at least be consistent with the other timestamps 
+        # but it should at least be consistent with the other timestamps
         # used in the system.
         if 'dest-from-settings' in options:
-            if LOCAL_EXPORT in settings.PEARSON:
-                dest = settings.PEARSON[LOCAL_EXPORT]
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
-                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
         elif 'destination' in options:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
-                
 
         if not os.path.isdir(dest):
             os.makedirs(dest)
 
         destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
-        
+
         # strings must be in latin-1 format.  CSV parser will
         # otherwise convert unicode objects to ascii.
         def ensure_encoding(value):
@@ -85,8 +85,8 @@ class Command(BaseCommand):
                 return value.encode('iso-8859-1')
             else:
                 return value
-            
-        dump_all = kwargs['dump_all']
+
+        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -104,6 +104,3 @@ class Command(BaseCommand):
                     writer.writerow(record)
                     tcu.uploaded_at = uploaded_at
                     tcu.save()
-
-
-        

common/djangoapps/student/management/commands/pearson_export_ead.py

 import csv
+import os
 from collections import OrderedDict
 from datetime import datetime
-from os.path import isdir, join
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
 from student.models import TestCenterRegistration
 
+
 class Command(BaseCommand):
-    
+
     CSV_TO_MODEL_FIELDS = OrderedDict([
         ('AuthorizationTransactionType', 'authorization_transaction_type'),
         ('AuthorizationID', 'authorization_id'),
@@ -20,7 +21,7 @@ class Command(BaseCommand):
         ('Accommodations', 'accommodation_code'),
         ('EligibilityApptDateFirst', 'eligibility_appointment_date_first'),
         ('EligibilityApptDateLast', 'eligibility_appointment_date_last'),
-        ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
+        ("LastUpdate", "user_updated_at"),  # in UTC, so same as what we store
     ])
 
     option_list = BaseCommand.option_list + (
@@ -37,11 +38,11 @@ class Command(BaseCommand):
         make_option('--dump_all',
                     action='store_true',
                     dest='dump_all',
-        ),
+                    ),
         make_option('--force_add',
                     action='store_true',
                     dest='force_add',
-        ),
+                    ),
     )
 
     def handle(self, **options):
@@ -56,12 +57,12 @@ class Command(BaseCommand):
         # but it should at least be consistent with the other timestamps
         # used in the system.
         if 'dest-from-settings' in options:
-            if LOCAL_EXPORT in settings.PEARSON:
-                dest = settings.PEARSON[LOCAL_EXPORT]
+            if 'LOCAL_EXPORT' in settings.PEARSON:
+                dest = settings.PEARSON['LOCAL_EXPORT']
             else:
                 raise CommandError('--dest-from-settings was enabled but the'
-                        'PEARSON[LOCAL_EXPORT] setting was not set.')
-        elif destinations in options:
+                                   'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif 'destinations' in options:
             dest = options['destination']
         else:
             raise CommandError('--destination or --dest-from-settings must be used')
@@ -71,7 +72,7 @@ class Command(BaseCommand):
 
         destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
-        dump_all = kwargs['dump_all']
+        dump_all = options['dump_all']
 
         with open(destfile, "wb") as outfile:
             writer = csv.DictWriter(outfile,
@@ -88,13 +89,9 @@ class Command(BaseCommand):
                     record["LastUpdate"] = record["LastUpdate"].strftime("%Y/%m/%d %H:%M:%S")
                     record["EligibilityApptDateFirst"] = record["EligibilityApptDateFirst"].strftime("%Y/%m/%d")
                     record["EligibilityApptDateLast"] = record["EligibilityApptDateLast"].strftime("%Y/%m/%d")
-                    if kwargs['force_add']:
+                    if options['force_add']:
                         record['AuthorizationTransactionType'] = 'Add'
 
                     writer.writerow(record)
                     tcr.uploaded_at = uploaded_at
                     tcr.save()
-
-
-
-        

common/djangoapps/student/management/commands/pearson_transfer.py

 from optparse import make_option
 
-from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand, CommandError
-import re
 from dogapi import dog_http_api, dog_stats_api
 import paramiko
 import boto
+import os
 
 dog_http_api.api_key = settings.DATADOG_API
 
@@ -25,16 +24,26 @@ class Command(BaseCommand):
         if not settings.PEARSON:
             raise CommandError('No PEARSON entries in auth/env.json.')
 
+        for value in ['LOCAL_IMPORT', 'SFTP_IMPORT', 'BUCKET', 'LOCAL_EXPORT',
+                      'SFTP_EXPORT']:
+            if value not in settings.PEARSON:
+                raise CommandError('No entry in the PEARSON settings'
+                                   '(env/auth.json) for {0}'.format(value))
+
         def import_pearson():
-            sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
-            s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
+            sftp(settings.PEARSON['SFTP_IMPORT'],
+                 settings.PEARSON['LOCAL_IMPORT'], options['mode'])
+            s3(settings.PEARSON['LOCAL_IMPORT'],
+               settings.PEARSON['BUCKET'], options['mode'])
             call_command('pearson_import', 'dest_from_settings')
 
         def export_pearson():
             call_command('pearson_export_ccd', 'dest_from_settings')
             call_command('pearson_export_ead', 'dest_from_settings')
-            sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
-            s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
+            sftp(settings.PEARSON['LOCAL_EXPORT'],
+                 settings.PEARSON['SFTP_EXPORT'], options['mode'])
+            s3(settings.PEARSON['LOCAL_EXPORT'],
+               settings.PEARSON['BUCKET'], options['mode'])
 
         if options['mode'] == 'export':
             export_pearson()
@@ -44,44 +53,44 @@ class Command(BaseCommand):
             export_pearson()
             import_pearson()
 
-
-        def sftp(files_from, files_to):
+        def sftp(files_from, files_to, mode):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
                 try:
-                    t = paramiko.Transport((hostname, 22))
-                    t.connect(username=settings.PEARSON[SFTP_USERNAME],
-                              password=settings.PEARSON[SFTP_PASSWORD])
+                    t = paramiko.Transport((settings.PEARSON['SFTP_HOSTNAME'], 22))
+                    t.connect(username=settings.PEARSON['SFTP_USERNAME'],
+                              password=settings.PEARSON['SFTP_PASSWORD'])
                     sftp = paramiko.SFTPClient.from_transport(t)
                     if os.path.isdir(files_from):
                         for filename in os.listdir(files_from):
-                            sftp.put(files_from+'/'+filename,
-                                     files_to+'/'+filename)
+                            sftp.put(files_from + '/' + filename,
+                                     files_to + '/' + filename)
                     else:
                         for filename in sftp.listdir(files_from):
-                            sftp.get(files_from+'/'+filename,
-                                     files_to+'/'+filename)
+                            sftp.get(files_from + '/' + filename,
+                                     files_to + '/' + filename)
                     t.close()
                 except:
                     dog_http_api.event('pearson {0}'.format(mode),
-                                       'sftp uploading failed', alert_type='error')
+                                       'sftp uploading failed',
+                                       alert_type='error')
                     raise
 
-        def s3(files_from, bucket):
+        def s3(files_from, bucket, mode):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
                 try:
-                    for filename in os.listdir(files):
-                        upload_file_to_s3(bucket, files_from+'/'+filename)
+                    for filename in os.listdir(files_from):
+                        upload_file_to_s3(bucket, files_from + '/' + filename)
                 except:
-                    dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed')
+                    dog_http_api.event('pearson {0}'.format(mode),
+                                       's3 archiving failed')
                     raise
 
-
         def upload_file_to_s3(bucket, filename):
             """
             Upload file to S3
             """
             s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
-                    settings.AWS_SECRET_ACCESS_KEY)
+                                 settings.AWS_SECRET_ACCESS_KEY)
             from boto.s3.key import Key
             b = s3.get_bucket(bucket)
             k = Key(b)