Various changes thanks to feedback from Brian to make the existing

export commands handle --dest-from-settings and --destination and fail
unless one is provided as well as rename pearson.py to pearson_transfer
and allow is to call the import/export commands directly.

I've set it to die in pearson_transfer.py if the django PEARSON settings
aren't available.  I don't want to try and provide defaults, these must
exist or it simply fails.

common/djangoapps/student/management/commands/pearson_export_cdd.py

@@ -37,7 +37,20 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-    def handle(self, **kwargs):
+    option_list = BaseCommand.option_list + (
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django? True/False'),
+        make_option('--destination',
+                    action='store_true',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files')
+    )
+
+    def handle(self, **options):
         # update time should use UTC in order to be comparable to the user_updated_at 
         # field
         uploaded_at = datetime.utcnow()
@@ -48,13 +61,23 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps 
         # used in the system.
-        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
-            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+        if options['dest-from-settings'] is True:
+            if settings.PEARSON[LOCAL_EXPORT]:
+                dest = settings.PEARSON[LOCAL_EXPORT]
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif options['destination']:
+            dest = options['destination']
+        else:
+            raise ComamndError('--destination or --dest-from-settings must be used')
+                
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
+            destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
 
         
         # strings must be in latin-1 format.  CSV parser will

common/djangoapps/student/management/commands/pearson_export_ead.py

@@ -23,24 +23,29 @@ class Command(BaseCommand):
         ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
     ])
 
-
     option_list = BaseCommand.option_list + (
-        make_option(
-            '--dump_all',
-            action='store_true',
-            dest='dump_all',
+        make_option('--dest-from-settings',
+                    action='store_true',
+                    dest='dest-from-settings',
+                    default=False,
+                    help='Retrieve the destination to export to from django? True/False'),
+        make_option('--destination',
+                    action='store_true',
+                    dest='destination',
+                    default=None,
+                    help='Where to store the exported files'),
+        make_option('--dump_all',
+                    action='store_true',
+                    dest='dump_all',
         ),
-        make_option(
-            '--force_add',
-            action='store_true',
-            dest='force_add',
+        make_option('--force_add',
+                    action='store_true',
+                    dest='force_add',
         ),
     )
-    
-    
-    def handle(self, **kwargs):
 
-        # update time should use UTC in order to be comparable to the user_updated_at 
+    def handle(self, **options):
+        # update time should use UTC in order to be comparable to the user_updated_at
         # field
         uploaded_at = datetime.utcnow()
 
@@ -50,13 +55,22 @@ class Command(BaseCommand):
         # Name will use timestamp -- this is UTC, so it will look funny,
         # but it should at least be consistent with the other timestamps
         # used in the system.
-        if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]):
-            os.makedirs(settings.PEARSON[LOCAL_EXPORT])
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+        if options['dest-from-settings'] is True:
+            if settings.PEARSON[LOCAL_EXPORT]:
+                dest = settings.PEARSON[LOCAL_EXPORT]
+            else:
+                raise CommandError('--dest-from-settings was enabled but the'
+                        'PEARSON[LOCAL_EXPORT] setting was not set.')
+        elif options['destination']:
+            dest = options['destination']
+        else:
+            raise ComamndError('--destination or --dest-from-settings must be used')
+
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
         else:
-            destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT],
-                    uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
+            destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
 
         dump_all = kwargs['dump_all']
 

common/djangoapps/student/management/commands/pearson.py → common/djangoapps/student/management/commands/pearson_transfer.py

@@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API
 
 class Command(BaseCommand):
 
-    option_list = BaseCommand.option_list
-    args = '<mode>'
-    help = """
-    Mode should be import or export depending on if you're fetching from pearson or
-    sending to them.
-    """
+    option_list = BaseCommand.option_list + (
+        make_option('--mode',
+                    action='store_true',
+                    dest='mode',
+                    default='both',
+                    help='mode is import, export, or both'),
+    )
 
-    def handle(self, *args):
-        if len(args) < 1:
-            raise CommandError('Usage is pearson {0}'.format(self.args))
+    def handle(self, **options):
+
+        if not settings.PEARSON:
+            raise CommandError('No PEARSON entries in auth/env.json.')
+
+        def import_pearson():
+            sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
+            s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
+            call_command('pearson_import', 'dest_from_settings=True')
+
+        def export_pearson():
+            call_command('pearson_export_ccd', 'dest_from_settings=True')
+            call_command('pearson_export_ead', 'dest_from_settings=True')
+            sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
+            s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
+
+        if options['mode'] == 'both':
+            export_pearson()
+            import_pearson()
+        elif options['mode'] == 'export':
+            export_pearson()
+        elif options['mode'] == 'import':
+            import_pearson()
+        else:
+            print("ERROR:  Mode must be export or import.")
 
-        for mode in args:
-            if mode == 'export':
-                sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
-                s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
-            elif mode == 'import':
-                sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
-                s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
-            else:
-                print("ERROR:  Mode must be export or import.")
 
         def sftp(files_from, files_to):
             with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):