Commit 378b1ff0 by Ashley Penney

Various changes thanks to feedback from Brian to make the existing

export commands handle --dest-from-settings and --destination and fail
unless one is provided as well as rename pearson.py to pearson_transfer
and allow is to call the import/export commands directly.

I've set it to die in pearson_transfer.py if the django PEARSON settings
aren't available.  I don't want to try and provide defaults, these must
exist or it simply fails.
parent ced29a25
...@@ -37,7 +37,20 @@ class Command(BaseCommand): ...@@ -37,7 +37,20 @@ class Command(BaseCommand):
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
]) ])
def handle(self, **kwargs): option_list = BaseCommand.option_list + (
make_option('--dest-from-settings',
action='store_true',
dest='dest-from-settings',
default=False,
help='Retrieve the destination to export to from django? True/False'),
make_option('--destination',
action='store_true',
dest='destination',
default=None,
help='Where to store the exported files')
)
def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at # update time should use UTC in order to be comparable to the user_updated_at
# field # field
uploaded_at = datetime.utcnow() uploaded_at = datetime.utcnow()
...@@ -48,13 +61,23 @@ class Command(BaseCommand): ...@@ -48,13 +61,23 @@ class Command(BaseCommand):
# Name will use timestamp -- this is UTC, so it will look funny, # Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps # but it should at least be consistent with the other timestamps
# used in the system. # used in the system.
if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]): if options['dest-from-settings'] is True:
os.makedirs(settings.PEARSON[LOCAL_EXPORT]) if settings.PEARSON[LOCAL_EXPORT]:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], dest = settings.PEARSON[LOCAL_EXPORT]
uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat")) else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif options['destination']:
dest = options['destination']
else:
raise ComamndError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
else: else:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], destfile = os.path.join(dest, uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
uploaded_at.strftime("cdd-%Y%m%d-%H%M%S.dat"))
# strings must be in latin-1 format. CSV parser will # strings must be in latin-1 format. CSV parser will
......
...@@ -23,24 +23,29 @@ class Command(BaseCommand): ...@@ -23,24 +23,29 @@ class Command(BaseCommand):
("LastUpdate", "user_updated_at"), # in UTC, so same as what we store ("LastUpdate", "user_updated_at"), # in UTC, so same as what we store
]) ])
option_list = BaseCommand.option_list + ( option_list = BaseCommand.option_list + (
make_option( make_option('--dest-from-settings',
'--dump_all', action='store_true',
action='store_true', dest='dest-from-settings',
dest='dump_all', default=False,
help='Retrieve the destination to export to from django? True/False'),
make_option('--destination',
action='store_true',
dest='destination',
default=None,
help='Where to store the exported files'),
make_option('--dump_all',
action='store_true',
dest='dump_all',
), ),
make_option( make_option('--force_add',
'--force_add', action='store_true',
action='store_true', dest='force_add',
dest='force_add',
), ),
) )
def handle(self, **kwargs):
# update time should use UTC in order to be comparable to the user_updated_at def handle(self, **options):
# update time should use UTC in order to be comparable to the user_updated_at
# field # field
uploaded_at = datetime.utcnow() uploaded_at = datetime.utcnow()
...@@ -50,13 +55,22 @@ class Command(BaseCommand): ...@@ -50,13 +55,22 @@ class Command(BaseCommand):
# Name will use timestamp -- this is UTC, so it will look funny, # Name will use timestamp -- this is UTC, so it will look funny,
# but it should at least be consistent with the other timestamps # but it should at least be consistent with the other timestamps
# used in the system. # used in the system.
if not os.path.isdir(settings.PEARSON[LOCAL_EXPORT]): if options['dest-from-settings'] is True:
os.makedirs(settings.PEARSON[LOCAL_EXPORT]) if settings.PEARSON[LOCAL_EXPORT]:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], dest = settings.PEARSON[LOCAL_EXPORT]
uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat")) else:
raise CommandError('--dest-from-settings was enabled but the'
'PEARSON[LOCAL_EXPORT] setting was not set.')
elif options['destination']:
dest = options['destination']
else:
raise ComamndError('--destination or --dest-from-settings must be used')
if not os.path.isdir(dest):
os.makedirs(dest)
destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
else: else:
destfile = os.path.join(settings.PEARSON[LOCAL_EXPORT], destfile = os.path.join(dest, uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
uploaded_at.strftime("ead-%Y%m%d-%H%M%S.dat"))
dump_all = kwargs['dump_all'] dump_all = kwargs['dump_all']
......
...@@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API ...@@ -12,26 +12,40 @@ dog_http_api.api_key = settings.DATADOG_API
class Command(BaseCommand): class Command(BaseCommand):
option_list = BaseCommand.option_list option_list = BaseCommand.option_list + (
args = '<mode>' make_option('--mode',
help = """ action='store_true',
Mode should be import or export depending on if you're fetching from pearson or dest='mode',
sending to them. default='both',
""" help='mode is import, export, or both'),
)
def handle(self, *args): def handle(self, **options):
if len(args) < 1:
raise CommandError('Usage is pearson {0}'.format(self.args)) if not settings.PEARSON:
raise CommandError('No PEARSON entries in auth/env.json.')
def import_pearson():
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
call_command('pearson_import', 'dest_from_settings=True')
def export_pearson():
call_command('pearson_export_ccd', 'dest_from_settings=True')
call_command('pearson_export_ead', 'dest_from_settings=True')
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
if options['mode'] == 'both':
export_pearson()
import_pearson()
elif options['mode'] == 'export':
export_pearson()
elif options['mode'] == 'import':
import_pearson()
else:
print("ERROR: Mode must be export or import.")
for mode in args:
if mode == 'export':
sftp(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[SFTP_EXPORT])
s3(settings.PEARSON[LOCAL_EXPORT], settings.PEARSON[BUCKET])
elif mode == 'import':
sftp(settings.PEARSON[SFTP_IMPORT], settings.PEARSON[LOCAL_IMPORT])
s3(settings.PEARSON[LOCAL_IMPORT], settings.PEARSON[BUCKET])
else:
print("ERROR: Mode must be export or import.")
def sftp(files_from, files_to): def sftp(files_from, files_to):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment