Skip to content

E
edx-platform
Commit d0ecae30 by Ashley Penney
Options

Fix datadog logging to use .name on the file objects and add an additional logging line.

parent 274cb8d8
Showing with 27 additions and 20 deletions
common/djangoapps/student/management/commands/pearson_import_conf_zip.py
......@@ -33,7 +33,9 @@ class Command(BaseCommand):
source_zip = args[0]
if not is_zipfile(source_zip):
raise CommandError("Input file is not a zipfile: \"{}\"".format(source_zip))
error = "Input file is not a zipfile: \"{}\"".format(source_zip)
Command.datadog_error(error, source_zip)
raise CommandError(error)
# loop through all files in zip, and process them based on filename prefix:
with ZipFile(source_zip, 'r') as zipfile:
......@@ -55,9 +57,9 @@ class Command(BaseCommand):
client_authorization_id = row['ClientAuthorizationID']
if not client_authorization_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile)
Command.datadog_error("Error in EAD file processing ({}): {}".format(row['Date'], row['Message']), eacfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), eacfile)
Command.datadog_error("Encountered bad record: {}".format(row), eacfile.name)
else:
try:
registration = TestCenterRegistration.objects.get(client_authorization_id=client_authorization_id)
......@@ -68,18 +70,18 @@ class Command(BaseCommand):
try:
registration.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile)
Command.datadog_error("Bad Date value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
# store the authorization Id if one is provided. (For debugging)
if row['AuthorizationID']:
try:
registration.authorization_id = int(row['AuthorizationID'])
except ValueError as ve:
Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile)
Command.datadog_error("Bad AuthorizationID value found for {}: message {}".format(client_authorization_id, ve), eacfile.name)
registration.confirmed_at = datetime.utcnow()
registration.save()
except TestCenterRegistration.DoesNotExist:
Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile)
Command.datadog_error("Failed to find record for client_auth_id {}".format(client_authorization_id), eacfile.name)
def process_vcdc(self, vcdcfile):
print "processing vcdc"
......@@ -88,28 +90,28 @@ class Command(BaseCommand):
client_candidate_id = row['ClientCandidateID']
if not client_candidate_id:
if row['Status'] == 'Error':
Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile)
Command.datadog_error("Error in CDD file processing ({}): {}".format(row['Date'], row['Message']), vcdcfile.name)
else:
Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile)
Command.datadog_error("Encountered bad record: {}".format(row), vcdcfile.name)
else:
try:
tcuser = TestCenterUser.objects.get(client_candidate_id=client_candidate_id)
Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile)
Command.datadog_error("Found demographics record for user {}".format(tcuser.user.username), vcdcfile.name)
# now update the record:
tcuser.upload_status = row['Status']
tcuser.upload_error_message = row['Message']
try:
tcuser.processed_at = strftime('%Y-%m-%d %H:%M:%S', strptime(row['Date'], '%Y/%m/%d %H:%M:%S'))
except ValueError as ve:
Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
Command.datadog_error("Bad Date value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
# store the candidate Id if one is provided. (For debugging)
if row['CandidateID']:
try:
tcuser.candidate_id = int(row['CandidateID'])
except ValueError as ve:
Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile)
Command.datadog_error("Bad CandidateID value found for {}: message {}".format(client_candidate_id, ve), vcdcfile.name)
tcuser.confirmed_at = datetime.utcnow()
tcuser.save()
except TestCenterUser.DoesNotExist:
Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile)
Command.datadog_error(" Failed to find record for client_candidate_id {}".format(client_candidate_id), vcdcfile.name)
common/djangoapps/student/management/commands/pearson_transfer.py
......@@ -39,14 +39,18 @@ class Command(BaseCommand):
'(env/auth.json) for {0}'.format(value))
def import_pearson():
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
call_command('pearson_import_conf_zip',
settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
os.remove(file)
try:
sftp(settings.PEARSON['SFTP_IMPORT'],
settings.PEARSON['LOCAL_IMPORT'], options['mode'])
s3(settings.PEARSON['LOCAL_IMPORT'],
settings.PEARSON['BUCKET'], options['mode'])
except Exception as e:
dog_http_api.event('Pearson Import failure', str(e))
else:
for file in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
call_command('pearson_import_conf_zip',
settings.PEARSON['LOCAL_IMPORT'] + '/' + file)
os.remove(file)
def export_pearson():
call_command('pearson_export_cdd', 'dest_from_settings')
......@@ -79,6 +83,7 @@ class Command(BaseCommand):
for filename in sftp.listdir(files_from):
sftp.get(files_from + '/' + filename,
files_to + '/' + filename)
sftp.remove(files_from + '/' + filename)
t.close()
except:
dog_http_api.event('pearson {0}'.format(mode),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment