reduce TB load for step stats

This commit is contained in:
Eren Golge 2019-08-22 15:28:11 +02:00
parent 549dbba128
commit 0a717faf1c
1 changed files with 9 additions and 7 deletions

View File

@ -212,13 +212,15 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
avg_loader_time += loader_time
# Plot Training Iter Stats
iter_stats = {"loss_posnet": postnet_loss.item(),
"loss_decoder": decoder_loss.item(),
"lr": current_lr,
"grad_norm": grad_norm,
"grad_norm_st": grad_norm_st,
"step_time": step_time}
tb_logger.tb_train_iter_stats(global_step, iter_stats)
# reduce TB load
if global_step % 10 == 0:
iter_stats = {"loss_posnet": postnet_loss.item(),
"loss_decoder": decoder_loss.item(),
"lr": current_lr,
"grad_norm": grad_norm,
"grad_norm_st": grad_norm_st,
"step_time": step_time}
tb_logger.tb_train_iter_stats(global_step, iter_stats)
if global_step % c.save_step == 0:
if c.checkpoint: