From 0a717faf1cacffe66f3bed41095b5093b199785c Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Thu, 22 Aug 2019 15:28:11 +0200 Subject: [PATCH] reduce TB load for step stats --- train.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/train.py b/train.py index 5c45181b..71cbb398 100644 --- a/train.py +++ b/train.py @@ -212,13 +212,15 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler, avg_loader_time += loader_time # Plot Training Iter Stats - iter_stats = {"loss_posnet": postnet_loss.item(), - "loss_decoder": decoder_loss.item(), - "lr": current_lr, - "grad_norm": grad_norm, - "grad_norm_st": grad_norm_st, - "step_time": step_time} - tb_logger.tb_train_iter_stats(global_step, iter_stats) + # reduce TB load + if global_step % 10 == 0: + iter_stats = {"loss_posnet": postnet_loss.item(), + "loss_decoder": decoder_loss.item(), + "lr": current_lr, + "grad_norm": grad_norm, + "grad_norm_st": grad_norm_st, + "step_time": step_time} + tb_logger.tb_train_iter_stats(global_step, iter_stats) if global_step % c.save_step == 0: if c.checkpoint: