From a108d0ee812ef9458276afb5cb868b8a2be9a87e Mon Sep 17 00:00:00 2001 From: erogol Date: Mon, 2 Nov 2020 13:12:19 +0100 Subject: [PATCH] check nan loss in glow-tts loss --- TTS/tts/layers/losses.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TTS/tts/layers/losses.py b/TTS/tts/layers/losses.py index c1bc85b5..50809de5 100644 --- a/TTS/tts/layers/losses.py +++ b/TTS/tts/layers/losses.py @@ -402,4 +402,9 @@ class GlowTTSLoss(torch.nn.Module): return_dict['loss'] = log_mle + loss_dur return_dict['log_mle'] = log_mle return_dict['loss_dur'] = loss_dur + + # check if any loss is NaN + for key, loss in return_dict.items(): + if torch.isnan(loss): + raise RuntimeError(f" [!] NaN loss with {key}.") return return_dict \ No newline at end of file