From 30561d0b5ff4fa3dc47ffa407efb07f4eb4636fc Mon Sep 17 00:00:00 2001 From: Eren G Date: Fri, 27 Jul 2018 13:47:13 +0200 Subject: [PATCH] Save best model based on train loss --- .compute | 5 +++-- train.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.compute b/.compute index 98303c6c..bc3fd4bf 100644 --- a/.compute +++ b/.compute @@ -3,5 +3,6 @@ # ls /snakepit/jobs/650/keep/ source /snakepit/jobs/650/keep/venv/bin/activate # source /snakepit/jobs/560/tmp/venv/bin/activate -python extract_feats.py --data_path /snakepit/shared/data/keithito/LJSpeech-1.1/wavs --out_path /snakepit/shared/data/keithito/LJSpeech-1.1/loader_data/ --config config.json --num_proc 8 -python train.py --config_path config.json --debug true +# python extract_feats.py --data_path /snakepit/shared/data/keithito/LJSpeech-1.1/wavs --out_path /snakepit/shared/data/keithito/LJSpeech-1.1/loader_data/ --config config.json --num_proc 32 +CUDA_VISIBLE_DEVICES="1" python train.py --config_path config.json --debug true +# nvidia-smi diff --git a/train.py b/train.py index f1e35e15..0b824ee4 100644 --- a/train.py +++ b/train.py @@ -410,7 +410,7 @@ def main(args): train_loss, current_step = train(model, criterion, criterion_st, train_loader, optimizer, optimizer_st, ap, epoch) val_loss = evaluate(model, criterion, criterion_st, val_loader, ap, current_step) print(" | > Train Loss: {:.5f} Validation Loss: {:.5f}".format(train_loss, val_loss)) - best_loss = save_best_model(model, optimizer, val_loss, + best_loss = save_best_model(model, optimizer, train_loss, best_loss, OUT_PATH, current_step, epoch)