Save best model based on train loss

This commit is contained in:
Eren G 2018-07-27 13:47:13 +02:00
parent b7be415396
commit d1d7cd9977
2 changed files with 4 additions and 3 deletions

View File

@ -3,5 +3,6 @@
# ls /snakepit/jobs/650/keep/ # ls /snakepit/jobs/650/keep/
source /snakepit/jobs/650/keep/venv/bin/activate source /snakepit/jobs/650/keep/venv/bin/activate
# source /snakepit/jobs/560/tmp/venv/bin/activate # source /snakepit/jobs/560/tmp/venv/bin/activate
python extract_feats.py --data_path /snakepit/shared/data/keithito/LJSpeech-1.1/wavs --out_path /snakepit/shared/data/keithito/LJSpeech-1.1/loader_data/ --config config.json --num_proc 8 # python extract_feats.py --data_path /snakepit/shared/data/keithito/LJSpeech-1.1/wavs --out_path /snakepit/shared/data/keithito/LJSpeech-1.1/loader_data/ --config config.json --num_proc 32
python train.py --config_path config.json --debug true CUDA_VISIBLE_DEVICES="1" python train.py --config_path config.json --debug true
# nvidia-smi

View File

@ -410,7 +410,7 @@ def main(args):
train_loss, current_step = train(model, criterion, criterion_st, train_loader, optimizer, optimizer_st, ap, epoch) train_loss, current_step = train(model, criterion, criterion_st, train_loader, optimizer, optimizer_st, ap, epoch)
val_loss = evaluate(model, criterion, criterion_st, val_loader, ap, current_step) val_loss = evaluate(model, criterion, criterion_st, val_loader, ap, current_step)
print(" | > Train Loss: {:.5f} Validation Loss: {:.5f}".format(train_loss, val_loss)) print(" | > Train Loss: {:.5f} Validation Loss: {:.5f}".format(train_loss, val_loss))
best_loss = save_best_model(model, optimizer, val_loss, best_loss = save_best_model(model, optimizer, train_loss,
best_loss, OUT_PATH, best_loss, OUT_PATH,
current_step, epoch) current_step, epoch)