From 8dfedb691e6be36e045868555276b2394ee62aa3 Mon Sep 17 00:00:00 2001
From: root <root@sp-mlc3-5423-0.mlc>
Date: Wed, 15 Jan 2020 12:07:19 +0100
Subject: [PATCH] fix noam schedule

---
 config.json | 7 +++----
 train.py    | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/config.json b/config.json
index 91863c4c..ef999fa9 100644
--- a/config.json
+++ b/config.json
@@ -38,7 +38,7 @@
     "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
     "eval_batch_size":16,   
     "r": 7,                 // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled.  
-    "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], // ONLY TACOTRON - set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled.  
+    "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed.
     "loss_masking": true,         // enable / disable loss masking against the sequence padding.
 
     // VALIDATION
@@ -47,11 +47,10 @@
     "test_sentences_file": null,  // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
 
     // OPTIMIZER
-    "noam_schedule": false,
+    "noam_schedule": false,        // use noam warmup and lr schedule.
     "grad_clip": 1,                // upper limit for gradients for clipping.
     "epochs": 1000,                // total number of epochs to train.
     "lr": 0.0001,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
-    "lr_decay": false,             // if true, Noam learning rate decaying is applied through training.
     "wd": 0.000001,         // Weight decay weight.
     "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
     
@@ -61,7 +60,7 @@
     "prenet_dropout": true,        // enable/disable dropout at prenet. 
 
     // ATTENTION
-    "attention_type": "graves",  // 'original' or 'graves'
+    "attention_type": "original",  // 'original' or 'graves'
     "attention_heads": 5,          // number of attention heads (only for 'graves')
     "attention_norm": "sigmoid",   // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron.
     "windowing": false,            // Enables attention windowing. Used only in eval mode.
diff --git a/train.py b/train.py
index ed2085b9..81bc2c72 100644
--- a/train.py
+++ b/train.py
@@ -151,7 +151,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
         global_step += 1
 
         # setup lr
-        if c.lr_decay:
+        if c.noam_schedule:
             scheduler.step()
         optimizer.zero_grad()
         if optimizer_st: