Adapt TTS for TacotronGST and some changes for Audio.py , better config.json naming

config_tacotron.json

@@ -40,11 +40,12 @@
         "windowing": false,            // Enables attention windowing. Used only in eval mode.
         "memory_size": 5,              // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. 
         "attention_norm": "sigmoid",   // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron.
-        "prenet_type": "bn",     // ONLY TACOTRON2 - "original" or "bn".
+        "prenet_type": "original",     // "original" or "bn".
-        "prenet_dropout": true,        // ONLY TACOTRON2 - enable/disable dropout at prenet. 
+        "prenet_dropout": true,        // enable/disable dropout at prenet. 
-        "use_forward_attn": true,      // ONLY TACOTRON2 - if it uses forward attention. In general, it aligns faster.
+        "use_forward_attn": true,      // if it uses forward attention. In general, it aligns faster.
-        "transition_agent": true,     // ONLY TACOTRON2 - enable/disable transition agent of forward attention.
+        "forward_attn_mask": false,    // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well.
-        "location_attn": false,        // ONLY TACOTRON2 - enable_disable location sensitive attention. It is enabled for TACOTRON by default.
+        "transition_agent": true,     // enable/disable transition agent of forward attention.
+        "location_attn": false,        // enable_disable location sensitive attention. It is enabled for TACOTRON by default.
         "loss_masking": true,         // enable / disable loss masking against the sequence padding.
         "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
         "stopnet": true,               // Train stopnet predicting the end of synthesis. 

config_tacotron2.json

@@ -42,6 +42,7 @@
         "prenet_type": "original",     // ONLY TACOTRON2 - "original" or "bn".
         "prenet_dropout": true,        // ONLY TACOTRON2 - enable/disable dropout at prenet. 
         "use_forward_attn": true,      // ONLY TACOTRON2 - if it uses forward attention. In general, it aligns faster.
+        "forward_attn_mask": false,    // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well.
         "transition_agent": false,     // ONLY TACOTRON2 - enable/disable transition agent of forward attention.
         "location_attn": false,        // ONLY TACOTRON2 - enable_disable location sensitive attention. It is enabled for TACOTRON by default.
         "loss_masking": true,          // enable / disable loss masking against the sequence padding.

config_tacotron_gst.json

@@ -0,0 +1,81 @@
+{
+        "run_name": "mozilla-tacotron-gst",
+        "run_description": "",
+    
+        "audio":{
+            // Audio processing parameters
+            "num_mels": 80,         // size of the mel spec frame. 
+            "num_freq": 1025,       // number of stft frequency levels. Size of the linear spectogram frame.
+            "sample_rate": 22050,   // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled.
+            "frame_length_ms": 50,  // stft window length in ms.
+            "frame_shift_ms": 12.5, // stft window hop-lengh in ms.
+            "preemphasis": 0.98,    // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
+            "min_level_db": -100,   // normalization range
+            "ref_level_db": 20,     // reference level db, theoretically 20db is the sound of air.
+            "power": 1.5,           // value to sharpen wav signals after GL algorithm.
+            "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
+            // Normalization parameters
+            "signal_norm": true,    // normalize the spec values in range [0, 1]
+            "symmetric_norm": false, // move normalization to range [-1, 1]
+            "max_norm": 1,          // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
+            "clip_norm": true,      // clip normalized values into the range.
+            "mel_fmin": 0.0,         // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
+            "mel_fmax": 8000.0,        // maximum freq level for mel-spec. Tune for dataset!!
+            "do_trim_silence": true  // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true)
+        },
+    
+        "distributed":{
+            "backend": "nccl",
+            "url": "tcp:\/\/localhost:54321"
+        },
+    
+        "reinit_layers": [],
+    
+        "model": "TacotronGST",          // one of the model in models/    
+        "grad_clip": 1,                // upper limit for gradients for clipping.
+        "epochs": 10000,                // total number of epochs to train.
+        "lr": 0.0001,                  // Initial learning rate. If Noam decay is active, maximum learning rate.
+        "lr_decay": false,             // if true, Noam learning rate decaying is applied through training.
+        "warmup_steps": 4000,          // Noam decay steps to increase the learning rate from 0 to "lr"
+        "windowing": false,            // Enables attention windowing. Used only in eval mode.
+        "memory_size": 5,              // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. 
+        "attention_norm": "sigmoid",   // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron.
+        "prenet_type": "original",           // "original" or "bn".
+        "prenet_dropout": true,        // enable/disable dropout at prenet. 
+        "use_forward_attn": true,      // if it uses forward attention. In general, it aligns faster.
+        "forward_attn_mask": false,    // Apply forward attention mask af inference to prevent bad modes. Try it if your model does not align well.
+        "transition_agent": true,     // enable/disable transition agent of forward attention.
+        "location_attn": false,        // enable_disable location sensitive attention. It is enabled for TACOTRON by default.
+        "loss_masking": true,         // enable / disable loss masking against the sequence padding.
+        "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars.
+        "stopnet": true,               // Train stopnet predicting the end of synthesis. 
+        "separate_stopnet": true,     // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER.
+        "tb_model_param_stats": false,     // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. 
+        
+        "batch_size": 32,       // Batch size for training. Lower values than 32 might cause hard to learn attention.
+        "eval_batch_size":16,   
+        "r": 5,                 // Number of frames to predict for step.
+        "wd": 0.000001,         // Weight decay weight.
+        "checkpoint": true,     // If true, it saves checkpoints per "save_step"
+        "save_step": 1000,      // Number of training steps expected to save traning stats and checkpoints.
+        "print_step": 10,       // Number of steps to log traning on console.
+        "batch_group_size": 0,  //Number of batches to shuffle after bucketing.
+    
+        "run_eval": true,
+        "test_delay_epochs": 5,  //Until attention is aligned, testing only wastes computation time.
+        "test_sentences_file": null, 
+        "data_path": "/media/erogol/data_ssd/Data/Mozilla/",  // DATASET-RELATED: can overwritten from command argument
+        "meta_file_train": "metadata_train.txt",      // DATASET-RELATED: metafile for training dataloader.
+        "meta_file_val": "metadata_val.txt",    // DATASET-RELATED: metafile for evaluation dataloader.
+        "dataset": "mozilla",      // DATASET-RELATED: one of TTS.dataset.preprocessors depending on your target dataset. Use "tts_cache" for pre-computed dataset by extract_features.py
+        "min_seq_len": 0,       // DATASET-RELATED: minimum text length to use in training
+        "max_seq_len": 150,     // DATASET-RELATED: maximum text length
+        "output_path": "../keep/",      // DATASET-RELATED: output path for all training outputs.
+        "num_loader_workers": 4,        // number of training data loader processes. Don't set it too big. 4-8 are good values.
+        "num_val_loader_workers": 4,    // number of evaluation data loader processes.
+        "phoneme_cache_path": "mozilla_us_phonemes",  // phoneme computation is slow, therefore, it caches results in the given folder.
+        "use_phonemes": true,           // use phonemes instead of raw characters. It is suggested for better pronounciation.
+        "phoneme_language": "en-us",     // depending on your target language, pick one from  https://github.com/bootphon/phonemizer#languages
+        "text_cleaner": "phoneme_cleaners"
+    }
+    

train.py

@@ -89,7 +89,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
         # setup input data
         text_input = data[0]
         text_lengths = data[1]
-        linear_input = data[2] if c.model == "Tacotron" else None
+        linear_input = data[2] if c.model in ["Tacotron", "TacotronGST"] else None
         mel_input = data[3]
         mel_lengths = data[4]
         stop_targets = data[5]
@@ -116,7 +116,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
             text_lengths = text_lengths.cuda(non_blocking=True)
             mel_input = mel_input.cuda(non_blocking=True)
             mel_lengths = mel_lengths.cuda(non_blocking=True)
-            linear_input = linear_input.cuda(non_blocking=True) if c.model == "Tacotron" else None
+            linear_input = linear_input.cuda(non_blocking=True) if c.model in ["Tacotron", "TacotronGST"] else None
             stop_targets = stop_targets.cuda(non_blocking=True)
 
         # forward pass model
@@ -127,13 +127,13 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
         stop_loss = criterion_st(stop_tokens, stop_targets) if c.stopnet else torch.zeros(1)
         if c.loss_masking:
             decoder_loss = criterion(decoder_output, mel_input, mel_lengths)
-            if c.model == "Tacotron":
+            if c.model in ["Tacotron", "TacotronGST"]:
                 postnet_loss = criterion(postnet_output, linear_input, mel_lengths)
             else:
                 postnet_loss = criterion(postnet_output, mel_input, mel_lengths)
         else:
             decoder_loss = criterion(decoder_output, mel_input)
-            if c.model == "Tacotron":
+            if c.model in ["Tacotron", "TacotronGST"]:
                 postnet_loss = criterion(postnet_output, linear_input)
             else:
                 postnet_loss = criterion(postnet_output, mel_input)
@@ -199,7 +199,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
 
                 # Diagnostic visualizations
                 const_spec = postnet_output[0].data.cpu().numpy()
-                gt_spec =  linear_input[0].data.cpu().numpy() if c.model == "Tacotron" else  mel_input[0].data.cpu().numpy()
+                gt_spec =  linear_input[0].data.cpu().numpy() if c.model in ["Tacotron", "TacotronGST"] else  mel_input[0].data.cpu().numpy()
                 align_img = alignments[0].data.cpu().numpy()
 
                 figures = {
@@ -210,7 +210,7 @@ def train(model, criterion, criterion_st, optimizer, optimizer_st, scheduler,
                 tb_logger.tb_train_figures(current_step, figures)
 
                 # Sample audio
-                if c.model == "Tacotron":
+                if c.model in ["Tacotron", "TacotronGST"]:
                     train_audio = ap.inv_spectrogram(const_spec.T)
                 else:
                     train_audio = ap.inv_mel_spectrogram(const_spec.T)
@@ -273,7 +273,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch):
                 # setup input data
                 text_input = data[0]
                 text_lengths = data[1]
-                linear_input = data[2] if c.model == "Tacotron" else None
+                linear_input = data[2] if c.model in ["Tacotron", "TacotronGST"] else None
                 mel_input = data[3]
                 mel_lengths = data[4]
                 stop_targets = data[5]
@@ -289,7 +289,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch):
                     text_input = text_input.cuda()
                     mel_input = mel_input.cuda()
                     mel_lengths = mel_lengths.cuda()
-                    linear_input = linear_input.cuda() if c.model == "Tacotron" else None
+                    linear_input = linear_input.cuda() if c.model in ["Tacotron", "TacotronGST"] else None
                     stop_targets = stop_targets.cuda()
 
                 # forward pass
@@ -300,13 +300,13 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch):
                 stop_loss = criterion_st(stop_tokens, stop_targets) if c.stopnet else torch.zeros(1)
                 if c.loss_masking:
                     decoder_loss = criterion(decoder_output, mel_input, mel_lengths)
-                    if c.model == "Tacotron":
+                    if c.model in ["Tacotron", "TacotronGST"]:
                         postnet_loss = criterion(postnet_output, linear_input, mel_lengths)
                     else:
                         postnet_loss = criterion(postnet_output, mel_input, mel_lengths)
                 else:
                     decoder_loss = criterion(decoder_output, mel_input)
-                    if c.model == "Tacotron":
+                    if c.model in ["Tacotron", "TacotronGST"]:
                         postnet_loss = criterion(postnet_output, linear_input)
                     else:
                         postnet_loss = criterion(postnet_output, mel_input)
@@ -339,7 +339,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch):
                 # Diagnostic visualizations
                 idx = np.random.randint(mel_input.shape[0])
                 const_spec = postnet_output[idx].data.cpu().numpy()
-                gt_spec = linear_input[idx].data.cpu().numpy() if c.model == "Tacotron" else  mel_input[idx].data.cpu().numpy()
+                gt_spec = linear_input[idx].data.cpu().numpy() if c.model in ["Tacotron", "TacotronGST"] else  mel_input[idx].data.cpu().numpy()
                 align_img = alignments[idx].data.cpu().numpy()
 
                 eval_figures = {
@@ -350,7 +350,7 @@ def evaluate(model, criterion, criterion_st, ap, current_step, epoch):
                 tb_logger.tb_eval_figures(current_step, eval_figures)
 
                 # Sample audio
-                if c.model == "Tacotron":
+                if c.model in ["Tacotron", "TacotronGST"]:
                     eval_audio = ap.inv_spectrogram(const_spec.T)
                 else:
                     eval_audio = ap.inv_mel_spectrogram(const_spec.T)
@@ -410,9 +410,9 @@ def main(args):
         optimizer_st = None
 
     if c.loss_masking:
-        criterion = L1LossMasked() if c.model == "Tacotron" else MSELossMasked()
+        criterion = L1LossMasked() if c.model in ["Tacotron", "TacotronGST"] else MSELossMasked()
     else:
-        criterion = nn.L1Loss() if c.model == "Tacotron" else nn.MSELoss()
+        criterion = nn.L1Loss() if c.model in ["Tacotron", "TacotronGST"] else nn.MSELoss()
     criterion_st = nn.BCEWithLogitsLoss() if c.stopnet else None
 
     if args.restore_path:

utils/audio.py

@@ -10,7 +10,6 @@ from scipy import signal, io
 
 class AudioProcessor(object):
     def __init__(self,
-                 bits=None,
                  sample_rate=None,
                  num_mels=None,
                  min_level_db=None,
@@ -32,7 +31,6 @@ class AudioProcessor(object):
 
         print(" > Setting up Audio Processor...")
 
-        self.bits = bits
         self.sample_rate = sample_rate
         self.num_mels = num_mels
         self.min_level_db = min_level_db
@@ -218,25 +216,29 @@ class AudioProcessor(object):
         return librosa.effects.trim(
             wav, top_db=40, frame_length=1024, hop_length=256)[0]
 
-    # WaveRNN repo specific functions
+    def mulaw_encode(self, wav, qc):
-    # def mulaw_encode(self, wav, qc):
+        mu = 2 ** qc - 1
-    #     mu = qc - 1
         # wav_abs = np.minimum(np.abs(wav), 1.0)
-    #     magnitude = np.log(1 + mu * wav_abs) / np.log(1. + mu)
+        signal = np.sign(wav) * np.log(1 + mu * np.abs(wav)) / np.log(1. + mu)
-    #     signal = np.sign(wav) * magnitude
+        # Quantize signal to the specified number of levels.
-    #     # Quantize signal to the specified number of levels.
+        signal = (signal + 1) / 2 * mu + 0.5
-    #     signal = (signal + 1) / 2 * mu + 0.5
+        return np.floor(signal,)
-    #     return signal.astype(np.int32)
 
-    # def mulaw_decode(self, wav, qc):
+    @staticmethod
-    #     """Recovers waveform from quantized values."""
+    def mulaw_decode(wav, qc):
-    #     mu = qc - 1
+        """Recovers waveform from quantized values."""
+        # from IPython.core.debugger import set_trace
+        # set_trace()
+        mu = 2 ** qc - 1
+        x = np.sign(wav) / mu * ((1 + mu) ** np.abs(wav) - 1)
+        return x
+        # mu = 2 ** qc - 1.
         # # Map values back to [-1, 1].
-    #     casted = wav.astype(np.float32)
+        # # casted = wav.astype(np.float32)
-    #     signal = 2 * (casted / mu) - 1
+        # # signal = 2 * casted / mu - 1
         # # Perform inverse of mu-law transformation.
-    #     magnitude = (1 / mu) * ((1 + mu) ** abs(signal) - 1)
+        # magnitude = (1 / mu) * ((1 + mu) ** abs(wav) - 1)
-    #     return np.sign(signal) * magnitude
+        # return np.sign(wav) * magnitude
 
     def load_wav(self, filename, encode=False):
         x, sr = sf.read(filename)
@@ -249,8 +251,8 @@ class AudioProcessor(object):
     def encode_16bits(self, x):
         return np.clip(x * 2**15, -2**15, 2**15 - 1).astype(np.int16)
 
-    def quantize(self, x):
+    def quantize(self, x, bits):
-        return (x + 1.) * (2**self.bits - 1) / 2
+        return (x + 1.) * (2**bits - 1) / 2
 
-    def dequantize(self, x):
+    def dequantize(self, x, bits):
-        return 2 * x / (2**self.bits - 1) - 1
+        return 2 * x / (2**bits - 1) - 1

utils/generic_utils.py

@@ -247,7 +247,7 @@ def setup_model(num_chars, c):
     print(" > Using model: {}".format(c.model))
     MyModel = importlib.import_module('models.' + c.model.lower())
     MyModel = getattr(MyModel, c.model)
-    if c.model.lower() == "tacotron":
+    if c.model.lower() in ["tacotron", "tacotrongst"]:
         model = MyModel(
             num_chars=num_chars,
             r=c.r,