From af46727517e63147554753e6958b348fad5fe616 Mon Sep 17 00:00:00 2001 From: gerazov Date: Fri, 12 Feb 2021 02:12:00 +0100 Subject: [PATCH 1/5] loading last checkpoint/best_model works, deleting last best models options added, loading last best_loss added --- TTS/bin/train_glow_tts.py | 18 +- TTS/bin/train_speedy_speech.py | 14 +- TTS/bin/train_tacotron.py | 12 +- TTS/bin/train_vocoder_gan.py | 15 +- TTS/bin/train_vocoder_wavegrad.py | 12 +- TTS/bin/train_vocoder_wavernn.py | 14 +- TTS/tts/configs/config.json | 346 +++++++++--------- TTS/tts/configs/glow_tts_gated_conv.json | 2 + TTS/tts/configs/glow_tts_ljspeech.json | 2 + .../ljspeech_tacotron2_dynamic_conv_attn.json | 344 ++++++++--------- TTS/tts/configs/speedy_speech_ljspeech.json | 2 + TTS/utils/arguments.py | 73 ++-- .../multiband-melgan_and_rwd_config.json | 2 + .../configs/multiband_melgan_config.json | 2 + .../multiband_melgan_config_mozilla.json | 2 + .../configs/parallel_wavegan_config.json | 2 + .../configs/universal_fullband_melgan.json | 2 + TTS/vocoder/configs/wavegrad_libritts.json | 2 + TTS/vocoder/configs/wavernn_config.json | 2 + TTS/vocoder/utils/io.py | 30 +- 20 files changed, 507 insertions(+), 391 deletions(-) diff --git a/TTS/bin/train_glow_tts.py b/TTS/bin/train_glow_tts.py index 9db2381e..14a20149 100644 --- a/TTS/bin/train_glow_tts.py +++ b/TTS/bin/train_glow_tts.py @@ -538,8 +538,16 @@ def main(args): # pylint: disable=redefined-outer-name num_params = count_parameters(model) print("\n > Model has {} parameters".format(num_params), flush=True) - if 'best_loss' not in locals(): + if args.restore_step == 0 or not args.best_path: best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -549,7 +557,8 @@ def main(args): # pylint: disable=redefined-outer-name model = data_depended_init(train_loader, model) for epoch in range(0, c.epochs): c_logger.print_epoch_start(epoch, c.epochs) - train_avg_loss_dict, global_step = train(train_loader, model, criterion, optimizer, + train_avg_loss_dict, global_step = train(train_loader, model, + criterion, optimizer, scheduler, ap, global_step, epoch) eval_avg_loss_dict = evaluate(eval_loader, model, criterion, ap, @@ -558,8 +567,9 @@ def main(args): # pylint: disable=redefined-outer-name target_loss = train_avg_loss_dict['avg_loss'] if c.run_eval: target_loss = eval_avg_loss_dict['avg_loss'] - best_loss = save_best_model(target_loss, best_loss, model, optimizer, global_step, epoch, c.r, - OUT_PATH) + best_loss = save_best_model(target_loss, best_loss, model, optimizer, + global_step, epoch, c.r, OUT_PATH, + keep_best=keep_best, keep_after=keep_after) if __name__ == '__main__': diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index a9a83bbf..4e521451 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -502,8 +502,16 @@ def main(args): # pylint: disable=redefined-outer-name num_params = count_parameters(model) print("\n > Model has {} parameters".format(num_params), flush=True) - if 'best_loss' not in locals(): + if args.restore_step == 0 or not args.best_path: best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -522,8 +530,8 @@ def main(args): # pylint: disable=redefined-outer-name if c.run_eval: target_loss = eval_avg_loss_dict['avg_loss'] best_loss = save_best_model(target_loss, best_loss, model, optimizer, - global_step, epoch, c.r, - OUT_PATH) + global_step, epoch, c.r, OUT_PATH, + keep_best=keep_best, keep_after=keep_after) if __name__ == '__main__': diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index 0a53f2a1..cdc68c94 100644 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -581,8 +581,16 @@ def main(args): # pylint: disable=redefined-outer-name num_params = count_parameters(model) print("\n > Model has {} parameters".format(num_params), flush=True) - if 'best_loss' not in locals(): + if args.restore_step == 0 or not args.best_path: best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False # define data loaders train_loader = setup_loader(ap, @@ -634,6 +642,8 @@ def main(args): # pylint: disable=redefined-outer-name epoch, c.r, OUT_PATH, + keep_best=keep_best, + keep_after=keep_after, scaler=scaler.state_dict() if c.mixed_precision else None ) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index 1f2beb70..ecc33288 100644 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -545,8 +545,16 @@ def main(args): # pylint: disable=redefined-outer-name num_params = count_parameters(model_disc) print(" > Discriminator has {} parameters".format(num_params), flush=True) - if 'best_loss' not in locals(): + if args.restore_step == 0 or not args.best_path: best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -571,7 +579,10 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, - model_losses=eval_avg_loss_dict) + keep_best=keep_best, + keep_after=keep_after, + model_losses=eval_avg_loss_dict, + ) if __name__ == '__main__': diff --git a/TTS/bin/train_vocoder_wavegrad.py b/TTS/bin/train_vocoder_wavegrad.py index d8dc88e1..7846aae5 100644 --- a/TTS/bin/train_vocoder_wavegrad.py +++ b/TTS/bin/train_vocoder_wavegrad.py @@ -393,8 +393,16 @@ def main(args): # pylint: disable=redefined-outer-name num_params = count_parameters(model) print(" > WaveGrad has {} parameters".format(num_params), flush=True) - if 'best_loss' not in locals(): + if args.restore_step == 0 or not args.best_path: best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -416,6 +424,8 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, + keep_best=keep_best, + keep_after=keep_after, model_losses=eval_avg_loss_dict, scaler=scaler.state_dict() if c.mixed_precision else None ) diff --git a/TTS/bin/train_vocoder_wavernn.py b/TTS/bin/train_vocoder_wavernn.py index b4ffe143..44ffef14 100644 --- a/TTS/bin/train_vocoder_wavernn.py +++ b/TTS/bin/train_vocoder_wavernn.py @@ -416,8 +416,16 @@ def main(args): # pylint: disable=redefined-outer-name num_parameters = count_parameters(model_wavernn) print(" > Model has {} parameters".format(num_parameters), flush=True) - if "best_loss" not in locals(): - best_loss = float("inf") + if args.restore_step == 0 or not args.best_path: + best_loss = float('inf') + print(" > Starting with inf best loss.") + else: + print(args.best_path) + best_loss = torch.load(args.best_path, + map_location='cpu')['model_loss'] + print(f" > Starting with loaded last best loss {best_loss}.") + keep_best = c.get('keep_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -440,6 +448,8 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, + keep_best=keep_best, + keep_after=keep_after, model_losses=eval_avg_loss_dict, scaler=scaler.state_dict() if c.mixed_precision else None ) diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json index 48f20e8f..5bd249d9 100644 --- a/TTS/tts/configs/config.json +++ b/TTS/tts/configs/config.json @@ -1,172 +1,174 @@ -{ - "model": "Tacotron2", - "run_name": "ljspeech-ddc", - "run_description": "tacotron2 with DDC and differential spectral loss.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": false, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "sigmoid", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - "use_noise_augment": true, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } - ] -} - +{ + "model": "Tacotron2", + "run_name": "ljspeech-ddc", + "run_description": "tacotron2 with DDC and differential spectral loss.", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 1, + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // VOCABULARY PARAMETERS + // if custom character set is not defined, + // default set in symbols.py is used + // "characters":{ + // "pad": "_", + // "eos": "~", + // "bos": "^", + // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + // "punctuations":"!'(),-.:;? ", + // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + // }, + + // DISTRIBUTED TRAINING + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54321" + }, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + "eval_batch_size":16, + "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. + "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. + + // LOSS SETTINGS + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled + "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled + "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled + "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled + "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. + "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. + + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": false, // use noam warmup and lr schedule. + "grad_clip": 1.0, // upper limit for gradients for clipping. + "epochs": 1000, // total number of epochs to train. + "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "wd": 0.000001, // Weight decay weight. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. + + // TACOTRON PRENET + "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. + "prenet_type": "original", // "original" or "bn". + "prenet_dropout": false, // enable/disable dropout at prenet. + + // TACOTRON ATTENTION + "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' + "attention_heads": 4, // number of attention heads (only for 'graves') + "attention_norm": "sigmoid", // softmax or sigmoid. + "windowing": false, // Enables attention windowing. Used only in eval mode. + "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. + "transition_agent": false, // enable/disable transition agent of forward attention. + "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. + "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. + "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ + "ddc_r": 7, // reduction rate for coarse decoder. + + // STOPNET + "stopnet": true, // Train stopnet predicting the end of synthesis. + "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log training on console. + "tb_plot_step": 100, // Number of steps to plot TB training figures. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "text_cleaner": "phoneme_cleaners", + "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "batch_group_size": 4, //Number of batches to shuffle after bucketing. + "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training + "max_seq_len": 153, // DATASET-RELATED: maximum text length + "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. + "use_noise_augment": true, + + // PATHS + "output_path": "/home/erogol/Models/LJSpeech/", + + // PHONEMES + "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages + + // MULTI-SPEAKER and GST + "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. + "use_gst": false, // use global style tokens + "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 + "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 + "gst": { // gst parameter if gst is enabled + "gst_style_input": null, // Condition the style input either on a + // -> wave file [path to wave] or + // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} + // with the dictionary being len(dict) <= len(gst_style_tokens). + "gst_embedding_dim": 512, + "gst_num_heads": 4, + "gst_style_tokens": 10, + "gst_use_speaker_embedding": false + }, + + // DATASETS + "datasets": // List of datasets. They all merged and they get different speaker_ids. + [ + { + "name": "ljspeech", + "path": "/home/erogol/Data/LJSpeech-1.1/", + "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers + "meta_file_val": null + } + ] +} + diff --git a/TTS/tts/configs/glow_tts_gated_conv.json b/TTS/tts/configs/glow_tts_gated_conv.json index d34fbaf0..865c6f29 100644 --- a/TTS/tts/configs/glow_tts_gated_conv.json +++ b/TTS/tts/configs/glow_tts_gated_conv.json @@ -93,6 +93,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. "apex_amp_level": null, diff --git a/TTS/tts/configs/glow_tts_ljspeech.json b/TTS/tts/configs/glow_tts_ljspeech.json index 636d9313..6e15de10 100644 --- a/TTS/tts/configs/glow_tts_ljspeech.json +++ b/TTS/tts/configs/glow_tts_ljspeech.json @@ -105,6 +105,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json index cd5ad8ab..3cf66870 100644 --- a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json +++ b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json @@ -1,171 +1,173 @@ -{ - "model": "Tacotron2", - "run_name": "ljspeech-dcattn", - "run_description": "tacotron2 with dynamic convolution attention.", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 1, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":16, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 0.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": false, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "dynamic_convolution", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "softmax", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": false, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "batch_group_size": 4, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/", - - // PHONEMES - "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": false, // use global style tokens - "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) <= len(gst_style_tokens). - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_style_tokens": 10, - "gst_use_speaker_embedding": false - }, - - // DATASETS - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "/home/erogol/Data/LJSpeech-1.1/", - "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers - "meta_file_val": null - } - ] -} - +{ + "model": "Tacotron2", + "run_name": "ljspeech-dcattn", + "run_description": "tacotron2 with dynamic convolution attention.", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 50.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 7600.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 1, + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": "/home/erogol/Data/LJSpeech-1.1/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // VOCABULARY PARAMETERS + // if custom character set is not defined, + // default set in symbols.py is used + // "characters":{ + // "pad": "_", + // "eos": "~", + // "bos": "^", + // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + // "punctuations":"!'(),-.:;? ", + // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + // }, + + // DISTRIBUTED TRAINING + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54321" + }, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + "eval_batch_size":16, + "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. + "gradual_training": [[0, 7, 64], [1, 5, 64], [50000, 3, 32], [130000, 2, 32], [290000, 1, 32]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "mixed_precision": true, // level of optimization with NVIDIA's apex feature for automatic mixed FP16/FP32 precision (AMP), NOTE: currently only O1 is supported, and use "O1" to activate. + + // LOSS SETTINGS + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled + "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled + "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled + "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled + "ga_alpha": 0.0, // weight for guided attention loss. If > 0, guided attention is enabled. + "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. + + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": false, // use noam warmup and lr schedule. + "grad_clip": 1.0, // upper limit for gradients for clipping. + "epochs": 1000, // total number of epochs to train. + "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "wd": 0.000001, // Weight decay weight. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. + + // TACOTRON PRENET + "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. + "prenet_type": "original", // "original" or "bn". + "prenet_dropout": false, // enable/disable dropout at prenet. + + // TACOTRON ATTENTION + "attention_type": "dynamic_convolution", // 'original' , 'graves', 'dynamic_convolution' + "attention_heads": 4, // number of attention heads (only for 'graves') + "attention_norm": "softmax", // softmax or sigmoid. + "windowing": false, // Enables attention windowing. Used only in eval mode. + "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. + "transition_agent": false, // enable/disable transition agent of forward attention. + "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. + "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. + "double_decoder_consistency": false, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ + "ddc_r": 7, // reduction rate for coarse decoder. + + // STOPNET + "stopnet": true, // Train stopnet predicting the end of synthesis. + "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log training on console. + "tb_plot_step": 100, // Number of steps to plot TB training figures. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "text_cleaner": "phoneme_cleaners", + "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "batch_group_size": 4, //Number of batches to shuffle after bucketing. + "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training + "max_seq_len": 153, // DATASET-RELATED: maximum text length + "compute_input_seq_cache": false, // if true, text sequences are computed before starting training. If phonemes are enabled, they are also computed at this stage. + + // PATHS + "output_path": "/home/erogol/Models/LJSpeech/", + + // PHONEMES + "phoneme_cache_path": "/home/erogol/Models/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages + + // MULTI-SPEAKER and GST + "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. + "use_gst": false, // use global style tokens + "use_external_speaker_embedding_file": false, // if true, forces the model to use external embedding per sample instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 + "external_speaker_embedding_file": "../../speakers-vctk-en.json", // if not null and use_external_speaker_embedding_file is true, it is used to load a specific embedding file and thus uses these embeddings instead of nn.embeddings, that is, it supports external embeddings such as those used at: https://arxiv.org/abs /1806.04558 + "gst": { // gst parameter if gst is enabled + "gst_style_input": null, // Condition the style input either on a + // -> wave file [path to wave] or + // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} + // with the dictionary being len(dict) <= len(gst_style_tokens). + "gst_embedding_dim": 512, + "gst_num_heads": 4, + "gst_style_tokens": 10, + "gst_use_speaker_embedding": false + }, + + // DATASETS + "datasets": // List of datasets. They all merged and they get different speaker_ids. + [ + { + "name": "ljspeech", + "path": "/home/erogol/Data/LJSpeech-1.1/", + "meta_file_train": "metadata.csv", // for vtck if list, ignore speakers id in list for train, its useful for test cloning with new speakers + "meta_file_val": null + } + ] +} + diff --git a/TTS/tts/configs/speedy_speech_ljspeech.json b/TTS/tts/configs/speedy_speech_ljspeech.json index bd511470..9f1d3f8b 100644 --- a/TTS/tts/configs/speedy_speech_ljspeech.json +++ b/TTS/tts/configs/speedy_speech_ljspeech.json @@ -109,6 +109,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n "mixed_precision": false, diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 948c90d3..d05936dc 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -43,6 +43,11 @@ def parse_arguments(argv): type=str, help="Model file to be restored. Use to finetune a model.", default="") + parser.add_argument( + "--best_path", + type=str, + help="Best model file to be used for extracting best loss.", + default="") parser.add_argument( "--config_path", type=str, @@ -67,11 +72,11 @@ def parse_arguments(argv): return parser.parse_args() -def get_last_checkpoint(path): - """Get latest checkpoint from a list of filenames. +def get_last_models(path): + """Get latest checkpoint or/and best model in path. It is based on globbing for `*.pth.tar` and the RegEx - `checkpoint_([0-9]+)`. + `(checkpoint|best_model)_([0-9]+)`. Parameters ---------- @@ -81,7 +86,7 @@ def get_last_checkpoint(path): Raises ------ ValueError - If no checkpoint files are found. + If no checkpoint or best_model files are found. Returns ------- @@ -89,22 +94,37 @@ def get_last_checkpoint(path): Last checkpoint filename. """ - last_checkpoint_num = 0 - last_checkpoint = None - filenames = glob.glob( - os.path.join(path, "/*.pth.tar")) - for filename in filenames: - try: - checkpoint_num = int( - re.search(r"checkpoint_([0-9]+)", filename).groups()[0]) - if checkpoint_num > last_checkpoint_num: - last_checkpoint_num = checkpoint_num - last_checkpoint = filename - except AttributeError: # if there's no match in the filename - pass - if last_checkpoint is None: - raise ValueError(f"No checkpoints in {path}!") - return last_checkpoint + file_names = glob.glob(os.path.join(path, "*.pth.tar")) + last_models = {} + last_model_nums = {} + for key in ['checkpoint', 'best_model']: + last_model_num = 0 + last_model = None + for file_name in file_names: + try: + model_num = int(re.search( + f"{key}_([0-9]+)", file_name).groups()[0]) + if model_num > last_model_num: + last_model_num = model_num + last_model = file_name + except AttributeError: # if there's no match in the filename + continue + last_models[key] = last_model + last_model_nums[key] = last_model_num + + # check what models were found + if not last_models: + raise ValueError(f"No models found in continue path {path}!") + elif 'checkpoint' not in last_models: # no checkpoint just best model + last_models['checkpoint'] = last_models['best_model'] + elif 'best_model' not in last_models: # no best model + # this shouldn't happen, but let's handle it just in case + last_models['best_model'] = None + # finally check if last best model is more recent than checkpoint + elif last_model_nums['best_model'] > last_model_nums['checkpoint']: + last_models['checkpoint'] = last_models['best_model'] + + return last_models['checkpoint'], last_models['best_model'] def process_args(args, model_type): @@ -143,15 +163,12 @@ def process_args(args, model_type): Class that does the TensorBoard loggind. """ - if args.continue_path != "": + if args.continue_path: args.output_path = args.continue_path args.config_path = os.path.join(args.continue_path, "config.json") - list_of_files = glob.glob( - os.path.join(args.continue_path, "*.pth.tar") - ) # * means all if need specific format then *.csv - args.restore_path = max(list_of_files, key=os.path.getctime) - # checkpoint number based continuing - # args.restore_path = get_last_checkpoint(args.continue_path) + args.restore_path, best_model = get_last_models(args.continue_path) + if not args.best_path: + args.best_path = best_model print(f" > Training continues for {args.restore_path}") # setup output paths and read configs @@ -178,7 +195,7 @@ def process_args(args, model_type): print(" > Mixed precision mode is ON") out_path = args.continue_path - if args.continue_path == "": + if not out_path: out_path = create_experiment_folder(c.output_path, c.run_name, args.debug) diff --git a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json index 0b751854..b4d42f4b 100644 --- a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -138,6 +138,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/multiband_melgan_config.json b/TTS/vocoder/configs/multiband_melgan_config.json index 7a5a13e3..af2af8a3 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.json +++ b/TTS/vocoder/configs/multiband_melgan_config.json @@ -128,6 +128,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/multiband_melgan_config_mozilla.json b/TTS/vocoder/configs/multiband_melgan_config_mozilla.json index 4978d42f..0f133fa7 100644 --- a/TTS/vocoder/configs/multiband_melgan_config_mozilla.json +++ b/TTS/vocoder/configs/multiband_melgan_config_mozilla.json @@ -141,6 +141,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/parallel_wavegan_config.json b/TTS/vocoder/configs/parallel_wavegan_config.json index fcd765bd..85e659f4 100644 --- a/TTS/vocoder/configs/parallel_wavegan_config.json +++ b/TTS/vocoder/configs/parallel_wavegan_config.json @@ -130,6 +130,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/universal_fullband_melgan.json b/TTS/vocoder/configs/universal_fullband_melgan.json index fe4433c2..efb6f3cd 100644 --- a/TTS/vocoder/configs/universal_fullband_melgan.json +++ b/TTS/vocoder/configs/universal_fullband_melgan.json @@ -124,6 +124,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/wavegrad_libritts.json b/TTS/vocoder/configs/wavegrad_libritts.json index a271ce33..9107d556 100644 --- a/TTS/vocoder/configs/wavegrad_libritts.json +++ b/TTS/vocoder/configs/wavegrad_libritts.json @@ -103,6 +103,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 5000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/wavernn_config.json b/TTS/vocoder/configs/wavernn_config.json index effb103b..220904c9 100644 --- a/TTS/vocoder/configs/wavernn_config.json +++ b/TTS/vocoder/configs/wavernn_config.json @@ -89,6 +89,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/utils/io.py b/TTS/vocoder/utils/io.py index 5c42dfca..232d972d 100644 --- a/TTS/vocoder/utils/io.py +++ b/TTS/vocoder/utils/io.py @@ -1,4 +1,5 @@ import os +import glob import torch import datetime import pickle as pickle_tts @@ -61,12 +62,13 @@ def save_checkpoint(model, optimizer, scheduler, model_disc, optimizer_disc, scheduler_disc, current_step, epoch, checkpoint_path, **kwargs) -def save_best_model(target_loss, best_loss, model, optimizer, scheduler, +def save_best_model(current_loss, best_loss, model, optimizer, scheduler, model_disc, optimizer_disc, scheduler_disc, current_step, - epoch, output_folder, **kwargs): - if target_loss < best_loss: - file_name = 'best_model.pth.tar' - checkpoint_path = os.path.join(output_folder, file_name) + epoch, out_path, keep_best=False, keep_after=10000, + **kwargs): + if current_loss < best_loss: + best_model_name = f'best_model_{current_step}.pth.tar' + checkpoint_path = os.path.join(out_path, best_model_name) print(" > BEST MODEL : {}".format(checkpoint_path)) save_model(model, optimizer, @@ -77,7 +79,21 @@ def save_best_model(target_loss, best_loss, model, optimizer, scheduler, current_step, epoch, checkpoint_path, - model_loss=target_loss, + model_loss=current_loss, **kwargs) - best_loss = target_loss + # only delete previous if current is saved successfully + if not keep_best or (current_step < keep_after): + model_names = glob.glob( + os.path.join(out_path, 'best_model*.pth.tar')) + for model_name in model_names: + if os.path.basename(model_name) == best_model_name: + continue + os.remove(model_name) + # create symlink to best model for convinience + link_name = 'best_model.pth.tar' + link_path = os.path.join(out_path, link_name) + if os.path.islink(link_path) or os.path.isfile(link_path): + os.remove(link_path) + os.symlink(best_model_name, os.path.join(out_path, link_name)) + best_loss = current_loss return best_loss From 310d18325e8f4bc4b9c46c27fd6d4572aecb4577 Mon Sep 17 00:00:00 2001 From: gerazov Date: Fri, 12 Feb 2021 10:55:45 +0100 Subject: [PATCH 2/5] brushed up printing model load path and best loss path --- TTS/bin/train_glow_tts.py | 6 ++++-- TTS/bin/train_speedy_speech.py | 4 +++- TTS/bin/train_tacotron.py | 10 ++++++---- TTS/bin/train_vocoder_gan.py | 8 +++++--- TTS/bin/train_vocoder_wavegrad.py | 4 +++- TTS/bin/train_vocoder_wavernn.py | 4 +++- TTS/utils/arguments.py | 4 +--- 7 files changed, 25 insertions(+), 15 deletions(-) diff --git a/TTS/bin/train_glow_tts.py b/TTS/bin/train_glow_tts.py index 14a20149..072ad41b 100644 --- a/TTS/bin/train_glow_tts.py +++ b/TTS/bin/train_glow_tts.py @@ -497,6 +497,7 @@ def main(args): # pylint: disable=redefined-outer-name criterion = GlowTTSLoss() if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)} ...") checkpoint = torch.load(args.restore_path, map_location='cpu') try: # TODO: fix optimizer init, model.cuda() needs to be called before @@ -514,7 +515,7 @@ def main(args): # pylint: disable=redefined-outer-name for group in optimizer.param_groups: group['initial_lr'] = c.lr - print(" > Model restored from step %d" % checkpoint['step'], + print(f" > Model restored from step {checkpoint['step']:d}", flush=True) args.restore_step = checkpoint['step'] else: @@ -542,7 +543,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index 4e521451..8e9dbc2e 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -461,6 +461,7 @@ def main(args): # pylint: disable=redefined-outer-name criterion = SpeedySpeechLoss(c) if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)} ...") checkpoint = torch.load(args.restore_path, map_location='cpu') try: # TODO: fix optimizer init, model.cuda() needs to be called before @@ -506,7 +507,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index cdc68c94..2382c0be 100644 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -534,12 +534,13 @@ def main(args): # pylint: disable=redefined-outer-name # setup criterion criterion = TacotronLoss(c, stopnet_pos_weight=c.stopnet_pos_weight, ga_sigma=0.4) if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location='cpu') try: - print(" > Restoring Model.") + print(" > Restoring Model...") model.load_state_dict(checkpoint['model']) # optimizer restore - print(" > Restoring Optimizer.") + print(" > Restoring Optimizer...") optimizer.load_state_dict(checkpoint['optimizer']) if "scaler" in checkpoint and c.mixed_precision: print(" > Restoring AMP Scaler...") @@ -547,7 +548,7 @@ def main(args): # pylint: disable=redefined-outer-name if c.reinit_layers: raise RuntimeError except (KeyError, RuntimeError): - print(" > Partial model initialization.") + print(" > Partial model initialization...") model_dict = model.state_dict() model_dict = set_init_dict(model_dict, checkpoint['model'], c) # torch.save(model_dict, os.path.join(OUT_PATH, 'state_dict.pt')) @@ -585,7 +586,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index ecc33288..9043a560 100644 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -485,6 +485,7 @@ def main(args): # pylint: disable=redefined-outer-name criterion_disc = DiscriminatorLoss(c) if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location='cpu') try: print(" > Restoring Generator Model...") @@ -523,7 +524,7 @@ def main(args): # pylint: disable=redefined-outer-name for group in optimizer_disc.param_groups: group['lr'] = c.lr_disc - print(" > Model restored from step %d" % checkpoint['step'], + print(f" > Model restored from step {checkpoint['step']:d}", flush=True) args.restore_step = checkpoint['step'] else: @@ -549,10 +550,11 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] - print(f" > Starting with loaded last best loss {best_loss}.") + print(f" > Starting with best loss of {best_loss}.") keep_best = c.get('keep_best', False) keep_after = c.get('keep_after', 10000) # void if keep_best False diff --git a/TTS/bin/train_vocoder_wavegrad.py b/TTS/bin/train_vocoder_wavegrad.py index 7846aae5..271e8d4c 100644 --- a/TTS/bin/train_vocoder_wavegrad.py +++ b/TTS/bin/train_vocoder_wavegrad.py @@ -354,6 +354,7 @@ def main(args): # pylint: disable=redefined-outer-name criterion.cuda() if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location='cpu') try: print(" > Restoring Model...") @@ -397,7 +398,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") diff --git a/TTS/bin/train_vocoder_wavernn.py b/TTS/bin/train_vocoder_wavernn.py index 44ffef14..5fde5025 100644 --- a/TTS/bin/train_vocoder_wavernn.py +++ b/TTS/bin/train_vocoder_wavernn.py @@ -383,6 +383,7 @@ def main(args): # pylint: disable=redefined-outer-name # restore any checkpoint if args.restore_path: + print(f" > Restoring from {os.path.basename(args.restore_path)}...") checkpoint = torch.load(args.restore_path, map_location="cpu") try: print(" > Restoring Model...") @@ -420,7 +421,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = float('inf') print(" > Starting with inf best loss.") else: - print(args.best_path) + print(" > Restoring best loss from " + f"{os.path.basename(args.best_path)} ...") best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index d05936dc..44345dd5 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -169,7 +169,6 @@ def process_args(args, model_type): args.restore_path, best_model = get_last_models(args.continue_path) if not args.best_path: args.best_path = best_model - print(f" > Training continues for {args.restore_path}") # setup output paths and read configs c = load_config(args.config_path) @@ -184,8 +183,7 @@ def process_args(args, model_type): if model_class == "TTS": check_config_tts(c) elif model_class == "VOCODER": - print("Vocoder config checker not implemented, " - "skipping ...") + print("Vocoder config checker not implemented, skipping ...") else: raise ValueError(f"model type {model_type} not recognized!") From 702dff3edcefd4b9e7d43cc8ec618f7c21c8d75a Mon Sep 17 00:00:00 2001 From: gerazov Date: Fri, 12 Feb 2021 11:03:52 +0100 Subject: [PATCH 3/5] added keep_best and keep_after to test configs. --- tests/inputs/test_glow_tts.json | 2 + tests/inputs/test_speedy_speech.json | 2 + tests/inputs/test_train_config.json | 352 +++++++++--------- .../test_vocoder_multiband_melgan_config.json | 2 + tests/inputs/test_vocoder_wavegrad.json | 2 + tests/inputs/test_vocoder_wavernn_config.json | 2 + 6 files changed, 187 insertions(+), 175 deletions(-) diff --git a/tests/inputs/test_glow_tts.json b/tests/inputs/test_glow_tts.json index e7d86eef..338ed8ec 100644 --- a/tests/inputs/test_glow_tts.json +++ b/tests/inputs/test_glow_tts.json @@ -106,6 +106,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. "apex_amp_level": null, diff --git a/tests/inputs/test_speedy_speech.json b/tests/inputs/test_speedy_speech.json index ae4b8b2d..4f9f36bc 100644 --- a/tests/inputs/test_speedy_speech.json +++ b/tests/inputs/test_speedy_speech.json @@ -111,6 +111,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n "mixed_precision": false, diff --git a/tests/inputs/test_train_config.json b/tests/inputs/test_train_config.json index cfd33669..8c9e20d3 100644 --- a/tests/inputs/test_train_config.json +++ b/tests/inputs/test_train_config.json @@ -1,175 +1,177 @@ -{ - "model": "Tacotron2", - "run_name": "test_sample_dataset_run", - "run_description": "sample dataset test run", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // VOCABULARY PARAMETERS - // if custom character set is not defined, - // default set in symbols.py is used - // "characters":{ - // "pad": "_", - // "eos": "~", - // "bos": "^", - // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", - // "punctuations":"!'(),-.:;? ", - // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" - // }, - - // DISTRIBUTED TRAINING - "distributed":{ - "backend": "nccl", - "url": "tcp:\/\/localhost:54321" - }, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "eval_batch_size":1, - "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. - "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "mixed_precision": false, - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // LOSS SETTINGS - "loss_masking": true, // enable / disable loss masking against the sequence padding. - "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled - "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled - "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled - "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled - "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled - "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. - "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. - "epochs": 1, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "wd": 0.000001, // Weight decay weight. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. - - // TACOTRON PRENET - "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "bn", // "original" or "bn". - "prenet_dropout": false, // enable/disable dropout at prenet. - - // TACOTRON ATTENTION - "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' - "attention_heads": 4, // number of attention heads (only for 'graves') - "attention_norm": "sigmoid", // softmax or sigmoid. - "windowing": false, // Enables attention windowing. Used only in eval mode. - "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. - "transition_agent": false, // enable/disable transition agent of forward attention. - "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. - "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. - "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ - "ddc_r": 7, // reduction rate for coarse decoder. - - // STOPNET - "stopnet": true, // Train stopnet predicting the end of synthesis. - "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. - - // TENSORBOARD and LOGGING - "print_step": 1, // Number of steps to log training on console. - "tb_plot_step": 100, // Number of steps to plot TB training figures. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "text_cleaner": "phoneme_cleaners", - "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "num_loader_workers": 0, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 0, // number of evaluation data loader processes. - "batch_group_size": 0, //Number of batches to shuffle after bucketing. - "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training - "max_seq_len": 153, // DATASET-RELATED: maximum text length - "compute_input_seq_cache": true, - - // PATHS - "output_path": "tests/train_outputs/", - - // PHONEMES - "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. - "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages - - // MULTI-SPEAKER and GST - "use_external_speaker_embedding_file": false, - "external_speaker_embedding_file": null, - "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. - "use_gst": true, // use global style tokens - "gst": { // gst parameter if gst is enabled - "gst_style_input": null, // Condition the style input either on a - // -> wave file [path to wave] or - // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} - // with the dictionary being len(dict) == len(gst_style_tokens). - "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. - "gst_embedding_dim": 512, - "gst_num_heads": 4, - "gst_style_tokens": 10 - }, - - // DATASETS - "train_portion": 0.1, // dataset portion used for training. It is mainly for internal experiments. - "eval_portion": 0.1, // dataset portion used for training. It is mainly for internal experiments. - "datasets": // List of datasets. They all merged and they get different speaker_ids. - [ - { - "name": "ljspeech", - "path": "tests/data/ljspeech/", - "meta_file_train": "metadata.csv", - "meta_file_val": "metadata.csv" - } - ] - -} - +{ + "model": "Tacotron2", + "run_name": "test_sample_dataset_run", + "run_description": "sample dataset test run", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 20.0, + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // VOCABULARY PARAMETERS + // if custom character set is not defined, + // default set in symbols.py is used + // "characters":{ + // "pad": "_", + // "eos": "~", + // "bos": "^", + // "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ", + // "punctuations":"!'(),-.:;? ", + // "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ" + // }, + + // DISTRIBUTED TRAINING + "distributed":{ + "backend": "nccl", + "url": "tcp:\/\/localhost:54321" + }, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 1, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + "eval_batch_size":1, + "r": 7, // Number of decoder frames to predict per iteration. Set the initial values if gradual training is enabled. + "gradual_training": [[0, 7, 4]], //set gradual training steps [first_step, r, batch_size]. If it is null, gradual training is disabled. For Tacotron, you might need to reduce the 'batch_size' as you proceeed. + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "ga_alpha": 10.0, // weight for guided attention loss. If > 0, guided attention is enabled. + "mixed_precision": false, + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 0, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // LOSS SETTINGS + "loss_masking": true, // enable / disable loss masking against the sequence padding. + "decoder_loss_alpha": 0.5, // original decoder loss weight. If > 0, it is enabled + "postnet_loss_alpha": 0.25, // original postnet loss weight. If > 0, it is enabled + "postnet_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_diff_spec_alpha": 0.25, // differential spectral loss weight. If > 0, it is enabled + "decoder_ssim_alpha": 0.5, // decoder ssim loss weight. If > 0, it is enabled + "postnet_ssim_alpha": 0.25, // postnet ssim loss weight. If > 0, it is enabled + "ga_alpha": 5.0, // weight for guided attention loss. If > 0, guided attention is enabled. + "stopnet_pos_weight": 15.0, // pos class weight for stopnet loss since there are way more negative samples than positive samples. + + // OPTIMIZER + "noam_schedule": false, // use noam warmup and lr schedule. + "grad_clip": 1.0, // upper limit for gradients for clipping. + "epochs": 1, // total number of epochs to train. + "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "wd": 0.000001, // Weight decay weight. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "seq_len_norm": false, // Normalize eash sample loss with its length to alleviate imbalanced datasets. Use it if your dataset is small or has skewed distribution of sequence lengths. + + // TACOTRON PRENET + "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. + "prenet_type": "bn", // "original" or "bn". + "prenet_dropout": false, // enable/disable dropout at prenet. + + // TACOTRON ATTENTION + "attention_type": "original", // 'original' , 'graves', 'dynamic_convolution' + "attention_heads": 4, // number of attention heads (only for 'graves') + "attention_norm": "sigmoid", // softmax or sigmoid. + "windowing": false, // Enables attention windowing. Used only in eval mode. + "use_forward_attn": false, // if it uses forward attention. In general, it aligns faster. + "forward_attn_mask": false, // Additional masking forcing monotonicity only in eval mode. + "transition_agent": false, // enable/disable transition agent of forward attention. + "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. + "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. + "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ + "ddc_r": 7, // reduction rate for coarse decoder. + + // STOPNET + "stopnet": true, // Train stopnet predicting the end of synthesis. + "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. + + // TENSORBOARD and LOGGING + "print_step": 1, // Number of steps to log training on console. + "tb_plot_step": 100, // Number of steps to plot TB training figures. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "text_cleaner": "phoneme_cleaners", + "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. + "num_loader_workers": 0, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 0, // number of evaluation data loader processes. + "batch_group_size": 0, //Number of batches to shuffle after bucketing. + "min_seq_len": 6, // DATASET-RELATED: minimum text length to use in training + "max_seq_len": 153, // DATASET-RELATED: maximum text length + "compute_input_seq_cache": true, + + // PATHS + "output_path": "tests/train_outputs/", + + // PHONEMES + "phoneme_cache_path": "tests/train_outputs/phoneme_cache/", // phoneme computation is slow, therefore, it caches results in the given folder. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages + + // MULTI-SPEAKER and GST + "use_external_speaker_embedding_file": false, + "external_speaker_embedding_file": null, + "use_speaker_embedding": false, // use speaker embedding to enable multi-speaker learning. + "use_gst": true, // use global style tokens + "gst": { // gst parameter if gst is enabled + "gst_style_input": null, // Condition the style input either on a + // -> wave file [path to wave] or + // -> dictionary using the style tokens {'token1': 'value', 'token2': 'value'} example {"0": 0.15, "1": 0.15, "5": -0.15} + // with the dictionary being len(dict) == len(gst_style_tokens). + "gst_use_speaker_embedding": true, // if true pass speaker embedding in attention input GST. + "gst_embedding_dim": 512, + "gst_num_heads": 4, + "gst_style_tokens": 10 + }, + + // DATASETS + "train_portion": 0.1, // dataset portion used for training. It is mainly for internal experiments. + "eval_portion": 0.1, // dataset portion used for training. It is mainly for internal experiments. + "datasets": // List of datasets. They all merged and they get different speaker_ids. + [ + { + "name": "ljspeech", + "path": "tests/data/ljspeech/", + "meta_file_train": "metadata.csv", + "meta_file_val": "metadata.csv" + } + ] + +} + diff --git a/tests/inputs/test_vocoder_multiband_melgan_config.json b/tests/inputs/test_vocoder_multiband_melgan_config.json index 9540b32b..e1d201ab 100644 --- a/tests/inputs/test_vocoder_multiband_melgan_config.json +++ b/tests/inputs/test_vocoder_multiband_melgan_config.json @@ -131,6 +131,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/tests/inputs/test_vocoder_wavegrad.json b/tests/inputs/test_vocoder_wavegrad.json index fc8059ec..5a068751 100644 --- a/tests/inputs/test_vocoder_wavegrad.json +++ b/tests/inputs/test_vocoder_wavegrad.json @@ -101,6 +101,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 10000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/tests/inputs/test_vocoder_wavernn_config.json b/tests/inputs/test_vocoder_wavernn_config.json index d477a66b..4239e8bd 100644 --- a/tests/inputs/test_vocoder_wavernn_config.json +++ b/tests/inputs/test_vocoder_wavernn_config.json @@ -97,6 +97,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" + "keep_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING From 0e78e31dbf98380395060593095ecebabd2bed0e Mon Sep 17 00:00:00 2001 From: gerazov Date: Fri, 12 Feb 2021 11:36:01 +0100 Subject: [PATCH 4/5] reformated docstrings in arguments.py --- TTS/utils/arguments.py | 85 ++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 53 deletions(-) diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 44345dd5..0a06b562 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -19,16 +19,11 @@ from TTS.tts.utils.generic_utils import check_config_tts def parse_arguments(argv): """Parse command line arguments of training scripts. - Parameters - ---------- - argv : list - This is a list of input arguments as given by sys.argv - - Returns - ------- - argparse.Namespace - Parsed arguments. + Args: + argv (list): This is a list of input arguments as given by sys.argv + Returns: + argparse.Namespace: Parsed arguments. """ parser = argparse.ArgumentParser() parser.add_argument( @@ -46,7 +41,8 @@ def parse_arguments(argv): parser.add_argument( "--best_path", type=str, - help="Best model file to be used for extracting best loss.", + help=("Best model file to be used for extracting best loss." + "If not specified, the latest best model in continue path is used"), default="") parser.add_argument( "--config_path", @@ -78,21 +74,14 @@ def get_last_models(path): It is based on globbing for `*.pth.tar` and the RegEx `(checkpoint|best_model)_([0-9]+)`. - Parameters - ---------- - path : list - Path to files to be compared. + Args: + path (list): Path to files to be compared. - Raises - ------ - ValueError - If no checkpoint or best_model files are found. - - Returns - ------- - last_checkpoint : str - Last checkpoint filename. + Raises: + ValueError: If no checkpoint or best_model files are found. + Returns: + last_checkpoint (str): Last checkpoint filename. """ file_names = glob.glob(os.path.join(path, "*.pth.tar")) last_models = {} @@ -130,38 +119,28 @@ def get_last_models(path): def process_args(args, model_type): """Process parsed comand line arguments. - Parameters - ---------- - args : argparse.Namespace or dict like - Parsed input arguments. - model_type : str - Model type used to check config parameters and setup the TensorBoard - logger. One of: - - tacotron - - glow_tts - - speedy_speech - - gan - - wavegrad - - wavernn + Args: + args (argparse.Namespace or dict like): Parsed input arguments. + model_type (str): Model type used to check config parameters and setup + the TensorBoard logger. One of: + - tacotron + - glow_tts + - speedy_speech + - gan + - wavegrad + - wavernn - Raises - ------ - ValueError - If `model_type` is not one of implemented choices. - - Returns - ------- - c : TTS.utils.io.AttrDict - Config paramaters. - out_path : str - Path to save models and logging. - audio_path : str - Path to save generated test audios. - c_logger : TTS.utils.console_logger.ConsoleLogger - Class that does logging to the console. - tb_logger : TTS.utils.tensorboard.TensorboardLogger - Class that does the TensorBoard loggind. + Raises: + ValueError: If `model_type` is not one of implemented choices. + Returns: + c (TTS.utils.io.AttrDict): Config paramaters. + out_path (str): Path to save models and logging. + audio_path (str): Path to save generated test audios. + c_logger (TTS.utils.console_logger.ConsoleLogger): Class that does + logging to the console. + tb_logger (TTS.utils.tensorboard.TensorboardLogger): Class that does + the TensorBoard loggind. """ if args.continue_path: args.output_path = args.continue_path From 61c88beb943db7c9a79176c17aef59757a67dbee Mon Sep 17 00:00:00 2001 From: gerazov Date: Mon, 15 Feb 2021 18:40:17 +0100 Subject: [PATCH 5/5] refactored keep_all_best --- TTS/bin/train_glow_tts.py | 6 +++--- TTS/bin/train_speedy_speech.py | 6 +++--- TTS/bin/train_tacotron.py | 6 +++--- TTS/bin/train_vocoder_gan.py | 6 +++--- TTS/bin/train_vocoder_wavegrad.py | 6 +++--- TTS/bin/train_vocoder_wavernn.py | 6 +++--- TTS/tts/configs/config.json | 4 ++-- TTS/tts/configs/glow_tts_gated_conv.json | 4 ++-- TTS/tts/configs/glow_tts_ljspeech.json | 4 ++-- TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json | 4 ++-- TTS/tts/configs/speedy_speech_ljspeech.json | 4 ++-- TTS/utils/arguments.py | 4 ++-- TTS/vocoder/configs/multiband-melgan_and_rwd_config.json | 4 ++-- TTS/vocoder/configs/multiband_melgan_config.json | 4 ++-- TTS/vocoder/configs/multiband_melgan_config_mozilla.json | 4 ++-- TTS/vocoder/configs/parallel_wavegan_config.json | 4 ++-- TTS/vocoder/configs/universal_fullband_melgan.json | 4 ++-- TTS/vocoder/configs/wavegrad_libritts.json | 4 ++-- TTS/vocoder/configs/wavernn_config.json | 4 ++-- TTS/vocoder/utils/io.py | 4 ++-- tests/inputs/test_glow_tts.json | 4 ++-- tests/inputs/test_speedy_speech.json | 4 ++-- tests/inputs/test_train_config.json | 4 ++-- tests/inputs/test_vocoder_multiband_melgan_config.json | 4 ++-- tests/inputs/test_vocoder_wavegrad.json | 4 ++-- tests/inputs/test_vocoder_wavernn_config.json | 4 ++-- 26 files changed, 58 insertions(+), 58 deletions(-) diff --git a/TTS/bin/train_glow_tts.py b/TTS/bin/train_glow_tts.py index 072ad41b..68471559 100644 --- a/TTS/bin/train_glow_tts.py +++ b/TTS/bin/train_glow_tts.py @@ -548,8 +548,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -571,7 +571,7 @@ def main(args): # pylint: disable=redefined-outer-name target_loss = eval_avg_loss_dict['avg_loss'] best_loss = save_best_model(target_loss, best_loss, model, optimizer, global_step, epoch, c.r, OUT_PATH, - keep_best=keep_best, keep_after=keep_after) + keep_all_best=keep_all_best, keep_after=keep_after) if __name__ == '__main__': diff --git a/TTS/bin/train_speedy_speech.py b/TTS/bin/train_speedy_speech.py index 8e9dbc2e..ebe78b21 100644 --- a/TTS/bin/train_speedy_speech.py +++ b/TTS/bin/train_speedy_speech.py @@ -512,8 +512,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False # define dataloaders train_loader = setup_loader(ap, 1, is_val=False, verbose=True) @@ -533,7 +533,7 @@ def main(args): # pylint: disable=redefined-outer-name target_loss = eval_avg_loss_dict['avg_loss'] best_loss = save_best_model(target_loss, best_loss, model, optimizer, global_step, epoch, c.r, OUT_PATH, - keep_best=keep_best, keep_after=keep_after) + keep_all_best=keep_all_best, keep_after=keep_after) if __name__ == '__main__': diff --git a/TTS/bin/train_tacotron.py b/TTS/bin/train_tacotron.py index 2382c0be..615901e1 100644 --- a/TTS/bin/train_tacotron.py +++ b/TTS/bin/train_tacotron.py @@ -591,8 +591,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False # define data loaders train_loader = setup_loader(ap, @@ -644,7 +644,7 @@ def main(args): # pylint: disable=redefined-outer-name epoch, c.r, OUT_PATH, - keep_best=keep_best, + keep_all_best=keep_all_best, keep_after=keep_after, scaler=scaler.state_dict() if c.mixed_precision else None ) diff --git a/TTS/bin/train_vocoder_gan.py b/TTS/bin/train_vocoder_gan.py index 9043a560..708bf350 100644 --- a/TTS/bin/train_vocoder_gan.py +++ b/TTS/bin/train_vocoder_gan.py @@ -555,8 +555,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with best loss of {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -581,7 +581,7 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, - keep_best=keep_best, + keep_all_best=keep_all_best, keep_after=keep_after, model_losses=eval_avg_loss_dict, ) diff --git a/TTS/bin/train_vocoder_wavegrad.py b/TTS/bin/train_vocoder_wavegrad.py index 271e8d4c..51a31509 100644 --- a/TTS/bin/train_vocoder_wavegrad.py +++ b/TTS/bin/train_vocoder_wavegrad.py @@ -403,8 +403,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -426,7 +426,7 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, - keep_best=keep_best, + keep_all_best=keep_all_best, keep_after=keep_after, model_losses=eval_avg_loss_dict, scaler=scaler.state_dict() if c.mixed_precision else None diff --git a/TTS/bin/train_vocoder_wavernn.py b/TTS/bin/train_vocoder_wavernn.py index 5fde5025..8e9c6a8b 100644 --- a/TTS/bin/train_vocoder_wavernn.py +++ b/TTS/bin/train_vocoder_wavernn.py @@ -426,8 +426,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = torch.load(args.best_path, map_location='cpu')['model_loss'] print(f" > Starting with loaded last best loss {best_loss}.") - keep_best = c.get('keep_best', False) - keep_after = c.get('keep_after', 10000) # void if keep_best False + keep_all_best = c.get('keep_all_best', False) + keep_after = c.get('keep_after', 10000) # void if keep_all_best False global_step = args.restore_step for epoch in range(0, c.epochs): @@ -450,7 +450,7 @@ def main(args): # pylint: disable=redefined-outer-name global_step, epoch, OUT_PATH, - keep_best=keep_best, + keep_all_best=keep_all_best, keep_after=keep_after, model_losses=eval_avg_loss_dict, scaler=scaler.state_dict() if c.mixed_precision else None diff --git a/TTS/tts/configs/config.json b/TTS/tts/configs/config.json index 5bd249d9..ba33acc5 100644 --- a/TTS/tts/configs/config.json +++ b/TTS/tts/configs/config.json @@ -121,8 +121,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/tts/configs/glow_tts_gated_conv.json b/TTS/tts/configs/glow_tts_gated_conv.json index 865c6f29..c4d7b1e5 100644 --- a/TTS/tts/configs/glow_tts_gated_conv.json +++ b/TTS/tts/configs/glow_tts_gated_conv.json @@ -93,8 +93,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. "apex_amp_level": null, diff --git a/TTS/tts/configs/glow_tts_ljspeech.json b/TTS/tts/configs/glow_tts_ljspeech.json index 6e15de10..5a4c47c2 100644 --- a/TTS/tts/configs/glow_tts_ljspeech.json +++ b/TTS/tts/configs/glow_tts_ljspeech.json @@ -105,8 +105,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json index 3cf66870..11e42259 100644 --- a/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json +++ b/TTS/tts/configs/ljspeech_tacotron2_dynamic_conv_attn.json @@ -121,8 +121,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/tts/configs/speedy_speech_ljspeech.json b/TTS/tts/configs/speedy_speech_ljspeech.json index 9f1d3f8b..f61f35cd 100644 --- a/TTS/tts/configs/speedy_speech_ljspeech.json +++ b/TTS/tts/configs/speedy_speech_ljspeech.json @@ -109,8 +109,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n "mixed_precision": false, diff --git a/TTS/utils/arguments.py b/TTS/utils/arguments.py index 0a06b562..87c0c885 100644 --- a/TTS/utils/arguments.py +++ b/TTS/utils/arguments.py @@ -68,7 +68,7 @@ def parse_arguments(argv): return parser.parse_args() -def get_last_models(path): +def get_last_checkpoint(path): """Get latest checkpoint or/and best model in path. It is based on globbing for `*.pth.tar` and the RegEx @@ -145,7 +145,7 @@ def process_args(args, model_type): if args.continue_path: args.output_path = args.continue_path args.config_path = os.path.join(args.continue_path, "config.json") - args.restore_path, best_model = get_last_models(args.continue_path) + args.restore_path, best_model = get_last_checkpoint(args.continue_path) if not args.best_path: args.best_path = best_model diff --git a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json index b4d42f4b..2670c0f3 100644 --- a/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/TTS/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -138,8 +138,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/multiband_melgan_config.json b/TTS/vocoder/configs/multiband_melgan_config.json index af2af8a3..807f0836 100644 --- a/TTS/vocoder/configs/multiband_melgan_config.json +++ b/TTS/vocoder/configs/multiband_melgan_config.json @@ -128,8 +128,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/multiband_melgan_config_mozilla.json b/TTS/vocoder/configs/multiband_melgan_config_mozilla.json index 0f133fa7..255315c8 100644 --- a/TTS/vocoder/configs/multiband_melgan_config_mozilla.json +++ b/TTS/vocoder/configs/multiband_melgan_config_mozilla.json @@ -141,8 +141,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/parallel_wavegan_config.json b/TTS/vocoder/configs/parallel_wavegan_config.json index 85e659f4..193b1f7b 100644 --- a/TTS/vocoder/configs/parallel_wavegan_config.json +++ b/TTS/vocoder/configs/parallel_wavegan_config.json @@ -130,8 +130,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/universal_fullband_melgan.json b/TTS/vocoder/configs/universal_fullband_melgan.json index efb6f3cd..511ae70e 100644 --- a/TTS/vocoder/configs/universal_fullband_melgan.json +++ b/TTS/vocoder/configs/universal_fullband_melgan.json @@ -124,8 +124,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/wavegrad_libritts.json b/TTS/vocoder/configs/wavegrad_libritts.json index 9107d556..ade20a8f 100644 --- a/TTS/vocoder/configs/wavegrad_libritts.json +++ b/TTS/vocoder/configs/wavegrad_libritts.json @@ -103,8 +103,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 5000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/configs/wavernn_config.json b/TTS/vocoder/configs/wavernn_config.json index 220904c9..aa2d7b9f 100644 --- a/TTS/vocoder/configs/wavernn_config.json +++ b/TTS/vocoder/configs/wavernn_config.json @@ -89,8 +89,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": false, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": false, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/TTS/vocoder/utils/io.py b/TTS/vocoder/utils/io.py index 232d972d..60def72a 100644 --- a/TTS/vocoder/utils/io.py +++ b/TTS/vocoder/utils/io.py @@ -64,7 +64,7 @@ def save_checkpoint(model, optimizer, scheduler, model_disc, optimizer_disc, def save_best_model(current_loss, best_loss, model, optimizer, scheduler, model_disc, optimizer_disc, scheduler_disc, current_step, - epoch, out_path, keep_best=False, keep_after=10000, + epoch, out_path, keep_all_best=False, keep_after=10000, **kwargs): if current_loss < best_loss: best_model_name = f'best_model_{current_step}.pth.tar' @@ -82,7 +82,7 @@ def save_best_model(current_loss, best_loss, model, optimizer, scheduler, model_loss=current_loss, **kwargs) # only delete previous if current is saved successfully - if not keep_best or (current_step < keep_after): + if not keep_all_best or (current_step < keep_after): model_names = glob.glob( os.path.join(out_path, 'best_model*.pth.tar')) for model_name in model_names: diff --git a/tests/inputs/test_glow_tts.json b/tests/inputs/test_glow_tts.json index 338ed8ec..0ee9395b 100644 --- a/tests/inputs/test_glow_tts.json +++ b/tests/inputs/test_glow_tts.json @@ -106,8 +106,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. "apex_amp_level": null, diff --git a/tests/inputs/test_speedy_speech.json b/tests/inputs/test_speedy_speech.json index 4f9f36bc..c4e27737 100644 --- a/tests/inputs/test_speedy_speech.json +++ b/tests/inputs/test_speedy_speech.json @@ -111,8 +111,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 5000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.:set n "mixed_precision": false, diff --git a/tests/inputs/test_train_config.json b/tests/inputs/test_train_config.json index 8c9e20d3..14449867 100644 --- a/tests/inputs/test_train_config.json +++ b/tests/inputs/test_train_config.json @@ -122,8 +122,8 @@ "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/tests/inputs/test_vocoder_multiband_melgan_config.json b/tests/inputs/test_vocoder_multiband_melgan_config.json index e1d201ab..92deaee4 100644 --- a/tests/inputs/test_vocoder_multiband_melgan_config.json +++ b/tests/inputs/test_vocoder_multiband_melgan_config.json @@ -131,8 +131,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/tests/inputs/test_vocoder_wavegrad.json b/tests/inputs/test_vocoder_wavegrad.json index 5a068751..f6208e8d 100644 --- a/tests/inputs/test_vocoder_wavegrad.json +++ b/tests/inputs/test_vocoder_wavegrad.json @@ -101,8 +101,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 10000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": true, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING diff --git a/tests/inputs/test_vocoder_wavernn_config.json b/tests/inputs/test_vocoder_wavernn_config.json index 4239e8bd..decafa70 100644 --- a/tests/inputs/test_vocoder_wavernn_config.json +++ b/tests/inputs/test_vocoder_wavernn_config.json @@ -97,8 +97,8 @@ "print_eval": false, // If True, it prints loss values for each step in eval run. "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" - "keep_best": true, // If true, keeps all best_models after keep_after steps - "keep_after": 10000, // Global step after which to keep best models if keep_best is true + "keep_all_best": true, // If true, keeps all best_models after keep_after steps + "keep_after": 10000, // Global step after which to keep best models if keep_all_best is true "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. // DATA LOADING