diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index a39bfccf..055062fe 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -11,8 +11,8 @@ import torch from torch.utils.data import DataLoader from TTS.speaker_encoder.dataset import MyDataset -from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss, SoftmaxLoss, SoftmaxAngleProtoLoss -from TTS.speaker_encoder.utils.generic_utils import check_config_speaker_encoder, save_best_model, save_checkpoint, setup_model +from TTS.speaker_encoder.losses import AngleProtoLoss, GE2ELoss, SoftmaxAngleProtoLoss +from TTS.speaker_encoder.utils.generic_utils import check_config_speaker_encoder, save_best_model, setup_model from TTS.speaker_encoder.utils.visual import plot_embeddings from TTS.tts.datasets.preprocess import load_meta_data from TTS.utils.audio import AudioProcessor @@ -51,9 +51,8 @@ def setup_loader(ap: AudioProcessor, is_val: bool = False, verbose: bool = False storage_size=c.storage["storage_size"], sample_from_storage_p=c.storage["sample_from_storage_p"], verbose=verbose, - augmentation_config=getattr(c, "audio_augmentation", None) + augmentation_config=getattr(c, "audio_augmentation", None) ) - # sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader( dataset, @@ -65,8 +64,7 @@ def setup_loader(ap: AudioProcessor, is_val: bool = False, verbose: bool = False return loader, dataset.get_num_speakers() -def train(model, optimizer, scheduler, criterion, data_loader, ap, global_step): - +def train(model, optimizer, scheduler, criterion, data_loader, global_step): model.train() epoch_time = 0 best_loss = float("inf") @@ -80,7 +78,6 @@ def train(model, optimizer, scheduler, criterion, data_loader, ap, global_step): # setup input data inputs, labels = data - loader_time = time.time() - end_time global_step += 1 @@ -139,7 +136,6 @@ def train(model, optimizer, scheduler, criterion, data_loader, ap, global_step): ), flush=True, ) - avg_loss_all += avg_loss if global_step % c.save_step == 0: @@ -215,8 +211,7 @@ def main(args): # pylint: disable=redefined-outer-name criterion.cuda() global_step = args.restore_step - # save_checkpoint(model, optimizer, criterion, 0.9, '../', global_step, 1) - _, global_step = train(model, optimizer, scheduler, criterion, data_loader, ap, global_step) + _, global_step = train(model, optimizer, scheduler, criterion, data_loader, global_step) if __name__ == "__main__": diff --git a/TTS/speaker_encoder/configs/config_resnet_angleproto.json b/TTS/speaker_encoder/configs/config_resnet_angleproto.json new file mode 100644 index 00000000..7cae1b25 --- /dev/null +++ b/TTS/speaker_encoder/configs/config_resnet_angleproto.json @@ -0,0 +1,956 @@ +{ + "model_name": "resnet", + "run_name": "speaker_encoder", + "run_description": "resnet speaker encoder trained with commonvoice all languages dev and train, Voxceleb 1 dev and Voxceleb 2 dev", + // AUDIO PARAMETERS + "audio":{ + // Audio processing parameters + "num_mels": 80, // size of the mel spec frame. + "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. + "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "min_level_db": -100, // normalization range + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + "stft_pad_mode": "reflect", + // Normalization parameters + "signal_norm": true, // normalize the spec values in range [0, 1] + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + "spec_gain": 20.0, + "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + "reinit_layers": [], + + "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss, "angleproto" to use Angular Prototypical loss and "softmaxproto" to use Softmax with Angular Prototypical loss + "grad_clip": 3.0, // upper limit for gradients for clipping. + "epochs": 1000, // total number of epochs to train. + "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_decay": false, // if true, Noam learning rate decaying is applied through training. + "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + "steps_plot_stats": 100, // number of steps to plot embeddings. + + // Speakers config + "num_speakers_in_batch": 200, // Batch size for training. + "num_utters_per_speaker": 2, // + "skip_speakers": true, // skip speakers with samples less than "num_utters_per_speaker" + + "voice_len": 2, // number of seconds for each training instance + + "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "wd": 0.000001, // Weight decay weight. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "save_step": 1000, // Number of training steps expected to save the best checkpoints in training. + "print_step": 50, // Number of steps to log traning on console. + "output_path": "../../../checkpoints/speaker_encoder/resnet_voxceleb1_and_voxceleb2-and-common-voice-all-using-angleproto/", // DATASET-RELATED: output path for all training outputs. + + "audio_augmentation": { + "p": 0.5, // propability of apply this method, 0 is disable rir and additive noise augmentation + "rir":{ + "rir_path": "/workspace/store/ecasanova/ComParE/RIRS_NOISES/simulated_rirs/", + "conv_mode": "full" + }, + "additive":{ + "sounds_path": "/workspace/store/ecasanova/ComParE/musan/", + // list of each of the directories in your data augmentation, if a directory is in "sounds_path" but is not listed here it will be ignored + "speech":{ + "min_snr_in_db": 13, + "max_snr_in_db": 20, + "min_num_noises": 2, + "max_num_noises": 3 + }, + "noise":{ + "min_snr_in_db": 0, + "max_snr_in_db": 15, + "min_num_noises": 1, + "max_num_noises": 1 + }, + "music":{ + "min_snr_in_db": 5, + "max_snr_in_db": 15, + "min_num_noises": 1, + "max_num_noises": 1 + } + }, + //add a gaussian noise to the data in order to increase robustness + "gaussian":{ // as the insertion of Gaussian noise is quick to be calculated, we added it after loading the wav file, this way, even audios that were reused with the cache can receive this noise + "p": 0.5, // propability of apply this method, 0 is disable + "min_amplitude": 0.0, + "max_amplitude": 1e-5 + } + }, + "model": { + "input_dim": 80, + "proj_dim": 512 + }, + "storage": { + "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage + "storage_size": 35 // the size of the in-memory storage with respect to a single batch + }, + "datasets": + [ + { + "name": "voxceleb2", + "path": "/workspace/scratch/ecasanova/datasets/VoxCeleb/vox2_dev_aac/", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "voxceleb1", + "path": "/workspace/scratch/ecasanova/datasets/VoxCeleb/vox1_dev_wav/", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fi", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fi", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-CN", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-CN", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rm-sursilv", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rm-sursilv", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lt", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lt", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ka", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ka", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sv-SE", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sv-SE", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pl", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pl", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ru", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ru", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/mn", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/mn", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/nl", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/nl", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sl", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sl", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/es", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/es", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pt", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pt", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hi", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hi", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ja", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ja", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ia", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ia", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/br", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/br", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/id", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/id", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/dv", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/dv", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ta", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ta", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/or", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/or", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-HK", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-HK", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/de", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/de", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/uk", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/uk", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/en", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/en", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fa", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fa", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/vi", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/vi", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ab", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ab", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sah", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/sah", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/vot", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/vot", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fr", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fr", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/tr", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/tr", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lg", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lg", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/mt", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/mt", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rw", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rw", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hu", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hu", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rm-vallader", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/rm-vallader", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/el", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/el", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/tt", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/tt", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-TW", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/zh-TW", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/et", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/et", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fy-NL", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/fy-NL", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cs", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cs", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/as", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/as", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ro", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ro", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/eo", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/eo", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pa-IN", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pa-IN", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/th", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/th", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/it", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/it", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ga-IE", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ga-IE", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cnh", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cnh", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ky", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ky", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ar", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ar", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/eu", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/eu", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ca", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ca", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/kab", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/kab", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cy", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cy", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cv", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/cv", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hsb", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/hsb", + "meta_file_train": "dev.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lv", + "meta_file_train": "train.tsv", + "meta_file_val": null + }, + + { + "name": "common_voice", + "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/lv", + "meta_file_train": "dev.tsv", + "meta_file_val": null + } + + ] +} \ No newline at end of file diff --git a/TTS/speaker_encoder/configs/config_resnet_commonvoice.json b/TTS/speaker_encoder/configs/config_resnet_softmax_angleproto.json similarity index 99% rename from TTS/speaker_encoder/configs/config_resnet_commonvoice.json rename to TTS/speaker_encoder/configs/config_resnet_softmax_angleproto.json index b3223824..4baca49a 100644 --- a/TTS/speaker_encoder/configs/config_resnet_commonvoice.json +++ b/TTS/speaker_encoder/configs/config_resnet_softmax_angleproto.json @@ -54,7 +54,7 @@ "checkpoint": true, // If true, it saves checkpoints per "save_step" "save_step": 1000, // Number of training steps expected to save the best checkpoints in training. "print_step": 50, // Number of steps to log traning on console. - "output_path": "../../../checkpoints/speaker_encoder/resnet_voxceleb1_and_voxceleb2-and-common-voice-all-continue/", // DATASET-RELATED: output path for all training outputs. + "output_path": "../../../checkpoints/speaker_encoder/resnet_voxceleb1_and_voxceleb2-and-common-voice-all/", // DATASET-RELATED: output path for all training outputs. "audio_augmentation": { "p": 0.5, // propability of apply this method, 0 is disable rir and additive noise augmentation diff --git a/TTS/speaker_encoder/configs/config_resnet_voxcebel_trainer.json b/TTS/speaker_encoder/configs/config_resnet_voxcebel_trainer.json deleted file mode 100644 index bf9d1c4c..00000000 --- a/TTS/speaker_encoder/configs/config_resnet_voxcebel_trainer.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "model_name": "resnet", - "run_name": "speaker_encoder", - "run_description": "train speaker encoder with VoxCeleb", - "audio":{ - // Audio processing parameters - "num_mels": 64, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "stft_pad_mode": "reflect", - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, - "do_trim_silence": true, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - "reinit_layers": [], - - "loss": "softmaxproto", // "ge2e" to use Generalized End-to-End loss, "angleproto" to use Angular Prototypical loss and "softmaxproto" to use Softmax with Angular Prototypical loss - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - - // Speakers config - "num_speakers_in_batch": 256, // Batch size for training. - "num_utters_per_speaker": 2, // - "skip_speakers": true, // skip speakers with samples less than "num_utters_per_speaker" - - "voice_len": 2, // number of seconds for each training instance - - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 2000, // Number of training steps expected to save traning stats and checkpoints. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../../checkpoints/speaker_encoder/continue-training-voxceleb-trainer/", // DATASET-RELATED: output path for all training outputs. - - "audio_augmentation": { - "p": 0.75, // propability of apply this method, 0 is disable rir and additive noise augmentation - "rir":{ - "rir_path": "/workspace/store/ecasanova/ComParE/RIRS_NOISES/simulated_rirs/", - "conv_mode": "full" - }, - "additive":{ - "sounds_path": "/workspace/store/ecasanova/ComParE/musan/", - // list of each of the directories in your data augmentation, if a directory is in "sounds_path" but is not listed here it will be ignored - "speech":{ - "min_snr_in_db": 13, - "max_snr_in_db": 20, - "min_num_noises": 3, - "max_num_noises": 7 - }, - "noise":{ - "min_snr_in_db": 0, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - }, - "music":{ - "min_snr_in_db": 5, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - } - }, - //add a gaussian noise to the data in order to increase robustness - "gaussian":{ // as the insertion of Gaussian noise is quick to be calculated, we added it after loading the wav file, this way, even audios that were reused with the cache can receive this noise - "p": 0.5, // propability of apply this method, 0 is disable - "min_amplitude": 0.0, - "max_amplitude": 1e-5 - } - }, - "model": { - "input_dim": 64, - "proj_dim": 512 - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 25 // the size of the in-memory storage with respect to a single batch - }, - "datasets": - [ - { - "name": "voxceleb2", - "path": "/workspace/scratch/ecasanova/datasets/VoxCeleb/vox2_dev_aac/", - "meta_file_train": null, - "meta_file_val": null - } - ] -} \ No newline at end of file diff --git a/TTS/speaker_encoder/configs/config_resnet_voxceleb1and2.json b/TTS/speaker_encoder/configs/config_resnet_voxceleb1and2.json deleted file mode 100644 index 3fafd165..00000000 --- a/TTS/speaker_encoder/configs/config_resnet_voxceleb1and2.json +++ /dev/null @@ -1,117 +0,0 @@ - -{ - "model_name": "resnet", - "run_name": "speaker_encoder", - "run_description": "resnet speaker encoder trained with commonvoice all languages dev and train, Voxceleb dev and Voxceleb 2 dev", - // AUDIO PARAMETERS - "audio":{ - // Audio processing parameters - "num_mels": 80, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "stft_pad_mode": "reflect", - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, - "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - "reinit_layers": [], - - "loss": "softmaxproto", // "ge2e" to use Generalized End-to-End loss, "angleproto" to use Angular Prototypical loss and "softmaxproto" to use Softmax with Angular Prototypical loss - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - - // Speakers config - "num_speakers_in_batch": 200, // Batch size for training. - "num_utters_per_speaker": 2, // - "skip_speakers": true, // skip speakers with samples less than "num_utters_per_speaker" - - "voice_len": 2, // number of seconds for each training instance - - "num_loader_workers": 1, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 2000, // Number of training steps expected to save the best checkpoints in training. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../../checkpoints/speaker_encoder/resnet_voxceleb12-pre-training/", // DATASET-RELATED: output path for all training outputs. - - "audio_augmentation": { - "p": 0.75, // propability of apply this method, 0 is disable rir and additive noise augmentation - "rir":{ - "rir_path": "/workspace/store/ecasanova/ComParE/RIRS_NOISES/simulated_rirs/", - "conv_mode": "full" - }, - "additive":{ - "sounds_path": "/workspace/store/ecasanova/ComParE/musan/", - // list of each of the directories in your data augmentation, if a directory is in "sounds_path" but is not listed here it will be ignored - "speech":{ - "min_snr_in_db": 13, - "max_snr_in_db": 20, - "min_num_noises": 3, - "max_num_noises": 7 - }, - "noise":{ - "min_snr_in_db": 0, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - }, - "music":{ - "min_snr_in_db": 5, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - } - }, - //add a gaussian noise to the data in order to increase robustness - "gaussian":{ // as the insertion of Gaussian noise is quick to be calculated, we added it after loading the wav file, this way, even audios that were reused with the cache can receive this noise - "p": 1, // propability of apply this method, 0 is disable - "min_amplitude": 0.0, - "max_amplitude": 1e-5 - } - }, - "model": { - "input_dim": 80, - "proj_dim": 512 - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 35 // the size of the in-memory storage with respect to a single batch - }, - "datasets": - [ - { - "name": "voxceleb2", - "path": "/workspace/scratch/ecasanova/datasets/VoxCeleb/vox2_dev_aac/", - "meta_file_train": null, - "meta_file_val": null - }, - { - "name": "voxceleb1", - "path": "/workspace/scratch/ecasanova/datasets/VoxCeleb/vox1_dev_wav/", - "meta_file_train": null, - "meta_file_val": null - } - ] -} \ No newline at end of file diff --git a/TTS/speaker_encoder/configs/config_softmaxproto.json b/TTS/speaker_encoder/configs/config_softmaxproto.json deleted file mode 100644 index 2283eb9c..00000000 --- a/TTS/speaker_encoder/configs/config_softmaxproto.json +++ /dev/null @@ -1,117 +0,0 @@ - -{ - "model_name": "resnet", - "run_name": "speaker_encoder", - "run_description": "train speaker encoder with VCTK", - "audio":{ - // Audio processing parameters - "num_mels": 80, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 16000, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "stft_pad_mode": "reflect", - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, - "do_trim_silence": true, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - "reinit_layers": [], - - "loss": "softmaxproto", // "ge2e" to use Generalized End-to-End loss, "angleproto" to use Angular Prototypical loss and "softmaxproto" to use Softmax with Angular Prototypical loss - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - - // Speakers config - "num_speakers_in_batch": 128, // Batch size for training. - "num_utters_per_speaker": 2, // - "skip_speakers": true, // skip speakers with samples less than "num_utters_per_speaker" - - "voice_len": 2, // number of seconds for each training instance - - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../../checkpoints/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. - - "audio_augmentation": { - "p": 0.75, // propability of apply this method, 0 is disable rir and additive noise augmentation - "rir":{ - "rir_path": "/workspace/store/ecasanova/ComParE/RIRS_NOISES/simulated_rirs/", - "conv_mode": "full" - }, - "additive":{ - "sounds_path": "/workspace/store/ecasanova/ComParE/musan/", - // list of each of the directories in your data augmentation, if a directory is in "sounds_path" but is not listed here it will be ignored - "speech":{ - "min_snr_in_db": 13, - "max_snr_in_db": 20, - "min_num_noises": 3, - "max_num_noises": 7 - }, - "noise":{ - "min_snr_in_db": 0, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - }, - "music":{ - "min_snr_in_db": 5, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - } - }, - //add a gaussian noise to the data in order to increase robustness - "gaussian":{ // as the insertion of Gaussian noise is quick to be calculated, we added it after loading the wav file, this way, even audios that were reused with the cache can receive this noise - "p": 1, // propability of apply this method, 0 is disable - "min_amplitude": 0.0, - "max_amplitude": 1e-5 - } - }, - "model": { - "input_dim": 80, - "proj_dim": 512 - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 1 // the size of the in-memory storage with respect to a single batch - }, - "datasets": - [ - - { - "name": "common_voice", - "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/ab/", - "meta_file_train": "train.tsv", - "meta_file_val": "test.tsv" - }, - { - "name": "vctk", - "path": "/workspace/store/ecasanova/datasets/VCTK-Corpus-removed-silence/", - "meta_file_train": null, - "meta_file_val": null - } - ] -} \ No newline at end of file diff --git a/TTS/speaker_encoder/configs/config_temp.json b/TTS/speaker_encoder/configs/config_temp.json deleted file mode 100644 index 737b16d8..00000000 --- a/TTS/speaker_encoder/configs/config_temp.json +++ /dev/null @@ -1,117 +0,0 @@ - -{ - "model_name": "resnet", - "run_name": "speaker_encoder", - "run_description": "resnet speaker encoder trained with commonvoice all languages dev and train, Voxceleb dev and Voxceleb 2 dev", - // AUDIO PARAMETERS - "audio":{ - // Audio processing parameters - "num_mels": 80, // size of the mel spec frame. - "fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame. - "sample_rate": 22050, //22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - "preemphasis": 0.98, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "min_level_db": -100, // normalization range - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - "stft_pad_mode": "reflect", - // Normalization parameters - "signal_norm": true, // normalize the spec values in range [0, 1] - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "spec_gain": 20.0, - "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - "reinit_layers": [], - - "loss": "softmaxproto", // "ge2e" to use Generalized End-to-End loss, "angleproto" to use Angular Prototypical loss and "softmaxproto" to use Softmax with Angular Prototypical loss - "grad_clip": 3.0, // upper limit for gradients for clipping. - "epochs": 1000, // total number of epochs to train. - "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_decay": false, // if true, Noam learning rate decaying is applied through training. - "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "steps_plot_stats": 10, // number of steps to plot embeddings. - - // Speakers config - "num_speakers_in_batch": 256, // Batch size for training. - "num_utters_per_speaker": 2, // - "skip_speakers": true, // skip speakers with samples less than "num_utters_per_speaker" - - "voice_len": 2, // number of seconds for each training instance - - "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "wd": 0.000001, // Weight decay weight. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "save_step": 5000, // Number of training steps expected to save the best checkpoints in training. - "print_step": 20, // Number of steps to log traning on console. - "output_path": "../../../checkpoints/speaker_encoder/continue-training-voxceleb-trainer-test/", // DATASET-RELATED: output path for all training outputs. - - "audio_augmentation": { - "p": 0.75, // propability of apply this method, 0 is disable rir and additive noise augmentation - "rir":{ - "rir_path": "/workspace/store/ecasanova/ComParE/RIRS_NOISES/simulated_rirs/", - "conv_mode": "full" - }, - "additive":{ - "sounds_path": "/workspace/store/ecasanova/ComParE/musan/", - // list of each of the directories in your data augmentation, if a directory is in "sounds_path" but is not listed here it will be ignored - "speech":{ - "min_snr_in_db": 13, - "max_snr_in_db": 20, - "min_num_noises": 3, - "max_num_noises": 7 - }, - "noise":{ - "min_snr_in_db": 0, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - }, - "music":{ - "min_snr_in_db": 5, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - } - }, - //add a gaussian noise to the data in order to increase robustness - "gaussian":{ // as the insertion of Gaussian noise is quick to be calculated, we added it after loading the wav file, this way, even audios that were reused with the cache can receive this noise - "p": 1, // propability of apply this method, 0 is disable - "min_amplitude": 0.0, - "max_amplitude": 1e-5 - } - }, - "model": { - "input_dim": 80, - "proj_dim": 512 - }, - "storage": { - "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage - "storage_size": 35 // the size of the in-memory storage with respect to a single batch - }, - "datasets": - [ - { - "name": "common_voice", - "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pt", - "meta_file_train": "dev.tsv", - "meta_file_val": null - }, - { - "name": "common_voice", - "path": "/workspace/scratch/ecasanova/datasets/common-voice/cv-corpus-6.1-2020-12-11_16khz/pt", - "meta_file_train": "dev.tsv", - "meta_file_val": null - } - ] -} \ No newline at end of file diff --git a/TTS/speaker_encoder/dataset.py b/TTS/speaker_encoder/dataset.py index 9c308a73..0b673753 100644 --- a/TTS/speaker_encoder/dataset.py +++ b/TTS/speaker_encoder/dataset.py @@ -8,18 +8,17 @@ from TTS.speaker_encoder.utils.generic_utils import AugmentWAV, Storage class MyDataset(Dataset): def __init__( - self, - ap, - meta_data, - voice_len=1.6, - num_speakers_in_batch=64, - storage_size=1, - sample_from_storage_p=0.5, - additive_noise= 1e-5, - num_utter_per_speaker=10, - skip_speakers=False, - verbose=False, - augmentation_config=None + self, + ap, + meta_data, + voice_len=1.6, + num_speakers_in_batch=64, + storage_size=1, + sample_from_storage_p=0.5, + num_utter_per_speaker=10, + skip_speakers=False, + verbose=False, + augmentation_config=None ): """ Args: @@ -105,23 +104,6 @@ class MyDataset(Dataset): self.speakers = [k for (k, v) in self.speaker_to_utters.items()] - # def __parse_items(self): - # """ - # Find unique speaker ids and create a dict mapping utterances from speaker id - # """ - # speakers = list({item[-1] for item in self.items}) - # self.speaker_to_utters = {} - # self.speakers = [] - # for speaker in speakers: - # speaker_utters = [item[1] for item in self.items if item[2] == speaker] - # if len(speaker_utters) < self.num_utter_per_speaker and self.skip_speakers: - # print( - # f" [!] Skipped speaker {speaker}. Not enough utterances {self.num_utter_per_speaker} vs {len(speaker_utters)}." - # ) - # else: - # self.speakers.append(speaker) - # self.speaker_to_utters[speaker] = speaker_utters - def __len__(self): return int(1e10) @@ -197,22 +179,26 @@ class MyDataset(Dataset): labels = [] feats = [] speakers = set() - from_disk = 0 - from_storage = 0 + for speaker, speaker_id in batch: speaker_id = int(speaker_id) # ensure that an speaker appears only once in the batch if speaker_id in speakers: + + # remove current speaker + if speaker_id in speakers_id_in_batch: + speakers_id_in_batch.remove(speaker_id) + speaker, _ = self.__sample_speaker(ignore_speakers=speakers_id_in_batch) speaker_id = self.speakerid_to_classid[speaker] + speakers_id_in_batch.add(speaker_id) if random.random() < self.sample_from_storage_p and self.storage.full(): # sample from storage (if full) - # print(help(self.storage)) wavs_, labels_ = self.storage.get_random_sample_fast() - from_storage += 1 - # force choose the current speaker or other not in batch + + # force choose the current speaker or other not in batch # It's necessary for ideal training with AngleProto and GE2E losses if labels_[0] in speakers_id_in_batch and labels_[0] != speaker_id: attempts = 0 @@ -225,13 +211,10 @@ class MyDataset(Dataset): # Try 5 times after that load from disk if attempts >= 5: wavs_, labels_ = self.__load_from_disk_and_storage(speaker) - from_storage -= 1 - from_disk += 1 break else: # don't sample from storage, but from HDD wavs_, labels_ = self.__load_from_disk_and_storage(speaker) - from_disk += 1 # append speaker for control speakers.add(labels_[0]) @@ -258,7 +241,7 @@ class MyDataset(Dataset): feats.extend(feats_) if self.num_speakers_in_batch != len(speakers): - raise ValueError('Speakers appear more than once on the Batch. This cannot happen because the loss functions AngleProto and GE2E consider these samples to be from another speaker.') + raise ValueError('Error: Speakers appear more than once on the Batch. This cannot happen because the loss functions AngleProto and GE2E consider these samples to be from another speaker.') feats = torch.stack(feats) labels = torch.stack(labels) diff --git a/TTS/speaker_encoder/losses.py b/TTS/speaker_encoder/losses.py index e521fe81..f0165739 100644 --- a/TTS/speaker_encoder/losses.py +++ b/TTS/speaker_encoder/losses.py @@ -103,18 +103,18 @@ class GE2ELoss(nn.Module): L.append(L_row) return torch.stack(L) - def forward(self, dvecs, label=None): + def forward(self, x, label=None): """ Calculates the GE2E loss for an input of dimensions (num_speakers, num_utts_per_speaker, dvec_feats) """ assert x.size()[1] >= 2 - centroids = torch.mean(dvecs, 1) - cos_sim_matrix = self.calc_cosine_sim(dvecs, centroids) + centroids = torch.mean(x, 1) + cos_sim_matrix = self.calc_cosine_sim(x, centroids) torch.clamp(self.w, 1e-6) cos_sim_matrix = self.w * cos_sim_matrix + self.b - L = self.embed_loss(dvecs, cos_sim_matrix) + L = self.embed_loss(x, cos_sim_matrix) return L.mean() @@ -141,6 +141,7 @@ class AngleProtoLoss(nn.Module): print(" > Initialised Angular Prototypical loss") + # pylint: disable=W0613 def forward(self, x, label=None): """ Calculates the AngleProto loss for an input of dimensions (num_speakers, num_utts_per_speaker, dvec_feats) diff --git a/TTS/speaker_encoder/models/resnet.py b/TTS/speaker_encoder/models/resnet.py index 23464527..0cb7e0a8 100644 --- a/TTS/speaker_encoder/models/resnet.py +++ b/TTS/speaker_encoder/models/resnet.py @@ -1,18 +1,16 @@ import torch import numpy as np import torch.nn as nn -import torch.nn.functional as F -import numpy as np class SELayer(nn.Module): def __init__(self, channel, reduction=8): super(SELayer, self).__init__() self.avg_pool = nn.AdaptiveAvgPool2d(1) self.fc = nn.Sequential( - nn.Linear(channel, channel // reduction), - nn.ReLU(inplace=True), - nn.Linear(channel // reduction, channel), - nn.Sigmoid() + nn.Linear(channel, channel // reduction), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), + nn.Sigmoid() ) def forward(self, x): @@ -57,16 +55,17 @@ class ResNetSpeakerEncoder(nn.Module): """Implementation of the model H/ASP without batch normalization in speaker embedding. This model was proposed in: https://arxiv.org/abs/2009.14153 Adapted from: https://github.com/clovaai/voxceleb_trainer """ + # pylint: disable=W0102 def __init__(self, input_dim=64, proj_dim=512, layers=[3, 4, 6, 3], num_filters=[32, 64, 128, 256], encoder_type='ASP', log_input=False): super(ResNetSpeakerEncoder, self).__init__() self.encoder_type = encoder_type self.input_dim = input_dim self.log_input = log_input - self.conv1 = nn.Conv2d(1, num_filters[0] , kernel_size=3, stride=1, padding=1) + self.conv1 = nn.Conv2d(1, num_filters[0], kernel_size=3, stride=1, padding=1) self.relu = nn.ReLU(inplace=True) self.bn1 = nn.BatchNorm2d(num_filters[0]) - + self.inplanes = num_filters[0] self.layer1 = self.create_layer(SEBasicBlock, num_filters[0], layers[0]) self.layer2 = self.create_layer(SEBasicBlock, num_filters[1], layers[1], stride=(2, 2)) @@ -116,11 +115,12 @@ class ResNetSpeakerEncoder(nn.Module): layers = [] layers.append(block(self.inplanes, planes, stride, downsample)) self.inplanes = planes * block.expansion - for i in range(1, blocks): + for _ in range(1, blocks): layers.append(block(self.inplanes, planes)) return nn.Sequential(*layers) + # pylint: disable=R0201 def new_parameter(self, *size): out = nn.Parameter(torch.FloatTensor(*size)) nn.init.xavier_normal_(out) @@ -130,7 +130,8 @@ class ResNetSpeakerEncoder(nn.Module): x = x.transpose(1, 2) with torch.no_grad(): with torch.cuda.amp.autocast(enabled=False): - if self.log_input: x = (x+1e-6).log() + if self.log_input: + x = (x+1e-6).log() x = self.instancenorm(x).unsqueeze(1) x = self.conv1(x) @@ -150,8 +151,8 @@ class ResNetSpeakerEncoder(nn.Module): x = torch.sum(x * w, dim=2) elif self.encoder_type == "ASP": mu = torch.sum(x * w, dim=2) - sg = torch.sqrt((torch.sum((x**2) * w, dim=2) - mu ** 2 ).clamp(min=1e-5) ) - x = torch.cat((mu, sg),1) + sg = torch.sqrt((torch.sum((x**2) * w, dim=2) - mu ** 2).clamp(min=1e-5)) + x = torch.cat((mu, sg), 1) x = x.view(x.size()[0], -1) x = self.fc(x) @@ -177,11 +178,11 @@ class ResNetSpeakerEncoder(nn.Module): for offset in offsets: offset = int(offset) end_offset = int(offset+num_frames) - frames = x[:,offset:end_offset] + frames = x[:, offset:end_offset] embed = self.forward(frames, training=False) embeddings.append(embed) embeddings = torch.stack(embeddings) if return_mean: embeddings = torch.mean(embeddings, dim=0) - return embeddings \ No newline at end of file + return embeddings diff --git a/TTS/speaker_encoder/utils/generic_utils.py b/TTS/speaker_encoder/utils/generic_utils.py index 78e7cb49..ff8f0447 100644 --- a/TTS/speaker_encoder/utils/generic_utils.py +++ b/TTS/speaker_encoder/utils/generic_utils.py @@ -52,47 +52,18 @@ class Storage(object): storage_size -= self.num_speakers_in_batch return self.storage[random.randint(0, storage_size)] + def get_random_sample_fast(self): '''Call this method only when storage is full''' return self.storage[random.randint(0, self.safe_storage_size)] - + class AugmentWAV(object): def __init__(self, ap, augmentation_config): self.ap = ap - - '''augmentation_config = { - "p": 1, - "rir":{ - "rir_path": "rir_path/" - "conv_mode": "full" - }, - "additive":{ - "sounds_path": "musan/", - # directorys in sounds_path - "speech":{ - "min_snr_in_db": 13, - "max_snr_in_db": 20, - "min_num_noises": 3, - "max_num_noises": 7 - }, - "noise":{ - "min_snr_in_db": 0, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - }, - "music":{ - "min_snr_in_db": 5, - "max_snr_in_db": 15, - "min_num_noises": 1, - "max_num_noises": 1 - } - } - }''' - self.use_additive_noise = False + if 'additive' in augmentation_config.keys(): self.additive_noise_config = augmentation_config['additive'] additive_path = self.additive_noise_config['sounds_path'] @@ -104,7 +75,7 @@ class AugmentWAV(object): if isinstance(self.additive_noise_config[key], dict): self.additive_noise_types.append(key) - additive_files = glob.glob(os.path.join(additive_path,'**/*.wav'), recursive=True) + additive_files = glob.glob(os.path.join(additive_path, '**/*.wav'), recursive=True) self.noise_list = {} @@ -118,12 +89,13 @@ class AugmentWAV(object): self.noise_list[noise_dir].append(wav_file) print(f" | > Using Additive Noise Augmentation: with {len(additive_files)} audios instances from {self.additive_noise_types}") - + self.use_rir = False + if 'rir' in augmentation_config.keys(): self.rir_config = augmentation_config['rir'] if self.rir_config['rir_path']: - self.rir_files = glob.glob(os.path.join(self.rir_config['rir_path'],'**/*.wav'), recursive=True) + self.rir_files = glob.glob(os.path.join(self.rir_config['rir_path'], '**/*.wav'), recursive=True) self.use_rir = True print(f" | > Using RIR Noise Augmentation: with {len(self.rir_files)} audios instances") @@ -161,9 +133,8 @@ class AugmentWAV(object): else: noises_wav += noise_wav - # if all possibel files is less than audio, choose other files + # if all possible files is less than audio, choose other files if noises_wav is None: - print("audio ignorado") return self.additive_noise(noise_type, audio) return audio + noises_wav @@ -172,24 +143,21 @@ class AugmentWAV(object): audio_len = audio.shape[0] rir_file = random.choice(self.rir_files) - rir = self.ap.load_wav(rir_file, sr=self.ap.sample_rate) rir = rir / np.sqrt(np.sum(rir ** 2)) return signal.convolve(audio, rir, mode=self.rir_config['conv_mode'])[:audio_len] def apply_one(self, audio): - return self.reverberate(audio) noise_type = random.choice(self.global_noise_list) if noise_type == "RIR_AUG": return self.reverberate(audio) - else: - return self.additive_noise(noise_type, audio) + + return self.additive_noise(noise_type, audio) def to_camel(text): text = text.capitalize() return re.sub(r"(?!^)_([a-zA-Z])", lambda m: m.group(1).upper(), text) - def setup_model(c): if c.model_name.lower() == 'lstm': model = LSTMSpeakerEncoder(c.model["input_dim"], c.model["proj_dim"], c.model["lstm_dim"], c.model["num_lstm_layers"]) @@ -286,8 +254,9 @@ def check_config_speaker_encoder(c): check_argument("model", c, restricted=True, val_type=dict) check_argument("model_name", c, restricted=True, val_type=str) check_argument("input_dim", c["model"], restricted=True, val_type=int) + check_argument("proj_dim", c["model"], restricted=True, val_type=int) + if c.model_name.lower() == 'lstm': - check_argument("proj_dim", c["model"], restricted=True, val_type=int) check_argument("lstm_dim", c["model"], restricted=True, val_type=int) check_argument("num_lstm_layers", c["model"], restricted=True, val_type=int) check_argument("use_lstm_with_projection", c["model"], restricted=True, val_type=bool)