diff --git a/recipes/ljspeech/xtts_v1/train_xtts.py b/recipes/ljspeech/xtts_v1/train_xtts.py index 6c07053b..641d050c 100644 --- a/recipes/ljspeech/xtts_v1/train_xtts.py +++ b/recipes/ljspeech/xtts_v1/train_xtts.py @@ -1,11 +1,29 @@ +import os + from trainer import Trainer, TrainerArgs from TTS.config.shared_configs import BaseDatasetConfig from TTS.tts.datasets import load_tts_samples from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig -# Define here the dataset used -config_ljspeech = BaseDatasetConfig( +# Logging parameters +RUN_NAME = "GPT_XTTS_LJSpeech_FT" +PROJECT_NAME = "XTTS_trainer" +DASHBOARD_LOGGER = "tensorboard" +LOGGER_URI = None + +# Set here the path that the checkpoints will be saved. Default: ./run/training/ +OUT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "run", "training") + +# Training Parameters +OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False +START_WITH_EVAL = True # if True it will star with evaluation +BATCH_SIZE = 3 # set here the batch size +GRAD_ACUMM_STEPS = 84 # set here the grad accumulation steps +# Note: we recommend that BATCH_SIZE * GRAD_ACUMM_STEPS need to be at least 252 for more efficient training. You can increase/decrease BATCH_SIZE but then set GRAD_ACUMM_STEPS accordingly. + +# Define here the dataset that you want to use for the fine tuning +config_dataset = BaseDatasetConfig( formatter="ljspeech", dataset_name="ljspeech", path="/raid/datasets/LJSpeech-1.1_24khz/", @@ -13,11 +31,26 @@ config_ljspeech = BaseDatasetConfig( language="en", ) -DATASETS_CONFIG_LIST = [config_ljspeech] +DATASETS_CONFIG_LIST = [config_dataset] + +# ToDo: update with the latest released checkpoints + +# DVAE parameters: For the training we need the dvae to extract the dvae tokens, given that you must provide the paths for this model +DVAE_CHECKPOINT = "/raid/datasets/xtts_models/dvae.pth" # DVAE checkpoint +MEL_NORM_FILE = ( + "/raid/datasets/xtts_models/mel_stats.pth" # Mel spectrogram norms, required for dvae mel spectrogram extraction +) + +# XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning. +TOKENIZER_FILE = "/raid/edresson/dev/Checkpoints/XTTS_evaluation/xtts_style_emb_repetition_fix_gt/tokenizer_merged_5.json" # vocab.json file +XTTS_CHECKPOINT = "/raid/edresson/dev/Checkpoints/XTTS_evaluation/xtts_style_emb_repetition_fix_gt/132500_gpt_ema_coqui_tts_with_enhanced_hifigan.pth" # model.pth file -def freeze_layers(trainer): - pass +# Training sentences generations +SPEAKER_REFERENCE = ( + "./tests/data/ljspeech/wavs/LJ001-0002.wav" # speaker reference to be used in training test sentences +) +LANGUAGE = config_dataset.language def main(): @@ -28,18 +61,18 @@ def main(): debug_loading_failures=False, max_wav_length=255995, # ~11.6 seconds max_text_length=200, - mel_norm_file="/raid/datasets/xtts_models/mel_stats.pth", - dvae_checkpoint="/raid/datasets/xtts_models/dvae.pth", + mel_norm_file=MEL_NORM_FILE, + dvae_checkpoint=DVAE_CHECKPOINT, # tokenizer_file="/raid/datasets/xtts_models/vocab.json", # vocab path of the model that you want to fine-tune # xtts_checkpoint="https://huggingface.co/coqui/XTTS-v1/resolve/hifigan/model.pth", - xtts_checkpoint="/raid/edresson/dev/Checkpoints/XTTS_evaluation/xtts_style_emb_repetition_fix_gt/132500_gpt_ema_coqui_tts_with_enhanced_hifigan.pth", # checkpoint path of the model that you want to fine-tune - tokenizer_file="/raid/edresson/dev/Checkpoints/XTTS_evaluation/xtts_style_emb_repetition_fix_gt/tokenizer_merged_5.json", + xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune + tokenizer_file=TOKENIZER_FILE, gpt_num_audio_tokens=8194, gpt_start_audio_token=8192, gpt_stop_audio_token=8193, ) audio_config = XttsAudioConfig( - sample_rate=22050, dvae_sample_rate=22050, diffusion_sample_rate=24000, output_sample_rate=24000 # GPT SR + sample_rate=22050, dvae_sample_rate=22050, diffusion_sample_rate=24000, output_sample_rate=24000 ) config = GPTTrainerConfig( output_path=OUT_PATH, @@ -67,7 +100,7 @@ def main(): print_eval=False, # Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters. optimizer="AdamW", - optimizer_wd_only_on_weights=True, # for multi-gpu training turn it off + optimizer_wd_only_on_weights=OPTIMIZER_WD_ONLY_ON_WEIGHTS, optimizer_params={"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": 1e-2}, lr=5e-06, # learning rate lr_scheduler="MultiStepLR", @@ -76,18 +109,13 @@ def main(): test_sentences=[ { "text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", - "speaker_wav": "/raid/edresson/dev/ref-ljspeech.wav", - "language": "en", + "speaker_wav": SPEAKER_REFERENCE, + "language": LANGUAGE, }, { "text": "This cake is great. It's so delicious and moist.", - "speaker_wav": "/raid/edresson/dev/ref-ljspeech.wav", - "language": "en", - }, - { - "text": "Levei muito tempo para desenvolver uma voz e agora que a tenho não vou ficar calado .", - "speaker_wav": "/raid/edresson/dev/ref-ljspeech.wav", - "language": "pt", + "speaker_wav": SPEAKER_REFERENCE, + "language": LANGUAGE, }, ], ) @@ -106,8 +134,8 @@ def main(): # init the trainer and 🚀 trainer = Trainer( TrainerArgs( - restore_path=RESTORE_PATH, - skip_train_epoch=SKIP_TRAIN_EPOCH, + restore_path=None, # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter + skip_train_epoch=False, start_with_eval=START_WITH_EVAL, grad_accum_steps=GRAD_ACUMM_STEPS, ), @@ -116,30 +144,9 @@ def main(): model=model, train_samples=train_samples, eval_samples=eval_samples, - callbacks={"on_epoch_start": freeze_layers}, ) trainer.fit() if __name__ == "__main__": - RUN_NAME = "GPT_XTTS_LJSpeech_fixed" - PROJECT_NAME = "XTTS_trainer" - OUT_PATH = "/raid/edresson/dev/Checkpoints/XTTS_v1_FT/" - # DASHBOARD_LOGGER = "clearml" - # LOGGER_URI = "s3://coqui-ai-models/TTS/Checkpoints/XTTS_v1/" - DASHBOARD_LOGGER = "tensorboard" - LOGGER_URI = None - RESTORE_PATH = None - SKIP_TRAIN_EPOCH = False - START_WITH_EVAL = True - BATCH_SIZE = 3 - GRAD_ACUMM_STEPS = 28 * 3 - - # debug - # DASHBOARD_LOGGER = "tensorboard" - # LOGGER_URI = None - # RESTORE_PATH = None - # BATCH_SIZE = 2 - # GRAD_ACUMM_STEPS = 1 - main() diff --git a/recipes/multilingual/xtts_v1/train_xtts.py b/recipes/multilingual/xtts_v1/train_xtts.py deleted file mode 100644 index fa13d8d4..00000000 --- a/recipes/multilingual/xtts_v1/train_xtts.py +++ /dev/null @@ -1,376 +0,0 @@ -from trainer import Trainer, TrainerArgs - -from TTS.config.shared_configs import BaseDatasetConfig -from TTS.tts.datasets import load_tts_samples -from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig - -config_coqui_MLS_metadata_train_with_previous_audio_key_de = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_german", - meta_file_train="metadata_train_with_previous_audio_key.csv", - language="de", -) - - -config_coqui_MLS_metadata_test_with_previous_audio_key_de = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_german", - meta_file_train="metadata_test_with_previous_audio_key.csv", - language="de", -) - - -config_coqui_MLS_metadata_dev_with_previous_audio_key_de = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_german", - meta_file_train="metadata_dev_with_previous_audio_key.csv", - language="de", -) - - -config_coqui_mls_french_metadata_with_previous_audio_key_fr = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_french/", - meta_file_train="metadata_with_previous_audio_key.csv", - language="fr", -) - - -config_coqui_mls_spanish_metadata_with_previous_audio_key_es = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_spanish/", - meta_file_train="/raid/datasets/MLS/mls_spanish/metadata_with_previous_audio_key.csv", - language="es", -) - - -config_coqui_mls_italian_metadata_with_previous_audio_key_it = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_italian/", - meta_file_train="/raid/datasets/MLS/mls_italian/metadata_with_previous_audio_key.csv", - language="it", -) - - -config_coqui_mls_portuguese_metadata_with_previous_audio_key_pt = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_portuguese/", - meta_file_train="/raid/datasets/MLS/mls_portuguese/metadata_with_previous_audio_key.csv", - language="pt", -) - - -config_coqui_mls_polish_metadata_with_previous_audio_key_pl = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/MLS/mls_polish/", - meta_file_train="/raid/datasets/MLS/mls_polish/metadata_with_previous_audio_key.csv", - language="pl", -) - - -config_coqui_common_voice_metafile_it_train_with_scores_it = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_it_train_with_scores.csv", - language="it", -) - - -config_coqui_common_voice_metafile_it_test_with_scores_it = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_it_test_with_scores.csv", - language="it", -) - - -config_coqui_common_voice_metafile_it_dev_with_scores_it = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_it_dev_with_scores.csv", - language="it", -) - - -config_coqui_common_voice_metafile_pt_train_with_scores_pt = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_pt_train_with_scores.csv", - language="pt", -) - - -config_coqui_common_voice_metafile_pt_test_with_scores_pt = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_pt_test_with_scores.csv", - language="pt", -) - - -config_coqui_common_voice_metafile_pt_dev_with_scores_pt = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_pt_dev_with_scores.csv", - language="pt", -) - - -config_coqui_common_voice_metafile_en_train_en = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_en_train.csv", - language="en", -) - - -config_coqui_common_voice_metafile_en_test_en = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_en_test.csv", - language="en", -) - - -config_coqui_common_voice_metafile_en_dev_en = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_en_dev.csv", - language="en", -) - - -config_coqui_common_voice_metafile_tr_validated_tr = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_tr_validated.csv", - language="tr", -) - - -config_coqui_common_voice_metafile_ru_validated_ru = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_ru_validated.csv", - language="ru", -) - - -config_coqui_common_voice_metafile_nl_validated_nl = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_nl_validated.csv", - language="nl", -) - - -config_coqui_common_voice_metafile_cs_validated_cs = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_cs_validated.csv", - language="cs", -) - - -config_coqui_common_voice_metafile_fr_validated_fr = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_fr_validated.csv", - language="fr", -) - - -config_coqui_common_voice_metafile_es_validated_es = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_es_validated.csv", - language="es", -) - - -config_coqui_common_voice_metafile_pl_validated_pl = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_pl_validated.csv", - language="pl", -) - - -config_coqui_common_voice_metafile_ar_validated_ar = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_ar_validated.csv", - language="ar", -) - - -config_coqui_common_voice_metafile_zh_CN_validated_zh_cn = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_zh-CN_validated.csv", - language="zh-cn", -) - - -config_coqui_common_voice_metafile_ja_validated_ja = BaseDatasetConfig( - formatter="coqui", - dataset_name="coqui", - path="/raid/datasets/common_voice/", - meta_file_train="/raid/datasets/common_voice/metafile_ja_validated.csv", - language="ja", -) - -# DATASETS_CONFIG_LIST=[config_coqui_MLS_metadata_train_with_previous_audio_key_de, config_coqui_MLS_metadata_test_with_previous_audio_key_de, config_coqui_MLS_metadata_dev_with_previous_audio_key_de, config_coqui_mls_french_metadata_with_previous_audio_key_fr, config_coqui_mls_spanish_metadata_with_previous_audio_key_es, config_coqui_mls_italian_metadata_with_previous_audio_key_it, config_coqui_mls_portuguese_metadata_with_previous_audio_key_pt, config_coqui_mls_polish_metadata_with_previous_audio_key_pl, config_coqui_common_voice_metafile_it_train_with_scores_it, config_coqui_common_voice_metafile_it_test_with_scores_it, config_coqui_common_voice_metafile_it_dev_with_scores_it, config_coqui_common_voice_metafile_pt_train_with_scores_pt, config_coqui_common_voice_metafile_pt_test_with_scores_pt, config_coqui_common_voice_metafile_pt_dev_with_scores_pt, config_coqui_common_voice_metafile_en_train_en, config_coqui_common_voice_metafile_en_test_en, config_coqui_common_voice_metafile_en_dev_en, config_coqui_common_voice_metafile_tr_validated_tr, config_coqui_common_voice_metafile_ru_validated_ru, config_coqui_common_voice_metafile_nl_validated_nl, config_coqui_common_voice_metafile_cs_validated_cs, config_coqui_common_voice_metafile_fr_validated_fr, config_coqui_common_voice_metafile_es_validated_es, config_coqui_common_voice_metafile_pl_validated_pl, config_coqui_common_voice_metafile_ar_validated_ar, config_coqui_common_voice_metafile_zh_CN_validated_zh_cn, config_coqui_common_voice_metafile_ja_validated_ja] - -# DATASETS_CONFIG_LIST = [config_coqui_mls_french_metadata_with_previous_audio_key_fr, config_coqui_MLS_metadata_test_with_previous_audio_key_de, config_coqui_mls_spanish_metadata_with_previous_audio_key_es, config_coqui_mls_italian_metadata_with_previous_audio_key_it] - -DATASETS_CONFIG_LIST = [ - config_coqui_MLS_metadata_test_with_previous_audio_key_de, - config_coqui_mls_italian_metadata_with_previous_audio_key_it, -] - - -def freeze_layers(trainer): - pass - - -def main(): - # init args and config - model_args = GPTArgs( - max_conditioning_length=132300, # 6 secs - min_conditioning_length=66150, # 3 secs - debug_loading_failures=False, - max_wav_length=255995, # ~11.6 seconds - max_text_length=200, - mel_norm_file="/raid/datasets/xtts_models/mel_stats.pth", - dvae_checkpoint="/raid/datasets/xtts_models/dvae.pth", - tokenizer_file="/raid/datasets/xtts_models/vocab.json", # vocab path of the model that you want to fine-tune - xtts_checkpoint="https://huggingface.co/coqui/XTTS-v1/resolve/hifigan/model.pth", # checkpoint path of the model that you want to fine-tune - gpt_num_audio_tokens=8194, - gpt_start_audio_token=8192, - gpt_stop_audio_token=8193, - ) - audio_config = XttsAudioConfig( - sample_rate=22050, dvae_sample_rate=22050, diffusion_sample_rate=24000, output_sample_rate=24000 # GPT SR - ) - config = GPTTrainerConfig( - output_path=OUT_PATH, - model_args=model_args, - run_name=RUN_NAME, - project_name=PROJECT_NAME, - run_description=""" - GPT XTTS training - """, - dashboard_logger=DASHBOARD_LOGGER, - logger_uri=LOGGER_URI, - audio=audio_config, - batch_size=BATCH_SIZE, - batch_group_size=48, - eval_batch_size=BATCH_SIZE, - num_loader_workers=8, - eval_split_max_size=256, - print_step=50, - plot_step=100, - log_model_step=1000, - save_step=10000, - save_n_checkpoints=1, - save_checkpoints=True, - # target_loss="loss", - print_eval=False, - # Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters. - optimizer="AdamW", - optimizer_wd_only_on_weights=True, # for multi-gpu training turn it off - optimizer_params={"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": 1e-2}, - lr=5e-06, # learning rate - lr_scheduler="MultiStepLR", - # it was adjusted accordly for the new step scheme - lr_scheduler_params={"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}, - test_sentences=[ - { - "text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", - "speaker_wav": "/raid/edresson/dev/ref.wav", - "language": "en", - }, - { - "text": "This cake is great. It's so delicious and moist.", - "speaker_wav": "/raid/edresson/dev/ref.wav", - "language": "en", - }, - ], - ) - - # init the model from config - model = GPTTrainer.init_from_config(config) - - # load training samples - train_samples, eval_samples = load_tts_samples( - DATASETS_CONFIG_LIST, - eval_split=True, - eval_split_max_size=config.eval_split_max_size, - eval_split_size=config.eval_split_size, - ) - - # init the trainer and 🚀 - trainer = Trainer( - TrainerArgs( - restore_path=RESTORE_PATH, - skip_train_epoch=SKIP_TRAIN_EPOCH, - start_with_eval=START_WITH_EVAL, - grad_accum_steps=GRAD_ACUMM_STEPS, - ), - config, - output_path=OUT_PATH, - model=model, - train_samples=train_samples, - eval_samples=eval_samples, - callbacks={"on_epoch_start": freeze_layers}, - ) - trainer.fit() - - -if __name__ == "__main__": - RUN_NAME = "GPT_XTTS" - PROJECT_NAME = "XTTS_trainer" - OUT_PATH = "/raid/edresson/dev/Checkpoints/XTTS_style_emb/" - DASHBOARD_LOGGER = "clearml" - LOGGER_URI = "s3://coqui-ai-models/TTS/Checkpoints/XTTS_style_emb/" - RESTORE_PATH = None - SKIP_TRAIN_EPOCH = False - START_WITH_EVAL = True - BATCH_SIZE = 9 - GRAD_ACUMM_STEPS = 28 - - # debug - # DASHBOARD_LOGGER = "tensorboard" - # LOGGER_URI = None - # RESTORE_PATH = None - BATCH_SIZE = 2 - GRAD_ACUMM_STEPS = 1 - - main()