Medix-AI/coqui-tts
Medix-AI
/
coqui-tts
mirror of https://github.com/coqui-ai/TTS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update `synthesis.py` for the trainer

This commit is contained in:
Eren Gölge 2021-05-25 14:40:25 +02:00
parent 130781dab6
commit f4f83b6379
2 changed files with 97 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
TTS/tts/utils/speakers.py
View File

@ -34,10 +34,6 @@ def save_speaker_mapping(out_path, speaker_mapping):
json.dump(speaker_mapping, f, indent=4) json.dump(speaker_mapping, f, indent=4)
def get_speakers(items):
def parse_speakers(c, args, meta_data_train, OUT_PATH): def parse_speakers(c, args, meta_data_train, OUT_PATH):
"""Returns number of speakers, speaker embedding shape and speaker mapping""" """Returns number of speakers, speaker embedding shape and speaker mapping"""
if c.use_speaker_embedding: if c.use_speaker_embedding:
@ -135,7 +131,7 @@ class SpeakerManager:
): ):
self.data_items = [] self.data_items = []
self.x_vectors = [] self.x_vectors = {}
self.speaker_ids = [] self.speaker_ids = []
self.clip_ids = [] self.clip_ids = []
self.speaker_encoder = None self.speaker_encoder = None
@ -171,7 +167,7 @@ class SpeakerManager:
def x_vector_dim(self): def x_vector_dim(self):
return len(self.x_vectors[list(self.x_vectors.keys())[0]]["embedding"]) return len(self.x_vectors[list(self.x_vectors.keys())[0]]["embedding"])
def parser_speakers_from_items(self, items: list): def parse_speakers_from_items(self, items: list):
speakers = sorted({item[2] for item in items}) speakers = sorted({item[2] for item in items})
self.speaker_ids = {name: i for i, name in enumerate(speakers)} self.speaker_ids = {name: i for i, name in enumerate(speakers)}
num_speakers = len(self.speaker_ids) num_speakers = len(self.speaker_ids)

184
TTS/tts/utils/synthesis.py
View File

@ -13,7 +13,7 @@ if "tensorflow" in installed or "tensorflow-gpu" in installed:
import tensorflow as tf import tensorflow as tf
def text_to_seqvec(text, CONFIG): def text_to_seq(text, CONFIG):
text_cleaner = [CONFIG.text_cleaner] text_cleaner = [CONFIG.text_cleaner]
# text ot phonemes to sequence vector # text ot phonemes to sequence vector
if CONFIG.use_phonemes: if CONFIG.use_phonemes:
@ -59,81 +59,82 @@ def numpy_to_tf(np_array, dtype):
def compute_style_mel(style_wav, ap, cuda=False): def compute_style_mel(style_wav, ap, cuda=False):
style_mel = torch.FloatTensor(ap.melspectrogram(ap.load_wav(style_wav, sr=ap.sample_rate))).unsqueeze(0) style_mel = torch.FloatTensor(
ap.melspectrogram(ap.load_wav(style_wav,
sr=ap.sample_rate))).unsqueeze(0)
if cuda: if cuda:
return style_mel.cuda() return style_mel.cuda()
return style_mel return style_mel
def run_model_torch(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None, speaker_embeddings=None): def run_model_torch(model,
if "tacotron" in CONFIG.model.lower(): inputs,
if CONFIG.gst: speaker_id=None,
decoder_output, postnet_output, alignments, stop_tokens = model.inference( style_mel=None,
inputs, style_mel=style_mel, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings x_vector=None):
) outputs = model.inference(inputs,
else: cond_input={
if truncated: 'speaker_ids': speaker_id,
decoder_output, postnet_output, alignments, stop_tokens = model.inference_truncated( 'x_vector': x_vector,
inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings 'style_mel': style_mel
) })
else: # elif "glow" in CONFIG.model.lower():
decoder_output, postnet_output, alignments, stop_tokens = model.inference( # inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable
inputs, speaker_ids=speaker_id, speaker_embeddings=speaker_embeddings # if hasattr(model, "module"):
) # # distributed model
elif "glow" in CONFIG.model.lower(): # postnet_output, _, _, _, alignments, _, _ = model.module.inference(
inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable # inputs,
if hasattr(model, "module"): # inputs_lengths,
# distributed model # g=speaker_id if speaker_id is not None else speaker_embeddings)
postnet_output, _, _, _, alignments, _, _ = model.module.inference( # else:
inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings # postnet_output, _, _, _, alignments, _, _ = model.inference(
) # inputs,
else: # inputs_lengths,
postnet_output, _, _, _, alignments, _, _ = model.inference( # g=speaker_id if speaker_id is not None else speaker_embeddings)
inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings # postnet_output = postnet_output.permute(0, 2, 1)
) # # these only belong to tacotron models.
postnet_output = postnet_output.permute(0, 2, 1) # decoder_output = None
# these only belong to tacotron models. # stop_tokens = None
decoder_output = None # elif CONFIG.model.lower() in ["speedy_speech", "align_tts"]:
stop_tokens = None # inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable
elif CONFIG.model.lower() in ["speedy_speech", "align_tts"]: # if hasattr(model, "module"):
inputs_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) # pylint: disable=not-callable # # distributed model
if hasattr(model, "module"): # postnet_output, alignments = model.module.inference(
# distributed model # inputs,
postnet_output, alignments = model.module.inference( # inputs_lengths,
inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings # g=speaker_id if speaker_id is not None else speaker_embeddings)
) # else:
else: # postnet_output, alignments = model.inference(
postnet_output, alignments = model.inference( # inputs,
inputs, inputs_lengths, g=speaker_id if speaker_id is not None else speaker_embeddings # inputs_lengths,
) # g=speaker_id if speaker_id is not None else speaker_embeddings)
postnet_output = postnet_output.permute(0, 2, 1) # postnet_output = postnet_output.permute(0, 2, 1)
# these only belong to tacotron models. # # these only belong to tacotron models.
decoder_output = None # decoder_output = None
stop_tokens = None # stop_tokens = None
else: # else:
raise ValueError("[!] Unknown model name.") # raise ValueError("[!] Unknown model name.")
return decoder_output, postnet_output, alignments, stop_tokens return outputs
def run_model_tf(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): def run_model_tf(model, inputs, CONFIG, speaker_id=None, style_mel=None):
if CONFIG.gst and style_mel is not None: if CONFIG.gst and style_mel is not None:
raise NotImplementedError(" [!] GST inference not implemented for TF") raise NotImplementedError(" [!] GST inference not implemented for TF")
if truncated:
raise NotImplementedError(" [!] Truncated inference not implemented for TF")
if speaker_id is not None: if speaker_id is not None:
raise NotImplementedError(" [!] Multi-Speaker not implemented for TF") raise NotImplementedError(" [!] Multi-Speaker not implemented for TF")
# TODO: handle multispeaker case # TODO: handle multispeaker case
decoder_output, postnet_output, alignments, stop_tokens = model(inputs, training=False) decoder_output, postnet_output, alignments, stop_tokens = model(
inputs, training=False)
return decoder_output, postnet_output, alignments, stop_tokens return decoder_output, postnet_output, alignments, stop_tokens
def run_model_tflite(model, inputs, CONFIG, truncated, speaker_id=None, style_mel=None): def run_model_tflite(model, inputs, CONFIG, speaker_id=None, style_mel=None):
if CONFIG.gst and style_mel is not None: if CONFIG.gst and style_mel is not None:
raise NotImplementedError(" [!] GST inference not implemented for TfLite") raise NotImplementedError(
if truncated: " [!] GST inference not implemented for TfLite")
raise NotImplementedError(" [!] Truncated inference not implemented for TfLite")
if speaker_id is not None: if speaker_id is not None:
raise NotImplementedError(" [!] Multi-Speaker not implemented for TfLite") raise NotImplementedError(
" [!] Multi-Speaker not implemented for TfLite")
# get input and output details # get input and output details
input_details = model.get_input_details() input_details = model.get_input_details()
output_details = model.get_output_details() output_details = model.get_output_details()
@ -152,9 +153,11 @@ def run_model_tflite(model, inputs, CONFIG, truncated, speaker_id=None, style_me
return decoder_output, postnet_output, None, None return decoder_output, postnet_output, None, None
def parse_outputs_torch(postnet_output, decoder_output, alignments, stop_tokens): def parse_outputs_torch(postnet_output, decoder_output, alignments,
stop_tokens):
postnet_output = postnet_output[0].data.cpu().numpy() postnet_output = postnet_output[0].data.cpu().numpy()
decoder_output = None if decoder_output is None else decoder_output[0].data.cpu().numpy() decoder_output = None if decoder_output is None else decoder_output[
0].data.cpu().numpy()
alignment = alignments[0].cpu().data.numpy() alignment = alignments[0].cpu().data.numpy()
stop_tokens = None if stop_tokens is None else stop_tokens[0].cpu().numpy() stop_tokens = None if stop_tokens is None else stop_tokens[0].cpu().numpy()
return postnet_output, decoder_output, alignment, stop_tokens return postnet_output, decoder_output, alignment, stop_tokens
@ -186,23 +189,23 @@ def inv_spectrogram(postnet_output, ap, CONFIG):
return wav return wav
def id_to_torch(speaker_id, cuda=False): def speaker_id_to_torch(speaker_id, cuda=False):
if speaker_id is not None: if speaker_id is not None:
speaker_id = np.asarray(speaker_id) speaker_id = np.asarray(speaker_id)
# TODO: test this for tacotron models
speaker_id = torch.from_numpy(speaker_id) speaker_id = torch.from_numpy(speaker_id)
if cuda: if cuda:
return speaker_id.cuda() return speaker_id.cuda()
return speaker_id return speaker_id
def embedding_to_torch(speaker_embedding, cuda=False): def embedding_to_torch(x_vector, cuda=False):
if speaker_embedding is not None: if x_vector is not None:
speaker_embedding = np.asarray(speaker_embedding) x_vector = np.asarray(x_vector)
speaker_embedding = torch.from_numpy(speaker_embedding).unsqueeze(0).type(torch.FloatTensor) x_vector = torch.from_numpy(x_vector).unsqueeze(
0).type(torch.FloatTensor)
if cuda: if cuda:
return speaker_embedding.cuda() return x_vector.cuda()
return speaker_embedding return x_vector
# TODO: perform GL with pytorch for batching # TODO: perform GL with pytorch for batching
@ -216,7 +219,8 @@ def apply_griffin_lim(inputs, input_lens, CONFIG, ap):
""" """
wavs = [] wavs = []
for idx, spec in enumerate(inputs): for idx, spec in enumerate(inputs):
wav_len = (input_lens[idx] * ap.hop_length) - ap.hop_length # inverse librosa padding wav_len = (input_lens[idx] *
ap.hop_length) - ap.hop_length # inverse librosa padding
wav = inv_spectrogram(spec, ap, CONFIG) wav = inv_spectrogram(spec, ap, CONFIG)
# assert len(wav) == wav_len, f" [!] wav lenght: {len(wav)} vs expected: {wav_len}" # assert len(wav) == wav_len, f" [!] wav lenght: {len(wav)} vs expected: {wav_len}"
wavs.append(wav[:wav_len]) wavs.append(wav[:wav_len])
@ -231,11 +235,10 @@ def synthesis(
ap, ap,
speaker_id=None, speaker_id=None,
style_wav=None, style_wav=None,
truncated=False,
enable_eos_bos_chars=False, # pylint: disable=unused-argument enable_eos_bos_chars=False, # pylint: disable=unused-argument
use_griffin_lim=False, use_griffin_lim=False,
do_trim_silence=False, do_trim_silence=False,
speaker_embedding=None, x_vector=None,
backend="torch", backend="torch",
): ):
"""Synthesize voice for the given text. """Synthesize voice for the given text.
@ -249,8 +252,6 @@ def synthesis(
model outputs. model outputs.
speaker_id (int): id of speaker speaker_id (int): id of speaker
style_wav (str | Dict[str, float]): Uses for style embedding of GST. style_wav (str | Dict[str, float]): Uses for style embedding of GST.
truncated (bool): keep model states after inference. It can be used
for continuous inference at long texts.
enable_eos_bos_chars (bool): enable special chars for end of sentence and start of sentence. enable_eos_bos_chars (bool): enable special chars for end of sentence and start of sentence.
do_trim_silence (bool): trim silence after synthesis. do_trim_silence (bool): trim silence after synthesis.
backend (str): tf or torch backend (str): tf or torch
@ -263,14 +264,15 @@ def synthesis(
else: else:
style_mel = compute_style_mel(style_wav, ap, cuda=use_cuda) style_mel = compute_style_mel(style_wav, ap, cuda=use_cuda)
# preprocess the given text # preprocess the given text
inputs = text_to_seqvec(text, CONFIG) inputs = text_to_seq(text, CONFIG)
# pass tensors to backend # pass tensors to backend
if backend == "torch": if backend == "torch":
if speaker_id is not None: if speaker_id is not None:
speaker_id = id_to_torch(speaker_id, cuda=use_cuda) speaker_id = speaker_id_to_torch(speaker_id, cuda=use_cuda)
if speaker_embedding is not None: if x_vector is not None:
speaker_embedding = embedding_to_torch(speaker_embedding, cuda=use_cuda) x_vector = embedding_to_torch(x_vector,
cuda=use_cuda)
if not isinstance(style_mel, dict): if not isinstance(style_mel, dict):
style_mel = numpy_to_torch(style_mel, torch.float, cuda=use_cuda) style_mel = numpy_to_torch(style_mel, torch.float, cuda=use_cuda)
@ -287,24 +289,26 @@ def synthesis(
inputs = tf.expand_dims(inputs, 0) inputs = tf.expand_dims(inputs, 0)
# synthesize voice # synthesize voice
if backend == "torch": if backend == "torch":
decoder_output, postnet_output, alignments, stop_tokens = run_model_torch( outputs = run_model_torch(model,
model, inputs, CONFIG, truncated, speaker_id, style_mel, speaker_embeddings=speaker_embedding inputs,
) speaker_id,
style_mel,
x_vector=x_vector)
postnet_output, decoder_output, alignments, stop_tokens = \
outputs['postnet_outputs'], outputs['decoder_outputs'],\
outputs['alignments'], outputs['stop_tokens']
postnet_output, decoder_output, alignment, stop_tokens = parse_outputs_torch( postnet_output, decoder_output, alignment, stop_tokens = parse_outputs_torch(
postnet_output, decoder_output, alignments, stop_tokens postnet_output, decoder_output, alignments, stop_tokens)
)
elif backend == "tf": elif backend == "tf":
decoder_output, postnet_output, alignments, stop_tokens = run_model_tf( decoder_output, postnet_output, alignments, stop_tokens = run_model_tf(
model, inputs, CONFIG, truncated, speaker_id, style_mel model, inputs, CONFIG, speaker_id, style_mel)
)
postnet_output, decoder_output, alignment, stop_tokens = parse_outputs_tf( postnet_output, decoder_output, alignment, stop_tokens = parse_outputs_tf(
postnet_output, decoder_output, alignments, stop_tokens postnet_output, decoder_output, alignments, stop_tokens)
)
elif backend == "tflite": elif backend == "tflite":
decoder_output, postnet_output, alignment, stop_tokens = run_model_tflite( decoder_output, postnet_output, alignment, stop_tokens = run_model_tflite(
model, inputs, CONFIG, truncated, speaker_id, style_mel model, inputs, CONFIG, speaker_id, style_mel)
) postnet_output, decoder_output = parse_outputs_tflite(
postnet_output, decoder_output = parse_outputs_tflite(postnet_output, decoder_output) postnet_output, decoder_output)
# convert outputs to numpy # convert outputs to numpy
# plot results # plot results
wav = None wav = None