Medix-AI/coqui-tts
Medix-AI
/
coqui-tts
mirror of https://github.com/coqui-ai/TTS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use tacotron abstract for multispeaker common definitions

This commit is contained in:
Edresson 2020-08-04 14:43:31 -03:00 committed by erogol
parent 26beea0e1b
commit 1d782487f5
4 changed files with 53 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mozilla_voice_tts/bin/train_tts.py
View File

@ -536,6 +536,7 @@ def main(args): # pylint: disable=redefined-outer-name
else: else:
num_speakers = 0 num_speakers = 0
speaker_embedding_dim = None speaker_embedding_dim = None
speaker_mapping = None
model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim) model = setup_model(num_chars, num_speakers, c, speaker_embedding_dim)

43
mozilla_voice_tts/tts/models/tacotron.py
View File

@ -27,6 +27,8 @@ class Tacotron(TacotronAbstract):
bidirectional_decoder=False, bidirectional_decoder=False,
double_decoder_consistency=False, double_decoder_consistency=False,
ddc_r=None, ddc_r=None,
encoder_in_features=256,
decoder_in_features=256,
speaker_embedding_dim=None, speaker_embedding_dim=None,
gst=False, gst=False,
gst_embedding_dim=256, gst_embedding_dim=256,
@ -40,39 +42,28 @@ class Tacotron(TacotronAbstract):
forward_attn, trans_agent, forward_attn_mask, forward_attn, trans_agent, forward_attn_mask,
location_attn, attn_K, separate_stopnet, location_attn, attn_K, separate_stopnet,
bidirectional_decoder, double_decoder_consistency, bidirectional_decoder, double_decoder_consistency,
ddc_r, gst, gst_embedding_dim, gst_num_heads, gst_style_tokens) ddc_r, encoder_in_features, decoder_in_features,
speaker_embedding_dim, gst, gst_embedding_dim,
gst_num_heads, gst_style_tokens)
# init layer dims # speaker embedding layers
decoder_in_features = 256 if self.num_speakers > 1:
encoder_in_features = 256 if not self.embeddings_per_sample:
speaker_embedding_dim = 256
if speaker_embedding_dim is None: self.speaker_embedding = nn.Embedding(self.num_speakers, speaker_embedding_dim)
# if speaker_embedding_dim is None we need use the nn.Embedding, with default speaker_embedding_dim self.speaker_embedding.weight.data.normal_(0, 0.3)
self.embeddings_per_sample = False
speaker_embedding_dim = 256
else:
# if speaker_embedding_dim is not None we need use speaker embedding per sample
self.embeddings_per_sample = True
# speaker and gst embeddings is concat in decoder input # speaker and gst embeddings is concat in decoder input
if num_speakers > 1: if self.num_speakers > 1:
decoder_in_features = decoder_in_features + speaker_embedding_dim # add speaker embedding dim self.decoder_in_features += speaker_embedding_dim # add speaker embedding dim
if self.gst:
decoder_in_features = decoder_in_features + gst_embedding_dim # add gst embedding dim
# embedding layer # embedding layer
self.embedding = nn.Embedding(num_chars, 256, padding_idx=0) self.embedding = nn.Embedding(num_chars, 256, padding_idx=0)
self.embedding.weight.data.normal_(0, 0.3)
# speaker embedding layers
if num_speakers > 1:
if not self.embeddings_per_sample:
self.speaker_embedding = nn.Embedding(num_speakers, speaker_embedding_dim)
self.speaker_embedding.weight.data.normal_(0, 0.3)
# base model layers # base model layers
self.embedding.weight.data.normal_(0, 0.3) self.encoder = Encoder(self.encoder_in_features)
self.encoder = Encoder(encoder_in_features) self.decoder = Decoder(self.decoder_in_features, decoder_output_dim, r,
self.decoder = Decoder(decoder_in_features, decoder_output_dim, r,
memory_size, attn_type, attn_win, attn_norm, memory_size, attn_type, attn_win, attn_norm,
prenet_type, prenet_dropout, forward_attn, prenet_type, prenet_dropout, forward_attn,
trans_agent, forward_attn_mask, location_attn, trans_agent, forward_attn_mask, location_attn,
@ -93,7 +84,7 @@ class Tacotron(TacotronAbstract):
# setup DDC # setup DDC
if self.double_decoder_consistency: if self.double_decoder_consistency:
self.coarse_decoder = Decoder( self.coarse_decoder = Decoder(
decoder_in_features, decoder_output_dim, ddc_r, memory_size, self.decoder_in_features, decoder_output_dim, ddc_r, memory_size,
attn_type, attn_win, attn_norm, prenet_type, prenet_dropout, attn_type, attn_win, attn_norm, prenet_type, prenet_dropout,
forward_attn, trans_agent, forward_attn_mask, location_attn, forward_attn, trans_agent, forward_attn_mask, location_attn,
attn_K, separate_stopnet) attn_K, separate_stopnet)

47
mozilla_voice_tts/tts/models/tacotron2.py
View File

@ -33,6 +33,8 @@ class Tacotron2(TacotronAbstract):
bidirectional_decoder=False, bidirectional_decoder=False,
double_decoder_consistency=False, double_decoder_consistency=False,
ddc_r=None, ddc_r=None,
encoder_in_features=512,
decoder_in_features=512,
speaker_embedding_dim=None, speaker_embedding_dim=None,
gst=False, gst=False,
gst_embedding_dim=512, gst_embedding_dim=512,
@ -45,38 +47,27 @@ class Tacotron2(TacotronAbstract):
forward_attn, trans_agent, forward_attn_mask, forward_attn, trans_agent, forward_attn_mask,
location_attn, attn_K, separate_stopnet, location_attn, attn_K, separate_stopnet,
bidirectional_decoder, double_decoder_consistency, bidirectional_decoder, double_decoder_consistency,
ddc_r, gst, gst_embedding_dim, gst_num_heads, gst_style_tokens) ddc_r, encoder_in_features, decoder_in_features,
speaker_embedding_dim, gst, gst_embedding_dim,
gst_num_heads, gst_style_tokens)
# init layer dims # speaker embedding layer
decoder_in_features = 512 if self.num_speakers > 1:
encoder_in_features = 512 if not self.embeddings_per_sample:
speaker_embedding_dim = 512
if speaker_embedding_dim is None: self.speaker_embedding = nn.Embedding(self.num_speakers, speaker_embedding_dim)
# if speaker_embedding_dim is None we need use the nn.Embedding, with default speaker_embedding_dim self.speaker_embedding.weight.data.normal_(0, 0.3)
self.embeddings_per_sample = False
speaker_embedding_dim = 512
else:
# if speaker_embedding_dim is not None we need use speaker embedding per sample
self.embeddings_per_sample = True
# speaker and gst embeddings is concat in decoder input # speaker and gst embeddings is concat in decoder input
if num_speakers > 1: if self.num_speakers > 1:
decoder_in_features = decoder_in_features + speaker_embedding_dim # add speaker embedding dim self.decoder_in_features += speaker_embedding_dim # add speaker embedding dim
if self.gst:
decoder_in_features = decoder_in_features + gst_embedding_dim # add gst embedding dim
# embedding layer # embedding layer
self.embedding = nn.Embedding(num_chars, 512, padding_idx=0) self.embedding = nn.Embedding(num_chars, 512, padding_idx=0)
# speaker embedding layer
if num_speakers > 1:
if not self.embeddings_per_sample:
self.speaker_embedding = nn.Embedding(num_speakers, speaker_embedding_dim)
self.speaker_embedding.weight.data.normal_(0, 0.3)
# base model layers # base model layers
self.encoder = Encoder(encoder_in_features) self.encoder = Encoder(self.encoder_in_features)
self.decoder = Decoder(decoder_in_features, self.decoder_output_dim, r, attn_type, attn_win, self.decoder = Decoder(self.decoder_in_features, self.decoder_output_dim, r, attn_type, attn_win,
attn_norm, prenet_type, prenet_dropout, attn_norm, prenet_type, prenet_dropout,
forward_attn, trans_agent, forward_attn_mask, forward_attn, trans_agent, forward_attn_mask,
location_attn, attn_K, separate_stopnet) location_attn, attn_K, separate_stopnet)
@ -85,16 +76,16 @@ class Tacotron2(TacotronAbstract):
# global style token layers # global style token layers
if self.gst: if self.gst:
self.gst_layer = GST(num_mel=80, self.gst_layer = GST(num_mel=80,
num_heads=gst_num_heads, num_heads=self.gst_num_heads,
num_style_tokens=gst_style_tokens, num_style_tokens=self.gst_style_tokens,
embedding_dim=gst_embedding_dim) embedding_dim=self.gst_embedding_dim)
# backward pass decoder # backward pass decoder
if self.bidirectional_decoder: if self.bidirectional_decoder:
self._init_backward_decoder() self._init_backward_decoder()
# setup DDC # setup DDC
if self.double_decoder_consistency: if self.double_decoder_consistency:
self.coarse_decoder = Decoder( self.coarse_decoder = Decoder(
decoder_in_features, self.decoder_output_dim, ddc_r, attn_type, self.decoder_in_features, self.decoder_output_dim, ddc_r, attn_type,
attn_win, attn_norm, prenet_type, prenet_dropout, forward_attn, attn_win, attn_norm, prenet_type, prenet_dropout, forward_attn,
trans_agent, forward_attn_mask, location_attn, attn_K, trans_agent, forward_attn_mask, location_attn, attn_K,
separate_stopnet) separate_stopnet)

15
mozilla_voice_tts/tts/models/tacotron_abstract.py
View File

@ -28,6 +28,9 @@ class TacotronAbstract(ABC, nn.Module):
bidirectional_decoder=False, bidirectional_decoder=False,
double_decoder_consistency=False, double_decoder_consistency=False,
ddc_r=None, ddc_r=None,
encoder_in_features=512,
decoder_in_features=512,
speaker_embedding_dim=None,
gst=False, gst=False,
gst_embedding_dim=512, gst_embedding_dim=512,
gst_num_heads=4, gst_num_heads=4,
@ -57,6 +60,9 @@ class TacotronAbstract(ABC, nn.Module):
self.location_attn = location_attn self.location_attn = location_attn
self.attn_K = attn_K self.attn_K = attn_K
self.separate_stopnet = separate_stopnet self.separate_stopnet = separate_stopnet
self.encoder_in_features = encoder_in_features
self.decoder_in_features = decoder_in_features
self.speaker_embedding_dim = speaker_embedding_dim
# layers # layers
self.embedding = None self.embedding = None
@ -64,8 +70,17 @@ class TacotronAbstract(ABC, nn.Module):
self.decoder = None self.decoder = None
self.postnet = None self.postnet = None
# multispeaker
if self.speaker_embedding_dim is None:
# if speaker_embedding_dim is None we need use the nn.Embedding, with default speaker_embedding_dim
self.embeddings_per_sample = False
else:
# if speaker_embedding_dim is not None we need use speaker embedding per sample
self.embeddings_per_sample = True
# global style token # global style token
if self.gst: if self.gst:
self.decoder_in_features += gst_embedding_dim # add gst embedding dim
self.gst_layer = None self.gst_layer = None
# model states # model states