Medix-AI/coqui-tts
Medix-AI
/
coqui-tts
mirror of https://github.com/coqui-ai/TTS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
coqui-tts/notebooks/TacotronPlayGround.ipynb

3.1 MiB
Raw Blame History

None <html lang="en"> <head> </head>
In [1]:
%load_ext autoreload
%autoreload 2
import os
import sys
import io
import torch 
import time
import numpy as np
from collections import OrderedDict

%pylab inline
rcParams["figure.figsize"] = (16,5)
sys.path.append('/home/erogol/projects/')

import librosa
import librosa.display

from torchviz import make_dot, make_dot_from_trace

from TTS.models.tacotron import Tacotron 
from TTS.layers import *
from TTS.utils.data import *
from TTS.utils.audio import AudioProcessor
from TTS.utils.generic_utils import load_config
from TTS.utils.text import text_to_sequence

import IPython
from IPython.display import Audio
from utils import *

Populating the interactive namespace from numpy and matplotlib
In [2]:
def tts(model, text, CONFIG, use_cuda, ap, figures=True):
    t_1 = time.time()
    waveform, alignment, spectrogram = create_speech(model, text, CONFIG, use_cuda, ap) 
    print(" >  Run-time: {}".format(time.time() - t_1))
    if figures:                                                                                                         
        visualize(alignment, spectrogram, CONFIG)                                                                       
    IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate))  
    return alignment, spectrogram
In [3]:
# Set constants
ROOT_PATH = '/data/shared/erogol_models/April-12-2018_06:00AM-06d4b23/'
MODEL_PATH = ROOT_PATH + '/checkpoint_118440.pth.tar'
CONFIG_PATH = ROOT_PATH + '/config.json'
OUT_FOLDER = ROOT_PATH + '/test/'
CONFIG = load_config(CONFIG_PATH)
use_cuda = False
In [4]:
# load the model
model = Tacotron(CONFIG.embedding_size, CONFIG.num_freq, CONFIG.num_mels, CONFIG.r)

# load the audio processor
ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db,
                    CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis,
                    CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=80)         


# load model state
if use_cuda:
    cp = torch.load(MODEL_PATH)
else:
    cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage)

# load the model
model.load_state_dict(cp['model'])
if use_cuda:
    model.cuda()
model.eval()

 | > Number of characted : 149
Out[4]:
Tacotron(
  (embedding): Embedding(149, 256)
  (encoder): Encoder(
    (prenet): Prenet(
      (layers): ModuleList(
        (0): Linear(in_features=256, out_features=256)
        (1): Linear(in_features=256, out_features=128)
      )
      (relu): ReLU()
      (dropout): Dropout(p=0.5)
    )
    (cbhg): CBHG(
      (relu): ReLU()
      (conv1d_banks): ModuleList(
        (0): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(1,), stride=(1,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (1): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (2): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (3): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (4): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (5): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (6): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (7): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (8): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (9): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(10,), stride=(1,), padding=(5,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (10): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(11,), stride=(1,), padding=(5,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (11): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(12,), stride=(1,), padding=(6,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (12): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(13,), stride=(1,), padding=(6,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (13): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(14,), stride=(1,), padding=(7,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (14): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(15,), stride=(1,), padding=(7,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (15): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(16,), stride=(1,), padding=(8,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
      )
      (max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
      (conv1d_projections): ModuleList(
        (0): BatchNormConv1d(
          (conv1d): Conv1d (2048, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
          (activation): ReLU()
        )
        (1): BatchNormConv1d(
          (conv1d): Conv1d (128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
          (bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
        )
      )
      (pre_highway): Linear(in_features=128, out_features=128)
      (highways): ModuleList(
        (0): Highway(
          (H): Linear(in_features=128, out_features=128)
          (T): Linear(in_features=128, out_features=128)
          (relu): ReLU()
          (sigmoid): Sigmoid()
        )
        (1): Highway(
          (H): Linear(in_features=128, out_features=128)
          (T): Linear(in_features=128, out_features=128)
          (relu): ReLU()
          (sigmoid): Sigmoid()
        )
        (2): Highway(
          (H): Linear(in_features=128, out_features=128)
          (T): Linear(in_features=128, out_features=128)
          (relu): ReLU()
          (sigmoid): Sigmoid()
        )
        (3): Highway(
          (H): Linear(in_features=128, out_features=128)
          (T): Linear(in_features=128, out_features=128)
          (relu): ReLU()
          (sigmoid): Sigmoid()
        )
      )
      (gru): GRU(128, 128, batch_first=True, bidirectional=True)
    )
  )
  (decoder): Decoder(
    (prenet): Prenet(
      (layers): ModuleList(
        (0): Linear(in_features=400, out_features=256)
        (1): Linear(in_features=256, out_features=128)
      )
      (relu): ReLU()
      (dropout): Dropout(p=0.5)
    )
    (attention_rnn): AttentionRNN(
      (rnn_cell): GRUCell(384, 256)
      (alignment_model): BahdanauAttention(
        (query_layer): Linear(in_features=256, out_features=256)
        (annot_layer): Linear(in_features=256, out_features=256)
        (v): Linear(in_features=256, out_features=1)
      )
    )
    (project_to_decoder_in): Linear(in_features=512, out_features=256)
    (decoder_rnns): ModuleList(
      (0): GRUCell(256, 256)
      (1): GRUCell(256, 256)
    )
    (proj_to_mel): Linear(in_features=256, out_features=400)
  )
  (postnet): CBHG(
    (relu): ReLU()
    (conv1d_banks): ModuleList(
      (0): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(1,), stride=(1,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (1): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (2): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (3): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (4): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (5): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (6): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (7): BatchNormConv1d(
        (conv1d): Conv1d (80, 80, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
    )
    (max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
    (conv1d_projections): ModuleList(
      (0): BatchNormConv1d(
        (conv1d): Conv1d (640, 256, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
        (bn): BatchNorm1d(256, eps=0.001, momentum=0.99, affine=True)
        (activation): ReLU()
      )
      (1): BatchNormConv1d(
        (conv1d): Conv1d (256, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
        (bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
      )
    )
    (pre_highway): Linear(in_features=80, out_features=80)
    (highways): ModuleList(
      (0): Highway(
        (H): Linear(in_features=80, out_features=80)
        (T): Linear(in_features=80, out_features=80)
        (relu): ReLU()
        (sigmoid): Sigmoid()
      )
      (1): Highway(
        (H): Linear(in_features=80, out_features=80)
        (T): Linear(in_features=80, out_features=80)
        (relu): ReLU()
        (sigmoid): Sigmoid()
      )
      (2): Highway(
        (H): Linear(in_features=80, out_features=80)
        (T): Linear(in_features=80, out_features=80)
        (relu): ReLU()
        (sigmoid): Sigmoid()
      )
      (3): Highway(
        (H): Linear(in_features=80, out_features=80)
        (T): Linear(in_features=80, out_features=80)
        (relu): ReLU()
        (sigmoid): Sigmoid()
      )
    )
    (gru): GRU(80, 80, batch_first=True, bidirectional=True)
  )
  (last_linear): Linear(in_features=160, out_features=1025)
)

EXAMPLES FROM TRAINING SET

In [5]:
import pandas as pd
df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata_val.csv', delimiter='|')
In [6]:
sentence = df.iloc[2, 1]
print(sentence)
model.decoder.max_decoder_steps = len(sentence)
align, spec = tts(model, sentence, CONFIG, use_cuda, ap)

Latona's findings were also confirmed by Ronald G. Wittmus, another FBI fingerprint expert.
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
torch.Size([1, 400])
 >  Run-time: 8.135989665985107
Your browser does not support the audio element.
No description has been provided for this image
In [7]:
sentence =  "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent."
model.decoder.max_decoder_steps = 250
alignment = tts(model, sentence, CONFIG, use_cuda, ap)

 >  Run-time: 17.50665545463562
Your browser does not support the audio element.
No description has been provided for this image
In [8]:
sentence = "Be a voice, not an echo."  # 'echo' is not in training set. 
alignment = tts(model, sentence, CONFIG, use_cuda, ap)

 >  Run-time: 13.074267864227295
Your browser does not support the audio element.
No description has been provided for this image
In [27]:
sentence = "The human voice is the most perfect instrument of all."
alignment = tts(model, sentence, CONFIG, use_cuda, ap)

 >  Run-time: 12.174984216690063
Your browser does not support the audio element.
No description has been provided for this image
In [28]:
sentence = "I'm sorry Dave. I'm afraid I can't do that."
alignment = tts(model, sentence, CONFIG, use_cuda, ap)

 >  Run-time: 20.04722023010254
Your browser does not support the audio element.
No description has been provided for this image
In [34]:
sentence = "This cake is great. It's so delicious and moist."
alignment = tts(model, sentence, CONFIG, use_cuda, ap)

 >  Run-time: 19.729145765304565
Your browser does not support the audio element.
No description has been provided for this image
</html>