mirror of https://github.com/coqui-ai/TTS.git
6.1 MiB
6.1 MiB
None
<html lang="en">
<head>
</head>
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
![No description has been provided for this image]()
</html>
In [1]:
%load_ext autoreload %autoreload 2 import os import sys import io import torch import time import numpy as np from collections import OrderedDict from matplotlib import pylab as plt %pylab inline rcParams["figure.figsize"] = (16,5) sys.path.append('/home/erogol/projects/') import librosa import librosa.display from TTS.models.tacotron import Tacotron from TTS.layers import * from TTS.utils.data import * from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import load_config from TTS.utils.text import text_to_sequence import IPython from IPython.display import Audio from utils import *
Populating the interactive namespace from numpy and matplotlib
/home/erogol/miniconda3/envs/pytorch4/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['plt'] `%matplotlib` prevents importing * from pylab and numpy "\n`%matplotlib` prevents importing * from pylab and numpy"
In [2]:
ls /data/shared/erogol_models/May-14-2018_07:10PM-stop-token-separate-larger-lr-b2e09f9
best_model.pth.tar checkpoint_18800.pth.tar checkpoint_10152.pth.tar checkpoint_2256.pth.tar checkpoint_10528.pth.tar checkpoint_2632.pth.tar checkpoint_10904.pth.tar checkpoint_3008.pth.tar checkpoint_1128.pth.tar checkpoint_3384.pth.tar checkpoint_11280.pth.tar checkpoint_376.pth.tar checkpoint_11656.pth.tar checkpoint_3760.pth.tar checkpoint_12032.pth.tar checkpoint_4136.pth.tar checkpoint_12408.pth.tar checkpoint_4512.pth.tar checkpoint_12784.pth.tar checkpoint_4888.pth.tar checkpoint_13160.pth.tar checkpoint_5264.pth.tar checkpoint_13536.pth.tar checkpoint_5640.pth.tar checkpoint_13912.pth.tar checkpoint_6016.pth.tar checkpoint_14288.pth.tar checkpoint_6392.pth.tar checkpoint_14664.pth.tar checkpoint_6768.pth.tar checkpoint_1504.pth.tar checkpoint_7144.pth.tar checkpoint_15040.pth.tar checkpoint_752.pth.tar checkpoint_15416.pth.tar checkpoint_7520.pth.tar checkpoint_15792.pth.tar checkpoint_7896.pth.tar checkpoint_16168.pth.tar checkpoint_8272.pth.tar checkpoint_16544.pth.tar checkpoint_8648.pth.tar checkpoint_16920.pth.tar checkpoint_9024.pth.tar checkpoint_17296.pth.tar checkpoint_9400.pth.tar checkpoint_17672.pth.tar checkpoint_9776.pth.tar checkpoint_18048.pth.tar checkpoints/ checkpoint_18424.pth.tar config.json checkpoint_1880.pth.tar events.out.tfevents.1526350247.mlc1
In [3]:
def tts(model, text, CONFIG, use_cuda, ap, figures=True): t_1 = time.time() waveform, alignment, spectrogram, stop_tokens = create_speech(model, text, CONFIG, use_cuda, ap) print(" > Run-time: {}".format(time.time() - t_1)) if figures: visualize(alignment, spectrogram, stop_tokens, CONFIG) IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate)) return alignment, spectrogram, stop_tokens
In [4]:
# Set constants ROOT_PATH = '/data/shared/erogol_models/May-14-2018_07:10PM-stop-token-separate-larger-lr-b2e09f9/' MODEL_PATH = ROOT_PATH + '/checkpoint_16920.pth.tar' CONFIG_PATH = ROOT_PATH + '/config.json' OUT_FOLDER = ROOT_PATH + '/test/' CONFIG = load_config(CONFIG_PATH) use_cuda = True
In [5]:
# load the model model = Tacotron(CONFIG.embedding_size, CONFIG.num_freq, CONFIG.num_mels, CONFIG.r) # load the audio processor ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db, CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis, CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=30) # load model state if use_cuda: cp = torch.load(MODEL_PATH) else: cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage) # load the model model.load_state_dict(cp['model']) if use_cuda: model.cuda() model.eval()
| > Number of characters : 149
Out[5]:
Tacotron(
(embedding): Embedding(149, 256)
(encoder): Encoder(
(prenet): Prenet(
(layers): ModuleList(
(0): Linear(in_features=256, out_features=256, bias=True)
(1): Linear(in_features=256, out_features=128, bias=True)
)
(relu): ReLU()
(dropout): Dropout(p=0.5)
)
(cbhg): CBHG(
(relu): ReLU()
(conv1d_banks): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(1,), stride=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(2): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(3): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(4): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(5): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(6): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(7): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(8): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(9): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(10,), stride=(1,), padding=(5,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(10): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(11): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(12,), stride=(1,), padding=(6,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(12): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(13,), stride=(1,), padding=(6,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(13): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(14,), stride=(1,), padding=(7,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(14): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(15,), stride=(1,), padding=(7,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(15): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(16,), stride=(1,), padding=(8,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
)
(max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
(conv1d_projections): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d(2048, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
)
)
(pre_highway): Linear(in_features=128, out_features=128, bias=False)
(highways): ModuleList(
(0): Highway(
(H): Linear(in_features=128, out_features=128, bias=True)
(T): Linear(in_features=128, out_features=128, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(1): Highway(
(H): Linear(in_features=128, out_features=128, bias=True)
(T): Linear(in_features=128, out_features=128, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(2): Highway(
(H): Linear(in_features=128, out_features=128, bias=True)
(T): Linear(in_features=128, out_features=128, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(3): Highway(
(H): Linear(in_features=128, out_features=128, bias=True)
(T): Linear(in_features=128, out_features=128, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
)
(gru): GRU(128, 128, batch_first=True, bidirectional=True)
)
)
(decoder): Decoder(
(prenet): Prenet(
(layers): ModuleList(
(0): Linear(in_features=400, out_features=256, bias=True)
(1): Linear(in_features=256, out_features=128, bias=True)
)
(relu): ReLU()
(dropout): Dropout(p=0.5)
)
(attention_rnn): AttentionRNN(
(rnn_cell): GRUCell(384, 256)
(alignment_model): BahdanauAttention(
(query_layer): Linear(in_features=256, out_features=256, bias=True)
(annot_layer): Linear(in_features=256, out_features=256, bias=True)
(v): Linear(in_features=256, out_features=1, bias=False)
)
)
(project_to_decoder_in): Linear(in_features=512, out_features=256, bias=True)
(decoder_rnns): ModuleList(
(0): GRUCell(256, 256)
(1): GRUCell(256, 256)
)
(proj_to_mel): Linear(in_features=256, out_features=400, bias=True)
(stopnet): Sequential(
(0): Linear(in_features=656, out_features=80, bias=True)
(1): ReLU()
(2): Linear(in_features=80, out_features=656, bias=True)
(3): ReLU()
(4): Linear(in_features=656, out_features=1, bias=True)
(5): Sigmoid()
)
)
(postnet): CBHG(
(relu): ReLU()
(conv1d_banks): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(1,), stride=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(2): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(3): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(4): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(5): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(6): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(7): BatchNormConv1d(
(conv1d): Conv1d(80, 80, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
)
(max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
(conv1d_projections): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d(640, 256, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(256, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d(256, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)
)
)
(pre_highway): Linear(in_features=80, out_features=80, bias=False)
(highways): ModuleList(
(0): Highway(
(H): Linear(in_features=80, out_features=80, bias=True)
(T): Linear(in_features=80, out_features=80, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(1): Highway(
(H): Linear(in_features=80, out_features=80, bias=True)
(T): Linear(in_features=80, out_features=80, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(2): Highway(
(H): Linear(in_features=80, out_features=80, bias=True)
(T): Linear(in_features=80, out_features=80, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(3): Highway(
(H): Linear(in_features=80, out_features=80, bias=True)
(T): Linear(in_features=80, out_features=80, bias=True)
(relu): ReLU()
(sigmoid): Sigmoid()
)
)
(gru): GRU(80, 80, batch_first=True, bidirectional=True)
)
(last_linear): Linear(in_features=160, out_features=1025, bias=True)
)
EXAMPLES FROM TRAINING SET¶
In [6]:
import pandas as pd df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata_val.csv', delimiter='|')
In [7]:
sentence = df.iloc[175, 1] print(sentence) model.decoder.max_decoder_steps = 250 align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
Hosty took the necessary steps to have the Dallas office of the FBI, rather than the New Orleans office, reestablished as the office with principal responsibility.
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 5.755854845046997
Your browser does not support the audio element.
Comparision with https://mycroft.ai/blog/available-voices/¶
In [8]:
sentence = "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent." model.decoder.max_decoder_steps = 250 align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap, figures=True)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 4.917491436004639
Your browser does not support the audio element.
In [9]:
sentence = "Be a voice,not an echo." # 'echo' is not in training set. align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 1.1238880157470703
Your browser does not support the audio element.
In [10]:
sentence = "The human voice is the most perfect instrument of all." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 2.4607677459716797
Your browser does not support the audio element.
In [11]:
sentence = "I'm sorry Dave. I'm afraid I can't do that." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 6.053100109100342
Your browser does not support the audio element.
In [12]:
sentence = "This cake is great. It's so delicious and moist." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 2.6189444065093994
Your browser does not support the audio element.
Comparison with https://keithito.github.io/audio-samples/¶
In [13]:
sentence = "Generative adversarial network or variational auto-encoder." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
!! Decoder stopped with 'max_decoder_steps'. Something is probably wrong. > Run-time: 13.515002012252808
Your browser does not support the audio element.
In [14]:
sentence = "Scientists at the CERN laboratory say they have discovered a new particle." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 4.4747514724731445
Your browser does not support the audio element.
In [15]:
sentence = "here’s a way to measure the acute emotional intelligence that has never gone out of style." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 5.41440224647522
Your browser does not support the audio element.
In [16]:
sentence = "President Trump met with other leaders at the Group of 20 conference." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 2.631229877471924
Your browser does not support the audio element.
In [17]:
sentence = "The buses aren't the problem, they actually provide a solution." align, spec, stop_tokens = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/projects/TTS/notebooks/utils.py:19: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead. torch.from_numpy(seq), volatile=True).unsqueeze(0).cuda()
> Run-time: 3.031719923019409
Your browser does not support the audio element.