mirror of https://github.com/coqui-ai/TTS.git
1.9 MiB
1.9 MiB
None
<html lang="en">
<head>
</head>
![No description has been provided for this image]()
![No description has been provided for this image]()
</html>
In [1]:
%load_ext autoreload %autoreload 2 import os import sys import io import torch import time import numpy as np from collections import OrderedDict %pylab inline rcParams["figure.figsize"] = (16,5) sys.path.append('/home/erogol/projects/') import librosa import librosa.display from TTS.models.tacotron import Tacotron from TTS.layers import * from TTS.utils.data import * from TTS.utils.audio import AudioProcessor from TTS.utils.generic_utils import load_config from TTS.utils.text import text_to_sequence import IPython from IPython.display import Audio from utils import *
Populating the interactive namespace from numpy and matplotlib
In [2]:
def tts(model, text, CONFIG, use_cuda, ap, figures=True): t_1 = time.time() waveform, alignment, spectrogram = create_speech(model, text, CONFIG, use_cuda, ap) print(" > Run-time: {}".format(time.time() - t_1)) if figures: visualize(alignment, spectrogram, CONFIG) IPython.display.display(Audio(waveform, rate=CONFIG.sample_rate)) return alignment, spectrogram
In [3]:
# Set constants ROOT_PATH = '../result/February-13-2018_01:04AM/' MODEL_PATH = ROOT_PATH + '/best_model.pth.tar' CONFIG_PATH = ROOT_PATH + '/config.json' OUT_FOLDER = ROOT_PATH + '/test/' CONFIG = load_config(CONFIG_PATH) use_cuda = False
In [4]:
# load the model model = Tacotron(CONFIG.embedding_size, CONFIG.hidden_size, CONFIG.num_mels, CONFIG.num_freq, CONFIG.r) # load the audio processor ap = AudioProcessor(CONFIG.sample_rate, CONFIG.num_mels, CONFIG.min_level_db, CONFIG.frame_shift_ms, CONFIG.frame_length_ms, CONFIG.preemphasis, CONFIG.ref_level_db, CONFIG.num_freq, CONFIG.power, griffin_lim_iters=80) # load model state if use_cuda: cp = torch.load(MODEL_PATH) else: cp = torch.load(MODEL_PATH, map_location=lambda storage, loc: storage) # small trick to remove DataParallel wrapper new_state_dict = OrderedDict() for k, v in cp['model'].items(): name = k[7:] # remove `module.` new_state_dict[name] = v cp['model'] = new_state_dict # load the model model.load_state_dict(cp['model']) if use_cuda: model.cuda() # model.eval() # model.encoder.eval() # model. model.eval()
| > Embedding dim : 149
Out[4]:
Tacotron(
(embedding): Embedding(149, 256)
(encoder): Encoder(
(prenet): Prenet(
(layers): ModuleList(
(0): Linear(in_features=256, out_features=256)
(1): Linear(in_features=256, out_features=128)
)
(relu): ReLU()
(dropout): Dropout(p=0.5)
)
(cbhg): CBHG(
(relu): ReLU()
(conv1d_banks): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(1,), stride=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(2): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(3): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(4): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(5): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(6): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(7): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(8): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(9,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(9): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(10,), stride=(1,), padding=(5,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(10): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(11,), stride=(1,), padding=(5,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(11): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(12,), stride=(1,), padding=(6,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(12): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(13,), stride=(1,), padding=(6,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(13): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(14,), stride=(1,), padding=(7,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(14): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(15,), stride=(1,), padding=(7,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(15): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(16,), stride=(1,), padding=(8,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
)
(max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
(conv1d_projections): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d (2048, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d (128, 128, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(128, eps=0.001, momentum=0.99, affine=True)
)
)
(pre_highway): Linear(in_features=128, out_features=128)
(highways): ModuleList(
(0): Highway(
(H): Linear(in_features=128, out_features=128)
(T): Linear(in_features=128, out_features=128)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(1): Highway(
(H): Linear(in_features=128, out_features=128)
(T): Linear(in_features=128, out_features=128)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(2): Highway(
(H): Linear(in_features=128, out_features=128)
(T): Linear(in_features=128, out_features=128)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(3): Highway(
(H): Linear(in_features=128, out_features=128)
(T): Linear(in_features=128, out_features=128)
(relu): ReLU()
(sigmoid): Sigmoid()
)
)
(gru): GRU(128, 128, batch_first=True, bidirectional=True)
)
)
(decoder): Decoder(
(input_layer): Linear(in_features=256, out_features=256)
(prenet): Prenet(
(layers): ModuleList(
(0): Linear(in_features=400, out_features=256)
(1): Linear(in_features=256, out_features=128)
)
(relu): ReLU()
(dropout): Dropout(p=0.5)
)
(attention_rnn): AttentionWrapper(
(rnn_cell): GRUCell(384, 256)
(alignment_model): BahdanauAttention(
(query_layer): Linear(in_features=256, out_features=256)
(tanh): Tanh()
(v): Linear(in_features=256, out_features=1)
)
)
(project_to_decoder_in): Linear(in_features=512, out_features=256)
(decoder_rnns): ModuleList(
(0): GRUCell(256, 256)
(1): GRUCell(256, 256)
)
(proj_to_mel): Linear(in_features=256, out_features=400)
)
(postnet): CBHG(
(relu): ReLU()
(conv1d_banks): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(1,), stride=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(2,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(2): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(3): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(4,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(4): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(5): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(6,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(6): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(7): BatchNormConv1d(
(conv1d): Conv1d (80, 80, kernel_size=(8,), stride=(1,), padding=(4,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
)
(max_pool1d): MaxPool1d(kernel_size=2, stride=1, padding=1, dilation=1, ceil_mode=False)
(conv1d_projections): ModuleList(
(0): BatchNormConv1d(
(conv1d): Conv1d (640, 256, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(256, eps=0.001, momentum=0.99, affine=True)
(activation): ReLU()
)
(1): BatchNormConv1d(
(conv1d): Conv1d (256, 80, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
(bn): BatchNorm1d(80, eps=0.001, momentum=0.99, affine=True)
)
)
(pre_highway): Linear(in_features=80, out_features=80)
(highways): ModuleList(
(0): Highway(
(H): Linear(in_features=80, out_features=80)
(T): Linear(in_features=80, out_features=80)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(1): Highway(
(H): Linear(in_features=80, out_features=80)
(T): Linear(in_features=80, out_features=80)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(2): Highway(
(H): Linear(in_features=80, out_features=80)
(T): Linear(in_features=80, out_features=80)
(relu): ReLU()
(sigmoid): Sigmoid()
)
(3): Highway(
(H): Linear(in_features=80, out_features=80)
(T): Linear(in_features=80, out_features=80)
(relu): ReLU()
(sigmoid): Sigmoid()
)
)
(gru): GRU(80, 80, batch_first=True, bidirectional=True)
)
(last_linear): Linear(in_features=160, out_features=1025)
)
EXAMPLES FROM TRAINING SET¶
In [5]:
import pandas as pd df = pd.read_csv('/data/shared/KeithIto/LJSpeech-1.0/metadata.csv', delimiter='|')
In [6]:
sentence = df.iloc[120, 1].lower().replace(',','') print(sentence) sentence = "that he has a 5 an 8 before him unless the press work is of the best" align = tts(model, sentence, CONFIG, use_cuda, ap)
that he has a 5 an 8 or a 3 before him unless the press work is of the best:
/home/erogol/miniconda3/envs/pytorch/lib/python3.6/site-packages/librosa/util/utils.py:1725: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. if np.issubdtype(x.dtype, float) or np.issubdtype(x.dtype, complex):
> Run-time: 6.436546802520752
/home/erogol/miniconda3/envs/pytorch/lib/python3.6/site-packages/librosa/display.py:656: FutureWarning: Conversion of the second argument of issubdtype from `complex` to `np.complexfloating` is deprecated. In future, it will be treated as `np.complex128 == np.dtype(complex).type`. if np.issubdtype(data.dtype, np.complex):
Your browser does not support the audio element.
NEW EXAMPLES¶
In [12]:
sentence = "For many decades, we've enriched foreign industry at the expense of American industry; subsidized the armies of other countries while allowing for the very sad depletion of our military; we've defended other nation's borders while refusing to defend our own; and spent trillions of dollars overseas while America's infrastructure has fallen into disrepair and decay" model.decoder.max_decoder_steps = 300 alignment = tts(model, sentence, CONFIG, use_cuda, ap)
/home/erogol/miniconda3/envs/pytorch/lib/python3.6/site-packages/librosa/util/utils.py:1725: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. if np.issubdtype(x.dtype, float) or np.issubdtype(x.dtype, complex):
> Run-time: 18.219324588775635
/home/erogol/miniconda3/envs/pytorch/lib/python3.6/site-packages/librosa/display.py:656: FutureWarning: Conversion of the second argument of issubdtype from `complex` to `np.complexfloating` is deprecated. In future, it will be treated as `np.complex128 == np.dtype(complex).type`. if np.issubdtype(data.dtype, np.complex):
Your browser does not support the audio element.