Medix-AI/coqui-tts
Medix-AI
/
coqui-tts
mirror of https://github.com/coqui-ai/TTS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change logging for the new cluster system

This commit is contained in:
Eren 2018-07-05 17:30:42 +02:00
parent 59771eabb3
commit 4160e8fca1
1 changed files with 37 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
train.py
View File

@ -134,7 +134,7 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
grad_norm_st, skip_flag = check_update(model.module.decoder.stopnet, 0.5, 100)
if skip_flag:
optimizer_st.zero_grad()
print(" | > Iteration skipped fro stopnet!!")
print(" | | > Iteration skipped fro stopnet!!")
continue
optimizer_st.step()
@ -142,12 +142,23 @@ def train(model, criterion, criterion_st, data_loader, optimizer, optimizer_st,
epoch_time += step_time
# update
progbar.update(num_iter+1, values=[('total_loss', loss.item()),
('linear_loss', linear_loss.item()),
('mel_loss', mel_loss.item()),
('stop_loss', stop_loss.item()),
('grad_norm', grad_norm.item()),
('grad_norm_st', grad_norm_st.item())])
# progbar.update(num_iter+1, values=[('total_loss', loss.item()),
# ('linear_loss', linear_loss.item()),
# ('mel_loss', mel_loss.item()),
# ('stop_loss', stop_loss.item()),
# ('grad_norm', grad_norm.item()),
# ('grad_norm_st', grad_norm_st.item())])
if current_step % c.print_step == 0:
print(" | | > TotalLoss: {:.5f}\t LinearLoss: {:.5f}\t MelLoss: \
{:.5f}\t StopLoss: {:.5f}\t GradNorm: {:.5f}\t \
GradNormST: {:.5f}".format(loss.item(),
linear_loss.item(),
mel_loss.item(),
stop_loss.item(),
grad_norm.item(),
grad_norm_st.item()))
avg_linear_loss += linear_loss.item()
avg_mel_loss += mel_loss.item()
avg_stop_loss += stop_loss.item()
@ -219,7 +230,7 @@ def evaluate(model, criterion, criterion_st, data_loader, current_step):
avg_mel_loss = 0
avg_stop_loss = 0
print(" | > Validation")
progbar = Progbar(len(data_loader.dataset) / c.batch_size)
# progbar = Progbar(len(data_loader.dataset) / c.batch_size)
n_priority_freq = int(3000 / (c.sample_rate * 0.5) * c.num_freq)
with torch.no_grad():
for num_iter, data in enumerate(data_loader):
@ -262,10 +273,16 @@ def evaluate(model, criterion, criterion_st, data_loader, current_step):
epoch_time += step_time
# update
progbar.update(num_iter+1, values=[('total_loss', loss.item()),
('linear_loss', linear_loss.item()),
('mel_loss', mel_loss.item()),
('stop_loss', stop_loss.item())])
# progbar.update(num_iter+1, values=[('total_loss', loss.item()),
# ('linear_loss', linear_loss.item()),
# ('mel_loss', mel_loss.item()),
# ('stop_loss', stop_loss.item())])
if current_step % c.print_step == 0:
print(" | | > TotalLoss: {:.5f}\t LinearLoss: {:.5f}\t MelLoss: \
{:.5f}\t StopLoss: {:.5f}\t".format(loss.item(),
linear_loss.item(),
mel_loss.item(),
stop_loss.item()))
avg_linear_loss += linear_loss.item()
avg_mel_loss += mel_loss.item()
@ -405,6 +422,7 @@ def main(args):
train_loss, current_step = train(
model, criterion, criterion_st, train_loader, optimizer, optimizer_st, epoch)
val_loss = evaluate(model, criterion, criterion_st, val_loader, current_step)
print(" >>> Train Loss: {:.5f}\t Validation Loss: {:.5f}".format(train_loss, val_loss))
best_loss = save_best_model(model, optimizer, val_loss,
best_loss, OUT_PATH,
current_step, epoch)