From 72817438db4d805754d19dea818e6b4eb0ce425d Mon Sep 17 00:00:00 2001
From: root <root@sp-mlc3-5423-0.mlc>
Date: Wed, 15 Jan 2020 01:53:27 +0100
Subject: [PATCH] graves v2

---
 config.json             |  2 +-
 layers/common_layers.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/config.json b/config.json
index 0bf6c378..fc33d16a 100644
--- a/config.json
+++ b/config.json
@@ -1,6 +1,6 @@
 {
     "model": "Tacotron2",          // one of the model in models/  
-    "run_name": "ljspeech-graves",
+    "run_name": "ljspeech-gravesv2",
     "run_description": "tacotron2 wuth graves attention",
 
     // AUDIO PARAMETERS
diff --git a/layers/common_layers.py b/layers/common_layers.py
index a768e684..f27ecf56 100644
--- a/layers/common_layers.py
+++ b/layers/common_layers.py
@@ -118,7 +118,7 @@ class GravesAttention(nn.Module):
 
     def __init__(self, query_dim, K):
         super(GravesAttention, self).__init__()
-        self._mask_value = 0.0
+        self._mask_value = 1e-8
         self.K = K
         # self.attention_alignment = 0.05
         self.eps = 1e-5
@@ -165,12 +165,14 @@ class GravesAttention(nn.Module):
 
         # attention GMM parameters
         sig_t = torch.nn.functional.softplus(b_t) + self.eps
+
         mu_t = self.mu_prev + torch.nn.functional.softplus(k_t)
         g_t = torch.softmax(g_t, dim=-1) / sig_t + self.eps
+
         j = self.J[:inputs.size(1)+1]
 
         # attention weights
-        phi_t = g_t.unsqueeze(-1) * (1 / (1 + torch.exp((mu_t.unsqueeze(-1) - j) / sig_t.unsqueeze(-1))))
+        phi_t = g_t.unsqueeze(-1) * (1 / (1 + torch.sigmoid((mu_t.unsqueeze(-1) - j) / sig_t.unsqueeze(-1))))
 
         # discritize attention weights
         alpha_t = torch.sum(phi_t, 1)
@@ -182,8 +184,6 @@ class GravesAttention(nn.Module):
             alpha_t.data.masked_fill_(~mask, self._mask_value)
 
         context = torch.bmm(alpha_t.unsqueeze(1), inputs).squeeze(1)
-        # for better visualization
-        # self.attention_weights = torch.clamp(alpha_t, min=0)
         self.attention_weights = alpha_t
         self.mu_prev = mu_t
         return context
@@ -356,7 +356,7 @@ class OriginalAttention(nn.Module):
         if self.forward_attn:
             alignment = self.apply_forward_attention(alignment)
             self.alpha = alignment
-        
+
         context = torch.bmm(alignment.unsqueeze(1), inputs)
         context = context.squeeze(1)
         self.attention_weights = alignment