From 4431e04b482238eee5d581eb2c9ca6789ac0ff12 Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Wed, 16 Jan 2019 16:26:05 +0100 Subject: [PATCH] use sigmoid for attention --- layers/attention.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/layers/attention.py b/layers/attention.py index 054f792c..93d0d5a1 100644 --- a/layers/attention.py +++ b/layers/attention.py @@ -167,12 +167,12 @@ class AttentionRNNCell(nn.Module): alignment[:, :back_win] = -float("inf") if front_win < memory.shape[1]: alignment[:, front_win:] = -float("inf") - # Update the window - self.win_idx = torch.argmax(alignment,1).long()[0].item() + # Update the window + self.win_idx = torch.argmax(alignment,1).long()[0].item() # Normalize context weight - alignment = F.softmax(alignment, dim=-1) + # alignment = F.softmax(alignment, dim=-1) # alignment = 5 * alignment - # alignment = torch.sigmoid(alignment) / torch.sigmoid(alignment).sum(dim=1).unsqueeze(1) + alignment = torch.sigmoid(alignment) / torch.sigmoid(alignment).sum(dim=1).unsqueeze(1) # Attention context vector # (batch, 1, dim) # c_i = \sum_{j=1}^{T_x} \alpha_{ij} h_j