From f4fa155cd34d05d9545c816f0d0c6fc7d61fb3e9 Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Mon, 14 Jan 2019 15:08:35 +0100 Subject: [PATCH] Make attn windowing optional --- config.json | 1 + layers/attention.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 8b09b81c..44d52ad7 100644 --- a/config.json +++ b/config.json @@ -31,6 +31,7 @@ "lr": 0.001, // Initial learning rate. If Noam decay is active, maximum learning rate. "lr_decay": false, // if true, Noam learning rate decaying is applied through training. "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "windowing": true, // Enables attention windowing. Used only in eval mode. "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. "eval_batch_size":32, diff --git a/layers/attention.py b/layers/attention.py index 83ccb504..ecb38587 100644 --- a/layers/attention.py +++ b/layers/attention.py @@ -160,8 +160,7 @@ class AttentionRNNCell(nn.Module): mask = mask.view(memory.size(0), -1) alignment.masked_fill_(1 - mask, -float("inf")) # Windowing - if not self.training: - # print(" > Windowing active") + if not self.training and self.windowing: back_win = self.win_idx - self.win_back front_win = self.win_idx + self.win_front if back_win > 0: