From 1ed4978e6900c4a6a9432ef0fda1da1ee8cb8dce Mon Sep 17 00:00:00 2001
From: Eren Golge <egolge@mozilla.com>
Date: Wed, 27 Mar 2019 14:56:40 +0100
Subject: [PATCH] text processing update

---
 tests/text_processing_tests.py | 22 ++++++++++++++++++++++
 utils/text/__init__.py         |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 tests/text_processing_tests.py

diff --git a/tests/text_processing_tests.py b/tests/text_processing_tests.py
new file mode 100644
index 00000000..16ebb7db
--- /dev/null
+++ b/tests/text_processing_tests.py
@@ -0,0 +1,22 @@
+import unittest
+import torch as T
+
+from utils.text import *
+
+def test_phoneme_to_sequence():
+    text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
+    text_cleaner = ["phoneme_cleaners"]
+    lang = "en-us"
+    sequence = phoneme_to_sequence(text, text_cleaner, lang)
+    text_hat = sequence_to_phoneme(sequence)
+    gt = "^ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!"
+    assert text_hat == gt
+
+
+def test_text2phone():
+    text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
+    text_cleaner = ["phoneme_cleaners"]
+    gt = "ɹ|iː|s|ə|n|t| |ɹ|ɪ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i|| |ɪ|n|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ɪ|s|p|ɑː|n|s|ə|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|uː|l|eɪ|ʃ|ə|n||| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!"
+    lang = "en-us"
+    phonemes = text2phone(text, lang)
+    assert gt == phonemes
\ No newline at end of file
diff --git a/utils/text/__init__.py b/utils/text/__init__.py
index e8bea47a..32474a55 100644
--- a/utils/text/__init__.py
+++ b/utils/text/__init__.py
@@ -49,7 +49,7 @@ def phoneme_to_sequence(text, cleaner_names, language):
     phonemes = text2phone(clean_text, language)
     if phonemes is None:
         print("!! After phoneme conversion the result is None. -- {} ".format(clean_text))
-    # iterate by skipping empty strings
+    # iterate by skipping empty strings - NOTE: might be useful to keep it to have a better intonation.
     for phoneme in filter(None, phonemes.split('|')):
         sequence += _phoneme_to_sequence(phoneme)
     # Append EOS char