add unittest for vocabulary parameters

tests/test_config.json

@@ -19,6 +19,16 @@
         "mel_fmax": 7600,        // maximum freq level for mel-spec. Tune for dataset!!
         "do_trim_silence": false
     },
+
+    "characters":{
+        "pad": "_",
+        "eos": "~",
+        "bos": "^",
+        "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
+        "punctuations":"!'(),-.:;? ",
+        "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
+    },
+
     "hidden_size": 128,
     "embedding_size": 256,
     "text_cleaner": "english_cleaners",

tests/test_text_processing.py

@@ -1,7 +1,14 @@
+import os
+# pylint: disable=unused-wildcard-import
+# pylint: disable=wildcard-import
+# pylint: disable=unused-import
 import unittest
-import torch as T
-
 from TTS.utils.text import *
+from TTS.tests import get_tests_path
+from TTS.utils.generic_utils import load_config
+
+TESTS_PATH = get_tests_path()
+conf = load_config(os.path.join(TESTS_PATH, 'test_config.json'))
 
 def test_phoneme_to_sequence():
     text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
@@ -9,67 +16,80 @@ def test_phoneme_to_sequence():
     lang = "en-us"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!"
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt 
 
     # multiple punctuations
     text = "Be a voice, not an! echo?"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ?"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # not ending with punctuation
     text = "Be a voice, not an! echo"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # original
     text = "Be a voice, not an echo!"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # extra space after the sentence
     text = "Be a voice, not an! echo.  "
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ."
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # extra space after the sentence
     text = "Be a voice, not an! echo.  "
     sequence = phoneme_to_sequence(text, text_cleaner, lang, True)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "^biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ.~"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # padding char
     text = "_Be a _voice, not an! echo_"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
-
+    assert text_hat == text_hat_with_params == gt
 
 def test_text2phone():
     text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
     gt = "ɹ|iː|s|ə|n|t| |ɹ|ɪ|s|ɜː|tʃ| |æ|t| |h|ɑːɹ|v|ɚ|d| |h|ɐ|z| |ʃ|oʊ|n| |m|ɛ|d|ᵻ|t|eɪ|ɾ|ɪ|ŋ| |f|ɔː|ɹ| |æ|z| |l|ɪ|ɾ|əl| |æ|z| |eɪ|t| |w|iː|k|s| |k|æ|n| |æ|k|tʃ|uː|əl|i| |ɪ|n|k|ɹ|iː|s|,| |ð|ə| |ɡ|ɹ|eɪ| |m|æ|ɾ|ɚ|ɹ| |ɪ|n|ð|ə| |p|ɑːɹ|t|s| |ʌ|v|ð|ə| |b|ɹ|eɪ|n| |ɹ|ɪ|s|p|ɑː|n|s|ə|b|əl| |f|ɔː|ɹ| |ɪ|m|oʊ|ʃ|ə|n|əl| |ɹ|ɛ|ɡ|j|uː|l|eɪ|ʃ|ə|n| |æ|n|d| |l|ɜː|n|ɪ|ŋ|!"
     lang = "en-us"
     ph = text2phone(text, lang)
-    assert gt == ph, f"\n{phonemes} \n vs \n{gt}"
+    assert gt == ph, f"\n{phonemes} \n vs \n{gt}"

utils/text/symbols.py

@@ -5,9 +5,9 @@ Defines the set of symbols used in text input to the model.
 The default is a set of ASCII characters that works well for English or text that has been run
 through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details.
 '''
-def make_symbols(characters, phnms, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'):
+def make_symbols(characters, phonemes, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'):# pylint: disable=redefined-outer-name
     ''' Function to create symbols and phonemes '''
-    _phonemes_sorted = sorted(list(phnms))
+    _phonemes_sorted = sorted(list(phonemes))
 
     # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
     _arpabet = ['@' + s for s in _phonemes_sorted]