add unittest for vocabulary parameters

tests/test_config.json

@@ -19,6 +19,16 @@
         "mel_fmax": 7600,        // maximum freq level for mel-spec. Tune for dataset!!
         "do_trim_silence": false
     },
+
+    "characters":{
+        "pad": "_",
+        "eos": "~",
+        "bos": "^",
+        "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
+        "punctuations":"!'(),-.:;? ",
+        "phonemes":"iyɨʉɯuɪʏʊeøɘəɵɤoɛœɜɞʌɔæɐaɶɑɒᵻʘɓǀɗǃʄǂɠǁʛpbtdʈɖcɟkɡqɢʔɴŋɲɳnɱmʙrʀⱱɾɽɸβfvθðszʃʒʂʐçʝxɣχʁħʕhɦɬɮʋɹɻjɰlɭʎʟˈˌːˑʍwɥʜʢʡɕʑɺɧɚ˞ɫ"
+    },
+
     "hidden_size": 128,
     "embedding_size": 256,
     "text_cleaner": "english_cleaners",

tests/test_text_processing.py

@@ -1,7 +1,14 @@
+import os
+# pylint: disable=unused-wildcard-import
+# pylint: disable=wildcard-import
+# pylint: disable=unused-import
 import unittest
-import torch as T
-
 from TTS.utils.text import *
+from TTS.tests import get_tests_path
+from TTS.utils.generic_utils import load_config
+
+TESTS_PATH = get_tests_path()
+conf = load_config(os.path.join(TESTS_PATH, 'test_config.json'))
 
 def test_phoneme_to_sequence():
     text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"
@@ -9,63 +16,76 @@ def test_phoneme_to_sequence():
     lang = "en-us"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "ɹiːsənt ɹɪsɜːtʃ æt hɑːɹvɚd hɐz ʃoʊn mɛdᵻteɪɾɪŋ fɔːɹ æz lɪɾəl æz eɪt wiːks kæn æktʃuːəli ɪnkɹiːs, ðə ɡɹeɪ mæɾɚɹ ɪnðə pɑːɹts ʌvðə bɹeɪn ɹɪspɑːnsəbəl fɔːɹ ɪmoʊʃənəl ɹɛɡjuːleɪʃən ænd lɜːnɪŋ!"
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt 
 
     # multiple punctuations
     text = "Be a voice, not an! echo?"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ?"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # not ending with punctuation
     text = "Be a voice, not an! echo"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # original
     text = "Be a voice, not an echo!"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn ɛkoʊ!"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # extra space after the sentence
     text = "Be a voice, not an! echo.  "
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ."
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # extra space after the sentence
     text = "Be a voice, not an! echo.  "
     sequence = phoneme_to_sequence(text, text_cleaner, lang, True)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "^biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ.~"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
 
     # padding char
     text = "_Be a _voice, not an! echo_"
     sequence = phoneme_to_sequence(text, text_cleaner, lang)
     text_hat = sequence_to_phoneme(sequence)
+    sequence_with_params = phoneme_to_sequence(text, text_cleaner, lang, tp=conf.characters)
+    text_hat_with_params = sequence_to_phoneme(sequence, tp=conf.characters)
     gt = "biː ɐ vɔɪs, nɑːt ɐn! ɛkoʊ"
     print(text_hat)
     print(len(sequence))
-    assert text_hat == gt
+    assert text_hat == text_hat_with_params == gt
-
 
 def test_text2phone():
     text = "Recent research at Harvard has shown meditating for as little as 8 weeks can actually increase, the grey matter in the parts of the brain responsible for emotional regulation and learning!"

utils/text/symbols.py

@@ -5,9 +5,9 @@ Defines the set of symbols used in text input to the model.
 The default is a set of ASCII characters that works well for English or text that has been run
 through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details.
 '''
-def make_symbols(characters, phnms, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'):
+def make_symbols(characters, phonemes, punctuations='!\'(),-.:;? ', pad='_', eos='~', bos='^'):# pylint: disable=redefined-outer-name
     ''' Function to create symbols and phonemes '''
-    _phonemes_sorted = sorted(list(phnms))
+    _phonemes_sorted = sorted(list(phonemes))
 
     # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
     _arpabet = ['@' + s for s in _phonemes_sorted]