From 59f85a7122d85cb4a9d5afc0318774bf48d691a4 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 27 Sep 2023 00:54:19 +0300 Subject: [PATCH] Remove duplicate code from xtts.tokenizer --- TTS/tts/layers/xtts/tokenizer.py | 37 -------------------------------- 1 file changed, 37 deletions(-) diff --git a/TTS/tts/layers/xtts/tokenizer.py b/TTS/tts/layers/xtts/tokenizer.py index 0fad8133..8dd81fac 100644 --- a/TTS/tts/layers/xtts/tokenizer.py +++ b/TTS/tts/layers/xtts/tokenizer.py @@ -171,17 +171,6 @@ def multilingual_cleaners(text, lang): return text -def english_cleaners(text): - """Pipeline for English text, including number and abbreviation expansion.""" - text = convert_to_ascii(text) - text = lowercase(text) - text = expand_numbers(text) - text = expand_abbreviations(text) - text = collapse_whitespace(text) - text = text.replace('"', "") - return text - - def remove_extraneous_punctuation(word): replacement_punctuation = {"{": "(", "}": ")", "[": "(", "]": ")", "`": "'", "—": "-", "—": "-", "`": "'", "ʼ": "'"} replace = re.compile( @@ -195,32 +184,6 @@ def remove_extraneous_punctuation(word): return word -def expand_numbers(text): - return normalize_numbers(text) - - -def lowercase(text): - return text.lower() - - -_whitespace_re = re.compile(r"\s+") - - -def collapse_whitespace(text): - return re.sub(_whitespace_re, " ", text) - - -def convert_to_ascii(text): - return unidecode(text) - - -def basic_cleaners(text): - """Basic pipeline that lowercases and collapses whitespace without transliteration.""" - text = lowercase(text) - text = collapse_whitespace(text) - return text - - def arabic_cleaners(text): text = lowercase(text) text = collapse_whitespace(text)