From ccba431cdac1e6f44bd39b83d6a8ebdd166992a5 Mon Sep 17 00:00:00 2001
From: erogol <erogol@hotmail.com>
Date: Tue, 23 Jun 2020 13:29:55 +0200
Subject: [PATCH] handling dates better with the text cleaner
---
models/tacotron2.py | 3 -
utils/text/__init__.py | 1 -
utils/text/cleaners.py | 2 +-
utils/text/number_norm.py | 149 +++++++--------------
vocoder/tests/test_melgan_discriminator.py | 2 +-
5 files changed, 49 insertions(+), 108 deletions(-)
diff --git a/models/tacotron2.py b/models/tacotron2.py
index 7a56212d..d9570f8d 100644
--- a/models/tacotron2.py
+++ b/models/tacotron2.py
@@ -44,9 +44,6 @@ class Tacotron2(TacotronAbstract):
proj_speaker_dim = 80 if num_speakers > 1 else 0
# base layers
self.embedding = nn.Embedding(num_chars, 512, padding_idx=0)
- std = sqrt(2.0 / (num_chars + 512))
- val = sqrt(3.0) * std # uniform bounds for std
- self.embedding.weight.data.uniform_(-val, val)
if num_speakers > 1:
self.speaker_embedding = nn.Embedding(num_speakers, 512)
self.speaker_embedding.weight.data.normal_(0, 0.3)
diff --git a/utils/text/__init__.py b/utils/text/__init__.py
index 79069192..41aa6778 100644
--- a/utils/text/__init__.py
+++ b/utils/text/__init__.py
@@ -77,7 +77,6 @@ def phoneme_to_sequence(text, cleaner_names, language, enable_eos_bos=False, tp=
_phonemes_to_id = {s: i for i, s in enumerate(_phonemes)}
sequence = []
- text = text.replace(":", "")
clean_text = _clean_text(text, cleaner_names)
to_phonemes = text2phone(clean_text, language)
if to_phonemes is None:
diff --git a/utils/text/cleaners.py b/utils/text/cleaners.py
index 35da8aef..f0a66f57 100644
--- a/utils/text/cleaners.py
+++ b/utils/text/cleaners.py
@@ -71,7 +71,7 @@ def remove_aux_symbols(text):
def replace_symbols(text):
text = text.replace(';', ',')
text = text.replace('-', ' ')
- text = text.replace(':', ' ')
+ text = text.replace(':', ',')
text = text.replace('&', 'and')
return text
diff --git a/utils/text/number_norm.py b/utils/text/number_norm.py
index d3d9a46b..54fa1093 100644
--- a/utils/text/number_norm.py
+++ b/utils/text/number_norm.py
@@ -1,127 +1,72 @@
+# -*- coding: utf-8 -*-
+""" from https://github.com/keithito/tacotron """
+
+import inflect
import re
+
+_inflect = inflect.engine()
_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
-_ordinal_re = re.compile(r'([0-9]+)(st|nd|rd|th)')
+_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
_number_re = re.compile(r'[0-9]+')
-_units = [
- '', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
- 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen',
- 'seventeen', 'eighteen', 'nineteen'
-]
-
-_tens = [
- '',
- 'ten',
- 'twenty',
- 'thirty',
- 'forty',
- 'fifty',
- 'sixty',
- 'seventy',
- 'eighty',
- 'ninety',
-]
-
-_digit_groups = [
- '',
- 'thousand',
- 'million',
- 'billion',
- 'trillion',
- 'quadrillion',
-]
-
-_ordinal_suffixes = [
- ('one', 'first'),
- ('two', 'second'),
- ('three', 'third'),
- ('five', 'fifth'),
- ('eight', 'eighth'),
- ('nine', 'ninth'),
- ('twelve', 'twelfth'),
- ('ty', 'tieth'),
-]
-
def _remove_commas(m):
- return m.group(1).replace(',', '')
+ return m.group(1).replace(',', '')
def _expand_decimal_point(m):
- return m.group(1).replace('.', ' point ')
+ return m.group(1).replace('.', ' point ')
def _expand_dollars(m):
- match = m.group(1)
- parts = match.split('.')
- if len(parts) > 2:
- return match + ' dollars' # Unexpected format
- dollars = int(parts[0]) if parts[0] else 0
- cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
- if dollars and cents:
- dollar_unit = 'dollar' if dollars == 1 else 'dollars'
- cent_unit = 'cent' if cents == 1 else 'cents'
- return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
- if dollars:
- dollar_unit = 'dollar' if dollars == 1 else 'dollars'
- return '%s %s' % (dollars, dollar_unit)
- if cents:
- cent_unit = 'cent' if cents == 1 else 'cents'
- return '%s %s' % (cents, cent_unit)
+ match = m.group(1)
+ parts = match.split('.')
+ if len(parts) > 2:
+ return match + ' dollars' # Unexpected format
+ dollars = int(parts[0]) if parts[0] else 0
+ cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+ if dollars and cents:
+ dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+ cent_unit = 'cent' if cents == 1 else 'cents'
+ return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
+ elif dollars:
+ dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+ return '%s %s' % (dollars, dollar_unit)
+ elif cents:
+ cent_unit = 'cent' if cents == 1 else 'cents'
+ return '%s %s' % (cents, cent_unit)
+ else:
return 'zero dollars'
-def _standard_number_to_words(n, digit_group):
- parts = []
- if n >= 1000:
- # Format next higher digit group.
- parts.append(_standard_number_to_words(n // 1000, digit_group + 1))
- n = n % 1000
-
- if n >= 100:
- parts.append('%s hundred' % _units[n // 100])
- if n % 100 >= len(_units):
- parts.append(_tens[(n % 100) // 10])
- parts.append(_units[(n % 100) % 10])
- else:
- parts.append(_units[n % 100])
- if n > 0:
- parts.append(_digit_groups[digit_group])
- return ' '.join([x for x in parts if x])
-
-
-def _number_to_words(n):
- # Handle special cases first, then go to the standard case:
- if n >= 1000000000000000000:
- return str(n) # Too large, just return the digits
- if n == 0:
- return 'zero'
- if n % 100 == 0 and n % 1000 != 0 and n < 3000:
- return _standard_number_to_words(n // 100, 0) + ' hundred'
- return _standard_number_to_words(n, 0)
+def _expand_ordinal(m):
+ return _inflect.number_to_words(m.group(0))
def _expand_number(m):
- return _number_to_words(int(m.group(0)))
-
-
-def _expand_ordinal(m):
- num = _number_to_words(int(m.group(1)))
- for suffix, replacement in _ordinal_suffixes:
- if num.endswith(suffix):
- return num[:-len(suffix)] + replacement
- return num + 'th'
+ num = int(m.group(0))
+ if num > 1000 and num < 3000:
+ if num == 2000:
+ return 'two thousand'
+ elif num > 2000 and num < 2010:
+ return 'two thousand ' + _inflect.number_to_words(num % 100)
+ elif num % 100 == 0:
+ return _inflect.number_to_words(num // 100) + ' hundred'
+ else:
+ return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
+ else:
+ return _inflect.number_to_words(num, andword='')
def normalize_numbers(text):
- text = re.sub(_comma_number_re, _remove_commas, text)
- text = re.sub(_pounds_re, r'\1 pounds', text)
- text = re.sub(_dollars_re, _expand_dollars, text)
- text = re.sub(_decimal_number_re, _expand_decimal_point, text)
- text = re.sub(_ordinal_re, _expand_ordinal, text)
- text = re.sub(_number_re, _expand_number, text)
- return text
+ text = re.sub(_comma_number_re, _remove_commas, text)
+ text = re.sub(_pounds_re, r'\1 pounds', text)
+ text = re.sub(_dollars_re, _expand_dollars, text)
+ text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+ text = re.sub(_ordinal_re, _expand_ordinal, text)
+ text = re.sub(_number_re, _expand_number, text)
+ return text
\ No newline at end of file
diff --git a/vocoder/tests/test_melgan_discriminator.py b/vocoder/tests/test_melgan_discriminator.py
index a4564b56..a89dad5e 100644
--- a/vocoder/tests/test_melgan_discriminator.py
+++ b/vocoder/tests/test_melgan_discriminator.py
@@ -23,4 +23,4 @@ def test_melgan_multi_scale_discriminator():
assert np.all(scores[0].shape == (4, 1, 64))
assert np.all(feats[0][0].shape == (4, 16, 4096))
assert np.all(feats[0][1].shape == (4, 64, 1024))
- assert np.all(feats[0][2].shape == (4, 256, 256))
+ assert np.all(feats[0][2].shape == (4, 256, 256))
\ No newline at end of file