#include "engine/community_models/soprano_tts/text_normalizer.h" #include "test_assert.h" #include #include #include namespace { namespace soprano = engine::community_models::soprano_tts; struct Case { std::string input; std::string expected; std::string label; }; // Expected values generated by running the reference clean_text // (ekwek1/soprano soprano/utils/text_normalizer.py with inflect + unidecode) // on the same inputs. const std::vector & reference_cases() { static const std::vector cases = { // Punctuation canonicalization (regression cases). {"Hello world.", "hello world.", "plain sentence"}, {"Hello world .", "hello world.", "space before period"}, {"Hello.. world...", "hello. world...", "double dot and ellipsis"}, {"Wait, what?! Really?!", "wait, what? really?", "interrobang"}, {"It's a test!!", "it's a test!", "double exclamation"}, {"dots... then.. one.", "dots... then. one.", "mixed dot runs"}, {"hello,world, , , test", "hello,world, test.", "comma runs"}, {"mixed .,. .. ,!!?? marks", "mixed....? marks.", "mixed punctuation"}, {" trailing spaces ", "trailing spaces.", "trimmed whitespace"}, {"goooood morning", "good morning.", "triple letters"}, {"This is a state-of-the-art model.", "this is a state-of-the-art model.", "hyphenated words"}, // Numbers. {"1.5 million dollars.", "one point five million dollars.", "decimal"}, {"I have 3 apples, 12 oranges, and 1990 memories.", "i have three apples, twelve oranges, and nineteen ninety memories.", "plain and year numbers"}, {"The year 2005 was great, but 1234 was better.", "the year two thousand five was great, but twelve thirty-four was better.", "2000s year and group-2 number"}, {"Chapter 12, section 4, verse 1050.", "chapter twelve, section four, verse ten fifty.", "round year"}, {"Call 555-123-4567 now!", "call five five five, one two three, four five six seven now!", "phone number"}, {"Meet me at 3:00 or 8:05 pm.", "meet me at three o'clock or eight oh five pm.", "times"}, {"It was 3:25:10 exactly.", "it was three, 'twenty-five', ten exactly.", "h:m:s with reference set-repr quirk"}, {"It costs $2.47 or \xC2\xA3" "20.", "it costs two dollars, forty-seven cents or ps twenty.", "money and unidecode pound"}, {"1st 2nd 3rd 4th place", "first second third fourth place.", "ordinals"}, {"I have 1/2 a cup and 100K followers.", "i have one over two a cup and one hundred thousand followers.", "fraction and K suffix"}, {"Use #1, DS4, R2D2, and 4K video.", "use number one, ds four, r two d two, and four thousand video.", "hash prefix, alphanumerics, M suffix"}, {"1 + 2 * 3 - 4 / 5 equals a lot", "one plus two times three minus four over five equals a lot.", "arithmetic operators"}, // Special characters and symbols. {"Email me @ home & work ~ maybe", "email me at home and work about maybe.", "at, ampersand, tilde"}, {"C++ and a_b and x=y and 5<6 and 7>2 and a<=b", "c plus plus and a b and x equals y and five less than six and " "seven greater than two and a less than or equal to b.", "symbol verbalization"}, {"Visit https://example.com/path today.", "visit h t t p s colon slash slash example dot com slash path today.", "link and domain dots"}, {"The em\xE2\x80\x94" "dash and en\xE2\x80\x93" "dash test.", "the em, dash and en-dash test.", "dashes"}, // Brackets, abbreviations, casing. {"Remove #brackets [ok]", "remove brackets, ok.", "brackets"}, {"Mr. and Mrs. Smith, Dr. Who, and St. Peter met at 5th Ave.", "mister and misess smith, doctor who, and saint peter met at fifth " "avenue.", "dot abbreviations"}, {"The CPU, GPU, APIs, CLI and CPUs work. etc. Mon Jan Feb Dec", "the c p u, g p u, a p i's, c l i and c p u's work. et cetera. " "monday january february december.", "cased abbreviations"}, {"Dr. Smith went to Washington.", "doctor smith went to washington.", "doctor abbreviation"}, {"CamelCaseWord and HTMLParser and APIs", "camel case word and h t m l parser and a p i's.", "camel case"}, // ASCII folding. {"caf\xC3\xA9" " na\xC3\xAF" "ve", "cafe naive.", "diacritics"}, // Newlines. {"Line one\nLine two without punct\nLine three!", "line one. line two without punct. line three!", "newlines"}, // Adversarial extras. {"0:00 start and 13:00 hours", "zero start and thirteen minutes hours.", "midnight and 24h time"}, {"Version 1.17.1.1 released", "version one point one seven point one point one released.", "multi-dot version"}, {"Call (111) 111-1111 or 111.111.1111", "call one one one, one one one, one one one one or one one one, " "one one one, one one one one.", "phone variants"}, {"Email me@example.com now", "email me at example dot com now.", "email address"}, {"Hello <3 world", "hello less than three world.", "heart"}, {"100c 100x1 100m", "one hundred c one hundred x one one hundred million.", "number-letter mixes"}, {"I paid $0.27 and $1.00.", "i paid twenty-seven cents and one dollar.", "cents-only and zero cents"}, {"2.47023 and 20.47023", "two point four seven zero two three and twenty point four seven " "zero two three.", "long decimals"}, {"12:00 and 11:59 sharp", "twelve o'clock and eleven fifty-nine sharp.", "noon and sharp time"}, {"The 1990s music and the 2020s", "the nineteen ninety s music and the twenty twenty s.", "decades"}, {"-5 degrees today", "minus five degrees today.", "negative number"}, {"saving \xC2\xA3" "50 meant a lot", "saving ps fifty meant a lot.", "pound sign via unidecode"}, {"#42 was chosen", "number forty-two was chosen.", "hash number"}, {"GBs of data, 2TBs more", "gigabytes of data, two terabytes more.", "storage abbreviations"}, {"1,234,567 people", "one million, two hundred thirty-four thousand, five hundred " "sixty-seven people.", "millions with commas"}, }; return cases; } } // namespace int main() { try { for (const auto & test_case : reference_cases()) { engine::test::require_eq( soprano::clean_soprano_text(test_case.input), test_case.expected, test_case.label); } std::cout << "soprano_text_cleaning_test passed (" << reference_cases().size() << " reference cases)\n"; return 0; } catch (const std::exception & error) { std::cerr << "soprano_text_cleaning_test failed: " << error.what() << "\n"; return 1; } }