voice_audio_multi/tests/unittests/test_soprano_text_cleaning.cpp

#include "engine/community_models/soprano_tts/text_normalizer.h"

#include "test_assert.h"

#include <iostream>
#include <string>
#include <vector>

namespace {

namespace soprano = engine::community_models::soprano_tts;

struct Case {
    std::string input;
    std::string expected;
    std::string label;
};

// Expected values generated by running the reference clean_text
// (ekwek1/soprano soprano/utils/text_normalizer.py with inflect + unidecode)
// on the same inputs.
const std::vector<Case> & reference_cases() {
    static const std::vector<Case> cases = {
        // Punctuation canonicalization (regression cases).
        {"Hello world.", "hello world.", "plain sentence"},
        {"Hello world .", "hello world.", "space before period"},
        {"Hello.. world...", "hello. world...", "double dot and ellipsis"},
        {"Wait, what?! Really?!", "wait, what? really?", "interrobang"},
        {"It's   a  test!!", "it's a test!", "double exclamation"},
        {"dots... then.. one.", "dots... then. one.", "mixed dot runs"},
        {"hello,world, , , test", "hello,world, test.", "comma runs"},
        {"mixed .,. .. ,!!?? marks", "mixed....? marks.", "mixed punctuation"},
        {"  trailing spaces   ", "trailing spaces.", "trimmed whitespace"},
        {"goooood morning", "good morning.", "triple letters"},
        {"This is a state-of-the-art model.",
         "this is a state-of-the-art model.", "hyphenated words"},
        // Numbers.
        {"1.5 million dollars.", "one point five million dollars.", "decimal"},
        {"I have 3 apples, 12 oranges, and 1990 memories.",
         "i have three apples, twelve oranges, and nineteen ninety memories.",
         "plain and year numbers"},
        {"The year 2005 was great, but 1234 was better.",
         "the year two thousand five was great, but twelve thirty-four was better.",
         "2000s year and group-2 number"},
        {"Chapter 12, section 4, verse 1050.",
         "chapter twelve, section four, verse ten fifty.", "round year"},
        {"Call 555-123-4567 now!",
         "call five five five, one two three, four five six seven now!",
         "phone number"},
        {"Meet me at 3:00 or 8:05 pm.",
         "meet me at three o'clock or eight oh five pm.", "times"},
        {"It was 3:25:10 exactly.",
         "it was three, 'twenty-five', ten exactly.",
         "h:m:s with reference set-repr quirk"},
        {"It costs $2.47 or \xC2\xA3" "20.",
         "it costs two dollars, forty-seven cents or ps twenty.",
         "money and unidecode pound"},
        {"1st 2nd 3rd 4th place", "first second third fourth place.", "ordinals"},
        {"I have 1/2 a cup and 100K followers.",
         "i have one over two a cup and one hundred thousand followers.",
         "fraction and K suffix"},
        {"Use #1, DS4, R2D2, and 4K video.",
         "use number one, ds four, r two d two, and four thousand video.",
         "hash prefix, alphanumerics, M suffix"},
        {"1 + 2 * 3 - 4 / 5 equals a lot",
         "one plus two times three minus four over five equals a lot.",
         "arithmetic operators"},
        // Special characters and symbols.
        {"Email me @ home & work ~ maybe",
         "email me at home and work about maybe.", "at, ampersand, tilde"},
        {"C++ and a_b and x=y and 5<6 and 7>2 and a<=b",
         "c plus plus and a b and x equals y and five less than six and "
         "seven greater than two and a less than or equal to b.",
         "symbol verbalization"},
        {"Visit https://example.com/path today.",
         "visit h t t p s colon slash slash example dot com slash path today.",
         "link and domain dots"},
        {"The em\xE2\x80\x94" "dash and en\xE2\x80\x93" "dash test.",
         "the em, dash and en-dash test.", "dashes"},
        // Brackets, abbreviations, casing.
        {"Remove #brackets [ok]", "remove brackets, ok.", "brackets"},
        {"Mr. and Mrs. Smith, Dr. Who, and St. Peter met at 5th Ave.",
         "mister and misess smith, doctor who, and saint peter met at fifth "
         "avenue.", "dot abbreviations"},
        {"The CPU, GPU, APIs, CLI and CPUs work. etc. Mon Jan Feb Dec",
         "the c p u, g p u, a p i's, c l i and c p u's work. et cetera. "
         "monday january february december.", "cased abbreviations"},
        {"Dr. Smith went to Washington.", "doctor smith went to washington.",
         "doctor abbreviation"},
        {"CamelCaseWord and HTMLParser and APIs",
         "camel case word and h t m l parser and a p i's.", "camel case"},
        // ASCII folding.
        {"caf\xC3\xA9" " na\xC3\xAF" "ve", "cafe naive.", "diacritics"},
        // Newlines.
        {"Line one\nLine two without punct\nLine three!",
         "line one. line two without punct. line three!", "newlines"},
        // Adversarial extras.
        {"0:00 start and 13:00 hours",
         "zero start and thirteen minutes hours.", "midnight and 24h time"},
        {"Version 1.17.1.1 released",
         "version one point one seven point one point one released.",
         "multi-dot version"},
        {"Call (111) 111-1111 or 111.111.1111",
         "call one one one, one one one, one one one one or one one one, "
         "one one one, one one one one.", "phone variants"},
        {"Email me@example.com now", "email me at example dot com now.",
         "email address"},
        {"Hello <3 world", "hello less than three world.", "heart"},
        {"100c 100x1 100m", "one hundred c one hundred x one one hundred million.",
         "number-letter mixes"},
        {"I paid $0.27 and $1.00.", "i paid twenty-seven cents and one dollar.",
         "cents-only and zero cents"},
        {"2.47023 and 20.47023",
         "two point four seven zero two three and twenty point four seven "
         "zero two three.", "long decimals"},
        {"12:00 and 11:59 sharp", "twelve o'clock and eleven fifty-nine sharp.",
         "noon and sharp time"},
        {"The 1990s music and the 2020s",
         "the nineteen ninety s music and the twenty twenty s.", "decades"},
        {"-5 degrees today", "minus five degrees today.", "negative number"},
        {"saving \xC2\xA3" "50 meant a lot", "saving ps fifty meant a lot.",
         "pound sign via unidecode"},
        {"#42 was chosen", "number forty-two was chosen.", "hash number"},
        {"GBs of data, 2TBs more", "gigabytes of data, two terabytes more.",
         "storage abbreviations"},
        {"1,234,567 people",
         "one million, two hundred thirty-four thousand, five hundred "
         "sixty-seven people.", "millions with commas"},
    };
    return cases;
}

}  // namespace

int main() {
    try {
        for (const auto & test_case : reference_cases()) {
            engine::test::require_eq(
                soprano::clean_soprano_text(test_case.input),
                test_case.expected,
                test_case.label);
        }
        std::cout << "soprano_text_cleaning_test passed ("
                  << reference_cases().size() << " reference cases)\n";
        return 0;
    } catch (const std::exception & error) {
        std::cerr << "soprano_text_cleaning_test failed: " << error.what() << "\n";
        return 1;
    }
}