#include "engine/community_models/sopro_tts/reference.h" #include "engine/community_models/sopro_tts/semantic_encoder.h" #include "engine/community_models/sopro_tts/speaker_encoder.h" #include "engine/community_models/sopro_tts/vocoder.h" #include "engine/framework/assets/tensor_source.h" #include "engine/framework/audio/resampling.h" #include #include #include #include #include #include namespace engine::community_models::sopro_tts { namespace audio_ops { namespace { constexpr float kRefGainLimitDb = 30.0F; constexpr float kRefPeakCeiling = 0.95F; constexpr float kPauseMinSeconds = 0.10F; constexpr float kPauseKeepSeconds = 0.15F; constexpr float kCropForwardSeconds = 5.0F; constexpr float kCropBackwardSeconds = 5.0F; constexpr float kMinKeepFraction = 0.75F; constexpr float kRoomToneSeconds = 0.25F; constexpr float kFadeSeconds = 0.02F; constexpr float kOnsetThresholdDb = -45.0F; constexpr float kOnsetOverFloorDb = 15.0F; constexpr int64_t kOnsetWindowFrames = 6; constexpr int64_t kOnsetMinFrames = 5; // torch.unfold + RMS: floor((n - win) / hop) + 1 frames, floored at 1e-6. std::vector frame_rms( const std::vector & wav, int64_t window, int64_t hop) { const auto total = static_cast(wav.size()); std::vector out; if (window <= 0 || hop <= 0 || total < window) { return out; } const int64_t frames = (total - window) / hop + 1; out.resize(static_cast(frames), 0.0F); for (int64_t frame = 0; frame < frames; ++frame) { const float * values = wav.data() + static_cast(frame * hop); double sum = 0.0; for (int64_t i = 0; i < window; ++i) { sum += static_cast(values[i]) * static_cast(values[i]); } out[static_cast(frame)] = std::max( static_cast(std::sqrt(sum / static_cast(window))), 1.0e-6F); } return out; } // torch.quantile's default "linear" interpolation. float quantile(std::vector values, float q) { if (values.empty()) { return 0.0F; } std::sort(values.begin(), values.end()); const double position = static_cast(q) * static_cast(values.size() - 1); const auto lower = static_cast(std::floor(position)); const size_t upper = std::min(lower + 1, values.size() - 1); const double weight = position - static_cast(lower); return static_cast(values[lower] * (1.0 - weight) + values[upper] * weight); } // torch.median returns the lower of the two middle elements for even counts. float lower_median(std::vector values) { if (values.empty()) { return 0.0F; } const size_t index = (values.size() - 1) / 2; std::nth_element(values.begin(), values.begin() + static_cast(index), values.end()); return values[index]; } struct PauseRun { int64_t begin = 0; int64_t end = 0; }; std::vector pause_runs(const std::vector & quiet, int64_t min_run) { std::vector runs; const auto count = static_cast(quiet.size()); int64_t index = 0; while (index < count) { if (quiet[static_cast(index)] == 0) { ++index; continue; } int64_t end = index; while (end < count && quiet[static_cast(end)] != 0) { ++end; } if (end - index >= min_run) { runs.push_back({index, end}); } index = end; } return runs; } float onset_threshold(const std::vector & rms) { float threshold = std::pow(10.0F, kOnsetThresholdDb / 20.0F); if (rms.size() >= 30) { threshold = std::max( threshold, quantile(rms, 0.1F) * std::pow(10.0F, kOnsetOverFloorDb / 20.0F)); } return threshold; } // energy_onset / energy_fraction: `over_floor_db` above the 10th-percentile // frame, never below the absolute onset threshold. float energy_threshold(const std::vector & rms, float over_floor_db) { return std::max( quantile(rms, 0.1F) * std::pow(10.0F, over_floor_db / 20.0F), std::pow(10.0F, kOnsetThresholdDb / 20.0F)); } // x[:(n // win) * win].view(-1, win) RMS, i.e. non-overlapping windows. std::vector block_rms(const std::vector & wav, int64_t window) { std::vector out; if (window <= 0) { return out; } const int64_t frames = static_cast(wav.size()) / window; out.resize(static_cast(frames), 0.0F); for (int64_t frame = 0; frame < frames; ++frame) { const float * values = wav.data() + static_cast(frame * window); double sum = 0.0; for (int64_t i = 0; i < window; ++i) { sum += static_cast(values[i]) * static_cast(values[i]); } out[static_cast(frame)] = static_cast(std::sqrt(sum / static_cast(window))); } return out; } // First index where `min_hits` of the next `window` frames are above threshold. std::optional first_sustained_hit( const std::vector & rms, float threshold, int64_t window, int64_t min_hits) { const auto count = static_cast(rms.size()); if (count < window) { return std::nullopt; } for (int64_t start = 0; start + window <= count; ++start) { int64_t hits = 0; for (int64_t i = 0; i < window; ++i) { if (rms[static_cast(start + i)] > threshold) { ++hits; } } if (hits >= min_hits) { return start; } } return std::nullopt; } std::vector finish_with_room_tone( const std::vector & wav, int sample_rate, float floor_level, std::mt19937_64 & rng) { const auto fade = static_cast(kFadeSeconds * static_cast(sample_rate)); std::vector out(wav); if (fade > 0 && static_cast(out.size()) >= fade) { const auto offset = static_cast(out.size()) - fade; for (int64_t i = 0; i < fade; ++i) { const float ramp = fade == 1 ? 1.0F : 1.0F - static_cast(i) / static_cast(fade - 1); out[static_cast(offset + i)] *= ramp; } } const auto tone_samples = static_cast(kRoomToneSeconds * static_cast(sample_rate)); std::normal_distribution normal(0.0F, 1.0F); for (int64_t i = 0; i < tone_samples; ++i) { out.push_back(normal(rng) * floor_level); } return out; } } // namespace std::vector crop_on_pause( const std::vector & wav, float target_seconds, int sample_rate, std::mt19937_64 & rng) { const auto rate = static_cast(sample_rate); const auto window = static_cast(rate * 0.025F); const auto hop = static_cast(rate * 0.010F); const auto total = static_cast(wav.size()); if (window <= 0 || hop <= 0 || total < 4 * window) { return wav; } const auto rms = frame_rms(wav, window, hop); if (rms.empty()) { return wav; } const float floor_level = quantile(rms, 0.1F); std::vector quiet(rms.size(), 0); for (size_t i = 0; i < rms.size(); ++i) { quiet[i] = rms[i] < floor_level * 4.0F ? 1 : 0; } const auto runs = pause_runs( quiet, std::max(1, std::lround(kPauseMinSeconds / 0.010F))); const auto keep = static_cast(std::lround(kPauseKeepSeconds / 0.010F)); const auto target = static_cast(std::lround(target_seconds * rate)); const auto cut_at = [&](const PauseRun & run) { const int64_t end = (run.begin + std::min(run.end - run.begin, keep)) * hop + window; return std::vector( wav.begin(), wav.begin() + static_cast(std::min(end, total))); }; if (total <= target) { if (!runs.empty() && runs.back().end * hop + window >= total - hop) { return wav; } const auto boundary = static_cast(static_cast(total) * kMinKeepFraction); for (auto it = runs.rbegin(); it != runs.rend(); ++it) { if (it->begin * hop >= boundary) { return cut_at(*it); } } return finish_with_room_tone(wav, sample_rate, floor_level, rng); } const auto forward_limit = target + static_cast(kCropForwardSeconds * rate); for (const auto & run : runs) { if (run.begin * hop >= target && run.begin * hop <= forward_limit) { return cut_at(run); } } const auto backward_limit = target - static_cast(kCropBackwardSeconds * rate); for (auto it = runs.rbegin(); it != runs.rend(); ++it) { if (it->begin * hop < target && it->end * hop >= backward_limit) { return cut_at(*it); } } std::vector truncated( wav.begin(), wav.begin() + static_cast(std::min(target, total))); return finish_with_room_tone(truncated, sample_rate, floor_level, rng); } SpeechLevel speech_level_db(const std::vector & wav, int sample_rate) { const auto rate = static_cast(sample_rate); const auto window = static_cast(rate * 0.025F); const auto hop = static_cast(rate * 0.010F); SpeechLevel out; if (window <= 0 || static_cast(wav.size()) < window) { double sum = 0.0; for (const float value : wav) { sum += static_cast(value) * static_cast(value); } const double rms = wav.empty() ? 0.0 : std::sqrt(sum / static_cast(wav.size())); out.level_db = 20.0F * std::log10(std::max(static_cast(rms), 1.0e-6F)); return out; } const auto rms = frame_rms(wav, window, hop); const float threshold = quantile(rms, 0.2F) * 1.5F; std::vector active; active.reserve(rms.size()); for (const float value : rms) { if (value > threshold) { active.push_back(value); } } if (active.empty()) { active = rms; } out.level_db = 20.0F * std::log10(lower_median(active)); out.active_seconds = static_cast(active.size()) * static_cast(hop) / rate; return out; } NormalizedReference normalize_reference(const std::vector & wav, int sample_rate) { const auto level = speech_level_db(wav, sample_rate); // Boost only: a reference that is already hotter than the prompt level is // passed through untouched instead of being pulled down. float gain_db = std::min(std::max(kPromptLevelDb - level.level_db, 0.0F), kRefGainLimitDb); float peak = 0.0F; for (const float value : wav) { peak = std::max(peak, std::fabs(value)); } if (peak > 0.0F) { // Never let the boost clip: cap it at the headroom left below 0.95. gain_db = std::min(gain_db, std::max(0.0F, 20.0F * std::log10(kRefPeakCeiling / peak))); } const float gain = std::pow(10.0F, gain_db / 20.0F); NormalizedReference out; out.wav = wav; for (auto & value : out.wav) { value *= gain; } out.level_db = level.level_db + gain_db; return out; } float output_gain(float prompt_level_db) { return std::pow(10.0F, (kOutputLevelDb - prompt_level_db) / 20.0F); } void soft_limit(std::vector & wav, float knee) { const float span = 1.0F - knee; if (span <= 0.0F) { return; } for (auto & value : wav) { const float magnitude = std::fabs(value); if (magnitude > knee) { const float limited = knee + span * std::tanh((magnitude - knee) / span); value = value < 0.0F ? -limited : limited; } } } std::optional speech_onset(const std::vector & wav, int sample_rate) { const auto window = static_cast(static_cast(sample_rate) * 0.010F); if (window <= 0 || static_cast(wav.size()) < window * kOnsetWindowFrames) { return std::nullopt; } const auto rms = block_rms(wav, window); const auto hit = first_sustained_hit( rms, onset_threshold(rms), kOnsetWindowFrames, kOnsetMinFrames); if (!hit.has_value()) { return std::nullopt; } return *hit * window; } std::optional energy_onset( const std::vector & wav, int sample_rate, float over_floor_db, int64_t min_frames) { const auto window = static_cast(static_cast(sample_rate) * 0.010F); if (window <= 0 || static_cast(wav.size()) < window * min_frames) { return std::nullopt; } const auto rms = block_rms(wav, window); if (static_cast(rms.size()) < min_frames) { return std::nullopt; } const auto hit = first_sustained_hit(rms, energy_threshold(rms, over_floor_db), min_frames, min_frames); if (!hit.has_value()) { return std::nullopt; } return *hit * window; } float energy_fraction( const std::vector & wav, int sample_rate, int64_t start, int64_t end, float over_floor_db) { const auto window = static_cast(static_cast(sample_rate) * 0.010F); if (window <= 0 || end <= start || static_cast(wav.size()) < window) { return 0.0F; } const auto rms = block_rms(wav, window); const float threshold = energy_threshold(rms, over_floor_db); const int64_t first = std::min(start / window, static_cast(rms.size())); const int64_t last = std::min(std::max(start / window + 1, end / window), static_cast(rms.size())); if (last <= first) { return 0.0F; } int64_t above = 0; for (int64_t i = first; i < last; ++i) { above += rms[static_cast(i)] > threshold ? 1 : 0; } return static_cast(above) / static_cast(last - first); } int64_t lead_cut(int64_t onset, int sample_rate, float lead, float skip) { const auto rate = static_cast(sample_rate); const int64_t cut = std::max(onset - static_cast(lead * rate), static_cast(skip * rate)); return std::min(cut, std::max(0, onset - static_cast(0.02F * rate))); } std::vector trim_lead( const std::vector & wav, int sample_rate, float lead, float skip) { const auto onset = speech_onset(wav, sample_rate); if (!onset.has_value()) { return wav; } const int64_t cut = std::min(lead_cut(*onset, sample_rate, lead, skip), static_cast(wav.size())); return std::vector(wav.begin() + static_cast(cut), wav.end()); } std::vector trim_trail(const std::vector & wav, int sample_rate, float trail) { const auto window = static_cast(static_cast(sample_rate) * 0.010F); if (window <= 0 || static_cast(wav.size()) < window) { return wav; } const auto rms = block_rms(wav, window); const float threshold = onset_threshold(rms); int64_t last = -1; for (int64_t i = 0; i < static_cast(rms.size()); ++i) { if (rms[static_cast(i)] > threshold) { last = i; } } if (last < 0) { return wav; } const int64_t end = std::min( static_cast(wav.size()), (last + 1) * window + static_cast(trail * static_cast(sample_rate))); return std::vector(wav.begin(), wav.begin() + static_cast(end)); } void fade_edges( std::vector & wav, int sample_rate, bool fade_in, bool fade_out, float fade_seconds) { const auto fade = static_cast(fade_seconds * static_cast(sample_rate)); if (fade <= 1 || static_cast(wav.size()) <= 2 * fade) { return; } for (int64_t i = 0; i < fade; ++i) { const float ramp = static_cast(i) / static_cast(fade - 1); if (fade_in) { wav[static_cast(i)] *= ramp; } if (fade_out) { wav[wav.size() - static_cast(fade) + static_cast(i)] *= 1.0F - ramp; } } } std::vector join_segments(std::vector> parts, int sample_rate) { std::vector out; const auto count = static_cast(parts.size()); for (int64_t i = 0; i < count; ++i) { fade_edges(parts[static_cast(i)], sample_rate, i > 0, i + 1 < count); out.insert( out.end(), parts[static_cast(i)].begin(), parts[static_cast(i)].end()); } return out; } } // namespace audio_ops SoproReferenceBuilder::SoproReferenceBuilder( const SoproTTSAssets & assets, const SoproSpeakerEncoderRuntime & speaker_encoder, const SoproSemanticEncoderRuntime & semantic_encoder, const SoproVocosRuntime & vocoder) : config_(assets.config), speaker_encoder_(speaker_encoder), semantic_encoder_(semantic_encoder), vocoder_(vocoder), mel_mean_(assets.config.model.acoustic_mel_mean), mel_std_(assets.config.model.acoustic_mel_std) { const auto & source = *assets.model_weights; const int64_t in_dim = config_.model.cond_in_dim; const int64_t hidden = config_.model.cond_hidden_dim; cond_proj_w0 = source.require_f32("cond_proj.0.weight", {hidden, in_dim}); cond_proj_b0 = source.require_f32("cond_proj.0.bias", {hidden}); cond_proj_w3 = source.require_f32("cond_proj.3.weight", {hidden, hidden}); cond_proj_b3 = source.require_f32("cond_proj.3.bias", {hidden}); } SoproReference SoproReferenceBuilder::build( const std::vector & audio24, float ref_seconds, std::mt19937_64 & rng) const { if (audio24.empty()) { throw std::runtime_error("Sopro requires non-empty reference audio"); } const auto sample_rate = static_cast(config_.sample_rate); auto cropped = audio_ops::crop_on_pause(audio24, ref_seconds, sample_rate, rng); auto normalized = audio_ops::normalize_reference(cropped, sample_rate); const float reference_level_db = normalized.level_db; auto wav = std::move(normalized.wav); const auto speaker_rate = static_cast(config_.speaker_encoder.sample_rate); const auto wav16 = engine::audio::resample_mono_torchaudio_sinc_hann( wav, sample_rate, speaker_rate); const auto embeddings = speaker_encoder_.encode(wav16); const int64_t in_dim = config_.model.cond_in_dim; const int64_t hidden = config_.model.cond_hidden_dim; std::vector conditioning; conditioning.reserve(static_cast(in_dim)); conditioning.insert(conditioning.end(), embeddings.id_emb.begin(), embeddings.id_emb.end()); conditioning.insert(conditioning.end(), embeddings.style_emb.begin(), embeddings.style_emb.end()); conditioning.insert(conditioning.end(), embeddings.style_ctrl.begin(), embeddings.style_ctrl.end()); if (static_cast(conditioning.size()) != in_dim) { throw std::runtime_error( "Sopro speaker embeddings do not add up to model.cond_in_dim; check config.json"); } std::vector projected(static_cast(hidden), 0.0F); for (int64_t o = 0; o < hidden; ++o) { const float * row = cond_proj_w0.data() + static_cast(o * in_dim); double sum = cond_proj_b0[static_cast(o)]; for (int64_t i = 0; i < in_dim; ++i) { sum += static_cast(row[i]) * static_cast(conditioning[static_cast(i)]); } const auto value = static_cast(sum); projected[static_cast(o)] = value / (1.0F + std::exp(-value)); // SiLU } SoproReference out; out.cond_vec.assign(static_cast(hidden), 0.0F); for (int64_t o = 0; o < hidden; ++o) { const float * row = cond_proj_w3.data() + static_cast(o * hidden); double sum = cond_proj_b3[static_cast(o)]; for (int64_t i = 0; i < hidden; ++i) { sum += static_cast(row[i]) * static_cast(projected[static_cast(i)]); } out.cond_vec[static_cast(o)] = static_cast(sum); } out.semantic_tokens = semantic_encoder_.encode(wav); auto mel = vocoder_.log_mel(wav); const int64_t n_mels = vocoder_.n_mels(); if (n_mels <= 0 || mel.size() % static_cast(n_mels) != 0) { throw std::runtime_error("Sopro reference mel has an unexpected shape"); } out.mel_frames = static_cast(mel.size()) / n_mels; for (int64_t c = 0; c < n_mels; ++c) { const float mean = mel_mean_[static_cast(c)]; const float scale = mel_std_[static_cast(c)]; float * row = mel.data() + static_cast(c * out.mel_frames); for (int64_t t = 0; t < out.mel_frames; ++t) { row[t] = (row[t] - mean) / scale; } } out.mel = std::move(mel); out.level_db = reference_level_db; return out; } } // namespace engine::community_models::sopro_tts