#pragma once #include "engine/framework/runtime/cache_slots.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/index_tts2/assets.h" #include "engine/models/index_tts2/audio_features.h" #include "engine/models/index_tts2/gpt.h" #include "engine/models/index_tts2/qwen_emotion.h" #include "engine/models/index_tts2/request.h" #include "engine/models/index_tts2/s2mel.h" #include "engine/models/index_tts2/semantic_codec.h" #include "engine/models/index_tts2/semantic_encoder.h" #include "engine/models/index_tts2/style_encoder.h" #include "engine/models/index_tts2/tokenizer_text.h" #include "engine/models/index_tts2/vocoder.h" #include #include #include #include #include #include namespace engine::models::index_tts2 { struct IndexTTS2AudioIdentity { int sample_rate = 0; int channels = 0; uint64_t sample_count = 0; uint64_t sample_hash = 0; }; class IndexTTS2Session final : public runtime::RuntimeSessionBase , public runtime::IOfflineVoiceTaskSession { public: IndexTTS2Session( runtime::TaskSpec task, runtime::SessionOptions options, std::shared_ptr assets); std::string family() const override; runtime::VoiceTaskKind task_kind() const override; runtime::RunMode run_mode() const override; void prepare(const runtime::SessionPreparationRequest & request) override; runtime::TaskResult run(const runtime::TaskRequest & request) override; private: struct SpeakerState { IndexTTS2AudioIdentity identity; IndexTTS2SemanticEmbedding semantic; IndexTTS2MelOutput reference_mel; IndexTTS2StyleEmbedding style; IndexTTS2S2MelSequence prompt_condition; }; struct EmotionState { IndexTTS2AudioIdentity identity; IndexTTS2SemanticEmbedding semantic; }; struct AudioIdentityEqual { bool operator()( const IndexTTS2AudioIdentity & lhs, const IndexTTS2AudioIdentity & rhs) const; }; const SpeakerState & resolve_speaker_state(const runtime::AudioBuffer & audio); const EmotionState & resolve_emotion_state(const runtime::AudioBuffer & audio); std::vector resolve_emotion_vector( const IndexTTS2Request & request, const SpeakerState & speaker, const EmotionState & emotion); runtime::AudioBuffer synthesize_segment( const std::vector & text_tokens, int32_t lang_id, const SpeakerState & speaker, const EmotionState & emotion, const std::vector & emotion_vector, const IndexTTS2GenerationOptions & options, uint32_t segment_seed); std::vector explicit_emotion_matrix_vector( const std::vector & emotion_weights, const IndexTTS2StyleEmbedding & style, bool use_random, uint32_t seed) const; runtime::TaskSpec task_; std::shared_ptr assets_; size_t gpt_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t s2mel_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t reference_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t emotion_text_prefill_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; size_t emotion_text_decode_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t weight_context_bytes_ = 32ull * 1024ull * 1024ull; int64_t emotion_text_max_tokens_ = 256; engine::assets::TensorStorageType matmul_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType conv_weight_storage_type_ = engine::assets::TensorStorageType::Native; bool mem_saver_ = false; IndexTTS2TextTokenizer tokenizer_; std::unique_ptr semantic_encoder_; std::unique_ptr semantic_codec_; std::unique_ptr style_encoder_; std::unique_ptr gpt_; std::unique_ptr s2mel_; std::unique_ptr vocoder_; std::unique_ptr qwen_emotion_; std::vector speaker_matrix_; std::vector emotion_matrix_; runtime::CacheSlots speaker_cache_; runtime::CacheSlots emotion_cache_; runtime::CacheSlots> emotion_text_weights_cache_; std::optional uncached_speaker_state_; std::optional uncached_emotion_state_; }; } // namespace engine::models::index_tts2