#pragma once #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/assets/tensor_source.h" #include "engine/framework/modules/speech_encoders/whisper_embedding.h" #include #include #include #include #include namespace engine::models::vevo2 { struct Vevo2Qwen2ARConfig { int64_t vocab_size = 0; int64_t hidden_size = 0; int64_t intermediate_size = 0; int64_t num_hidden_layers = 0; int64_t num_attention_heads = 0; int64_t num_key_value_heads = 0; int64_t max_position_embeddings = 0; int64_t max_window_layers = 0; int64_t bos_token_id = 151643; int64_t eos_token_id = 151645; int64_t pad_token_id = 151643; float rms_norm_eps = 1.0e-6F; float rope_theta = 1000000.0F; bool tie_word_embeddings = true; std::string model_type; std::vector generation_eos_token_ids; int64_t generation_top_k = 25; float generation_top_p = 0.8F; float generation_temperature = 1.0F; float generation_repetition_penalty = 1.1F; bool generation_do_sample = true; }; struct Vevo2CocoTokenizerConfig { std::string coco_type; int64_t downsample_rate = 0; int64_t codebook_size = 0; int64_t hidden_size = 0; int64_t codebook_dim = 0; int64_t vocos_dim = 0; int64_t vocos_intermediate_dim = 0; int64_t vocos_num_layers = 0; bool use_normed_whisper = true; int64_t whisper_dim = 0; int64_t chromagram_dim = 0; }; struct Vevo2AcousticPreprocessConfig { int64_t sample_rate = 24000; int64_t hop_size = 480; int64_t n_fft = 1920; int64_t num_mels = 128; int64_t win_size = 1920; float fmin = 0.0F; float fmax = 12000.0F; float mel_var = 8.14F; float mel_mean = -4.92F; }; struct Vevo2FMConfig { Vevo2AcousticPreprocessConfig preprocess; int64_t mel_dim = 128; int64_t hidden_size = 1024; int64_t num_layers = 16; int64_t num_heads = 16; float cfg_scale = 0.2F; bool use_cond_code = true; int64_t cond_codebook_size = 16384; int64_t cond_scale_factor = 4; float sigma = 1.0e-5F; std::string time_scheduler = "cos"; int64_t cond_sample_rate = 16000; Vevo2CocoTokenizerConfig coco_content_style; bool use_text_as_condition = false; int64_t text_cond_codebook_size = 0; }; struct Vevo2VocoderConfig { Vevo2AcousticPreprocessConfig preprocess; int64_t input_channels = 128; int64_t dim = 1024; int64_t intermediate_dim = 4096; int64_t num_layers = 30; int64_t n_fft = 1920; int64_t hop_size = 480; std::string padding = "same"; }; struct Vevo2Config { Vevo2Qwen2ARConfig ar; Vevo2CocoTokenizerConfig prosody_tokenizer; Vevo2CocoTokenizerConfig content_style_tokenizer; Vevo2FMConfig fm; Vevo2FMConfig fm_text; Vevo2VocoderConfig vocoder; engine::modules::WhisperEmbeddingConfig whisper; }; struct Vevo2Assets { assets::ResourceBundle resources; Vevo2Config config; std::shared_ptr content_style_tokenizer_weights; std::shared_ptr prosody_tokenizer_weights; std::shared_ptr ar_weights; std::shared_ptr fm_weights; std::shared_ptr fm_whisper_stats; std::shared_ptr vocoder_weights_0; std::shared_ptr whisper_weights; }; std::shared_ptr load_vevo2_assets( const std::filesystem::path & model_path, const std::optional & whisper_model_path = std::nullopt); } // namespace engine::models::vevo2