#pragma once #include "engine/framework/assets/resource_bundle.h" #include #include #include #include #include namespace engine::assets { class TensorSource; } namespace engine::models::audio_flamingo { enum class AudioFlamingoVariant { V3, Next, }; struct AudioFlamingoFrontendConfig { int sample_rate = 16000; int64_t feature_size = 128; int64_t hop_length = 160; int64_t n_fft = 400; int64_t chunk_length_sec = 30; int64_t max_audio_length_sec = 600; }; struct AudioFlamingoAudioEncoderConfig { int64_t num_mel_bins = 128; int64_t num_hidden_layers = 32; int64_t num_attention_heads = 20; int64_t intermediate_size = 5120; int64_t hidden_size = 1280; int64_t max_source_positions = 1500; }; struct AudioFlamingoTextDecoderConfig { int64_t vocab_size = 0; int64_t hidden_size = 0; int64_t intermediate_size = 0; int64_t num_hidden_layers = 0; int64_t num_attention_heads = 0; int64_t num_key_value_heads = 0; int64_t head_dim = 0; int64_t max_position_embeddings = 0; int64_t audio_token_id = 0; int64_t audio_bos_token_id = 0; int64_t audio_eos_token_id = 0; std::vector eos_token_ids; float rms_norm_eps = 1.0e-6F; float rope_theta = 1000000.0F; }; struct AudioFlamingoRoTEConfig { int64_t max_position_embeddings = 1200; float rope_theta = 1200.0F; float partial_rotary_factor = 0.2F; float audio_frame_step = 0.01F; }; struct AudioFlamingoConfig { AudioFlamingoVariant variant = AudioFlamingoVariant::V3; int64_t max_new_tokens = 512; bool projector_bias = true; AudioFlamingoFrontendConfig frontend; AudioFlamingoAudioEncoderConfig audio_encoder; AudioFlamingoTextDecoderConfig text_decoder; AudioFlamingoRoTEConfig rote; }; struct AudioFlamingoAssets { assets::ResourceBundle resources; AudioFlamingoConfig config; std::shared_ptr model_weights; }; std::shared_ptr load_audio_flamingo_assets(const std::filesystem::path & model_path); std::shared_ptr load_audio_flamingo_adapter( std::shared_ptr assets, const std::filesystem::path & path, float strength); } // namespace engine::models::audio_flamingo