#pragma once // LFM2.5-Audio assets. The model ships as llama.cpp-format GGUF components in // one directory: the LFM2 backbone with its text tokenizer, an mmproj file // with the FastConformer encoder and the audio adapter, and for speech output // a vocoder file (depthformer) and a tokenizer file (audio detokenizer). The // session picks each component through a session option, so quantizations can // be mixed without renaming the published files. // // Reference implementation: liquid-audio v1.3.0, // https://github.com/Liquid4All/liquid-audio/tree/v1.3.0. File paths in the // lfm2_audio comments are relative to its src/liquid_audio/. #include "engine/framework/assets/tensor_source.h" #include #include #include #include #include namespace engine::community_models::lfm2_audio { // LFM2 hybrid backbone. A layer with zero KV heads is a gated short-conv // block; the others are GQA attention blocks. struct Lfm2BackboneConfig { int64_t vocab_size = 0; int64_t hidden_size = 0; int64_t intermediate_size = 0; int64_t num_attention_heads = 0; int64_t head_dim = 0; int64_t conv_kernel_size = 0; int64_t context_length = 0; std::vector kv_heads; // one entry per layer float rms_norm_eps = 1e-5f; float rope_theta = 1e6f; [[nodiscard]] int64_t num_layers() const noexcept { return static_cast(kv_heads.size()); } [[nodiscard]] bool is_attention_layer(int64_t layer) const { return kv_heads.at(static_cast(layer)) > 0; } }; // NeMo FastConformer encoder (dw_striding x8) followed by the audio adapter // MLP that maps encoder frames to backbone embeddings. struct Lfm2FastConformerEncoderConfig { int64_t n_mels = 0; int64_t hidden_size = 0; int64_t num_layers = 0; int64_t num_heads = 0; int64_t intermediate_size = 0; int64_t conv_kernel_size = 0; int64_t subsampling_channels = 0; int64_t adapter_hidden_size = 0; int64_t output_size = 0; float layer_norm_eps = 1e-5f; }; // The depthformer in the vocoder GGUF: from the backbone's hidden state it // predicts the codes of one audio frame, one codebook after another // (LFM2AudioModel._sample_audio_frame, model/lfm2_audio.py). Its blocks are // pre-norm GQA attention with QK-norm and interleaved RoPE, then SwiGLU, with // no final norm (RawLMBackbone(has_embedding=False), model/transformer.py). struct Lfm2DepthformerConfig { int64_t input_size = 0; // the backbone hidden size int64_t hidden_size = 0; int64_t num_layers = 0; int64_t num_heads = 0; int64_t num_kv_heads = 0; int64_t head_dim = 0; int64_t intermediate_size = 0; int64_t codebooks = 0; int64_t audio_vocab_size = 0; // the codes plus end-of-audio, which is the last float rms_norm_eps = 1e-5f; float rope_theta = 1e6f; [[nodiscard]] int32_t end_of_audio() const noexcept { return static_cast(audio_vocab_size - 1); } }; // The audio detokenizer (detokenizer.py): the mean of the frame's code // embeddings, repeated `upsample` times, an LFM2 hybrid with causal // sliding-window attention, and a linear head to log-magnitude and phase for // an ISTFT. struct Lfm2DetokenizerConfig { Lfm2BackboneConfig lfm; // vocab_size unused int64_t sliding_window = 0; int64_t output_size = 0; int64_t codebooks = 0; int64_t codebook_size = 0; int64_t upsample = 0; int64_t n_fft = 0; int64_t hop_length = 0; int sample_rate = 0; }; struct Lfm2TextVocabulary { std::vector tokens; std::vector merges; std::vector token_types; std::string pre_tokenizer; }; // The model root and whatever the loader can check without choosing a // component: every file the session could select lives under model_root. struct Lfm2AudioAssets { std::filesystem::path model_root; // Set when --model named a backbone GGUF file instead of a directory. std::string default_model_gguf; }; // The components chosen for one session. struct Lfm2AudioComponents { std::filesystem::path model_path; std::filesystem::path mmproj_path; std::shared_ptr model; std::shared_ptr mmproj; Lfm2BackboneConfig backbone; Lfm2FastConformerEncoderConfig encoder; Lfm2TextVocabulary vocabulary; std::vector languages; }; std::shared_ptr load_lfm2_audio_assets(const std::filesystem::path & model_path); // Whether `model_path` (a directory or a GGUF file) holds any LFM2-Audio // component: an LFM2 backbone or an lfm2a mmproj GGUF, complete or not. The // loader claims such paths so that load_lfm2_audio_assets can say what is // missing. Never throws. bool has_lfm2_audio_component(const std::filesystem::path & model_path); // Resolves the backbone and mmproj GGUFs. An empty name means the default: // the only backbone GGUF in the model root, and "mmproj-" or // else the only mmproj GGUF. std::shared_ptr load_lfm2_audio_components( const Lfm2AudioAssets & assets, const std::string & model_gguf, const std::string & mmproj_gguf); // Interleaved generation alternates blocks of this many text tokens and // audio frames. struct Lfm2InterleaveConfig { int64_t text_steps = 0; int64_t audio_steps = 0; }; // The components speech output adds: the vocoder GGUF (depthformer and the // detokenizer's code embedding) and the detokenizer GGUF. The backbone's // audio-frame input embedding stays in the mmproj file. struct Lfm2AudioOutputComponents { std::filesystem::path vocoder_path; std::filesystem::path detokenizer_path; std::shared_ptr vocoder; std::shared_ptr detokenizer; Lfm2DepthformerConfig depthformer; Lfm2DetokenizerConfig detokenizer_config; Lfm2InterleaveConfig interleave; }; // An empty name means "vocoder-" / "tokenizer-", // or else the only vocoder- / tokenizer- GGUF in the model root. std::shared_ptr load_lfm2_audio_output_components( const Lfm2AudioAssets & assets, const Lfm2AudioComponents & components, const std::string & vocoder_gguf, const std::string & detokenizer_gguf); } // namespace engine::community_models::lfm2_audio