#pragma once #include "engine/framework/core/backend.h" #include "engine/framework/runtime/session.h" #include "engine/community_models/voxcpm1/assets.h" #include "engine/community_models/voxcpm1/types.h" #include #include #include #include #include #include namespace engine::core { class ExecutionContext; } namespace engine::community_models::voxcpm1 { struct VoxCPM1AudioVAEDecoderConfig { size_t weight_context_bytes = 768ull * 1024ull * 1024ull; size_t graph_context_bytes = 1024ull * 1024ull * 1024ull; size_t encoder_graph_context_bytes = 1024ull * 1024ull * 1024ull; int64_t latent_frame_capacity = 0; int64_t encoder_sample_capacity = 240000; engine::assets::TensorStorageType weight_storage_type = engine::assets::TensorStorageType::F32; }; // Streaming decode state for maintaining convolution state across patches struct AudioVAEStreamingDecodeState { struct SlotSpec { int64_t frames = 0; int64_t channels = 0; std::string name; }; struct Slot { int64_t frames = 0; int64_t channels = 0; ggml_tensor* tensor = nullptr; std::string name; }; struct PendingUpdate { size_t slot_index = 0; ggml_tensor* tensor = nullptr; }; AudioVAEStreamingDecodeState() = default; ~AudioVAEStreamingDecodeState(); AudioVAEStreamingDecodeState(const AudioVAEStreamingDecodeState&) = delete; AudioVAEStreamingDecodeState& operator=(const AudioVAEStreamingDecodeState&) = delete; AudioVAEStreamingDecodeState(AudioVAEStreamingDecodeState&& other) noexcept; AudioVAEStreamingDecodeState& operator=(AudioVAEStreamingDecodeState&& other) noexcept; void reset(); void clear(); bool initialize(const std::vector& specs, core::ExecutionContext& execution_context); void begin_graph(); ggml_tensor* take_slot(int64_t frames, int64_t channels, const std::string& name); void queue_update(ggml_tensor* tensor); void build_update_graph(ggml_cgraph* graph) const; void publish_updates(core::ExecutionContext& execution_context); bool is_initialized() const { return !slots_.empty() && ctx_ != nullptr; } ggml_context* get_context() const { return ctx_.get(); } private: struct GgmlContextDeleter { void operator()(ggml_context* ctx) const noexcept { if (ctx != nullptr) { ggml_free(ctx); } } }; std::unique_ptr ctx_; ggml_backend_buffer_t buffer_ = nullptr; std::vector slots_; std::vector pending_updates_; size_t cursor_ = 0; }; class VoxCPM1AudioVAEDecoderRuntime final { public: VoxCPM1AudioVAEDecoderRuntime( std::shared_ptr assets, engine::core::ExecutionContext &execution_context, VoxCPM1AudioVAEDecoderConfig config = {}); ~VoxCPM1AudioVAEDecoderRuntime(); runtime::AudioBuffer decode_features(const std::vector &features, int64_t patches); // Streaming decode: initialize state once, then call decode_streaming_step per patch bool supports_streaming_decode() const; bool initialize_streaming_decode_state(AudioVAEStreamingDecodeState& state); runtime::AudioBuffer decode_streaming_step(const std::vector& patch_features, AudioVAEStreamingDecodeState& state); VoxCPM1EncodedPrompt encode_prompt_audio( const std::optional &prompt_audio, const std::string &prompt_text, const std::optional &reference_audio); void release_runtime_memory(); void release_encoder_graph(); private: class Impl; std::unique_ptr impl_; }; } // namespace engine::community_models::voxcpm1