#pragma once #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/audio/dsp.h" #include "engine/framework/audio/nemo_mel_frontend.h" #include "engine/framework/core/backend_weight_store.h" #include "engine/framework/core/execution_context.h" #include "engine/framework/modules/conformer_modules.h" #include "engine/framework/modules/transformers/transformer_blocks.h" #include "engine/framework/runtime/model.h" #include "engine/framework/tokenizers/sentencepiece.h" #include #include #include namespace engine::models::canary_asr { struct CanaryAssets { assets::ResourceBundle resources; std::shared_ptr source; std::vector vocabulary; std::vector window; audio::AudioTensor filterbank; std::shared_ptr frontend; int32_t special_token(const std::string & text) const; }; struct CanaryWeights { std::unique_ptr store; modules::DepthwiseConvSubsamplingWeights subsampling; modules::LinearWeights encoder_out, head; std::vector encoder; std::vector decoder; core::TensorValue embedding, positions; modules::NormWeights embedding_norm, decoder_norm; }; std::shared_ptr load_canary_assets(const std::filesystem::path & path); std::unique_ptr load_canary_weights( const CanaryAssets & assets, core::ExecutionContext & execution, assets::TensorStorageType type); struct CanaryFrontendFeatures { std::vector values; int64_t raw_frames = 0; int64_t valid_frames = 0; }; CanaryFrontendFeatures extract_canary_frontend( const std::vector & samples, const CanaryAssets & assets, size_t threads); class CanaryRuntime { public: CanaryRuntime(const CanaryAssets & assets, const CanaryWeights & weights, core::ExecutionContext & execution); ~CanaryRuntime(); std::vector transcribe(const std::vector & samples, const std::vector & prompt, int64_t max_tokens); private: struct Graphs; const CanaryAssets & assets_; const CanaryWeights & weights_; core::ExecutionContext & execution_; std::unique_ptr graphs_; }; std::shared_ptr make_canary_asr_loader(); } // namespace engine::models::canary_asr