#pragma once #include "engine/framework/runtime/streaming_asr_session.h" #include "engine/framework/model_spec/metadata.h" #include "engine/framework/runtime/session_base.h" #include "engine/community_models/parakeet_tdt/assets.h" #include "engine/community_models/parakeet_tdt/decoder.h" #include "engine/community_models/parakeet_tdt/encoder.h" #include "engine/community_models/parakeet_tdt/frontend.h" #include "engine/community_models/parakeet_tdt/weights.h" #include #include #include #include #include #include #include namespace engine::community_models::parakeet_tdt { std::shared_ptr make_parakeet_tdt_loader(); std::shared_ptr make_phonon2_asr_loader(); std::shared_ptr make_underdog_bark3_loader(); std::shared_ptr make_haas_asr_loader(); class ParakeetTDTSessionBase : public runtime::RuntimeSessionBase { public: ParakeetTDTSessionBase( runtime::TaskSpec task, runtime::SessionOptions options, std::shared_ptr assets, std::shared_ptr contract); ~ParakeetTDTSessionBase() override; protected: std::string family_impl() const; runtime::VoiceTaskKind task_kind_impl() const; runtime::RunMode run_mode_impl() const; ParakeetDecodeOptions decode_options_for_request(const runtime::TaskRequest & request) const; runtime::TaskSpec task_; std::shared_ptr assets_; std::shared_ptr contract_; std::string option_prefix_; std::shared_ptr weights_; size_t weight_context_bytes_ = 3072ull * 1024ull * 1024ull; size_t encoder_graph_arena_bytes_ = 1024ull * 1024ull * 1024ull; size_t decoder_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; engine::assets::TensorStorageType matmul_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType conv_weight_storage_type_ = engine::assets::TensorStorageType::Native; bool encoder_flash_attention_ = false; ParakeetFrontend frontend_; std::unique_ptr encoder_; std::unique_ptr decoder_; }; class ParakeetTDTOfflineSession final : public ParakeetTDTSessionBase , public runtime::IOfflineVoiceTaskSession { public: ParakeetTDTOfflineSession( runtime::TaskSpec task, runtime::SessionOptions options, std::shared_ptr assets, std::shared_ptr contract); std::string family() const override; runtime::VoiceTaskKind task_kind() const override; runtime::RunMode run_mode() const override; void prepare(const runtime::SessionPreparationRequest & request) override; runtime::TaskResult run(const runtime::TaskRequest & request) override; private: runtime::TaskResult run_long_form( const runtime::AudioBuffer & audio, const ParakeetDecodeOptions & options); runtime::TaskResult run_vad_chunks( const runtime::AudioBuffer & audio, const std::unordered_map & request_options, const ParakeetDecodeOptions & options); runtime::IOfflineVoiceTaskSession & vad_session(); std::string offline_mode_ = "full_context"; int64_t center_samples_ = 0; int64_t left_context_samples_ = 0; int64_t right_context_samples_ = 0; int64_t auto_full_context_max_samples_ = 0; std::filesystem::path vad_model_path_; std::unique_ptr vad_model_; std::unique_ptr vad_session_; }; class ParakeetTDTStreamingSession final : public ParakeetTDTSessionBase , public runtime::StreamingAsrSessionBase { public: ParakeetTDTStreamingSession( runtime::TaskSpec task, runtime::SessionOptions options, std::shared_ptr assets, std::shared_ptr contract); ~ParakeetTDTStreamingSession() override; std::string family() const override; runtime::VoiceTaskKind task_kind() const override; runtime::RunMode run_mode() const override; void prepare(const runtime::SessionPreparationRequest & request) override; runtime::StreamingPolicy streaming_policy() const override; private: void begin_stream(const runtime::TaskRequest & request) override; void reset_stream_state() override; void consume_audio(const runtime::AudioChunk & chunk) override; void complete_stream(runtime::TaskResult & result) override; ParakeetDecodedText process_ready_windows(bool flush_tail); bool process_center_window(int64_t center_end_sample, bool flush_tail); ParakeetDecodedText merged_decode() const; int64_t center_samples_ = 0; int64_t left_context_samples_ = 0; int64_t right_context_samples_ = 0; int64_t next_center_start_sample_ = 0; int64_t decoded_frame_offset_ = 0; int64_t buffer_start_sample_ = 0; int64_t received_samples_ = 0; runtime::AudioBuffer streaming_audio_; std::unique_ptr window_frontend_; ParakeetDecodeOptions streaming_decode_options_; std::vector token_ids_; std::vector token_frame_indices_; std::vector token_durations_; }; } // namespace engine::community_models::parakeet_tdt