#pragma once #include #include #include #include #include #include #include "engine/framework/core/backend.h" #include "frontend.h" #include "http.h" namespace minitts::server { constexpr uint64_t kDefaultMaxRequestBodyBytes = 2ull * 1024ull * 1024ull * 1024ull; // Per-model overrides of the server-level `live_ingest` block. Each field is // independently optional: a model that only needs a longer deadline says so and // inherits the rest, rather than restating the whole policy and drifting from it. struct LiveIngestOverrides { std::optional idle_timeout_ms; std::optional total_timeout_ms; std::optional max_body_bytes; std::optional max_chunk_bytes; std::optional send_timeout_ms; }; // Server policy with any per-model override applied on top. LiveIngestLimits resolve_live_ingest_limits( const LiveIngestLimits & base, const LiveIngestOverrides & overrides); struct ServerModelConfig { struct VoicePreset { std::optional voice_id; std::optional voice_ref; std::optional reference_text; // Request options the preset supplies, passed to the model as given. std::unordered_map options; }; std::string id; std::filesystem::path path; std::optional model_spec_override; std::string family; std::string task = "tts"; std::string mode = "offline"; bool lazy = false; // Overrides ServerConfig::device for this model. When absent the server-wide // device remains the default, preserving existing single-device configs. std::optional device; int slots = 1; // Capacity read only by the startup-selected parallel runtime. // Overrides ServerConfig::busy_timeout_ms for this model, and acts as the ceiling // a per-request busy_timeout_ms is clamped to. Model runtimes differ by orders of // magnitude (a short TTS clip vs. minutes of music generation), so one fleet-wide // bound is either too tight for the slow models or useless for the fast ones. std::optional busy_timeout_ms; // Only meaningful for a streaming model reachable over the live-ingest route; // ignored otherwise, since no other route delivers its body incrementally. LiveIngestOverrides live_ingest; std::optional config_id; std::optional weight_id; std::unordered_map load_options; std::unordered_map session_options; std::unordered_map default_request_options; std::unordered_map voice_presets; std::optional default_voice_preset; std::optional default_voice_preset_id; }; struct ServerConfig { std::string host = "127.0.0.1"; int port = 8080; std::string cors_origins = ""; bool ui_enabled = true; // Whether the UI was asked for (--ui, or "ui": true in the config file) rather // than on by default. bool ui_requested = false; bool ui_management = false; engine::core::BackendType backend = engine::core::BackendType::Cuda; std::string backend_impl; int device = 0; int threads = 1; bool lazy_load = false; bool log_request_body = false; uint64_t max_request_body_bytes = kDefaultMaxRequestBodyBytes; // A single model runs one request at a time (serialized on model.mutex). If a // running inference wedges the GPU -- a CUDA call that never returns cannot be // cancelled from userspace -- later requests would otherwise block forever, so // once the current run has held the lock this long a new request fails fast with // 503 instead of parking a worker thread. Must exceed the slowest legitimate // single inference (music generation can take minutes). 0 disables the guard and // restores unbounded waiting. int busy_timeout_ms = 300000; // Upper bound on how many models may be resident in memory at once. Loading a // model past the limit first unloads the least recently used idle model (its // next request reloads it), so a multi-model config can run on a device that // only fits a few of them. 1 enforces a single resident model. 0 disables the // limit and keeps the original behavior: once loaded, a model stays in memory // until it is unloaded explicitly or the server exits. int max_loaded_models = 0; // Unload every resident model once the server has been idle this long without // any model load/run (steady-clock ms). 0 disables idle unload. Complements // max_loaded_models: that bounds peak residency, this frees memory during // quiet periods. The next request reloads lazily. int idle_unload_ms = 0; // Minimum free memory (host and GPU, each) the server must retain after // loading a model, in MiB. Before every lazy load the server estimates the // model's resident footprint from its weights and refuses to load when // estimate + this headroom would not fit. 0 disables the memory guard // entirely (the default), so existing deployments see no behavior change // unless they opt in. int min_free_memory_mb = 0; // Memory for cached resampling kernels in MiB, 0 disables the cache; unset keeps the library default. std::optional resample_kernel_cache_mb; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; std::string frontend_listener; ServerFrontendOptions frontend_options; std::optional model_spec_override; // Session options for every model of a family that does not set them itself: // the configured models and the ones loaded later through POST /v1/models/load // (the WebUI). Keys are family-qualified, e.g. "irodori_tts.mem_saver". std::unordered_map default_session_options; // Voice library shared across all TTS models: *.wav files plus a `prompt_text` // mapping file (|). A request `voice` name that is not a // model preset resolves to /.wav as the cloning reference. std::optional voice_dir; std::vector models; }; engine::core::BackendType parse_server_backend(const std::string & value); ServerConfig load_server_config(const std::filesystem::path & path); ServerConfig load_server_config(const std::filesystem::path & path, bool parallel_jobs); // Adds the config.default_session_options entries for model.family that the model's // own session_options do not set. void apply_default_session_options(const ServerConfig & config, ServerModelConfig & model); // A WebUI load describes a model only by what its request carries. For an id the // server config defines, takes the fields such a request cannot carry from that // definition, while the load is still the same kind of model (family and task). void inherit_configured_model_fields(const ServerModelConfig & configured, ServerModelConfig & model); } // namespace minitts::server