#include "app/cli/request.h" #include "engine/framework/audio/output.h" #include "engine/framework/io/filesystem.h" #include "engine/framework/io/json.h" #include "engine/framework/runtime/registry.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef _OPENMP #include #endif namespace { using JsonValue = engine::io::json::Value; using JsonObject = engine::io::json::Value::Object; struct PerfCaseSpec { std::string id; std::string family; std::string case_id; std::string request_id; std::optional warmup_request_id; }; struct MeasurementSummary { std::vector wall_ms; double average_wall_ms = 0.0; double rtf = 0.0; double duration_sec = 0.0; }; struct CaseSummary { std::string id; std::string family; std::string case_id; std::string request_id; std::string task; std::string mode; MeasurementSummary cold; MeasurementSummary warm; }; struct RunResult { engine::runtime::TaskResult result; double wall_ms = 0.0; }; std::string optional_arg(int argc, char ** argv, const std::string & name, const std::string & fallback) { for (int i = 1; i + 1 < argc; ++i) { if (std::string_view(argv[i]) == name) { return argv[i + 1]; } } return fallback; } std::optional find_arg(int argc, char ** argv, const std::string & name) { for (int i = 1; i + 1 < argc; ++i) { if (std::string_view(argv[i]) == name) { return std::string(argv[i + 1]); } } return std::nullopt; } int parse_int_arg(int argc, char ** argv, const std::string & name, int fallback) { if (const auto value = find_arg(argc, argv, name)) { return std::stoi(*value); } return fallback; } std::string quote_json(const std::string & value) { return engine::io::json::stringify_string(value); } std::string lower_ascii(std::string value) { std::transform(value.begin(), value.end(), value.begin(), [](unsigned char ch) { return static_cast(std::tolower(ch)); }); return value; } std::string make_timestamp() { const auto now = std::chrono::system_clock::now(); const auto time = std::chrono::system_clock::to_time_t(now); std::tm tm = {}; #if defined(_WIN32) localtime_s(&tm, &time); #else localtime_r(&time, &tm); #endif std::ostringstream out; out << std::put_time(&tm, "%Y%m%d-%H%M%S"); return out.str(); } engine::core::BackendType parse_backend(const std::string & value) { const std::string lowered = lower_ascii(value); if (lowered == "cpu") { return engine::core::BackendType::Cpu; } if (lowered == "cuda") { return engine::core::BackendType::Cuda; } if (lowered == "vulkan") { return engine::core::BackendType::Vulkan; } if (lowered == "metal") { return engine::core::BackendType::Metal; } if (lowered == "best") { return engine::core::BackendType::BestAvailable; } throw std::runtime_error("unsupported backend: " + value); } std::string shorten_text(std::string text, size_t limit) { if (text.size() <= limit) { return text; } size_t cut = text.rfind(' ', limit); if (cut == std::string::npos || cut < (limit / 2)) { cut = limit; } text.resize(cut); while (!text.empty() && std::isspace(static_cast(text.back())) != 0) { text.pop_back(); } return text; } std::filesystem::path write_clipped_wav( const std::filesystem::path & source_path, const std::filesystem::path & temp_dir, double seconds) { const auto audio = minitts::cli::read_audio_buffer(source_path); const int64_t frame_limit = static_cast(seconds * static_cast(audio.sample_rate)); const int64_t sample_limit = std::max( 1, std::min( static_cast(audio.samples.size()), frame_limit * static_cast(audio.channels))); std::vector clipped( audio.samples.begin(), audio.samples.begin() + static_cast(sample_limit)); const auto target = temp_dir / source_path.filename(); engine::audio::WavPcm16Sink().write( target, engine::audio::AudioBuffer{ audio.sample_rate, audio.channels, std::move(clipped), }); return target; } JsonValue clip_audio_path_field( const JsonValue & value, const std::filesystem::path & repo_root, const std::filesystem::path & temp_dir, double seconds) { const auto absolute = std::filesystem::path(value.as_string()).is_absolute() ? std::filesystem::path(value.as_string()) : repo_root / value.as_string(); return JsonValue::make_string(write_clipped_wav(absolute, temp_dir, seconds).string()); } JsonValue derive_warmup_request( const JsonValue & request, const std::filesystem::path & repo_root, const std::filesystem::path & temp_dir) { JsonObject out = request.as_object(); const auto shorten_field = [&](const char * key, size_t limit) { const auto it = out.find(key); if (it != out.end() && it->second.is_string()) { it->second = JsonValue::make_string(shorten_text(it->second.as_string(), limit)); } }; shorten_field("text", 80); shorten_field("target_text", 80); shorten_field("style_ref_text", 80); shorten_field("reference_text", 80); shorten_field("lyrics", 80); shorten_field("instruct", 64); const auto clip_field = [&](const char * key) { const auto it = out.find(key); if (it != out.end() && it->second.is_string()) { it->second = clip_audio_path_field(it->second, repo_root, temp_dir, 2.0); } }; clip_field("audio"); clip_field("voice_ref"); clip_field("source_audio"); clip_field("target_voice"); clip_field("prosody_ref"); clip_field("style_ref"); const auto it_duration = out.find("duration_seconds"); if (it_duration != out.end() && it_duration->second.is_number()) { it_duration->second = JsonValue::make_number(std::min(4.0, it_duration->second.as_number())); } const auto it_max_tokens = out.find("max_tokens"); if (it_max_tokens != out.end() && it_max_tokens->second.is_number()) { it_max_tokens->second = JsonValue::make_number(std::min(64, it_max_tokens->second.as_i64())); } if (const auto options_it = out.find("options"); options_it != out.end() && options_it->second.is_object()) { JsonObject options = options_it->second.as_object(); const auto reduce_numeric_option = [&](const char * key, double cap) { const auto it = options.find(key); if (it != options.end() && it->second.is_number()) { it->second = JsonValue::make_number(std::min(cap, it->second.as_number())); } }; reduce_numeric_option("duration_seconds", 4.0); reduce_numeric_option("target_duration_seconds", 4.0); options_it->second = JsonValue::make_object(std::move(options)); } return JsonValue::make_object(std::move(out)); } const JsonValue & require_case(const JsonValue & root, const std::string & case_id) { for (const auto & item : root.require("cases").as_array()) { if (engine::io::json::require_string(item, "id") == case_id) { return item; } } throw std::runtime_error("perf harness could not find audiocpp_cli case: " + case_id); } const JsonValue & require_request(const JsonValue & case_value, const std::string & request_id) { for (const auto & request : case_value.require("requests").as_array()) { if (engine::io::json::require_string(request, "id") == request_id) { return request; } } throw std::runtime_error( "perf harness could not find request " + request_id + " in case " + engine::io::json::require_string(case_value, "id")); } std::unordered_map json_string_map(const JsonValue * value) { std::unordered_map out; if (value == nullptr || value->is_null()) { return out; } for (const auto & [key, child] : value->as_object()) { out.emplace(key, minitts::cli::json_option_string(child)); } return out; } PerfCaseSpec parse_perf_case(const JsonValue & value) { PerfCaseSpec out; out.id = engine::io::json::require_string(value, "id"); out.family = engine::io::json::require_string(value, "family"); out.case_id = engine::io::json::require_string(value, "case_id"); out.request_id = engine::io::json::require_string(value, "request_id"); if (const auto * warmup_request_id = value.find("warmup_request_id"); warmup_request_id != nullptr && warmup_request_id->is_string()) { out.warmup_request_id = warmup_request_id->as_string(); } return out; } engine::runtime::ModelLoadRequest build_load_request( const JsonValue & case_value, const std::filesystem::path & repo_root) { engine::runtime::ModelLoadRequest request; request.model_path = repo_root / engine::io::json::require_string(case_value, "model"); request.family_hint = engine::io::json::require_string(case_value, "family"); request.options = json_string_map(case_value.find("load_options")); return request; } engine::runtime::SessionOptions build_session_options( const JsonValue & case_value, engine::core::BackendType backend_type, int device, int threads) { engine::runtime::SessionOptions options; options.backend.type = backend_type; options.backend.device = device; options.backend.threads = threads; options.options = json_string_map(case_value.find("session_options")); return options; } engine::runtime::TaskSpec build_task_spec(const JsonValue & case_value) { return engine::runtime::TaskSpec{ engine::runtime::parse_voice_task_kind(engine::io::json::require_string(case_value, "task")), engine::runtime::parse_run_mode(engine::io::json::require_string(case_value, "mode")), }; } double audio_duration_sec(const engine::runtime::AudioBuffer & audio) { if (audio.sample_rate <= 0 || audio.channels <= 0) { return 0.0; } return static_cast(audio.samples.size()) / static_cast(audio.sample_rate * audio.channels); } double result_duration_sec( const engine::runtime::TaskRequest & request, const engine::runtime::TaskResult & result) { if (result.audio_output.has_value()) { return audio_duration_sec(*result.audio_output); } if (!result.named_audio_outputs.empty()) { return audio_duration_sec(result.named_audio_outputs.front().audio); } if (request.audio_input.has_value()) { return audio_duration_sec(*request.audio_input); } return 0.0; } RunResult run_impl( engine::runtime::IVoiceTaskSession & session, const engine::runtime::TaskSpec & task_spec, const engine::runtime::TaskRequest & request, bool prepare_first) { const auto started = std::chrono::steady_clock::now(); if (prepare_first) { session.prepare(engine::runtime::build_preparation_request(request)); } if (task_spec.mode == engine::runtime::RunMode::Offline) { auto * offline = dynamic_cast(&session); if (offline == nullptr) { throw std::runtime_error("selected model does not support offline mode"); } auto result = offline->run(request); const auto ended = std::chrono::steady_clock::now(); return { std::move(result), std::chrono::duration(ended - started).count(), }; } auto * streaming = dynamic_cast(&session); if (streaming == nullptr) { throw std::runtime_error("selected model does not support streaming mode"); } if (!request.audio_input.has_value()) { throw std::runtime_error("streaming perf request requires request.audio_input"); } streaming->reset(); const auto & audio = *request.audio_input; constexpr int kChunkSamples = 512; for (size_t offset = 0; offset < audio.samples.size(); offset += static_cast(kChunkSamples)) { const size_t available = std::min(static_cast(kChunkSamples), audio.samples.size() - offset); std::vector chunk(static_cast(kChunkSamples), 0.0f); std::copy( audio.samples.begin() + static_cast(offset), audio.samples.begin() + static_cast(offset + available), chunk.begin()); streaming->process_audio_chunk(engine::runtime::AudioChunk{ audio.sample_rate, audio.channels, static_cast(offset / static_cast(std::max(1, audio.channels))), std::move(chunk), }); } auto result = streaming->finalize(); const auto ended = std::chrono::steady_clock::now(); return { std::move(result), std::chrono::duration(ended - started).count(), }; } RunResult run_once( engine::runtime::IVoiceTaskSession & session, const engine::runtime::TaskSpec & task_spec, const engine::runtime::TaskRequest & request) { return run_impl(session, task_spec, request, true); } RunResult run_prepared_once( engine::runtime::IVoiceTaskSession & session, const engine::runtime::TaskSpec & task_spec, const engine::runtime::TaskRequest & request) { return run_impl(session, task_spec, request, false); } MeasurementSummary measure_cold( const engine::runtime::ModelRegistry & registry, const engine::runtime::ModelLoadRequest & load_request, const engine::runtime::TaskSpec & task_spec, const engine::runtime::SessionOptions & session_options, const engine::runtime::TaskRequest & request) { MeasurementSummary summary; for (int rep = 0; rep < 3; ++rep) { const auto started = std::chrono::steady_clock::now(); auto model = registry.load(load_request); auto session = model->create_task_session(task_spec, session_options); auto run = run_once(*session, task_spec, request); const auto ended = std::chrono::steady_clock::now(); summary.wall_ms.push_back(std::chrono::duration(ended - started).count()); summary.duration_sec = result_duration_sec(request, run.result); } summary.average_wall_ms = std::accumulate(summary.wall_ms.begin(), summary.wall_ms.end(), 0.0) / static_cast(summary.wall_ms.size()); if (summary.duration_sec > 0.0) { summary.rtf = (summary.average_wall_ms / 1000.0) / summary.duration_sec; } return summary; } MeasurementSummary measure_warm( const engine::runtime::ModelRegistry & registry, const engine::runtime::ModelLoadRequest & load_request, const engine::runtime::TaskSpec & task_spec, const engine::runtime::SessionOptions & session_options, const engine::runtime::TaskRequest & warmup_request, const engine::runtime::TaskRequest & request) { auto model = registry.load(load_request); auto session = model->create_task_session(task_spec, session_options); (void) run_once(*session, task_spec, warmup_request); session->prepare(engine::runtime::build_preparation_request(request)); MeasurementSummary summary; for (int rep = 0; rep < 3; ++rep) { auto run = run_prepared_once(*session, task_spec, request); summary.wall_ms.push_back(run.wall_ms); summary.duration_sec = result_duration_sec(request, run.result); } summary.average_wall_ms = std::accumulate(summary.wall_ms.begin(), summary.wall_ms.end(), 0.0) / static_cast(summary.wall_ms.size()); if (summary.duration_sec > 0.0) { summary.rtf = (summary.average_wall_ms / 1000.0) / summary.duration_sec; } return summary; } std::string measurement_json(const MeasurementSummary & measurement) { std::ostringstream out; out << "{" << "\"wall_ms\":["; for (size_t i = 0; i < measurement.wall_ms.size(); ++i) { if (i != 0) { out << ","; } out << measurement.wall_ms[i]; } out << "],\"average_wall_ms\":" << measurement.average_wall_ms << ",\"duration_sec\":" << measurement.duration_sec << ",\"rtf\":" << measurement.rtf << "}"; return out.str(); } std::string case_summary_json(const CaseSummary & summary) { std::ostringstream out; out << "{" << "\"id\":" << quote_json(summary.id) << ",\"family\":" << quote_json(summary.family) << ",\"case_id\":" << quote_json(summary.case_id) << ",\"request_id\":" << quote_json(summary.request_id) << ",\"task\":" << quote_json(summary.task) << ",\"mode\":" << quote_json(summary.mode) << ",\"cold\":" << measurement_json(summary.cold) << ",\"warm\":" << measurement_json(summary.warm) << "}"; return out.str(); } } // namespace int main(int argc, char ** argv) { try { std::cout.setf(std::ios::unitbuf); const std::filesystem::path repo_root = std::filesystem::current_path(); const std::filesystem::path perf_cases_path = optional_arg( argc, argv, "--perf-cases", "tests/perf/model_perf_cases.json"); const std::filesystem::path path_cases_path = optional_arg( argc, argv, "--request-cases", "tests/perf/model_perf_request_cases.json"); const std::string backend_name = optional_arg(argc, argv, "--backend", "cuda"); const int device = parse_int_arg(argc, argv, "--device", 0); const int threads = parse_int_arg(argc, argv, "--threads", 1); if (threads <= 0) { throw std::runtime_error("--threads must be positive"); } #ifdef _OPENMP omp_set_num_threads(threads); #endif const auto selected_id = find_arg(argc, argv, "--id"); const auto summary_out_arg = find_arg(argc, argv, "--summary-out"); const auto output_root = summary_out_arg.has_value() ? std::filesystem::path(*summary_out_arg).parent_path() : repo_root / "build" / "logs" / "perf" / ("cpp_" + lower_ascii(backend_name) + "_" + make_timestamp()); std::filesystem::create_directories(output_root); const auto perf_root = engine::io::json::parse_file(perf_cases_path); const auto path_root = engine::io::json::parse_file(path_cases_path); auto registry = engine::runtime::make_default_registry(); const auto backend_type = parse_backend(backend_name); std::vector summaries; for (const auto & perf_case_value : perf_root.require("cases").as_array()) { const auto perf_case = parse_perf_case(perf_case_value); if (selected_id.has_value() && *selected_id != perf_case.id) { continue; } const auto & case_value = require_case(path_root, perf_case.case_id); const auto & request_value = require_request(case_value, perf_case.request_id); const auto load_request = build_load_request(case_value, repo_root); const auto task_spec = build_task_spec(case_value); const auto session_options = build_session_options(case_value, backend_type, device, threads); const auto request = minitts::cli::build_request_from_json(request_value, repo_root); const auto temp_dir = output_root / (perf_case.id + "_warmup"); std::filesystem::create_directories(temp_dir); engine::runtime::TaskRequest warmup_request; if (perf_case.warmup_request_id.has_value()) { warmup_request = minitts::cli::build_request_from_json( require_request(case_value, *perf_case.warmup_request_id), repo_root); } else { warmup_request = minitts::cli::build_request_from_json( derive_warmup_request(request_value, repo_root, temp_dir), repo_root); } CaseSummary summary; summary.id = perf_case.id; summary.family = perf_case.family; summary.case_id = perf_case.case_id; summary.request_id = perf_case.request_id; summary.task = engine::runtime::to_string(task_spec.task); summary.mode = engine::runtime::to_string(task_spec.mode); summary.cold = measure_cold(registry, load_request, task_spec, session_options, request); summary.warm = measure_warm(registry, load_request, task_spec, session_options, warmup_request, request); summaries.push_back(summary); std::cout << perf_case.id << " cold_avg_ms=" << summary.cold.average_wall_ms << " cold_rtf=" << summary.cold.rtf << " warm_avg_ms=" << summary.warm.average_wall_ms << " warm_rtf=" << summary.warm.rtf << std::endl; } std::ostringstream summary_json; summary_json << "{\"backend\":" << quote_json(backend_name) << ",\"device\":" << device << ",\"threads\":" << threads << ",\"cases\":["; for (size_t i = 0; i < summaries.size(); ++i) { if (i != 0) { summary_json << ","; } summary_json << case_summary_json(summaries[i]); } summary_json << "]}"; const auto summary_path = summary_out_arg.has_value() ? std::filesystem::path(*summary_out_arg) : output_root / "summary.json"; std::ofstream(summary_path) << summary_json.str() << "\n"; std::cout << "summary_json=" << summary_path.string() << std::endl; return 0; } catch (const std::exception & ex) { std::cerr << "model_perf failed: " << ex.what() << "\n"; return 1; } }