/* * End-to-end smoke test for MOSS-VoiceGenerator: designs a voice from a written * instruction, speaks the given text in it, and writes a WAV. * * moss_voicegen_smoke --model --instruct "" --text "" \ * --language English --output out.wav [--weight-type bf16] [--seed 0] [--threads N] */ #include "engine/community_models/moss_voicegen/assets.h" #include "engine/framework/decoders/moss_tts_delay/backbone.h" #include "engine/framework/decoders/moss_tts_delay/delay_decoder.h" #include "engine/framework/decoders/moss_tts_delay/heads.h" #include "engine/community_models/moss_voicegen/tokenizer_text.h" #include "engine/framework/audio/wav_writer.h" #include "engine/framework/core/backend.h" #include "engine/framework/core/execution_context.h" #include "engine/framework/codecs/moss_audio_tokenizer_codec_runtime.h" #include "engine/framework/modules/multi_codebook_embedding.h" #include #include #include #include #include #include #include #include #include #include namespace { std::string arg_value(int argc, char ** argv, const std::string & name, const std::string & fallback) { for (int i = 1; i + 1 < argc; ++i) { if (argv[i] == name) { return argv[i + 1]; } } return fallback; } engine::assets::TensorStorageType parse_weight_type(const std::string & value) { if (value == "f32") { return engine::assets::TensorStorageType::F32; } if (value == "bf16") { return engine::assets::TensorStorageType::BF16; } if (value == "q8_0") { return engine::assets::TensorStorageType::Q8_0; } throw std::runtime_error("unsupported --weight-type: " + value); } } // namespace int main(int argc, char ** argv) { try { const std::string model_dir = arg_value(argc, argv, "--model", ""); const std::string instruction = arg_value(argc, argv, "--instruct", ""); const std::string text = arg_value(argc, argv, "--text", ""); const std::string language = arg_value(argc, argv, "--language", "English"); const std::string output = arg_value(argc, argv, "--output", "moss_voicegen.wav"); const auto weight_type = parse_weight_type(arg_value(argc, argv, "--weight-type", "bf16")); const auto seed = static_cast(std::stoul(arg_value(argc, argv, "--seed", "0"))); const int threads = std::stoi(arg_value(argc, argv, "--threads", "8")); if (model_dir.empty() || text.empty()) { std::cerr << "usage: moss_voicegen_smoke --model --text " " [--instruct ] [--output out.wav]\n"; return 2; } const auto assets = engine::models::moss_voicegen::load_moss_voicegen_assets(model_dir); const auto & config = assets->config; const int64_t n_vq = config.num_codebooks; const int64_t hidden_size = config.backbone.hidden_size; const engine::models::moss_voicegen::MossVoiceGenTextProcessor processor(assets); const auto prompt = processor.build_generation_prefix(text, instruction, language); const auto prompt_rows = static_cast(prompt.text_tokens.size()); // With no reference recording there is nothing anchoring the duration, so bound it // from the text. Roughly 12.5 frames a second at this codec's hop; the floor keeps // the model from collapsing on the first frame, the ceiling from rambling. const auto characters = static_cast(text.size()); const int64_t min_frames = std::max(8, characters / 6); const int64_t max_frames = std::max(min_frames * 4, characters / 2 + 50); const int64_t max_steps = max_frames + n_vq + 4; std::cout << "prompt rows=" << prompt_rows << " frame bounds=[" << min_frames << ", " << max_frames << "] max steps=" << max_steps << "\n"; engine::modules::MultiCodebookEmbeddingSpec codebook_spec; codebook_spec.hidden_size = hidden_size; codebook_spec.num_codebooks = n_vq; codebook_spec.vocab_size = config.audio_vocab_size + 1; codebook_spec.pad_token_id = config.audio_pad_code; codebook_spec.tensor_prefix = "emb_ext"; const engine::modules::MultiCodebookEmbedding codebooks(*assets->model_weights, codebook_spec); std::vector prompt_bias(static_cast(prompt_rows * hidden_size), 0.0F); for (int64_t row = 0; row < prompt_rows; ++row) { codebooks.add_bias( prompt.audio_codes.data() + static_cast(row * n_vq), prompt_bias.data() + static_cast(row * hidden_size)); } engine::core::BackendConfig backend_config; backend_config.type = engine::core::BackendType::Cpu; backend_config.device = 0; backend_config.threads = threads; engine::core::ExecutionContext execution_context(backend_config); const engine::decoders::MossTtsDelayBackboneRuntime backbone( assets->config, assets->model_weights, execution_context, 512ull * 1024ull * 1024ull, 8192ull * 1024ull * 1024ull, weight_type); const engine::decoders::MossTtsDelayHeadsRuntime heads( assets->config, assets->model_weights, execution_context, 256ull * 1024ull * 1024ull, 4096ull * 1024ull * 1024ull, weight_type); // Checkpoint defaults from the model card; this family degenerates at a generic preset. engine::decoders::MossTtsDelaySamplingOptions sampling; sampling.text_temperature = std::stof(arg_value(argc, argv, "--text-temperature", "1.5")); sampling.audio_temperature = std::stof(arg_value(argc, argv, "--audio-temperature", "1.5")); sampling.audio_top_p = std::stof(arg_value(argc, argv, "--audio-top-p", "0.6")); sampling.audio_top_k = std::stoi(arg_value(argc, argv, "--audio-top-k", "50")); sampling.audio_repetition_penalty = std::stof(arg_value(argc, argv, "--audio-repetition-penalty", "1.1")); engine::decoders::MossTtsDelayLengthBounds bounds; bounds.min_frames = min_frames; bounds.max_frames = max_frames; engine::decoders::MossTtsDelayDecoder decoder(config, sampling, seed, bounds); const auto generation_start = std::chrono::steady_clock::now(); backbone.begin_generation(prompt_rows + max_steps + 8); auto hidden = backbone.prefill(prompt.text_tokens, prompt_bias); engine::decoders::MossTtsDelayStepLogits logits; std::vector row_bias(static_cast(hidden_size), 0.0F); int64_t steps = 0; std::vector text_trace; for (; steps < max_steps; ++steps) { heads.evaluate(hidden, logits); const auto row = decoder.step(logits); text_trace.push_back(row.text_token); if (decoder.stopped()) { break; } std::fill(row_bias.begin(), row_bias.end(), 0.0F); codebooks.add_bias(row.codes.data(), row_bias.data()); hidden = backbone.step(row.text_token, row_bias); } const double generation_seconds = std::chrono::duration(std::chrono::steady_clock::now() - generation_start).count(); int64_t codebooks_out = 0; int64_t frames = 0; const auto flat_codes = decoder.extract_audio_codes(codebooks_out, frames); std::cout << "generated steps=" << steps << " frames=" << frames << " in " << generation_seconds << " s\n"; std::cout << "text token trace (start=" << config.audio_start_token_id << " gen=" << config.audio_assistant_gen_slot_token_id << " delay=" << config.audio_assistant_delay_slot_token_id << " end=" << config.audio_end_token_id << " im_end=" << config.im_end_token_id << "):\n "; for (const int32_t token : text_trace) { std::cout << " " << token; } std::cout << "\n"; if (frames <= 0) { std::cerr << "FAIL: no audio frames were produced\n"; return 1; } std::vector> codes(static_cast(codebooks_out)); for (int64_t codebook = 0; codebook < codebooks_out; ++codebook) { codes[static_cast(codebook)].assign( flat_codes.begin() + static_cast(codebook * frames), flat_codes.begin() + static_cast((codebook + 1) * frames)); } const std::string codes_dump = arg_value(argc, argv, "--dump-codes", ""); if (!codes_dump.empty()) { // Plain JSON so the reference codec can decode exactly these codes and the two // waveforms can be compared. std::ofstream out(codes_dump); out << "{\"codebooks\": " << codebooks_out << ", \"frames\": " << frames << ", \"codes\": ["; for (int64_t codebook = 0; codebook < codebooks_out; ++codebook) { out << (codebook == 0 ? "[" : ", ["); for (int64_t frame = 0; frame < frames; ++frame) { out << (frame == 0 ? "" : ", ") << codes[static_cast(codebook)][static_cast(frame)]; } out << "]"; } out << "]}\n"; std::cout << "wrote codes to " << codes_dump << "\n"; } const auto decode_start = std::chrono::steady_clock::now(); engine::codecs::MossAudioTokenizerCodecRuntime codec( assets->audio_tokenizer_weights, execution_context, codebooks_out, engine::codecs::MossAudioTokenizerCodecRuntimeOptions{ 4096ull * 1024ull * 1024ull, 2048ull * 1024ull * 1024ull, 2048ull * 1024ull * 1024ull, false, }, engine::codecs::moss_audio_tokenizer_v1_config()); const auto decoded = codec.decode(engine::codecs::MossAudioTokenizerCodes{ codes.empty() ? 0 : static_cast(codes.front().size()), std::move(codes), }); const double decode_seconds = std::chrono::duration(std::chrono::steady_clock::now() - decode_start).count(); const auto & waveform = decoded.channels.at(0); double peak = 0.0; double energy = 0.0; for (const float sample : waveform) { peak = std::max(peak, static_cast(std::fabs(sample))); energy += static_cast(sample) * sample; } const double rms = waveform.empty() ? 0.0 : std::sqrt(energy / static_cast(waveform.size())); const double duration = static_cast(waveform.size()) / static_cast(codec.sampling_rate()); engine::audio::write_pcm16_wav(output, static_cast(codec.sampling_rate()), 1, waveform); std::cout << "decoded " << waveform.size() << " samples (" << duration << " s at " << codec.sampling_rate() << " Hz) in " << decode_seconds << " s\n"; std::cout << "peak=" << peak << " rms=" << rms << "\n"; std::cout << "real-time factor: " << ((generation_seconds + decode_seconds) / duration) << "\n"; std::cout << "wrote " << output << "\n"; return peak > 0.0 ? 0 : 1; } catch (const std::exception & error) { std::cerr << "error: " << error.what() << "\n"; return 1; } }