#include "engine/framework/runtime/partial_text.h" #include "test_assert.h" #include #include #include namespace { using engine::runtime::PartialTextPublisher; using engine::runtime::transcript_publishable_end; using engine::test::require_eq; // What a consumer assembles by appending every delta, which is the contract: // partials concatenate into the transcript. std::string appended(const std::vector & decodes) { PartialTextPublisher publisher; std::string client; for (const auto & decode : decodes) { client += publisher.publish(decode); } return client; } void test_growing_transcript_yields_increments() { PartialTextPublisher publisher; require_eq(publisher.publish("Some call me nat"), std::string("Some call me nat"), "first"); require_eq(publisher.publish("Some call me nature. Others"), std::string("ure. Others"), "second"); require_eq(publisher.publish("Some call me nature. Others"), std::string(""), "unchanged"); } // The reason this is shared rather than reimplemented per family: a tokenizer // falls back to bytes for text its vocabulary does not cover, so a decode can // stop part way through a character. Publishing that puts half a code point on // the wire, where it reaches a JSON encoder as invalid UTF-8. void test_partial_character_is_held_until_complete() { const std::string cjk = "\xE4\xB8\x80"; // U+4E00, three bytes PartialTextPublisher publisher; require_eq(publisher.publish("ab" + cjk.substr(0, 1)), std::string("ab"), "lead byte held"); require_eq(publisher.publish("ab" + cjk.substr(0, 2)), std::string(""), "still incomplete"); require_eq(publisher.publish("ab" + cjk), cjk, "released whole"); require_eq(appended({"ab" + cjk.substr(0, 1), "ab" + cjk.substr(0, 2), "ab" + cjk}), "ab" + cjk, "assembled"); } void test_two_byte_characters_are_held_too() { const std::string ru = "\xD0\x9F\xD1\x80\xD0\xB8"; // При require_eq(appended({ru.substr(0, 1), ru.substr(0, 3), ru.substr(0, 5), ru}), ru, "cyrillic"); } void test_four_byte_character_is_held_until_complete() { const std::string emoji = "\xF0\x9F\x8E\xB5"; // U+1F3B5 PartialTextPublisher publisher; require_eq(publisher.publish(emoji.substr(0, 3)), std::string(""), "incomplete"); require_eq(publisher.publish(emoji), emoji, "released whole"); } // A revision cannot be retracted -- the delta has already gone out -- but the // next one must still start on a character boundary rather than inside one. void test_revision_resumes_on_a_character_boundary() { const std::string cjk = "\xE4\xB8\x80"; PartialTextPublisher publisher; require_eq(publisher.publish("ab" + cjk), "ab" + cjk, "published"); // Same first byte of the character, different continuation. const std::string revised = "ab\xE4\xB8\x81"; const std::string delta = publisher.publish(revised); require_eq(delta, std::string("\xE4\xB8\x81"), "whole character re-sent"); } // A decode that truncates mid-character must not un-publish the character it // cut: the decode that restores it would then send it twice, and a consumer // that appends every delta would show it twice. void test_truncated_decode_does_not_resend() { const std::string cjk = "\xE4\xB8\x80"; PartialTextPublisher publisher; require_eq(publisher.publish("ab" + cjk), "ab" + cjk, "published whole"); require_eq(publisher.publish("ab" + cjk.substr(0, 2)), std::string(""), "truncated"); require_eq(publisher.publish("ab" + cjk), std::string(""), "not resent"); require_eq(appended({"ab" + cjk, "ab" + cjk.substr(0, 2), "ab" + cjk}), "ab" + cjk, "consumer sees it once"); } // The agreement check is bounded, so state what that buys and what it gives up. // A revision near the end -- where a streaming decode actually revises -- is // seen, and the delta resumes from it. void test_revision_within_the_window_is_seen() { PartialTextPublisher publisher; require_eq(publisher.publish("the quick brown fox"), std::string("the quick brown fox"), "first"); const std::string delta = publisher.publish("the quick brown dog"); require_eq(delta, std::string("dog"), "resumes at the divergence"); } // A revision further back than the window is not seen, and cannot be: the text // it would correct has already gone out and nothing can retract it. The // publisher keeps going forward instead of re-sending a transcript the consumer // cannot un-append. void test_revision_behind_the_window_does_not_resend_history() { PartialTextPublisher publisher; std::string transcript(600, 'a'); require_eq(publisher.publish(transcript).size(), transcript.size(), "first"); std::string revised = transcript; revised[0] = 'b'; // far behind the 256-byte window revised += "tail"; require_eq(publisher.publish(revised), std::string("tail"), "only the new tail"); } void test_shrinking_transcript_publishes_nothing() { PartialTextPublisher publisher; require_eq(publisher.publish("abcdef"), std::string("abcdef"), "first"); require_eq(publisher.publish("abc"), std::string(""), "shrunk"); // And the text it dropped is not re-sent when it comes back, because the // consumer was never told to remove it. require_eq(publisher.publish("abcdef"), std::string(""), "not resent"); } void test_reset_forgets_the_published_prefix() { PartialTextPublisher publisher; require_eq(publisher.publish("hello"), std::string("hello"), "first"); publisher.reset(); require_eq(publisher.published(), std::string(""), "cleared"); require_eq(publisher.publish("hello"), std::string("hello"), "republished after reset"); } void test_empty_and_ascii_edges() { PartialTextPublisher publisher; require_eq(publisher.publish(""), std::string(""), "empty"); require_eq(publisher.publish("a"), std::string("a"), "single byte"); } // voxtral_realtime keeps its own byte offset and calls this directly, so pin it // down on its own: the emoji split below is how Tekken byte tokens emit one. void test_publishable_end_stops_before_an_incomplete_tail() { const std::string emoji = "\xF0\x9F\x98\x80"; // U+1F600, four bytes for (std::size_t n = 1; n < emoji.size(); ++n) { require_eq(transcript_publishable_end("hi " + emoji.substr(0, n)), std::size_t{3}, "incomplete emoji held"); } require_eq(transcript_publishable_end("hi " + emoji), std::size_t{7}, "complete emoji"); require_eq(transcript_publishable_end("\xC3"), std::size_t{0}, "lone two-byte lead"); require_eq(transcript_publishable_end("\xC3\xA9"), std::size_t{2}, "complete two-byte"); require_eq(transcript_publishable_end("abc"), std::size_t{3}, "ascii"); require_eq(transcript_publishable_end(""), std::size_t{0}, "empty"); } } // namespace int main() { try { test_growing_transcript_yields_increments(); test_partial_character_is_held_until_complete(); test_two_byte_characters_are_held_too(); test_four_byte_character_is_held_until_complete(); test_revision_resumes_on_a_character_boundary(); test_truncated_decode_does_not_resend(); test_revision_within_the_window_is_seen(); test_revision_behind_the_window_does_not_resend_history(); test_shrinking_transcript_publishes_nothing(); test_reset_forgets_the_published_prefix(); test_empty_and_ascii_edges(); test_publishable_end_stops_before_an_incomplete_tail(); std::cout << "partial_text_test passed\n"; } catch (const std::exception & ex) { std::cerr << "partial_text_test failed: " << ex.what() << "\n"; return 1; } return 0; }