From f00dcad3051998744cd185eb5273fa98865436ea Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 26 Jul 2026 23:44:19 +0000 Subject: [PATCH] backend(audio-cpp): stop a repeated lead byte from orphaning the next delta The first UTF-8 fix closed the cumulative half only. Rule 2 discards a fragment the known text already starts with, and when that fragment is the LEAD BYTE of a new character it looks exactly like a repeat of an older character beginning with the same byte. It was discarded rather than held, its continuation bytes then arrived alone and began the next delta, and utf8_complete_prefix_length only ever inspected the trailing sequence, so a delta invalid at the FRONT went out whole. Through a real Go proto.Unmarshal the review's four-character repro gave 3 deltas, 2 unmarshal failures and a lost transcript. Reachable from the incremental families, not only from voxtral: nemotron_asr's decoder cuts at a byte offset and vibevoice_asr's common_prefix_size compares bytes, so both split characters. Measured over 30,000 randomized incremental traces, 53.28% of Japanese traces and 9.52% of French ones carried at least one delta the Go runtime refuses. Two changes. Rule 2 no longer judges a fragment that ends mid-character, so the lead byte is held instead of swallowed and the character survives intact; the cost is a few duplicated bytes in a shrinking cumulative report, which no pinned family produces. release() additionally drops leading orphan continuation bytes, so no delta can begin mid-character whatever the rules above it decide. Losing a byte keeps the stream alive; emitting one ends the RPC and takes the final_result with it. Post-fix all 60,000 traces produce zero unmarshal failures, and the cumulative streams plus both pure-ASCII incremental streams are byte-identical to the previous commit, so nothing changed for the families already working. The weight-dtype allow list moves to family_gate, where it is stdlib-only and pinned by a test rather than only by a comment. Two comment citations corrected. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/cpp/audio-cpp/family_gate.cpp | 67 ++++++++++++++++++ backend/cpp/audio-cpp/family_gate.h | 27 ++++++++ backend/cpp/audio-cpp/family_gate_test.cpp | 49 +++++++++++++ backend/cpp/audio-cpp/loaded_model.cpp | 76 ++++----------------- backend/cpp/audio-cpp/stream_delta.cpp | 63 +++++++++++++---- backend/cpp/audio-cpp/stream_delta.h | 30 +++++--- backend/cpp/audio-cpp/stream_delta_test.cpp | 72 +++++++++++++++++-- 7 files changed, 297 insertions(+), 87 deletions(-) diff --git a/backend/cpp/audio-cpp/family_gate.cpp b/backend/cpp/audio-cpp/family_gate.cpp index 3405cc6fb..8f9ed4b29 100644 --- a/backend/cpp/audio-cpp/family_gate.cpp +++ b/backend/cpp/audio-cpp/family_gate.cpp @@ -63,4 +63,71 @@ FamilyDecision decide_family(bool path_is_gguf, const std::string &embedded_fami return decision; } +namespace { + +// Families that ABORT THE PROCESS on a weight dtype they cannot handle, and the +// dtypes they can. See the header for why this is a list of crashes rather than +// a list of preferences. +// +// supertonic: upstream's docs/gguf.md:90 records its 16-bit GGUF column as +// "---", i.e. NOT TESTED, and its q8_0 as "No (unsupported weight dtype)". Only +// the `orig` package is marked Pass, and its 698 weight tensors are f32 while +// its 72 index and shape constants are i64. Attributed rather than assumed: the +// abort is identical through the unary TTS RPC and through TTSStream, so it is +// the packaging and not the streaming path. +// +// TO REMOVE AN ENTRY: bump AUDIO_CPP_VERSION past a fix, load a package in the +// refused dtype, and synthesise. If audio comes out, delete the entry. No test +// can do that for you, which is exactly why it is written here: the test beside +// this file pins WHAT the table says, not whether upstream has moved on. Do not +// widen an entry without running that, because what it prevents is a process +// death rather than a wrong answer. +struct DtypeAllowList { + const char *family; + const char *allowed[3]; +}; + +constexpr DtypeAllowList kDtypeAllowLists[] = { + {"supertonic", {"f32", "i64", nullptr}}, +}; + +const DtypeAllowList *find_allow_list(const std::string &family) { + for (const auto &entry : kDtypeAllowLists) { + if (family == entry.family) { + return &entry; + } + } + return nullptr; +} + +} // namespace + +bool weight_dtype_is_supported(const std::string &family, const std::string &dtype) { + const DtypeAllowList *list = find_allow_list(family); + if (list == nullptr) { + return true; + } + for (const char *const *name = list->allowed; *name != nullptr; ++name) { + if (dtype == *name) { + return true; + } + } + return false; +} + +std::string supported_weight_dtypes(const std::string &family) { + const DtypeAllowList *list = find_allow_list(family); + if (list == nullptr) { + return {}; + } + std::string out; + for (const char *const *name = list->allowed; *name != nullptr; ++name) { + if (!out.empty()) { + out += ", "; + } + out += *name; + } + return out; +} + } // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/family_gate.h b/backend/cpp/audio-cpp/family_gate.h index cffaeb3d0..7c40da48d 100644 --- a/backend/cpp/audio-cpp/family_gate.h +++ b/backend/cpp/audio-cpp/family_gate.h @@ -32,4 +32,31 @@ struct FamilyDecision { FamilyDecision decide_family(bool path_is_gguf, const std::string &embedded_family, const std::string &configured_family); +// True when `family` can run weights stored as `dtype`, where dtype is the +// string a TensorMetadata carries ("f32", "f16", "q8_0", "i64", ...). +// +// This is a LIST OF FAMILIES THAT CRASH THE PROCESS, not a list of families that +// perform badly. It exists because the failure is not an exception: loading a +// supertonic package whose weights are f16 or q8_0 reaches ggml_concat with one +// f16 operand and one f32 one, and ggml_abort takes the backend down with +// SIGABRT on the FIRST request. Nothing upstream of the load can catch that, so +// an operator sees a model that loaded successfully and a backend that dies on +// every request with no status and no message. +// +// A family with no entry is unrestricted, which is every family but one. +// +// Split out of loaded_model.cpp, where the caller lives, so that the policy is +// stdlib-only and can be held by a test: the caller needs a real GGUF on disk +// and an engine, and neither is available to a unit test. What the test pins is +// that the table says what it is meant to say, so widening it is a deliberate +// act rather than a typo. It CANNOT pin the removal criterion, which is +// "upstream fixed it": no test can know that without downloading the package and +// synthesising, so that step stays a documented manual one at the table itself. +bool weight_dtype_is_supported(const std::string &family, const std::string &dtype); + +// The dtypes `family` is restricted to, as "f32, i64", or empty when it is not +// restricted at all. For the refusal message, so the operator is told what to +// look for rather than only what is wrong. +std::string supported_weight_dtypes(const std::string &family); + } // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/family_gate_test.cpp b/backend/cpp/audio-cpp/family_gate_test.cpp index e0b665cfb..1c6f90aea 100644 --- a/backend/cpp/audio-cpp/family_gate_test.cpp +++ b/backend/cpp/audio-cpp/family_gate_test.cpp @@ -22,6 +22,11 @@ static void check(bool ok, const std::string &name) { } } +static void check_eq(const std::string &got, const std::string &want, + const std::string &name) { + check(got == want, name + " (got \"" + got + "\" want \"" + want + "\")"); +} + using namespace audiocpp_backend; static void test_gguf_suffix_detection() { @@ -102,6 +107,49 @@ static void test_directory_ignores_embedded_family() { "a directory refusal does not blame GGUF metadata it could not have"); } +// Pins the weight-dtype allow list. Not a style preference: an entry here is a +// family that ABORTS THE PROCESS on the first request when handed the wrong +// dtype, so the model loads and then every request kills the backend with no +// status and no message. +// +// What this test can and cannot do, stated so the next reader does not expect +// more of it: it pins WHAT THE TABLE SAYS, so widening an entry is a deliberate +// act rather than a typo, and it pins that an unlisted family is unrestricted. +// It CANNOT pin the removal criterion, which is "upstream fixed it": knowing +// that needs the package downloaded and a synthesis run, so it stays a manual +// step documented at the table in family_gate.cpp. +static void test_weight_dtype_allow_list() { + // The entry that exists, and the exact reason it exists. + check(!weight_dtype_is_supported("supertonic", "f16"), + "supertonic refuses f16, the package that aborts the process"); + check(!weight_dtype_is_supported("supertonic", "q8_0"), + "supertonic refuses q8_0, which upstream records as unsupported"); + check(!weight_dtype_is_supported("supertonic", "bf16"), + "supertonic refuses bf16, which is untested rather than known good"); + check(weight_dtype_is_supported("supertonic", "f32"), + "supertonic accepts f32, which is what the orig package stores"); + check(weight_dtype_is_supported("supertonic", "i64"), + "supertonic accepts i64: the orig package carries 72 such tensors and " + "refusing them would refuse the artifact that works"); + + // Every other family is unrestricted, and must stay that way: this guard is + // for process death, not for quality. + check(weight_dtype_is_supported("nemotron_asr", "q8_0"), + "an unlisted family is not restricted"); + check(weight_dtype_is_supported("citrinet_asr", "f16"), + "an unlisted family is not restricted by another family's entry"); + check(weight_dtype_is_supported("", "anything"), + "an empty family name is not restricted"); + + // The message the operator reads has to name the remedy, so the refusal is + // actionable rather than only correct. + check_eq(supported_weight_dtypes("supertonic"), "f32, i64", + "the refusal can name what to look for"); + check_eq(supported_weight_dtypes("nemotron_asr"), "", + "an unlisted family reports no restriction, which is what makes " + "the caller skip the whole check"); +} + int main() { test_gguf_suffix_detection(); test_explicit_family_always_wins(); @@ -109,6 +157,7 @@ int main() { test_foreign_gguf_is_refused(); test_directory_without_family_is_refused(); test_directory_ignores_embedded_family(); + test_weight_dtype_allow_list(); if (failures) { fprintf(stderr, "%d check(s) failed\n", failures); return 1; diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp index fb80bee30..46a2c0886 100644 --- a/backend/cpp/audio-cpp/loaded_model.cpp +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -164,56 +164,20 @@ std::string require_family(const std::string &resolved_path, return decision.family; } -// Families that CRASH THE PROCESS on a weight dtype they cannot handle, and the -// dtypes they can. -// -// A guard rather than a note, because the failure is not an exception: loading a -// supertonic package whose weights are stored as f16 or q8_0 reaches -// ggml_concat with one f16 operand and one f32 one, and ggml_abort takes the -// whole backend process down with SIGABRT on the FIRST synthesis. Nothing -// upstream of here can catch that, so an operator sees a backend that dies on -// every request with no status and no message, and the model that killed it -// loaded successfully. -// -// Attributed rather than assumed: the abort is identical through the unary TTS -// RPC and through TTSStream, and upstream's own docs/gguf.md:90 records -// supertonic's 16-bit GGUF column as "---", i.e. NOT TESTED, and its q8_0 as -// "No (unsupported weight dtype)". Only the `orig` package, whose 698 weight -// tensors are f32, is marked Pass, and that is the one every verification of -// this backend has used. -// -// i64 is allowed alongside f32 because the orig package itself carries 72 i64 -// tensors (shape and index constants), so refusing them would refuse the -// artifact that works. -// -// TO REMOVE THIS: bump AUDIO_CPP_VERSION past a fix, load a supertonic f16 -// package, and synthesise. If it produces audio, delete the entry. Do not widen -// it without running that, because the failure it prevents is a process death -// rather than a wrong answer. -struct DtypeAllowList { - const char *family; - const char *allowed[3]; -}; - -constexpr DtypeAllowList kDtypeAllowLists[] = { - {"supertonic", {"f32", "i64", nullptr}}, -}; - // Refuses a GGUF whose weights are stored in a dtype the family cannot survive. // +// The POLICY lives in family_gate's weight_dtype_is_supported, which is +// stdlib-only and therefore testable; this is only the part that needs a file +// and an engine to read one. See the table there for why an entry exists and +// what has to be run before deleting it. +// // Only GGUF paths are inspected. A directory of safetensors carries its dtypes // per file and has not been tested against this failure, so it is passed // through rather than guessed at. void require_supported_weight_dtypes(const std::string &family, const std::string &resolved_path) { - const DtypeAllowList *list = nullptr; - for (const auto &entry : kDtypeAllowLists) { - if (family == entry.family) { - list = &entry; - break; - } - } - if (list == nullptr || !path_looks_like_gguf(resolved_path)) { + const std::string allowed_names = supported_weight_dtypes(family); + if (allowed_names.empty() || !path_looks_like_gguf(resolved_path)) { return; } @@ -226,14 +190,7 @@ void require_supported_weight_dtypes(const std::string &family, return; } for (const auto &tensor : source->tensors()) { - bool allowed = false; - for (const char *const *name = list->allowed; *name != nullptr; ++name) { - if (tensor.dtype == *name) { - allowed = true; - break; - } - } - if (!allowed) { + if (!weight_dtype_is_supported(family, tensor.dtype)) { offending_dtype = tensor.dtype; offending_tensor = tensor.name; break; @@ -249,13 +206,6 @@ void require_supported_weight_dtypes(const std::string &family, if (offending_dtype.empty()) { return; } - std::string allowed_names; - for (const char *const *name = list->allowed; *name != nullptr; ++name) { - if (!allowed_names.empty()) { - allowed_names += ", "; - } - allowed_names += *name; - } throw ConfigError( "audio-cpp: family '" + family + "' cannot run weights stored as '" + offending_dtype + "' (tensor '" + offending_tensor + "' in " + @@ -676,10 +626,12 @@ engine::runtime::TaskResult run_streaming_audio( // An interleaved buffer whose float count is not a whole number of frames // is a truncated input, and the integer division below would silently drop // the tail floats: they would never be fed, never reach the transcript, and - // nothing would say so. Upstream refuses it from the other side of this - // same call rather than tolerating it: vibevoice_asr's process_audio_chunk - // throws "VibeVoice-ASR streamed audio samples must be divisible by channel - // count" (session.cpp:74-77). + // nothing would say so. Upstream refuses the same condition rather than + // tolerating it, in two places: vibevoice_asr's audio_frame_count throws + // "VibeVoice-ASR audio samples must be divisible by channel count" + // (session.cpp:70-76), and its process_audio_chunk throws the same with + // "streamed" in the text about the chunks this driver hands it + // (session.cpp:742-747). // // ConfigError, i.e. INVALID_ARGUMENT, because the buffer came from the // caller's file. read_audio_file's positive-rate path always answers mono diff --git a/backend/cpp/audio-cpp/stream_delta.cpp b/backend/cpp/audio-cpp/stream_delta.cpp index 87d5ad64a..a328f7b90 100644 --- a/backend/cpp/audio-cpp/stream_delta.cpp +++ b/backend/cpp/audio-cpp/stream_delta.cpp @@ -11,21 +11,40 @@ bool starts_with(const std::string &text, const std::string &prefix) { text.compare(0, prefix.size(), prefix) == 0; } +// Number of leading bytes of `text` that cannot BEGIN a character, i.e. orphan +// continuation bytes with no lead byte in front of them. +// +// They are unrecoverable rather than early: the byte that would have led them +// has already gone past, and nothing can be prepended to a fragment after the +// fact. Holding them would stall the stream for good, and emitting them puts +// invalid UTF-8 on the wire, so the caller DROPS them. Losing a byte keeps the +// stream alive; emitting one ends it, and takes the final_result still to come +// with it. +std::size_t utf8_orphan_prefix_length(const std::string &text) { + std::size_t index = 0; + while (index < text.size() && + (static_cast(text[index]) & 0xC0) == 0x80) { + ++index; + } + return index; +} + // Length of the longest prefix of `text` that does NOT end inside a multi-byte // UTF-8 sequence, i.e. the most that can go on the wire without splitting a // character in half. // -// Only the TRAILING sequence is examined. Bytes further back are none of this -// function's business: a family that emitted a malformed sequence in the middle -// of its own transcript cannot be repaired here without deleting part of that -// transcript. +// Only the TRAILING sequence is examined; the leading side is +// utf8_orphan_prefix_length's job. Bytes in the MIDDLE are neither one's +// business: a family that emitted a malformed sequence inside its own text +// cannot be repaired here without deleting part of that transcript, and the +// stream is lost anyway, because final_result.text carries the same bytes +// through the same proto3 string field. // // Anything that can never complete is reported as complete, so it goes out -// rather than being held forever: a lone continuation byte with no lead, a lead -// byte the encoding does not define, and a run of five or more continuation -// bytes are all passed through. A tracker that stalled on undecodable input -// would turn one bad byte into a permanently silent stream, which is worse than -// the bad byte. +// rather than being held forever: a lead byte the encoding does not define, and +// a run of five or more continuation bytes, are both passed through. A tracker +// that stalled on undecodable input would turn one bad byte into a permanently +// silent stream, which is worse than the bad byte. std::size_t utf8_complete_prefix_length(const std::string &text) { std::size_t index = text.size(); std::size_t continuations = 0; @@ -65,7 +84,13 @@ std::size_t utf8_complete_prefix_length(const std::string &text) { std::string TranscriptDeltaTracker::release(const std::string &fragment) { // Held-back bytes go in FRONT of whatever arrived next, or the character // they begin is reassembled in the wrong order. - const std::string candidate = pending_ + fragment; + std::string candidate = pending_ + fragment; + // A fragment must not BEGIN mid-character either. pending_ always starts on + // a lead byte, so this only bites when nothing was held and the caller's + // rules dropped the lead byte somewhere upstream; it is the backstop that + // makes "no delta this class returns is ever invalid UTF-8" true of the + // FRONT as well as the back, independently of those rules being right. + candidate.erase(0, utf8_orphan_prefix_length(candidate)); const std::size_t cut = utf8_complete_prefix_length(candidate); pending_ = candidate.substr(cut); std::string emitted = candidate.substr(0, cut); @@ -84,7 +109,21 @@ std::string TranscriptDeltaTracker::observe(const std::string &partial_text) { const std::string known = assembled_ + pending_; // Already accounted for. Covers the repeat voxtral produces when the sink // and the return value carry the same event, and a hypothesis that shrank. - if (starts_with(known, partial_text)) { + // + // The second condition is not decoration, and it is the whole of the second + // half of the UTF-8 bug. A fragment that ENDS MID-CHARACTER is not judged + // here at all, because the two readings of it are indistinguishable by + // bytes: "\xC3" is a prefix of an already-delivered "\xC3\x9F", and it is + // also the lead byte of a brand new "\xC3\xA9". Discarding it loses a + // character whose continuation bytes then arrive alone and begin the next + // delta with an orphan; holding it costs, at worst, a few duplicated bytes + // in a shrinking cumulative report, which no pinned family produces (every + // cumulative reporter here is append-only, and voxtral provably so, since + // its decode is a byte concatenation). One reading kills the RPC, the other + // glitches a hypothetical. Measured before this condition existed: + // 33.74% of Japanese traces carried at least one invalid delta. + if (starts_with(known, partial_text) && + utf8_complete_prefix_length(partial_text) == partial_text.size()) { return {}; } if (starts_with(partial_text, known)) { @@ -95,7 +134,7 @@ std::string TranscriptDeltaTracker::observe(const std::string &partial_text) { // // A family that REWRITES its hypothesis lands here too, and the client's // view is then wrong in a way nothing downstream can fix. nemotron's - // decoder has such a branch (decoder.cpp:551-554): when the new text is not + // decoder has such a branch (decoder.cpp:552-554): when the new text is not // an extension of what it already emitted, it emits the whole new text. So // "the cat sat" followed by "the cat sap" leaves the client holding // "the cat satthe cat sap", and reconcile then correctly refuses to append diff --git a/backend/cpp/audio-cpp/stream_delta.h b/backend/cpp/audio-cpp/stream_delta.h index 4600fbe87..6fdfc3483 100644 --- a/backend/cpp/audio-cpp/stream_delta.h +++ b/backend/cpp/audio-cpp/stream_delta.h @@ -45,8 +45,17 @@ // wire format REQUIRES valid UTF-8: the C++ runtime serializes an invalid one // with at most a warning, but the Go runtime refuses to unmarshal it, and the // client loses every remaining delta AND the final_result. So no fragment this -// class returns ever ends inside a character; an incomplete trailing sequence is -// held back and merged into the next fragment. +// class returns ever BEGINS OR ENDS inside a character: an incomplete trailing +// sequence is held back and merged into the next fragment, and a leading orphan +// continuation byte, which nothing can ever complete, is dropped. +// +// It is NOT a voxtral-only concern, which is what the first attempt at this +// assumed. The incremental families split characters by the same arithmetic: +// nemotron_asr's decoder cuts at a BYTE offset (decoder.cpp:550) and +// vibevoice_asr's common_prefix_size compares BYTES (session.cpp:80-86). Nor is +// European text the worst case: a Japanese transcript, whose every character is +// three bytes, carried at least one invalid delta in 33.74% of traces until +// rule 2 below learned to leave an incomplete fragment alone. #include @@ -60,9 +69,13 @@ public: // The rules, in order, all of them against everything KNOWN (delivered // plus held back), never against the delivered text alone: // 1. An empty partial says nothing. - // 2. A partial the known text ALREADY STARTS WITH has been accounted for: - // nothing is emitted. This is what absorbs voxtral's repeat of the - // last event in each batch, and a cumulative hypothesis that shrinks. + // 2. A partial the known text ALREADY STARTS WITH, AND WHICH IS ITSELF A + // WHOLE NUMBER OF CHARACTERS, has been accounted for: nothing is + // emitted. This is what absorbs voxtral's repeat of the last event in + // each batch, and a cumulative hypothesis that shrinks. The second + // condition is what keeps a new character's lead byte from being + // mistaken for a repeat of an old character that starts with the same + // byte; see the note at the rule in the implementation. // 3. A partial that EXTENDS the known text is a cumulative report: only // its new suffix is emitted. // 4. Anything else is an incremental fragment: it is emitted whole and @@ -76,9 +89,10 @@ public: // does in practice. // // What comes back is the fragment MINUS any incomplete trailing UTF-8 - // sequence, which is carried into the next call. So an empty return can - // also mean "the only new bytes were half a character", and the caller - // needs no knowledge of that: writing nothing is exactly right. + // sequence, which is carried into the next call, and minus any leading + // orphan continuation byte, which is dropped. So an empty return can also + // mean "the only new bytes were half a character", and the caller needs no + // knowledge of that: writing nothing is exactly right. std::string observe(const std::string &partial_text); // Reconciles against TaskResult::text_output, which is authoritative, and diff --git a/backend/cpp/audio-cpp/stream_delta_test.cpp b/backend/cpp/audio-cpp/stream_delta_test.cpp index 65b4a400e..b769a2e16 100644 --- a/backend/cpp/audio-cpp/stream_delta_test.cpp +++ b/backend/cpp/audio-cpp/stream_delta_test.cpp @@ -339,12 +339,72 @@ static void test_undecodable_bytes_are_not_held_forever() { check_eq(invalid_run.observe(std::string("\xFE\x80\x80")), std::string("\xFE\x80\x80"), "an undefined lead with continuations is passed through"); - // Five continuation bytes with no lead in sight: the scan gives up rather - // than walking back through the whole buffer. + // Five continuation bytes with no lead in sight. They are DROPPED, not + // held: nothing can ever precede them, so holding would stall the stream + // for good, and emitting them would put invalid UTF-8 on the wire. See + // test_a_fragment_never_begins_mid_character. TranscriptDeltaTracker orphans; - check_eq(orphans.observe(std::string("\x80\x80\x80\x80\x80")), - std::string("\x80\x80\x80\x80\x80"), - "a run of orphan continuation bytes is passed through"); + check_eq(orphans.observe(std::string("\x80\x80\x80\x80\x80")), "", + "a run of orphan continuation bytes is dropped, not emitted"); + check_eq(orphans.observe("after"), "after", + "and the stream continues past them"); +} + +// THE SECOND HALF OF THE SAME BUG, and the one that survived fix round 1. +// +// Rule 2 discards a fragment the known text already starts with. When that +// fragment is the LEAD BYTE of a NEW character it looks exactly like a repeat of +// an earlier character beginning with the same byte, so it was discarded and +// never held. Its continuation bytes then arrived on their own and began the +// next delta, which is invalid UTF-8 at the FRONT, and utf8_complete_prefix_length +// only ever inspected the TRAILING sequence. +// +// Reachable from shipping families, not synthetic: nemotron_asr's +// decoder.cpp:550 cuts at a BYTE offset (current_text.substr(emitted_text.size())) +// and vibevoice_asr's common_prefix_size (session.cpp:80-86) compares BYTES, so +// both split characters mid-sequence. The trace below is exactly how they split +// "ssee" spelled with the German sharp s, an e-acute, a euro sign and an o-grave, +// three of which begin with the same 0xC3 lead byte. +static void test_a_repeated_lead_byte_is_not_swallowed() { + TranscriptDeltaTracker tracker; + std::vector emitted; + const std::string sharp_s = "\xC3\x9F"; // U+00DF + const std::string e_acute = "\xC3\xA9"; // U+00E9 + const std::string euro = "\xE2\x82\xAC"; // U+20AC + const std::string o_grave = "\xC3\xB2"; // U+00F2 + const std::string full = sharp_s + e_acute + euro + o_grave; + + const std::string view = client_view(tracker, + {sharp_s.substr(0, 1), sharp_s.substr(1), + e_acute.substr(0, 1), + e_acute.substr(1) + euro, + o_grave.substr(0, 1), o_grave.substr(1)}, + &emitted); + + for (size_t i = 0; i < emitted.size(); ++i) { + check(is_valid_utf8(emitted[i]), + "repeated-lead fragment " + std::to_string(i) + " is valid UTF-8"); + } + check_eq(view, full, "no character is lost to a repeated lead byte"); + check_eq(tracker.reconcile(full), "", "the final text adds nothing"); +} + +// The same shape one layer down, as a backstop: a fragment that BEGINS with +// orphan continuation bytes must never go on the wire, whatever produced it. +// Dropping bytes keeps the stream alive; emitting them ends it, and takes the +// final_result that was still to come with it. +static void test_a_fragment_never_begins_mid_character() { + TranscriptDeltaTracker tracker; + const std::string first = tracker.observe(std::string("\xA9") + "rest"); + check(is_valid_utf8(first), "a leading orphan continuation byte is not emitted"); + check_eq(first, "rest", "the rest of the fragment still goes out"); + + TranscriptDeltaTracker all_orphans; + check_eq(all_orphans.observe(std::string("\x82\xAC")), "", + "a fragment that is nothing but orphans emits nothing"); + check_eq(all_orphans.assembled(), "", + "and nothing is recorded as delivered"); + check_eq(all_orphans.observe("after"), "after", "the stream continues"); } int main() { @@ -365,6 +425,8 @@ int main() { test_held_bytes_join_the_next_fragment(); test_a_complete_character_is_not_held(); test_undecodable_bytes_are_not_held_forever(); + test_a_repeated_lead_byte_is_not_swallowed(); + test_a_fragment_never_begins_mid_character(); if (failures) { fprintf(stderr, "%d check(s) failed\n", failures); return 1;