From 5b0196c7d0f51668786df644fe06a8901bf9efaf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 26 Apr 2026 19:35:21 +0000 Subject: [PATCH] fix(whisper): scrub invalid UTF-8 from segment text before protobuf marshal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit whisper.cpp can emit bytes that are not valid UTF-8 — typically a multibyte codepoint split across token boundaries. protobuf string fields reject those at marshal time, which would surface as a transcribe failure. Run strings.ToValidUTF8 on the segment text before it leaves the cgo boundary so the bad byte gets replaced with U+FFFD. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-7 [Claude Code] --- backend/go/whisper/gowhisper.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/go/whisper/gowhisper.go b/backend/go/whisper/gowhisper.go index bf7cb7a45..56d794f0f 100644 --- a/backend/go/whisper/gowhisper.go +++ b/backend/go/whisper/gowhisper.go @@ -139,7 +139,10 @@ func (w *Whisper) AudioTranscription(opts *pb.TranscriptRequest) (pb.TranscriptR // segment start/end conversion factor taken from https://github.com/ggml-org/whisper.cpp/blob/master/examples/cli/cli.cpp#L895 s := CppGetSegmentStart(i) * (10000000) t := CppGetSegmentEnd(i) * (10000000) - txt := strings.Clone(CppGetSegmentText(i)) + // whisper.cpp can emit bytes that aren't valid UTF-8 (e.g. a multibyte + // codepoint split across token boundaries); protobuf string fields + // reject those at marshal time. Scrub before the value escapes cgo. + txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") tokens := make([]int32, CppNTokens(i)) if opts.Diarize && CppGetSegmentSpeakerTurnNext(i) {