From 985df46d246f29299c0988837b3c90992eaac0cd Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Fri, 2 Oct 2026 09:18:12 +0200 Subject: [PATCH] fix(whisperx): keep the transcript when diarization fails, report real errors (#12427) AudioTranscription caught every exception and returned an empty TranscriptResult. A failed diarization step therefore discarded a transcript that was already finished: with an HF token that has not accepted the terms of the gated pyannote pipeline, the download fails with 403 and every transcription came back as an empty text with HTTP 200. Diarization now degrades: if it fails, the transcript is returned without speaker labels and the reason is logged. Any other failure aborts the call with INTERNAL instead of pretending success with an empty text. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Stefan Walcz --- backend/python/whisperx/backend.py | 21 +++++++++++-------- .../python/whisperx/test_transcript_utils.py | 18 ++++++++++++++++ backend/python/whisperx/transcript_utils.py | 14 +++++++++++++ docs/content/features/audio-to-text.md | 2 ++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/backend/python/whisperx/backend.py b/backend/python/whisperx/backend.py index dc6202286..c3f900e3e 100644 --- a/backend/python/whisperx/backend.py +++ b/backend/python/whisperx/backend.py @@ -16,7 +16,7 @@ import grpc sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors -from transcript_utils import require_diarization_token, seconds_to_nanoseconds +from transcript_utils import diarize_or_keep, require_diarization_token, seconds_to_nanoseconds @@ -112,13 +112,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Diarize if requested and HF token is available if request.diarize and self.hf_token: - if self.diarize_pipeline is None: - self.diarize_pipeline = DiarizationPipeline( - token=self.hf_token, - device=self.device, - ) - diarize_segments = self.diarize_pipeline(audio) - transcript = whisperx.assign_word_speakers(diarize_segments, transcript) + def _diarize(t): + if self.diarize_pipeline is None: + self.diarize_pipeline = DiarizationPipeline( + token=self.hf_token, + device=self.device, + ) + return whisperx.assign_word_speakers(self.diarize_pipeline(audio), t) + + transcript = diarize_or_keep(transcript, _diarize, lambda m: print(m, file=sys.stderr)) # Build result segments for idx, seg in enumerate(transcript["segments"]): @@ -137,8 +139,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): text += seg_text except Exception as err: + # Report the failure instead of an empty, successful-looking result. print(f"Unexpected {err=}, {type(err)=}", file=sys.stderr) - return backend_pb2.TranscriptResult(segments=[], text="") + context.abort(grpc.StatusCode.INTERNAL, f"transcription failed: {err}") return backend_pb2.TranscriptResult(segments=resultSegments, text=text) diff --git a/backend/python/whisperx/test_transcript_utils.py b/backend/python/whisperx/test_transcript_utils.py index debe2ea6e..708d4bde2 100644 --- a/backend/python/whisperx/test_transcript_utils.py +++ b/backend/python/whisperx/test_transcript_utils.py @@ -21,5 +21,23 @@ class TestTranscriptUtils(unittest.TestCase): ) + def test_failed_diarization_keeps_the_transcript(self): + transcript = {"segments": [{"text": "Die Rechnung"}]} + logged = [] + + def refused(_): + raise RuntimeError("403 Client Error: gated repo") + + result = transcript_utils.diarize_or_keep(transcript, refused, logged.append) + self.assertIs(result, transcript) + self.assertIn("403", logged[0]) + + def test_successful_diarization_is_returned(self): + transcript = {"segments": [{"text": "Die Rechnung"}]} + with_speakers = {"segments": [{"text": "Die Rechnung", "speaker": "SPEAKER_00"}]} + result = transcript_utils.diarize_or_keep(transcript, lambda _: with_speakers, lambda _: None) + self.assertIs(result, with_speakers) + + if __name__ == "__main__": unittest.main() diff --git a/backend/python/whisperx/transcript_utils.py b/backend/python/whisperx/transcript_utils.py index a8ac57510..2211cf670 100644 --- a/backend/python/whisperx/transcript_utils.py +++ b/backend/python/whisperx/transcript_utils.py @@ -10,3 +10,17 @@ def require_diarization_token(diarize, token): def seconds_to_nanoseconds(seconds): """Convert WhisperX timestamps to the duration unit used by LocalAI.""" return int(seconds * 1_000_000_000) + + +def diarize_or_keep(transcript, diarize, log): + """Run diarization; if it fails, keep the transcript without speakers. + + Diarization is an add-on to a finished transcript. A refused download of + the gated pyannote pipeline (403) or any other diarization error must not + throw the transcript away. + """ + try: + return diarize(transcript) + except Exception as err: # noqa: BLE001 - any diarization failure degrades + log(f"Diarization failed, returning transcript without speakers: {err!r}") + return transcript diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 0a4b6ee65..e90423749 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -112,6 +112,8 @@ In addition to `file` and `model`, the endpoint accepts the following multipart | `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. | | `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. | +If speaker diarization fails after transcription succeeded, the WhisperX backend logs the error and returns the transcript without speaker labels. Other transcription failures return an error instead of an empty transcript. Diarization still requires `HF_TOKEN`. + The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled). ## Streaming transcriptions