fix(whisperx): keep the transcript when diarization fails, report real errors (#12427)

AudioTranscription caught every exception and returned an empty
TranscriptResult. A failed diarization step therefore discarded a
transcript that was already finished: with an HF token that has not
accepted the terms of the gated pyannote pipeline, the download fails
with 403 and every transcription came back as an empty text with
HTTP 200.

Diarization now degrades: if it fails, the transcript is returned
without speaker labels and the reason is logged. Any other failure
aborts the call with INTERNAL instead of pretending success with an
empty text.

Assisted-by: Claude:claude-opus-5-5

Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>

backend/python/whisperx/backend.py

@@ -16,7 +16,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
-from transcript_utils import require_diarization_token, seconds_to_nanoseconds
+from transcript_utils import diarize_or_keep, require_diarization_token, seconds_to_nanoseconds
 
 
 
@@ -112,13 +112,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
             # Diarize if requested and HF token is available
             if request.diarize and self.hf_token:
-                if self.diarize_pipeline is None:
-                    self.diarize_pipeline = DiarizationPipeline(
-                        token=self.hf_token,
-                        device=self.device,
-                    )
-                diarize_segments = self.diarize_pipeline(audio)
-                transcript = whisperx.assign_word_speakers(diarize_segments, transcript)
+                def _diarize(t):
+                    if self.diarize_pipeline is None:
+                        self.diarize_pipeline = DiarizationPipeline(
+                            token=self.hf_token,
+                            device=self.device,
+                        )
+                    return whisperx.assign_word_speakers(self.diarize_pipeline(audio), t)
+
+                transcript = diarize_or_keep(transcript, _diarize, lambda m: print(m, file=sys.stderr))
 
             # Build result segments
             for idx, seg in enumerate(transcript["segments"]):
@@ -137,8 +139,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 text += seg_text
 
         except Exception as err:
+            # Report the failure instead of an empty, successful-looking result.
             print(f"Unexpected {err=}, {type(err)=}", file=sys.stderr)
-            return backend_pb2.TranscriptResult(segments=[], text="")
+            context.abort(grpc.StatusCode.INTERNAL, f"transcription failed: {err}")
 
         return backend_pb2.TranscriptResult(segments=resultSegments, text=text)
 

backend/python/whisperx/test_transcript_utils.py

@@ -21,5 +21,23 @@ class TestTranscriptUtils(unittest.TestCase):
         )
 
 
+    def test_failed_diarization_keeps_the_transcript(self):
+        transcript = {"segments": [{"text": "Die Rechnung"}]}
+        logged = []
+
+        def refused(_):
+            raise RuntimeError("403 Client Error: gated repo")
+
+        result = transcript_utils.diarize_or_keep(transcript, refused, logged.append)
+        self.assertIs(result, transcript)
+        self.assertIn("403", logged[0])
+
+    def test_successful_diarization_is_returned(self):
+        transcript = {"segments": [{"text": "Die Rechnung"}]}
+        with_speakers = {"segments": [{"text": "Die Rechnung", "speaker": "SPEAKER_00"}]}
+        result = transcript_utils.diarize_or_keep(transcript, lambda _: with_speakers, lambda _: None)
+        self.assertIs(result, with_speakers)
+
+
 if __name__ == "__main__":
     unittest.main()

backend/python/whisperx/transcript_utils.py

@@ -10,3 +10,17 @@ def require_diarization_token(diarize, token):
 def seconds_to_nanoseconds(seconds):
     """Convert WhisperX timestamps to the duration unit used by LocalAI."""
     return int(seconds * 1_000_000_000)
+
+
+def diarize_or_keep(transcript, diarize, log):
+    """Run diarization; if it fails, keep the transcript without speakers.
+
+    Diarization is an add-on to a finished transcript. A refused download of
+    the gated pyannote pipeline (403) or any other diarization error must not
+    throw the transcript away.
+    """
+    try:
+        return diarize(transcript)
+    except Exception as err:  # noqa: BLE001 - any diarization failure degrades
+        log(f"Diarization failed, returning transcript without speakers: {err!r}")
+        return transcript

docs/content/features/audio-to-text.md

@@ -112,6 +112,8 @@ In addition to `file` and `model`, the endpoint accepts the following multipart
 | `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. |
 | `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. |
 
+If speaker diarization fails after transcription succeeded, the WhisperX backend logs the error and returns the transcript without speaker labels. Other transcription failures return an error instead of an empty transcript. Diarization still requires `HF_TOKEN`.
+
 The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled).
 
 ## Streaming transcriptions