From 9e6a7a077cf5b4fe9b6e709f11e39f2f2d325ddc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 3 Nov 2026 19:08:04 +0000 Subject: [PATCH] fix(realtime): leave speech-to-speech client commits at the session rate The speech-to-speech path labels its transcription WAV with the session rate for client commits, so resampling first handed it 16 kHz audio under a 24 kHz header. Resample only in the chained pipeline, and say so in the docs. Assisted-by: Claude:claude-sonnet-5-5 go-test Signed-off-by: Ettore Di Giacinto (cherry picked from commit 82cc411af54cf884e4a44a3e96292a323ba5611c) --- core/http/endpoints/openai/realtime.go | 4 +++- docs/content/features/openai-realtime.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 064658a7a7..d8546c30db 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2122,7 +2122,9 @@ func vadScanWindowSec(sv *types.RealtimeSessionSemanticVad, silenceThreshold flo // decoded at 16 kHz (1.5x speed/pitch, garbage or empty transcript, then // input_not_committed on the follow-up response). func commitUtterance(ctx context.Context, utt []byte, session *Session, conv *Conversation, t Transport, slot *commitSlot) { - if rate := session.InputSampleRate; rate > 0 && rate != localSampleRate && len(utt) > 0 { + // Speech-to-speech sessions keep the audio at the session rate and label + // the transcription WAV with that rate (see s2sCommitTurn). + if rate := session.InputSampleRate; session.s2s == nil && rate > 0 && rate != localSampleRate && len(utt) > 0 { utt = sound.Int16toBytesLE(sound.ResampleInt16(sound.BytesToInt16sLE(utt), rate, localSampleRate)) } commitUtteranceWithTranscript(ctx, utt, nil, nil, "", session, conv, t, slot) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index d628e5b893..b9a42859c2 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -589,7 +589,7 @@ ws://localhost:8080/v1/realtime?model=gpt-realtime Audio is sent and received as raw PCM in the WebSocket messages, following the OpenAI Realtime API protocol. -Committed user audio reaches the transcription backend as 16 kHz mono WAV, whichever way the turn was committed: server-side VAD and client `input_audio_buffer.commit` both resample from the session input sample rate (for example 24 kHz) first. +In the chained pipeline, committed user audio reaches the transcription backend as 16 kHz mono WAV, whichever way the turn was committed: server-side VAD and client `input_audio_buffer.commit` both resample from the session input sample rate (for example 24 kHz) first. ### WebRTC