From fbfb2ccadf3004e5c94659a3c57be3475ef52421 Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Sun, 25 Oct 2026 12:53:50 +0000 Subject: [PATCH] fix(realtime): resample client commits to 16 kHz before transcription The committed-utterance WAV is written with a fixed 16 kHz header. The VAD commit path stores audio that is already resampled to 16 kHz (vadCommit), but the client-commit path (input_audio_buffer.commit) passed the raw buffer at the session input rate. For a 24 kHz OpenAI Realtime session the backend decoded 24 kHz audio as 16 kHz (1.5x speed/pitch): garbage or empty transcripts, then input_not_committed on the follow-up response. commitUtterance now resamples from session.InputSampleRate to localSampleRate, so both paths hand 16 kHz audio to the 16 kHz header. This replaces the earlier approach of this PR (header at the session rate), which mislabelled the already-resampled VAD audio (review feedback). Tests go through the real paths with a 24 kHz session: commitUtterance and a server_vad commit via vadTick; both must produce a 16 kHz WAV of the resampled length. The client-commit spec fails without the fix. Fixes #12747 Signed-off-by: nexxtmobile.de Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 05ba891da6ded4abfe0c2aec517e6eb6c6a6ab55) (cherry picked from commit 56e2b6d0cc5cbf1944ef50735f7b499d998eb25c) --- core/http/endpoints/openai/realtime.go | 10 ++ .../openai/realtime_commit_wav_rate_test.go | 100 ++++++++++++++++++ docs/content/features/openai-realtime.md | 2 + 3 files changed, 112 insertions(+) create mode 100644 core/http/endpoints/openai/realtime_commit_wav_rate_test.go diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 951a75fafc..064658a7a7 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2114,7 +2114,17 @@ func vadScanWindowSec(sv *types.RealtimeSessionSemanticVad, silenceThreshold flo return window } +// commitUtterance commits a client-issued turn (input_audio_buffer.commit). +// The client buffer holds audio at the session input sample rate (e.g. +// 24 kHz for OpenAI Realtime clients), while every commit path writes the +// transcription WAV at localSampleRate: the VAD path stores audio that is +// already resampled (vadCommit). Resample here so a 24 kHz buffer is not +// decoded at 16 kHz (1.5x speed/pitch, garbage or empty transcript, then +// input_not_committed on the follow-up response). func commitUtterance(ctx context.Context, utt []byte, session *Session, conv *Conversation, t Transport, slot *commitSlot) { + if rate := session.InputSampleRate; rate > 0 && rate != localSampleRate && len(utt) > 0 { + utt = sound.Int16toBytesLE(sound.ResampleInt16(sound.BytesToInt16sLE(utt), rate, localSampleRate)) + } commitUtteranceWithTranscript(ctx, utt, nil, nil, "", session, conv, t, slot) } diff --git a/core/http/endpoints/openai/realtime_commit_wav_rate_test.go b/core/http/endpoints/openai/realtime_commit_wav_rate_test.go new file mode 100644 index 0000000000..ef3620cd32 --- /dev/null +++ b/core/http/endpoints/openai/realtime_commit_wav_rate_test.go @@ -0,0 +1,100 @@ +package openai + +import ( + "bytes" + "context" + "encoding/binary" + "os" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/endpoints/openai/types" + "github.com/mudler/LocalAI/core/schema" +) + +// wavCaptureModel records the sample rate and data size of every WAV file a +// commit hands to the transcription backend. +type wavCaptureModel struct { + *fakeModel + mu sync.Mutex + rates []int + dataLens []int +} + +func (m *wavCaptureModel) capture(path string) { + b, err := os.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + Expect(len(b)).To(BeNumerically(">=", 44)) + Expect(bytes.Equal(b[0:4], []byte("RIFF")) && bytes.Equal(b[8:12], []byte("WAVE"))).To(BeTrue()) + m.mu.Lock() + defer m.mu.Unlock() + m.rates = append(m.rates, int(binary.LittleEndian.Uint32(b[24:28]))) + m.dataLens = append(m.dataLens, len(b)-44) +} + +func (m *wavCaptureModel) Transcribe(ctx context.Context, audio, language string, translate, diarize bool, prompt string) (*schema.TranscriptionResult, error) { + m.capture(audio) + return m.fakeModel.Transcribe(ctx, audio, language, translate, diarize, prompt) +} + +func (m *wavCaptureModel) TranscribeStream(ctx context.Context, audio, language string, translate, diarize bool, prompt string, onDelta func(text string)) (*schema.TranscriptionResult, error) { + m.capture(audio) + return m.fakeModel.TranscribeStream(ctx, audio, language, translate, diarize, prompt, onDelta) +} + +// Regression: the committed-utterance WAV is always written with a 16 kHz +// header, so the audio inside must be 16 kHz on every commit path — also for +// a 24 kHz session. The client-commit path used to pass the raw 24 kHz buffer +// (decoded at 1.5x speed: garbage or empty transcript); the VAD path stores +// audio that is already resampled and must stay untouched. +var _ = Describe("committed-utterance WAV sample rate", func() { + const sessionRate = 24000 + + newModel := func() *wavCaptureModel { + return &wavCaptureModel{fakeModel: &fakeModel{ + vadSegments: []schema.VADSegment{{Start: 0.1, End: 0.6}}, + transcribeFinal: &schema.TranscriptionResult{Text: "hello"}, + }} + } + newSession := func(m *wavCaptureModel, td *types.TurnDetectionUnion) *Session { + return &Session{ + TranscriptionOnly: true, // stop after the transcript + TurnDetection: td, + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{}, + ModelInterface: m, + InputSampleRate: sessionRate, + respSink: newResponseSink(), + } + } + + It("resamples a 24 kHz client commit to 16 kHz", func() { + m := newModel() + session := newSession(m, nil) + utt := make([]byte, sessionRate*2) // 1 s of 16-bit mono PCM at 24 kHz + + commitUtterance(context.Background(), utt, session, &Conversation{}, &fakeTransport{}, session.nextCommitSlot()) + + Expect(m.rates).To(Equal([]int{localSampleRate})) + Expect(m.dataLens[0]).To(BeNumerically("~", localSampleRate*2, 4), "1 s at 16 kHz") + }) + + It("writes the VAD commit of a 24 kHz session as 16 kHz audio", func() { + m := newModel() + session := newSession(m, &types.TurnDetectionUnion{ServerVad: &types.ServerVad{SilenceDurationMs: 500}}) + tr := &fakeTransport{} + sink := newTurnSink(session, &Conversation{}, tr, newLiveTurnState(session, tr), context.Background(), time.Now()) + session.InputAudioBuffer = make([]byte, int(1.4*sessionRate)*2) // 1.4 s at 24 kHz + + vadTick(sink, 0.5) + session.respSink.wait() + + Expect(tr.countEvents(types.ServerEventTypeInputAudioBufferCommitted)).To(Equal(1)) + Expect(m.rates).To(Equal([]int{localSampleRate})) + Expect(m.dataLens[0]).To(BeNumerically("~", int(1.4*localSampleRate)*2, 4), "1.4 s at 16 kHz") + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 7aec72d11d..d628e5b893 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -589,6 +589,8 @@ ws://localhost:8080/v1/realtime?model=gpt-realtime Audio is sent and received as raw PCM in the WebSocket messages, following the OpenAI Realtime API protocol. +Committed user audio reaches the transcription backend as 16 kHz mono WAV, whichever way the turn was committed: server-side VAD and client `input_audio_buffer.commit` both resample from the session input sample rate (for example 24 kHz) first. + ### WebRTC The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: