fix(realtime): resample client commits to 16 kHz before transcription

The committed-utterance WAV is written with a fixed 16 kHz header. The VAD
commit path stores audio that is already resampled to 16 kHz (vadCommit),
but the client-commit path (input_audio_buffer.commit) passed the raw
buffer at the session input rate. For a 24 kHz OpenAI Realtime session the
backend decoded 24 kHz audio as 16 kHz (1.5x speed/pitch): garbage or empty
transcripts, then input_not_committed on the follow-up response.

commitUtterance now resamples from session.InputSampleRate to
localSampleRate, so both paths hand 16 kHz audio to the 16 kHz header.
This replaces the earlier approach of this PR (header at the session rate),
which mislabelled the already-resampled VAD audio (review feedback).

Tests go through the real paths with a 24 kHz session: commitUtterance and
a server_vad commit via vadTick; both must produce a 16 kHz WAV of the
resampled length. The client-commit spec fails without the fix.

Fixes #12747

Signed-off-by: nexxtmobile.de <git@nexxtmobile.de>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/http/endpoints/openai/realtime.go

@@ -2013,7 +2013,17 @@ func vadScanWindowSec(sv *types.RealtimeSessionSemanticVad, silenceThreshold flo
 	return window
 }
 
+// commitUtterance commits a client-issued turn (input_audio_buffer.commit).
+// The client buffer holds audio at the session input sample rate (e.g.
+// 24 kHz for OpenAI Realtime clients), while every commit path writes the
+// transcription WAV at localSampleRate: the VAD path stores audio that is
+// already resampled (vadCommit). Resample here so a 24 kHz buffer is not
+// decoded at 16 kHz (1.5x speed/pitch, garbage or empty transcript, then
+// input_not_committed on the follow-up response).
 func commitUtterance(ctx context.Context, utt []byte, session *Session, conv *Conversation, t Transport, slot *commitSlot) {
+	if rate := session.InputSampleRate; rate > 0 && rate != localSampleRate && len(utt) > 0 {
+		utt = sound.Int16toBytesLE(sound.ResampleInt16(sound.BytesToInt16sLE(utt), rate, localSampleRate))
+	}
 	commitUtteranceWithTranscript(ctx, utt, nil, nil, "", session, conv, t, slot)
 }
 

core/http/endpoints/openai/realtime_commit_wav_rate_test.go

@@ -0,0 +1,100 @@
+package openai
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"os"
+	"sync"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/endpoints/openai/types"
+	"github.com/mudler/LocalAI/core/schema"
+)
+
+// wavCaptureModel records the sample rate and data size of every WAV file a
+// commit hands to the transcription backend.
+type wavCaptureModel struct {
+	*fakeModel
+	mu       sync.Mutex
+	rates    []int
+	dataLens []int
+}
+
+func (m *wavCaptureModel) capture(path string) {
+	b, err := os.ReadFile(path)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(b)).To(BeNumerically(">=", 44))
+	Expect(bytes.Equal(b[0:4], []byte("RIFF")) && bytes.Equal(b[8:12], []byte("WAVE"))).To(BeTrue())
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.rates = append(m.rates, int(binary.LittleEndian.Uint32(b[24:28])))
+	m.dataLens = append(m.dataLens, len(b)-44)
+}
+
+func (m *wavCaptureModel) Transcribe(ctx context.Context, audio, language string, translate, diarize bool, prompt string) (*schema.TranscriptionResult, error) {
+	m.capture(audio)
+	return m.fakeModel.Transcribe(ctx, audio, language, translate, diarize, prompt)
+}
+
+func (m *wavCaptureModel) TranscribeStream(ctx context.Context, audio, language string, translate, diarize bool, prompt string, onDelta func(text string)) (*schema.TranscriptionResult, error) {
+	m.capture(audio)
+	return m.fakeModel.TranscribeStream(ctx, audio, language, translate, diarize, prompt, onDelta)
+}
+
+// Regression: the committed-utterance WAV is always written with a 16 kHz
+// header, so the audio inside must be 16 kHz on every commit path — also for
+// a 24 kHz session. The client-commit path used to pass the raw 24 kHz buffer
+// (decoded at 1.5x speed: garbage or empty transcript); the VAD path stores
+// audio that is already resampled and must stay untouched.
+var _ = Describe("committed-utterance WAV sample rate", func() {
+	const sessionRate = 24000
+
+	newModel := func() *wavCaptureModel {
+		return &wavCaptureModel{fakeModel: &fakeModel{
+			vadSegments:     []schema.VADSegment{{Start: 0.1, End: 0.6}},
+			transcribeFinal: &schema.TranscriptionResult{Text: "hello"},
+		}}
+	}
+	newSession := func(m *wavCaptureModel, td *types.TurnDetectionUnion) *Session {
+		return &Session{
+			TranscriptionOnly:       true, // stop after the transcript
+			TurnDetection:           td,
+			InputAudioTranscription: &types.AudioTranscription{},
+			ModelConfig:             &config.ModelConfig{},
+			ModelInterface:          m,
+			InputSampleRate:         sessionRate,
+			respSink:                newResponseSink(),
+		}
+	}
+
+	It("resamples a 24 kHz client commit to 16 kHz", func() {
+		m := newModel()
+		session := newSession(m, nil)
+		utt := make([]byte, sessionRate*2) // 1 s of 16-bit mono PCM at 24 kHz
+
+		commitUtterance(context.Background(), utt, session, &Conversation{}, &fakeTransport{}, session.nextCommitSlot())
+
+		Expect(m.rates).To(Equal([]int{localSampleRate}))
+		Expect(m.dataLens[0]).To(BeNumerically("~", localSampleRate*2, 4), "1 s at 16 kHz")
+	})
+
+	It("writes the VAD commit of a 24 kHz session as 16 kHz audio", func() {
+		m := newModel()
+		session := newSession(m, &types.TurnDetectionUnion{ServerVad: &types.ServerVad{SilenceDurationMs: 500}})
+		tr := &fakeTransport{}
+		sink := newTurnSink(session, &Conversation{}, tr, newLiveTurnState(session, tr), context.Background(), time.Now())
+		session.InputAudioBuffer = make([]byte, int(1.4*sessionRate)*2) // 1.4 s at 24 kHz
+
+		vadTick(sink, 0.5)
+		session.respSink.wait()
+
+		Expect(tr.countEvents(types.ServerEventTypeInputAudioBufferCommitted)).To(Equal(1))
+		Expect(m.rates).To(Equal([]int{localSampleRate}))
+		Expect(m.dataLens[0]).To(BeNumerically("~", int(1.4*localSampleRate)*2, 4), "1.4 s at 16 kHz")
+	})
+})

docs/content/features/openai-realtime.md

@@ -479,6 +479,8 @@ ws://localhost:8080/v1/realtime?model=gpt-realtime
 
 Audio is sent and received as raw PCM in the WebSocket messages, following the OpenAI Realtime API protocol.
 
+Committed user audio reaches the transcription backend as 16 kHz mono WAV, whichever way the turn was committed: server-side VAD and client `input_audio_buffer.commit` both resample from the session input sample rate (for example 24 kHz) first.
+
 ### WebRTC
 
 The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: