fix(realtime): keep sound detection for skipped turns

A VAD commit that is skipped because interrupt_response is false runs
with an already cancelled response context. Sound detection used that
context, so the model call failed and the turn never got its
sound-detection event. Run it under the session context, like the
transcription and the voice gate, so overlapping turns keep their
sound events.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto committed 2026-10-13 08:40:25 +00:00
1 parent f22fd45e21
commit 28c70be87e
3 files changed
+25 -2

No files matched your search

+4 -1
View File
@@ -2094,7 +2094,10 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt
// committed window and emit its scored AudioSet tags as a separate event.
// A failure here is logged but must never abort the turn.
if session.SoundDetectionEnabled {
if sderr := emitSoundDetection(ctx, t, session, generateItemID(), f.Name()); sderr != nil {
// Session context, like the transcription: a turn whose response
// context is already cancelled (a skipped overlapping turn, a barge-in)
// still gets its sound events.
if sderr := emitSoundDetection(sctx, t, session, generateItemID(), f.Name()); sderr != nil {
xlog.Error("sound detection failed", "error", sderr)
}
}
@@ -195,7 +195,10 @@ func (m *fakeModel) Transcribe(_ context.Context, _, _ string, _, diarize bool,
return m.transcribeFinal, m.transcribeErr
}
func (m *fakeModel) SoundDetection(context.Context, string, int, float32) (*schema.SoundClassificationResult, error) {
func (m *fakeModel) SoundDetection(ctx context.Context, _ string, _ int, _ float32) (*schema.SoundClassificationResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if m.soundDetectionErr != nil {
return nil, m.soundDetectionErr
}
@@ -77,6 +77,23 @@ var _ = Describe("realtime interrupt_response", func() {
Entry("server", "server_vad"), Entry("semantic", "semantic_vad"),
)
It("still emits sound detection for a skipped overlapping turn", func() {
s := &Session{TurnDetection: decode("server_vad", `,"interrupt_response":false`),
InputAudioTranscription: &types.AudioTranscription{}, SoundDetectionEnabled: true,
ModelInterface: &fakeModel{cfg: &config.ModelConfig{}, transcribeFinal: &schema.TranscriptionResult{Text: "next turn"},
soundDetectionResult: &schema.SoundClassificationResult{Detections: []schema.SoundClassification{{Label: "Dog", Score: 0.9}}}},
ModelConfig: &config.ModelConfig{}, respSink: newResponseSink()}
holdResponse(s)
conv, tr := &Conversation{}, &fakeTransport{}
committed := make(chan struct{})
s.issueCommit(context.Background(), respcoord.SourceVAD, func(turnCtx context.Context, slot *commitSlot) {
commitUtteranceWithTranscript(turnCtx, []byte{1, 2, 3, 4}, nil, nil, "", s, conv, tr, slot)
close(committed)
})
Eventually(committed).Should(BeClosed())
Expect(tr.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(1))
})
It("applies session updates to interruption and echoes false", func() {
s := &Session{TurnDetection: defaultTurnDetection(nil), ModelConfig: &config.ModelConfig{}}
var update types.SessionUpdateEvent