mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 17:18:49 -04:00
fix(realtime): keep sound detection for skipped turns
A VAD commit that is skipped because interrupt_response is false runs with an already cancelled response context. Sound detection used that context, so the model call failed and the turn never got its sound-detection event. Run it under the session context, like the transcription and the voice gate, so overlapping turns keep their sound events. Assisted-by: Claude Code:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
3 files changed
+25
-2
No files matched your search
@@ -2094,7 +2094,10 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt
|
||||
// committed window and emit its scored AudioSet tags as a separate event.
|
||||
// A failure here is logged but must never abort the turn.
|
||||
if session.SoundDetectionEnabled {
|
||||
if sderr := emitSoundDetection(ctx, t, session, generateItemID(), f.Name()); sderr != nil {
|
||||
// Session context, like the transcription: a turn whose response
|
||||
// context is already cancelled (a skipped overlapping turn, a barge-in)
|
||||
// still gets its sound events.
|
||||
if sderr := emitSoundDetection(sctx, t, session, generateItemID(), f.Name()); sderr != nil {
|
||||
xlog.Error("sound detection failed", "error", sderr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,7 +195,10 @@ func (m *fakeModel) Transcribe(_ context.Context, _, _ string, _, diarize bool,
|
||||
return m.transcribeFinal, m.transcribeErr
|
||||
}
|
||||
|
||||
func (m *fakeModel) SoundDetection(context.Context, string, int, float32) (*schema.SoundClassificationResult, error) {
|
||||
func (m *fakeModel) SoundDetection(ctx context.Context, _ string, _ int, _ float32) (*schema.SoundClassificationResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.soundDetectionErr != nil {
|
||||
return nil, m.soundDetectionErr
|
||||
}
|
||||
|
||||
@@ -77,6 +77,23 @@ var _ = Describe("realtime interrupt_response", func() {
|
||||
Entry("server", "server_vad"), Entry("semantic", "semantic_vad"),
|
||||
)
|
||||
|
||||
It("still emits sound detection for a skipped overlapping turn", func() {
|
||||
s := &Session{TurnDetection: decode("server_vad", `,"interrupt_response":false`),
|
||||
InputAudioTranscription: &types.AudioTranscription{}, SoundDetectionEnabled: true,
|
||||
ModelInterface: &fakeModel{cfg: &config.ModelConfig{}, transcribeFinal: &schema.TranscriptionResult{Text: "next turn"},
|
||||
soundDetectionResult: &schema.SoundClassificationResult{Detections: []schema.SoundClassification{{Label: "Dog", Score: 0.9}}}},
|
||||
ModelConfig: &config.ModelConfig{}, respSink: newResponseSink()}
|
||||
holdResponse(s)
|
||||
conv, tr := &Conversation{}, &fakeTransport{}
|
||||
committed := make(chan struct{})
|
||||
s.issueCommit(context.Background(), respcoord.SourceVAD, func(turnCtx context.Context, slot *commitSlot) {
|
||||
commitUtteranceWithTranscript(turnCtx, []byte{1, 2, 3, 4}, nil, nil, "", s, conv, tr, slot)
|
||||
close(committed)
|
||||
})
|
||||
Eventually(committed).Should(BeClosed())
|
||||
Expect(tr.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(1))
|
||||
})
|
||||
|
||||
It("applies session updates to interruption and echoes false", func() {
|
||||
s := &Session{TurnDetection: defaultTurnDetection(nil), ModelConfig: &config.ModelConfig{}}
|
||||
var update types.SessionUpdateEvent
|
||||
|
||||
Reference in new issue
Block a user