From d8a1e3c2e480fb8e38fcc4dad675a72828f2dcbc Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:17:31 +0200 Subject: [PATCH] fix(realtime): echo response.metadata on response.created and response.done (#11198) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit response.create accepts a metadata map and ResponseCreateParams has carried the field all along, but triggerResponse never copied it onto the Response it emits, so both terminals went out with metadata omitted. That field is the only thing tying a terminal event back to the response.create that asked for it. Our own doc comment on ResponseCreateEvent says so — "the metadata field is a good way to disambiguate multiple simultaneous Responses" — and it is what makes an out-of-band response (conversation: "none") usable at all: a client running one alongside the spoken conversation has no way to tell its own answer from the conversation's, so it waits for a reply it already received and gave away. Found from the client side: a headless text turn injected into a live session was answered correctly in about a second, and the caller still blocked until its own two-minute timeout because it could not recognise the answer. Carry the map on liveResponse so all three terminals (in_progress, cancelled, completed) report it, and leave it omitted when response.create sent none. Assisted-by: Claude:claude-opus-5 gofmt Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/http/endpoints/openai/realtime.go | 35 +++++++---- .../endpoints/openai/realtime_stream_test.go | 60 +++++++++++++++++++ docs/content/features/openai-realtime.md | 25 ++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index bee3b52ed..32b08b3fd 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2163,6 +2163,11 @@ type liveResponse struct { output []types.MessageItemUnion usage backend.TokenUsage outcome responseOutcome + // metadata is echoed back on response.created and response.done. It is the + // only thing tying a terminal event to the response.create that asked for + // it, which is what lets a client run an out-of-band response alongside the + // spoken conversation and still recognise its own answer. + metadata map[string]string } func (r *liveResponse) addItem(it types.MessageItemUnion) { r.output = append(r.output, it) } @@ -2192,12 +2197,16 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, // terminals the legacy code emitted (one response.done per turn, with empty // Output/Usage) are gone; tool turns are now internal to this single response. r := &liveResponse{id: generateUniqueID()} + if overrides != nil { + r.metadata = overrides.Metadata + } sendEvent(t, types.ResponseCreatedEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusInProgress, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusInProgress, + Metadata: r.metadata, }, }) @@ -2208,10 +2217,11 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCancelled, - Output: r.output, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCancelled, + Output: r.output, + Metadata: r.metadata, }, }) case outcomeFailed: @@ -2221,11 +2231,12 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCompleted, - Output: r.output, - Usage: responseUsage(r.usage), + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCompleted, + Output: r.output, + Usage: responseUsage(r.usage), + Metadata: r.metadata, }, }) } diff --git a/core/http/endpoints/openai/realtime_stream_test.go b/core/http/endpoints/openai/realtime_stream_test.go index 439f3240e..2d5d7d7a1 100644 --- a/core/http/endpoints/openai/realtime_stream_test.go +++ b/core/http/endpoints/openai/realtime_stream_test.go @@ -263,4 +263,64 @@ var _ = Describe("triggerResponse", func() { Expect(done.Response.Usage.OutputTokens).To(Equal(3)) Expect(done.Response.Usage.TotalTokens).To(Equal(8)) }) + + // response.metadata is the only thing tying a terminal event back to the + // response.create that asked for it. Without the echo, a client running an + // out-of-band response alongside the spoken conversation cannot tell its own + // answer from the conversation's, and blocks until it times out. + It("echoes response.create metadata back on response.created and response.done", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, &types.ResponseCreateParams{ + Metadata: map[string]string{"client_run": "abc123"}, + }) + + var created *types.ResponseCreatedEvent + var done *types.ResponseDoneEvent + for i := range t.events { + switch e := t.events[i].(type) { + case types.ResponseCreatedEvent: + created = &e + case types.ResponseDoneEvent: + done = &e + } + } + Expect(created).NotTo(BeNil()) + Expect(created.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + Expect(done).NotTo(BeNil()) + Expect(done.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + }) + + // Omitted rather than sent as an empty object, matching the omitempty tag. + It("sends no metadata when response.create carried none", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, nil) + + for i := range t.events { + if d, ok := t.events[i].(types.ResponseDoneEvent); ok { + Expect(d.Response.Metadata).To(BeEmpty()) + } + } + }) }) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 4a139c687..bf2a50b6f 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -347,6 +347,31 @@ By default a realtime session responds with audio plus a transcript. To make the The GA `output_modalities` wins when both are present. A response-level value overrides the session-level one, and when neither is set the session falls back to `["audio"]`. +### Out-of-band responses and `metadata` + +A response can be created outside the default conversation by setting `conversation` to `none`. The reply is not added to the conversation history, which is what makes it usable for a side channel — answering a chat message or a webhook while a spoken conversation is in progress. + +Because such a response arrives on the same socket as everything else, attach `metadata` to correlate it. LocalAI echoes the map back verbatim on both `response.created` and `response.done`: + +```json +{"type": "response.create", "response": { + "conversation": "none", + "output_modalities": ["text"], + "metadata": {"client_run": "abc123"}, + "input": [{"type": "message", "role": "user", + "content": [{"type": "input_text", "text": "Is the oven still on?"}]}] +}} +``` + +```json +{"type": "response.done", "response": { + "id": "resp_...", "status": "completed", + "metadata": {"client_run": "abc123"} +}} +``` + +Keys are strings up to 64 characters, values up to 512. A response created without `metadata` omits the field rather than sending an empty object. + ## Gating a realtime pipeline with voice recognition A pipeline realtime model can require speaker verification before it responds. Add a `voice_recognition` block under `pipeline`. When present, each committed utterance is verified against authorized speakers; unauthorized utterances are dropped before the LLM runs (no LLM call, no tool execution, no TTS). The session stays open.