diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index bee3b52ed..32b08b3fd 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2163,6 +2163,11 @@ type liveResponse struct { output []types.MessageItemUnion usage backend.TokenUsage outcome responseOutcome + // metadata is echoed back on response.created and response.done. It is the + // only thing tying a terminal event to the response.create that asked for + // it, which is what lets a client run an out-of-band response alongside the + // spoken conversation and still recognise its own answer. + metadata map[string]string } func (r *liveResponse) addItem(it types.MessageItemUnion) { r.output = append(r.output, it) } @@ -2192,12 +2197,16 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, // terminals the legacy code emitted (one response.done per turn, with empty // Output/Usage) are gone; tool turns are now internal to this single response. r := &liveResponse{id: generateUniqueID()} + if overrides != nil { + r.metadata = overrides.Metadata + } sendEvent(t, types.ResponseCreatedEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusInProgress, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusInProgress, + Metadata: r.metadata, }, }) @@ -2208,10 +2217,11 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCancelled, - Output: r.output, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCancelled, + Output: r.output, + Metadata: r.metadata, }, }) case outcomeFailed: @@ -2221,11 +2231,12 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCompleted, - Output: r.output, - Usage: responseUsage(r.usage), + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCompleted, + Output: r.output, + Usage: responseUsage(r.usage), + Metadata: r.metadata, }, }) } diff --git a/core/http/endpoints/openai/realtime_stream_test.go b/core/http/endpoints/openai/realtime_stream_test.go index 439f3240e..2d5d7d7a1 100644 --- a/core/http/endpoints/openai/realtime_stream_test.go +++ b/core/http/endpoints/openai/realtime_stream_test.go @@ -263,4 +263,64 @@ var _ = Describe("triggerResponse", func() { Expect(done.Response.Usage.OutputTokens).To(Equal(3)) Expect(done.Response.Usage.TotalTokens).To(Equal(8)) }) + + // response.metadata is the only thing tying a terminal event back to the + // response.create that asked for it. Without the echo, a client running an + // out-of-band response alongside the spoken conversation cannot tell its own + // answer from the conversation's, and blocks until it times out. + It("echoes response.create metadata back on response.created and response.done", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, &types.ResponseCreateParams{ + Metadata: map[string]string{"client_run": "abc123"}, + }) + + var created *types.ResponseCreatedEvent + var done *types.ResponseDoneEvent + for i := range t.events { + switch e := t.events[i].(type) { + case types.ResponseCreatedEvent: + created = &e + case types.ResponseDoneEvent: + done = &e + } + } + Expect(created).NotTo(BeNil()) + Expect(created.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + Expect(done).NotTo(BeNil()) + Expect(done.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + }) + + // Omitted rather than sent as an empty object, matching the omitempty tag. + It("sends no metadata when response.create carried none", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, nil) + + for i := range t.events { + if d, ok := t.events[i].(types.ResponseDoneEvent); ok { + Expect(d.Response.Metadata).To(BeEmpty()) + } + } + }) }) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 4a139c687..bf2a50b6f 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -347,6 +347,31 @@ By default a realtime session responds with audio plus a transcript. To make the The GA `output_modalities` wins when both are present. A response-level value overrides the session-level one, and when neither is set the session falls back to `["audio"]`. +### Out-of-band responses and `metadata` + +A response can be created outside the default conversation by setting `conversation` to `none`. The reply is not added to the conversation history, which is what makes it usable for a side channel — answering a chat message or a webhook while a spoken conversation is in progress. + +Because such a response arrives on the same socket as everything else, attach `metadata` to correlate it. LocalAI echoes the map back verbatim on both `response.created` and `response.done`: + +```json +{"type": "response.create", "response": { + "conversation": "none", + "output_modalities": ["text"], + "metadata": {"client_run": "abc123"}, + "input": [{"type": "message", "role": "user", + "content": [{"type": "input_text", "text": "Is the oven still on?"}]}] +}} +``` + +```json +{"type": "response.done", "response": { + "id": "resp_...", "status": "completed", + "metadata": {"client_run": "abc123"} +}} +``` + +Keys are strings up to 64 characters, values up to 512. A response created without `metadata` omits the field rather than sending an empty object. + ## Gating a realtime pipeline with voice recognition A pipeline realtime model can require speaker verification before it responds. Add a `voice_recognition` block under `pipeline`. When present, each committed utterance is verified against authorized speakers; unauthorized utterances are dropped before the LLM runs (no LLM call, no tool execution, no TTS). The session stays open.