fix(images): bundle uv for backend installs

Runtime backend installation creates Python virtual environments with uv, but the final images did not include it. Copy the multi-architecture uv binaries into the shared runtime base and verify them during the image build so minimal L4T images can reinstall backends.

Fixes #10720

Assisted-by: Codex:gpt-5

Dockerfile

@@ -6,12 +6,19 @@ ARG UBUNTU_CODENAME=noble
 ARG APT_MIRROR=""
 ARG APT_PORTS_MIRROR=""
 
+FROM ghcr.io/astral-sh/uv:0.8.22 AS uv
+
 FROM ${BASE_IMAGE} AS requirements
 
 ARG APT_MIRROR
 ARG APT_PORTS_MIRROR
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Installed backends create their virtual environments at runtime, including in
+# minimal L4T images where neither system Python nor pip is available.
+COPY --from=uv /uv /uvx /usr/local/bin/
+RUN uv --version
+
 # hwdata ships /usr/share/hwdata/pci.ids. Without it, the ghw library we use
 # for hardware detection cannot resolve PCI vendor IDs and fails to enumerate
 # GPUs at all, so the image reports "No GPU detected" (see issue #10941).

backend/python/chatterbox/requirements-cublas12.txt

@@ -1,6 +1,7 @@
-torch
-torchaudio
-transformers
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.6.0+cu124
+torchaudio==2.6.0+cu124
+transformers<5
 numpy>=1.24.0,<1.26.0
 # chatterbox-tts itself is installed with --no-deps in install.sh.
 # These are its real runtime deps, mirroring upstream's pyproject.toml
@@ -15,4 +16,4 @@ conformer
 safetensors
 spacy-pkuseg
 pykakasi==2.3.0
-accelerate
+accelerate

backend/python/chatterbox/requirements.txt

@@ -2,5 +2,5 @@ grpcio==1.71.0
 protobuf
 certifi
 packaging
-setuptools
+setuptools<81
 poetry

core/http/endpoints/openai/realtime.go

@@ -2163,6 +2163,11 @@ type liveResponse struct {
 	output  []types.MessageItemUnion
 	usage   backend.TokenUsage
 	outcome responseOutcome
+	// metadata is echoed back on response.created and response.done. It is the
+	// only thing tying a terminal event to the response.create that asked for
+	// it, which is what lets a client run an out-of-band response alongside the
+	// spoken conversation and still recognise its own answer.
+	metadata map[string]string
 }
 
 func (r *liveResponse) addItem(it types.MessageItemUnion) { r.output = append(r.output, it) }
@@ -2192,12 +2197,16 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 	// terminals the legacy code emitted (one response.done per turn, with empty
 	// Output/Usage) are gone; tool turns are now internal to this single response.
 	r := &liveResponse{id: generateUniqueID()}
+	if overrides != nil {
+		r.metadata = overrides.Metadata
+	}
 	sendEvent(t, types.ResponseCreatedEvent{
 		ServerEventBase: types.ServerEventBase{},
 		Response: types.Response{
-			ID:     r.id,
-			Object: "realtime.response",
-			Status: types.ResponseStatusInProgress,
+			ID:       r.id,
+			Object:   "realtime.response",
+			Status:   types.ResponseStatusInProgress,
+			Metadata: r.metadata,
 		},
 	})
 
@@ -2208,10 +2217,11 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 		sendEvent(t, types.ResponseDoneEvent{
 			ServerEventBase: types.ServerEventBase{},
 			Response: types.Response{
-				ID:     r.id,
-				Object: "realtime.response",
-				Status: types.ResponseStatusCancelled,
-				Output: r.output,
+				ID:       r.id,
+				Object:   "realtime.response",
+				Status:   types.ResponseStatusCancelled,
+				Output:   r.output,
+				Metadata: r.metadata,
 			},
 		})
 	case outcomeFailed:
@@ -2221,11 +2231,12 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 		sendEvent(t, types.ResponseDoneEvent{
 			ServerEventBase: types.ServerEventBase{},
 			Response: types.Response{
-				ID:     r.id,
-				Object: "realtime.response",
-				Status: types.ResponseStatusCompleted,
-				Output: r.output,
-				Usage:  responseUsage(r.usage),
+				ID:       r.id,
+				Object:   "realtime.response",
+				Status:   types.ResponseStatusCompleted,
+				Output:   r.output,
+				Usage:    responseUsage(r.usage),
+				Metadata: r.metadata,
 			},
 		})
 	}

core/http/endpoints/openai/realtime_stream_test.go

@@ -263,4 +263,64 @@ var _ = Describe("triggerResponse", func() {
 		Expect(done.Response.Usage.OutputTokens).To(Equal(3))
 		Expect(done.Response.Usage.TotalTokens).To(Equal(8))
 	})
+
+	// response.metadata is the only thing tying a terminal event back to the
+	// response.create that asked for it. Without the echo, a client running an
+	// out-of-band response alongside the spoken conversation cannot tell its own
+	// answer from the conversation's, and blocks until it times out.
+	It("echoes response.create metadata back on response.created and response.done", func() {
+		m := &fakeModel{
+			cfg:         &config.ModelConfig{},
+			predictResp: backend.LLMResponse{Response: "Hi there."},
+		}
+		session := &Session{
+			OutputSampleRate: 24000,
+			ModelInterface:   m,
+			ModelConfig:      &config.ModelConfig{},
+			OutputModalities: []types.Modality{types.ModalityText},
+		}
+		t := &fakeTransport{}
+
+		triggerResponse(context.Background(), session, &Conversation{}, t, &types.ResponseCreateParams{
+			Metadata: map[string]string{"client_run": "abc123"},
+		})
+
+		var created *types.ResponseCreatedEvent
+		var done *types.ResponseDoneEvent
+		for i := range t.events {
+			switch e := t.events[i].(type) {
+			case types.ResponseCreatedEvent:
+				created = &e
+			case types.ResponseDoneEvent:
+				done = &e
+			}
+		}
+		Expect(created).NotTo(BeNil())
+		Expect(created.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123"))
+		Expect(done).NotTo(BeNil())
+		Expect(done.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123"))
+	})
+
+	// Omitted rather than sent as an empty object, matching the omitempty tag.
+	It("sends no metadata when response.create carried none", func() {
+		m := &fakeModel{
+			cfg:         &config.ModelConfig{},
+			predictResp: backend.LLMResponse{Response: "Hi there."},
+		}
+		session := &Session{
+			OutputSampleRate: 24000,
+			ModelInterface:   m,
+			ModelConfig:      &config.ModelConfig{},
+			OutputModalities: []types.Modality{types.ModalityText},
+		}
+		t := &fakeTransport{}
+
+		triggerResponse(context.Background(), session, &Conversation{}, t, nil)
+
+		for i := range t.events {
+			if d, ok := t.events[i].(types.ResponseDoneEvent); ok {
+				Expect(d.Response.Metadata).To(BeEmpty())
+			}
+		}
+	})
 })

docs/content/features/openai-realtime.md

@@ -347,6 +347,31 @@ By default a realtime session responds with audio plus a transcript. To make the
 
 The GA `output_modalities` wins when both are present. A response-level value overrides the session-level one, and when neither is set the session falls back to `["audio"]`.
 
+### Out-of-band responses and `metadata`
+
+A response can be created outside the default conversation by setting `conversation` to `none`. The reply is not added to the conversation history, which is what makes it usable for a side channel — answering a chat message or a webhook while a spoken conversation is in progress.
+
+Because such a response arrives on the same socket as everything else, attach `metadata` to correlate it. LocalAI echoes the map back verbatim on both `response.created` and `response.done`:
+
+```json
+{"type": "response.create", "response": {
+  "conversation": "none",
+  "output_modalities": ["text"],
+  "metadata": {"client_run": "abc123"},
+  "input": [{"type": "message", "role": "user",
+             "content": [{"type": "input_text", "text": "Is the oven still on?"}]}]
+}}
+```
+
+```json
+{"type": "response.done", "response": {
+  "id": "resp_...", "status": "completed",
+  "metadata": {"client_run": "abc123"}
+}}
+```
+
+Keys are strings up to 64 characters, values up to 512. A response created without `metadata` omits the field rather than sending an empty object.
+
 ## Gating a realtime pipeline with voice recognition
 
 A pipeline realtime model can require speaker verification before it responds. Add a `voice_recognition` block under `pipeline`. When present, each committed utterance is verified against authorized speakers; unauthorized utterances are dropped before the LLM runs (no LLM call, no tool execution, no TTS). The session stays open.