diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index cdff7aec5..b22e1a8dd 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -72,6 +72,19 @@ func resolveOutputModalities(session, response []types.Modality) []types.Modalit return []types.Modality{types.ModalityAudio} } +// modalitiesWithAlias returns output_modalities, falling back to the legacy +// beta `modalities` field when only the alias was supplied. OpenAI's Realtime +// beta named this field `modalities`; the GA field is `output_modalities`. +// Accepting the alias keeps beta clients (and the large amount of community +// sample code that still sends `modalities`) from silently receiving audio +// when they asked for text-only. The GA field wins when both are present. +func modalitiesWithAlias(output, alias []types.Modality) []types.Modality { + if len(output) > 0 { + return output + } + return alias +} + // modalitiesContainAudio reports whether the resolved modalities include audio // output. func modalitiesContainAudio(m []types.Modality) bool { @@ -1232,8 +1245,8 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode session.MaxOutputTokens = rt.MaxOutputTokens } - if len(rt.OutputModalities) > 0 { - session.OutputModalities = rt.OutputModalities + if mods := modalitiesWithAlias(rt.OutputModalities, rt.Modalities); len(mods) > 0 { + session.OutputModalities = mods } return nil @@ -2218,7 +2231,7 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa canStream := len(tools) == 0 || config.TemplateConfig.UseTokenizerTemplate var respMods []types.Modality if overrides != nil { - respMods = overrides.OutputModalities + respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities) } if canStream && modalitiesContainAudio(resolveOutputModalities(session.OutputModalities, respMods)) { if streamLLMResponse(ctx, session, conv, t, r, conversationHistory, images, config, tools, toolChoice, toolTurn) { @@ -2417,7 +2430,7 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa _, isWebRTC := t.(*WebRTCTransport) var respMods []types.Modality if overrides != nil { - respMods = overrides.OutputModalities + respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities) } modalities := resolveOutputModalities(session.OutputModalities, respMods) if modalitiesContainAudio(modalities) { diff --git a/core/http/endpoints/openai/realtime_modality_test.go b/core/http/endpoints/openai/realtime_modality_test.go index 0a17d7a03..e0574b8f8 100644 --- a/core/http/endpoints/openai/realtime_modality_test.go +++ b/core/http/endpoints/openai/realtime_modality_test.go @@ -1,6 +1,8 @@ package openai import ( + "encoding/json" + "github.com/mudler/LocalAI/core/http/endpoints/openai/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,3 +39,79 @@ var _ = Describe("resolveOutputModalities", func() { Expect(modalitiesContainAudio([]types.Modality{types.ModalityText, types.ModalityAudio})).To(BeTrue()) }) }) + +var _ = Describe("modalitiesWithAlias", func() { + It("uses output_modalities when set", func() { + got := modalitiesWithAlias([]types.Modality{types.ModalityText}, nil) + Expect(got).To(ConsistOf(types.ModalityText)) + }) + + It("falls back to the legacy modalities alias when output is empty", func() { + got := modalitiesWithAlias(nil, []types.Modality{types.ModalityText}) + Expect(got).To(ConsistOf(types.ModalityText)) + }) + + It("prefers output_modalities over the alias when both are present", func() { + got := modalitiesWithAlias([]types.Modality{types.ModalityText}, []types.Modality{types.ModalityAudio}) + Expect(got).To(ConsistOf(types.ModalityText)) + }) + + It("returns empty when neither is set", func() { + Expect(modalitiesWithAlias(nil, nil)).To(BeEmpty()) + }) +}) + +// These tests exercise the actual JSON wire boundary rather than calling the +// helper directly: they decode representative session.update / response.create +// payloads and reproduce the exact expressions used in updateSession and +// triggerResponseAtTurn. This guards against a regression where a wrong JSON +// tag or a missed call site would let encoding/json silently drop the legacy +// `modalities` key and fall the session back to audio. +var _ = Describe("modalities JSON boundary", func() { + It("keeps a session.update text-only when only the legacy modalities alias is sent", func() { + var ev types.SessionUpdateEvent + Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{"modalities":["text"]}}`), &ev)).To(Succeed()) + Expect(ev.Session.Realtime).NotTo(BeNil()) + + // Mirrors updateSession: session.OutputModalities = modalitiesWithAlias(rt.OutputModalities, rt.Modalities) + rt := ev.Session.Realtime + effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil) + Expect(effective).To(ConsistOf(types.ModalityText)) + Expect(modalitiesContainAudio(effective)).To(BeFalse()) + }) + + It("still honors the GA output_modalities field on session.update", func() { + var ev types.SessionUpdateEvent + Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{"output_modalities":["text"]}}`), &ev)).To(Succeed()) + rt := ev.Session.Realtime + effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil) + Expect(effective).To(ConsistOf(types.ModalityText)) + }) + + It("defaults a session.update with no modalities to audio", func() { + var ev types.SessionUpdateEvent + Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{}}`), &ev)).To(Succeed()) + rt := ev.Session.Realtime + effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil) + Expect(modalitiesContainAudio(effective)).To(BeTrue()) + }) + + It("keeps a response.create text-only when only the legacy modalities alias is sent", func() { + var ev types.ResponseCreateEvent + Expect(json.Unmarshal([]byte(`{"type":"response.create","response":{"modalities":["text"]}}`), &ev)).To(Succeed()) + + // Mirrors triggerResponseAtTurn: respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities) + respMods := modalitiesWithAlias(ev.Response.OutputModalities, ev.Response.Modalities) + effective := resolveOutputModalities(nil, respMods) + Expect(effective).To(ConsistOf(types.ModalityText)) + Expect(modalitiesContainAudio(effective)).To(BeFalse()) + }) + + It("lets a response.create alias override an audio session", func() { + var ev types.ResponseCreateEvent + Expect(json.Unmarshal([]byte(`{"type":"response.create","response":{"modalities":["text"]}}`), &ev)).To(Succeed()) + respMods := modalitiesWithAlias(ev.Response.OutputModalities, ev.Response.Modalities) + effective := resolveOutputModalities([]types.Modality{types.ModalityAudio}, respMods) + Expect(effective).To(ConsistOf(types.ModalityText)) + }) +}) diff --git a/core/http/endpoints/openai/types/types.go b/core/http/endpoints/openai/types/types.go index 2f75486ad..21e78004f 100644 --- a/core/http/endpoints/openai/types/types.go +++ b/core/http/endpoints/openai/types/types.go @@ -933,6 +933,13 @@ type RealtimeSession struct { // The set of modalities the model can respond with. It defaults to ["audio"], indicating that the model will respond with audio plus a transcript. ["text"] can be used to make the model respond with text only. It is not possible to request both text and audio at the same time. OutputModalities []Modality `json:"output_modalities,omitempty"` + // Modalities is the legacy OpenAI Realtime *beta* field name for + // OutputModalities. Many beta clients and community sample code still send + // `modalities`; standard JSON decoding would otherwise drop the unknown key + // silently and the session would fall back to audio. Accepted only as an + // alias: OutputModalities (the GA field) wins when both are present. + Modalities []Modality `json:"modalities,omitempty"` + // Reference to a prompt template and its variables. Prompt *PromptReference `json:"prompt,omitempty"` @@ -1170,6 +1177,10 @@ type ResponseCreateParams struct { // The set of modalities the model used to respond, currently the only possible values are [\"audio\"], [\"text\"]. Audio output always include a text transcript. Setting the output to mode text will disable audio output from the model. OutputModalities []Modality `json:"output_modalities,omitempty"` + // Modalities is the legacy beta alias for OutputModalities on + // response.create; see RealtimeSession.Modalities. + Modalities []Modality `json:"modalities,omitempty"` + // Reference to a prompt template and its variables. // // See https://platform.openai.com/docs/guides/text?api-mode=responses#reusable-prompts. diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 9eb913a6b..491300a5c 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -240,6 +240,26 @@ most reliable fix for WebRTC connections that establish and then drop. The API follows the OpenAI Realtime API protocol for handling sessions, audio buffers, and conversation items. +### Response modalities (text-only sessions) + +By default a realtime session responds with audio plus a transcript. To make the model respond with **text only**, set the output modalities on either the session or an individual response: + +```json +{"type": "session.update", "session": {"output_modalities": ["text"]}} +``` + +```json +{"type": "response.create", "response": {"output_modalities": ["text"]}} +``` + +`output_modalities` is the GA field name. For compatibility, LocalAI also accepts the legacy Realtime *beta* field name `modalities` as an alias (a lot of community sample code still sends `modalities: ["text"]`): + +```json +{"type": "session.update", "session": {"modalities": ["text"]}} +``` + +The GA `output_modalities` wins when both are present. A response-level value overrides the session-level one, and when neither is set the session falls back to `["audio"]`. + ## Gating a realtime pipeline with voice recognition A pipeline realtime model can require speaker verification before it responds. Add a `voice_recognition` block under `pipeline`. When present, each committed utterance is verified against authorized speakers; unauthorized utterances are dropped before the LLM runs (no LLM call, no tool execution, no TTS). The session stays open.