fix(realtime): accept legacy 'modalities' alias for output_modalities (fixes #11103) (#11104)

* fix(realtime): accept legacy 'modalities' alias for output_modalities

OpenAI's Realtime *beta* used the field name `modalities`; the GA field is
`output_modalities`. LocalAI only binds `output_modalities`, so a client
sending the still-common beta field `modalities: ["text"]` has it silently
dropped by encoding/json and the session falls back to audio: TTS runs and the
client receives large response.output_audio.* frames even though it asked for
text-only.

Accept `modalities` as an alias on both session.update (RealtimeSession) and
response.create (ResponseCreateParams). The GA `output_modalities` wins when
both are present, so GA clients are unaffected. Applied at the two existing
resolution points via a small modalitiesWithAlias helper.

Fixes #11103

Signed-off-by: Anai-Guo <antai12232931@anaiguo.com>

* test(realtime): add JSON-boundary regression for modalities alias

Decode representative session.update and response.create payloads that
carry only the legacy beta `modalities` key and assert the effective
output modality resolves to text (not audio), reproducing the exact
expressions used in updateSession and triggerResponseAtTurn. This guards
against a wrong JSON tag or a missed call site letting encoding/json drop
the alias silently.

Also document output_modalities (and the accepted legacy modalities
alias) for text-only sessions in the realtime feature docs.

Signed-off-by: Tai An <antai12232931@outlook.com>

---------

Signed-off-by: Anai-Guo <antai12232931@anaiguo.com>
Signed-off-by: Tai An <antai12232931@outlook.com>
Co-authored-by: Anai-Guo <antai12232931@anaiguo.com>

core/http/endpoints/openai/realtime.go

@@ -72,6 +72,19 @@ func resolveOutputModalities(session, response []types.Modality) []types.Modalit
 	return []types.Modality{types.ModalityAudio}
 }
 
+// modalitiesWithAlias returns output_modalities, falling back to the legacy
+// beta `modalities` field when only the alias was supplied. OpenAI's Realtime
+// beta named this field `modalities`; the GA field is `output_modalities`.
+// Accepting the alias keeps beta clients (and the large amount of community
+// sample code that still sends `modalities`) from silently receiving audio
+// when they asked for text-only. The GA field wins when both are present.
+func modalitiesWithAlias(output, alias []types.Modality) []types.Modality {
+	if len(output) > 0 {
+		return output
+	}
+	return alias
+}
+
 // modalitiesContainAudio reports whether the resolved modalities include audio
 // output.
 func modalitiesContainAudio(m []types.Modality) bool {
@@ -1232,8 +1245,8 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 		session.MaxOutputTokens = rt.MaxOutputTokens
 	}
 
-	if len(rt.OutputModalities) > 0 {
-		session.OutputModalities = rt.OutputModalities
+	if mods := modalitiesWithAlias(rt.OutputModalities, rt.Modalities); len(mods) > 0 {
+		session.OutputModalities = mods
 	}
 
 	return nil
@@ -2218,7 +2231,7 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa
 		canStream := len(tools) == 0 || config.TemplateConfig.UseTokenizerTemplate
 		var respMods []types.Modality
 		if overrides != nil {
-			respMods = overrides.OutputModalities
+			respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities)
 		}
 		if canStream && modalitiesContainAudio(resolveOutputModalities(session.OutputModalities, respMods)) {
 			if streamLLMResponse(ctx, session, conv, t, r, conversationHistory, images, config, tools, toolChoice, toolTurn) {
@@ -2417,7 +2430,7 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa
 		_, isWebRTC := t.(*WebRTCTransport)
 		var respMods []types.Modality
 		if overrides != nil {
-			respMods = overrides.OutputModalities
+			respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities)
 		}
 		modalities := resolveOutputModalities(session.OutputModalities, respMods)
 		if modalitiesContainAudio(modalities) {

core/http/endpoints/openai/realtime_modality_test.go

@@ -1,6 +1,8 @@
 package openai
 
 import (
+	"encoding/json"
+
 	"github.com/mudler/LocalAI/core/http/endpoints/openai/types"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -37,3 +39,79 @@ var _ = Describe("resolveOutputModalities", func() {
 		Expect(modalitiesContainAudio([]types.Modality{types.ModalityText, types.ModalityAudio})).To(BeTrue())
 	})
 })
+
+var _ = Describe("modalitiesWithAlias", func() {
+	It("uses output_modalities when set", func() {
+		got := modalitiesWithAlias([]types.Modality{types.ModalityText}, nil)
+		Expect(got).To(ConsistOf(types.ModalityText))
+	})
+
+	It("falls back to the legacy modalities alias when output is empty", func() {
+		got := modalitiesWithAlias(nil, []types.Modality{types.ModalityText})
+		Expect(got).To(ConsistOf(types.ModalityText))
+	})
+
+	It("prefers output_modalities over the alias when both are present", func() {
+		got := modalitiesWithAlias([]types.Modality{types.ModalityText}, []types.Modality{types.ModalityAudio})
+		Expect(got).To(ConsistOf(types.ModalityText))
+	})
+
+	It("returns empty when neither is set", func() {
+		Expect(modalitiesWithAlias(nil, nil)).To(BeEmpty())
+	})
+})
+
+// These tests exercise the actual JSON wire boundary rather than calling the
+// helper directly: they decode representative session.update / response.create
+// payloads and reproduce the exact expressions used in updateSession and
+// triggerResponseAtTurn. This guards against a regression where a wrong JSON
+// tag or a missed call site would let encoding/json silently drop the legacy
+// `modalities` key and fall the session back to audio.
+var _ = Describe("modalities JSON boundary", func() {
+	It("keeps a session.update text-only when only the legacy modalities alias is sent", func() {
+		var ev types.SessionUpdateEvent
+		Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{"modalities":["text"]}}`), &ev)).To(Succeed())
+		Expect(ev.Session.Realtime).NotTo(BeNil())
+
+		// Mirrors updateSession: session.OutputModalities = modalitiesWithAlias(rt.OutputModalities, rt.Modalities)
+		rt := ev.Session.Realtime
+		effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil)
+		Expect(effective).To(ConsistOf(types.ModalityText))
+		Expect(modalitiesContainAudio(effective)).To(BeFalse())
+	})
+
+	It("still honors the GA output_modalities field on session.update", func() {
+		var ev types.SessionUpdateEvent
+		Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{"output_modalities":["text"]}}`), &ev)).To(Succeed())
+		rt := ev.Session.Realtime
+		effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil)
+		Expect(effective).To(ConsistOf(types.ModalityText))
+	})
+
+	It("defaults a session.update with no modalities to audio", func() {
+		var ev types.SessionUpdateEvent
+		Expect(json.Unmarshal([]byte(`{"type":"session.update","session":{}}`), &ev)).To(Succeed())
+		rt := ev.Session.Realtime
+		effective := resolveOutputModalities(modalitiesWithAlias(rt.OutputModalities, rt.Modalities), nil)
+		Expect(modalitiesContainAudio(effective)).To(BeTrue())
+	})
+
+	It("keeps a response.create text-only when only the legacy modalities alias is sent", func() {
+		var ev types.ResponseCreateEvent
+		Expect(json.Unmarshal([]byte(`{"type":"response.create","response":{"modalities":["text"]}}`), &ev)).To(Succeed())
+
+		// Mirrors triggerResponseAtTurn: respMods = modalitiesWithAlias(overrides.OutputModalities, overrides.Modalities)
+		respMods := modalitiesWithAlias(ev.Response.OutputModalities, ev.Response.Modalities)
+		effective := resolveOutputModalities(nil, respMods)
+		Expect(effective).To(ConsistOf(types.ModalityText))
+		Expect(modalitiesContainAudio(effective)).To(BeFalse())
+	})
+
+	It("lets a response.create alias override an audio session", func() {
+		var ev types.ResponseCreateEvent
+		Expect(json.Unmarshal([]byte(`{"type":"response.create","response":{"modalities":["text"]}}`), &ev)).To(Succeed())
+		respMods := modalitiesWithAlias(ev.Response.OutputModalities, ev.Response.Modalities)
+		effective := resolveOutputModalities([]types.Modality{types.ModalityAudio}, respMods)
+		Expect(effective).To(ConsistOf(types.ModalityText))
+	})
+})

core/http/endpoints/openai/types/types.go

@@ -933,6 +933,13 @@ type RealtimeSession struct {
 	// The set of modalities the model can respond with. It defaults to ["audio"], indicating that the model will respond with audio plus a transcript. ["text"] can be used to make the model respond with text only. It is not possible to request both text and audio at the same time.
 	OutputModalities []Modality `json:"output_modalities,omitempty"`
 
+	// Modalities is the legacy OpenAI Realtime *beta* field name for
+	// OutputModalities. Many beta clients and community sample code still send
+	// `modalities`; standard JSON decoding would otherwise drop the unknown key
+	// silently and the session would fall back to audio. Accepted only as an
+	// alias: OutputModalities (the GA field) wins when both are present.
+	Modalities []Modality `json:"modalities,omitempty"`
+
 	// Reference to a prompt template and its variables.
 	Prompt *PromptReference `json:"prompt,omitempty"`
 
@@ -1170,6 +1177,10 @@ type ResponseCreateParams struct {
 	// The set of modalities the model used to respond, currently the only possible values are [\"audio\"], [\"text\"]. Audio output always include a text transcript. Setting the output to mode text will disable audio output from the model.
 	OutputModalities []Modality `json:"output_modalities,omitempty"`
 
+	// Modalities is the legacy beta alias for OutputModalities on
+	// response.create; see RealtimeSession.Modalities.
+	Modalities []Modality `json:"modalities,omitempty"`
+
 	// Reference to a prompt template and its variables.
 	//
 	// See https://platform.openai.com/docs/guides/text?api-mode=responses#reusable-prompts.

docs/content/features/openai-realtime.md

@@ -240,6 +240,26 @@ most reliable fix for WebRTC connections that establish and then drop.
 
 The API follows the OpenAI Realtime API protocol for handling sessions, audio buffers, and conversation items.
 
+### Response modalities (text-only sessions)
+
+By default a realtime session responds with audio plus a transcript. To make the model respond with **text only**, set the output modalities on either the session or an individual response:
+
+```json
+{"type": "session.update", "session": {"output_modalities": ["text"]}}
+```
+
+```json
+{"type": "response.create", "response": {"output_modalities": ["text"]}}
+```
+
+`output_modalities` is the GA field name. For compatibility, LocalAI also accepts the legacy Realtime *beta* field name `modalities` as an alias (a lot of community sample code still sends `modalities: ["text"]`):
+
+```json
+{"type": "session.update", "session": {"modalities": ["text"]}}
+```
+
+The GA `output_modalities` wins when both are present. A response-level value overrides the session-level one, and when neither is set the session falls back to `["audio"]`.
+
 ## Gating a realtime pipeline with voice recognition
 
 A pipeline realtime model can require speaker verification before it responds. Add a `voice_recognition` block under `pipeline`. When present, each committed utterance is verified against authorized speakers; unauthorized utterances are dropped before the LLM runs (no LLM call, no tool execution, no TTS). The session stays open.