fix(realtime): accept GA WebRTC signaling (#11778)

OpenAI GA clients send multipart or raw SDP requests. They expect a bare
SDP answer. LocalAI only accepted its legacy JSON envelope, so signaling
failed before media setup.

Keep the JSON contract for existing clients. Accept both GA request
shapes and choose the matching response format.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/http/endpoints/openai/realtime_webrtc.go

@@ -1,6 +1,9 @@
 package openai
 
 import (
+	"encoding/json"
+	"io"
+	"mime"
 	"net/http"
 	"time"
 
@@ -27,6 +30,61 @@ type RealtimeCallResponse struct {
 	SessionID string `json:"session_id"`
 }
 
+func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) {
+	var req RealtimeCallRequest
+	mediaType := ""
+	contentType := c.Request().Header.Get(echo.HeaderContentType)
+	if contentType != "" {
+		var err error
+		mediaType, _, err = mime.ParseMediaType(contentType)
+		if err != nil {
+			return req, false, err
+		}
+	}
+
+	switch mediaType {
+	case echo.MIMEMultipartForm:
+		if err := c.Request().ParseMultipartForm(32 << 20); err != nil {
+			return req, true, err
+		}
+		req.SDP = c.FormValue("sdp")
+		var session struct {
+			Model            string `json:"model"`
+			LocalAIAssistant bool   `json:"localai_assistant,omitempty"`
+		}
+		if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil {
+			return req, true, err
+		}
+		req.Model = session.Model
+		req.LocalAIAssistant = session.LocalAIAssistant
+		return req, true, nil
+	case "application/sdp":
+		sdp, err := readRealtimeSDP(c.Request().Body)
+		req.SDP = sdp
+		req.Model = c.QueryParam("model")
+		return req, true, err
+	default:
+		err := c.Bind(&req)
+		return req, false, err
+	}
+}
+
+func readRealtimeSDP(body io.Reader) (string, error) {
+	data, err := io.ReadAll(body)
+	return string(data), err
+}
+
+func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error {
+	if plainSDPResponse {
+		return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp))
+	}
+
+	return c.JSON(http.StatusCreated, RealtimeCallResponse{
+		SDP:       sdp,
+		SessionID: sessionID,
+	})
+}
+
 // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling.
 func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 	se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig())
@@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 		if settingEngineErr != nil {
 			return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()})
 		}
-		var req RealtimeCallRequest
-		if err := c.Bind(&req); err != nil {
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(c)
+		if err != nil {
 			return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"})
 		}
 		if req.SDP == "" {
@@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 			runRealtimeSession(application, transport, req.Model, evaluator, opts)
 		}()
 
-		return c.JSON(http.StatusCreated, RealtimeCallResponse{
-			SDP:       localDesc.SDP,
-			SessionID: sessionID,
-		})
+		return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID)
 	}
 }
 

core/http/endpoints/openai/realtime_webrtc_request_test.go

@@ -0,0 +1,91 @@
+package openai
+
+import (
+	"bytes"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"net/textproto"
+
+	"github.com/labstack/echo/v4"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("decodeRealtimeCallRequest", func() {
+	It("decodes the legacy JSON request", func() {
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`))
+		request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
+		Expect(plainSDPResponse).To(BeFalse())
+	})
+
+	It("decodes the OpenAI multipart request", func() {
+		var body bytes.Buffer
+		writer := multipart.NewWriter(&body)
+		sdpHeader := make(textproto.MIMEHeader)
+		sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`)
+		sdpHeader.Set("Content-Type", "application/sdp")
+		sdpPart, err := writer.CreatePart(sdpHeader)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = sdpPart.Write([]byte("offer"))
+		Expect(err).NotTo(HaveOccurred())
+		sessionHeader := make(textproto.MIMEHeader)
+		sessionHeader.Set("Content-Disposition", `form-data; name="session"`)
+		sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON)
+		sessionPart, err := writer.CreatePart(sessionHeader)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(writer.Close()).To(Succeed())
+
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body)
+		request.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
+		Expect(plainSDPResponse).To(BeTrue())
+	})
+
+	It("decodes a raw SDP request with the model query parameter", func() {
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer"))
+		request.Header.Set(echo.HeaderContentType, "application/sdp")
+
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"}))
+		Expect(plainSDPResponse).To(BeTrue())
+	})
+})
+
+var _ = Describe("writeRealtimeCallResponse", func() {
+	It("writes the bare SDP answer for OpenAI request formats", func() {
+		response := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
+		context := echo.New().NewContext(request, response)
+
+		Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed())
+
+		Expect(response.Code).To(Equal(http.StatusCreated))
+		Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp"))
+		Expect(response.Body.String()).To(Equal("answer"))
+	})
+
+	It("preserves the JSON response for legacy requests", func() {
+		response := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
+		context := echo.New().NewContext(request, response)
+
+		Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed())
+
+		Expect(response.Code).To(Equal(http.StatusCreated))
+		Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON))
+		Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`))
+	})
+})

docs/content/features/openai-realtime.md

@@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O
 
 ### WebRTC
 
-The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint:
+The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter:
 
 ```
-POST http://localhost:8080/v1/realtime?model=gpt-realtime
+POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime
 Content-Type: application/sdp
 
 <SDP offer body>
 ```
 
-The response contains the SDP answer to complete the WebRTC handshake.
+The response has the `application/sdp` content type and contains the bare SDP answer.
+
+The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object:
+
+```bash
+curl http://localhost:8080/v1/realtime/calls \
+  -F "sdp=<offer.sdp;type=application/sdp" \
+  -F 'session={"type":"realtime","model":"gpt-realtime"};type=application/json'
+```
+
+LocalAI also accepts its original JSON request format for compatibility. A JSON request contains top-level `sdp` and `model` fields and receives a JSON response with `sdp` and `session_id` fields.
 
 #### Opus backend requirement