From 893a45141c5a3d94b9c86d2b79cbe7c4fc3dc835 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 29 Aug 2026 21:28:37 +0200 Subject: [PATCH] fix(realtime): accept GA WebRTC signaling (#11778) OpenAI GA clients send multipart or raw SDP requests. They expect a bare SDP answer. LocalAI only accepted its legacy JSON envelope, so signaling failed before media setup. Keep the JSON contract for existing clients. Accept both GA request shapes and choose the matching response format. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/http/endpoints/openai/realtime_webrtc.go | 67 ++++++++++++-- .../openai/realtime_webrtc_request_test.go | 91 +++++++++++++++++++ docs/content/features/openai-realtime.md | 16 +++- 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 core/http/endpoints/openai/realtime_webrtc_request_test.go diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 4f13862c8..eca3aa2c6 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -1,6 +1,9 @@ package openai import ( + "encoding/json" + "io" + "mime" "net/http" "time" @@ -27,6 +30,61 @@ type RealtimeCallResponse struct { SessionID string `json:"session_id"` } +func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) { + var req RealtimeCallRequest + mediaType := "" + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return req, false, err + } + } + + switch mediaType { + case echo.MIMEMultipartForm: + if err := c.Request().ParseMultipartForm(32 << 20); err != nil { + return req, true, err + } + req.SDP = c.FormValue("sdp") + var session struct { + Model string `json:"model"` + LocalAIAssistant bool `json:"localai_assistant,omitempty"` + } + if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil { + return req, true, err + } + req.Model = session.Model + req.LocalAIAssistant = session.LocalAIAssistant + return req, true, nil + case "application/sdp": + sdp, err := readRealtimeSDP(c.Request().Body) + req.SDP = sdp + req.Model = c.QueryParam("model") + return req, true, err + default: + err := c.Bind(&req) + return req, false, err + } +} + +func readRealtimeSDP(body io.Reader) (string, error) { + data, err := io.ReadAll(body) + return string(data), err +} + +func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error { + if plainSDPResponse { + return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp)) + } + + return c.JSON(http.StatusCreated, RealtimeCallResponse{ + SDP: sdp, + SessionID: sessionID, + }) +} + // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) @@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { if settingEngineErr != nil { return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) } - var req RealtimeCallRequest - if err := c.Bind(&req); err != nil { + req, plainSDPResponse, err := decodeRealtimeCallRequest(c) + if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) } if req.SDP == "" { @@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { runRealtimeSession(application, transport, req.Model, evaluator, opts) }() - return c.JSON(http.StatusCreated, RealtimeCallResponse{ - SDP: localDesc.SDP, - SessionID: sessionID, - }) + return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID) } } diff --git a/core/http/endpoints/openai/realtime_webrtc_request_test.go b/core/http/endpoints/openai/realtime_webrtc_request_test.go new file mode 100644 index 000000000..187555329 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_request_test.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("decodeRealtimeCallRequest", func() { + It("decodes the legacy JSON request", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeFalse()) + }) + + It("decodes the OpenAI multipart request", func() { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + sdpHeader := make(textproto.MIMEHeader) + sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`) + sdpHeader.Set("Content-Type", "application/sdp") + sdpPart, err := writer.CreatePart(sdpHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sdpPart.Write([]byte("offer")) + Expect(err).NotTo(HaveOccurred()) + sessionHeader := make(textproto.MIMEHeader) + sessionHeader.Set("Content-Disposition", `form-data; name="session"`) + sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON) + sessionPart, err := writer.CreatePart(sessionHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body) + request.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeTrue()) + }) + + It("decodes a raw SDP request with the model query parameter", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer")) + request.Header.Set(echo.HeaderContentType, "application/sdp") + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"})) + Expect(plainSDPResponse).To(BeTrue()) + }) +}) + +var _ = Describe("writeRealtimeCallResponse", func() { + It("writes the bare SDP answer for OpenAI request formats", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp")) + Expect(response.Body.String()).To(Equal("answer")) + }) + + It("preserves the JSON response for legacy requests", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON)) + Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`)) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 417fab331..54bec58ab 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O ### WebRTC -The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint: +The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: ``` -POST http://localhost:8080/v1/realtime?model=gpt-realtime +POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime Content-Type: application/sdp ``` -The response contains the SDP answer to complete the WebRTC handshake. +The response has the `application/sdp` content type and contains the bare SDP answer. + +The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object: + +```bash +curl http://localhost:8080/v1/realtime/calls \ + -F "sdp=