diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 4f13862c8..eca3aa2c6 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -1,6 +1,9 @@ package openai import ( + "encoding/json" + "io" + "mime" "net/http" "time" @@ -27,6 +30,61 @@ type RealtimeCallResponse struct { SessionID string `json:"session_id"` } +func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) { + var req RealtimeCallRequest + mediaType := "" + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return req, false, err + } + } + + switch mediaType { + case echo.MIMEMultipartForm: + if err := c.Request().ParseMultipartForm(32 << 20); err != nil { + return req, true, err + } + req.SDP = c.FormValue("sdp") + var session struct { + Model string `json:"model"` + LocalAIAssistant bool `json:"localai_assistant,omitempty"` + } + if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil { + return req, true, err + } + req.Model = session.Model + req.LocalAIAssistant = session.LocalAIAssistant + return req, true, nil + case "application/sdp": + sdp, err := readRealtimeSDP(c.Request().Body) + req.SDP = sdp + req.Model = c.QueryParam("model") + return req, true, err + default: + err := c.Bind(&req) + return req, false, err + } +} + +func readRealtimeSDP(body io.Reader) (string, error) { + data, err := io.ReadAll(body) + return string(data), err +} + +func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error { + if plainSDPResponse { + return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp)) + } + + return c.JSON(http.StatusCreated, RealtimeCallResponse{ + SDP: sdp, + SessionID: sessionID, + }) +} + // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) @@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { if settingEngineErr != nil { return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) } - var req RealtimeCallRequest - if err := c.Bind(&req); err != nil { + req, plainSDPResponse, err := decodeRealtimeCallRequest(c) + if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) } if req.SDP == "" { @@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { runRealtimeSession(application, transport, req.Model, evaluator, opts) }() - return c.JSON(http.StatusCreated, RealtimeCallResponse{ - SDP: localDesc.SDP, - SessionID: sessionID, - }) + return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID) } } diff --git a/core/http/endpoints/openai/realtime_webrtc_request_test.go b/core/http/endpoints/openai/realtime_webrtc_request_test.go new file mode 100644 index 000000000..187555329 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_request_test.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("decodeRealtimeCallRequest", func() { + It("decodes the legacy JSON request", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeFalse()) + }) + + It("decodes the OpenAI multipart request", func() { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + sdpHeader := make(textproto.MIMEHeader) + sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`) + sdpHeader.Set("Content-Type", "application/sdp") + sdpPart, err := writer.CreatePart(sdpHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sdpPart.Write([]byte("offer")) + Expect(err).NotTo(HaveOccurred()) + sessionHeader := make(textproto.MIMEHeader) + sessionHeader.Set("Content-Disposition", `form-data; name="session"`) + sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON) + sessionPart, err := writer.CreatePart(sessionHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body) + request.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeTrue()) + }) + + It("decodes a raw SDP request with the model query parameter", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer")) + request.Header.Set(echo.HeaderContentType, "application/sdp") + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"})) + Expect(plainSDPResponse).To(BeTrue()) + }) +}) + +var _ = Describe("writeRealtimeCallResponse", func() { + It("writes the bare SDP answer for OpenAI request formats", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp")) + Expect(response.Body.String()).To(Equal("answer")) + }) + + It("preserves the JSON response for legacy requests", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON)) + Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`)) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 417fab331..54bec58ab 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O ### WebRTC -The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint: +The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: ``` -POST http://localhost:8080/v1/realtime?model=gpt-realtime +POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime Content-Type: application/sdp ``` -The response contains the SDP answer to complete the WebRTC handshake. +The response has the `application/sdp` content type and contains the bare SDP answer. + +The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object: + +```bash +curl http://localhost:8080/v1/realtime/calls \ + -F "sdp=