diff --git a/core/cli/run.go b/core/cli/run.go index c70693186..7d35fb693 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -38,6 +38,7 @@ type RunCMD struct { ExternalBackends []string `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"` WebRTCNAT1To1IPs []string `env:"LOCALAI_WEBRTC_NAT_1TO1_IPS,WEBRTC_NAT_1TO1_IPS" help:"IPs advertised as the host ICE candidates for /v1/realtime WebRTC instead of every local interface. Set to the reachable host/LAN IP when running under Docker host networking or NAT, where pion otherwise offers unreachable bridge addresses and the connection drops after ICE consent checks fail." group:"api"` WebRTCICEInterfaces []string `env:"LOCALAI_WEBRTC_ICE_INTERFACES,WEBRTC_ICE_INTERFACES" help:"Restrict /v1/realtime WebRTC ICE candidate gathering to these network interfaces (e.g. eth0), filtering out docker0/veth noise." group:"api"` + WebRTCUDPPort int `env:"LOCALAI_WEBRTC_UDP_PORT" help:"Shared UDP port for /v1/realtime WebRTC ICE traffic. Publish this port as UDP and allow it through the firewall." group:"api" name:"web-rtc-udp-port"` BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"` BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"` ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"` @@ -311,6 +312,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { config.WithExternalBackends(r.ExternalBackends...), config.WithWebRTCNAT1To1IPs(r.WebRTCNAT1To1IPs...), config.WithWebRTCICEInterfaces(r.WebRTCICEInterfaces...), + config.WithWebRTCUDPPort(r.WebRTCUDPPort), config.WithOpaqueErrors(r.OpaqueErrors), config.WithEnforcedPredownloadScans(!r.DisablePredownloadScan), config.WithSubtleKeyComparison(r.UseSubtleKeyComparison), diff --git a/core/config/application_config.go b/core/config/application_config.go index b7ec9a316..009fc1cea 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -33,7 +33,10 @@ type ApplicationConfig struct { WebRTCNAT1To1IPs []string // WebRTCICEInterfaces, when set, restricts ICE candidate gathering to these // network interfaces (e.g. eth0), filtering out docker0/veth noise. - WebRTCICEInterfaces []string + WebRTCICEInterfaces []string + // WebRTCUDPPort, when positive, is the shared UDP port used by all WebRTC + // peer connections. Zero keeps pion's default ephemeral-port behavior. + WebRTCUDPPort int UploadLimitMB, Threads, ContextSize int ArtifactDownloadConcurrency int F16 bool @@ -367,6 +370,12 @@ func WithWebRTCICEInterfaces(interfaces ...string) AppOption { } } +func WithWebRTCUDPPort(port int) AppOption { + return func(o *ApplicationConfig) { + o.WebRTCUDPPort = port + } +} + func WithMachineTag(tag string) AppOption { return func(o *ApplicationConfig) { o.MachineTag = tag diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 26edf94ea..4f13862c8 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -29,7 +29,15 @@ type RealtimeCallResponse struct { // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { + se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) + if settingEngineErr != nil { + xlog.Error("failed to configure realtime WebRTC UDP listener", "error", settingEngineErr) + } + return func(c echo.Context) error { + if settingEngineErr != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) + } var req RealtimeCallRequest if err := c.Bind(&req); err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) @@ -48,7 +56,6 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { return c.JSON(http.StatusInternalServerError, map[string]string{"error": "codec registration failed"}) } - se := webRTCSettingEngine(application.ApplicationConfig()) api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithSettingEngine(se)) pc, err := api.NewPeerConnection(webrtc.Configuration{}) diff --git a/core/http/endpoints/openai/realtime_webrtc_ice.go b/core/http/endpoints/openai/realtime_webrtc_ice.go index 82e106c73..1b43bb919 100644 --- a/core/http/endpoints/openai/realtime_webrtc_ice.go +++ b/core/http/endpoints/openai/realtime_webrtc_ice.go @@ -1,6 +1,9 @@ package openai import ( + "fmt" + "net" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/xlog" "github.com/pion/webrtc/v4" @@ -14,10 +17,10 @@ import ( // connection often establishes on a good pair and then drops once ICE consent // checks fail on the unreachable ones. The two opt-in knobs below let an // operator advertise only the reachable address. -func webRTCSettingEngine(cfg *config.ApplicationConfig) webrtc.SettingEngine { +func webRTCSettingEngine(cfg *config.ApplicationConfig) (webrtc.SettingEngine, error) { s := webrtc.SettingEngine{} if cfg == nil { - return s + return s, nil } if len(cfg.WebRTCNAT1To1IPs) > 0 { s.SetNAT1To1IPs(cfg.WebRTCNAT1To1IPs, webrtc.ICECandidateTypeHost) @@ -27,7 +30,14 @@ func webRTCSettingEngine(cfg *config.ApplicationConfig) webrtc.SettingEngine { s.SetInterfaceFilter(filter) xlog.Debug("realtime webrtc: restricting ICE interfaces", "interfaces", cfg.WebRTCICEInterfaces) } - return s + if cfg.WebRTCUDPPort > 0 { + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: cfg.WebRTCUDPPort}) + if err != nil { + return s, fmt.Errorf("bind WebRTC UDP port %d: %w", cfg.WebRTCUDPPort, err) + } + s.SetICEUDPMux(webrtc.NewICEUDPMux(nil, conn)) + } + return s, nil } // iceInterfaceFilter returns an interface allow-list predicate for pion, or nil diff --git a/core/http/endpoints/openai/realtime_webrtc_ice_test.go b/core/http/endpoints/openai/realtime_webrtc_ice_test.go index fbda610cb..b17870510 100644 --- a/core/http/endpoints/openai/realtime_webrtc_ice_test.go +++ b/core/http/endpoints/openai/realtime_webrtc_ice_test.go @@ -1,6 +1,9 @@ package openai import ( + "net" + "runtime" + "github.com/mudler/LocalAI/core/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,8 +27,9 @@ var _ = Describe("webRTC ICE settings", func() { }) Describe("webRTCSettingEngine", func() { - It("does not panic on a nil config", func() { - Expect(func() { webRTCSettingEngine(nil) }).NotTo(Panic()) + It("uses pion's ephemeral-port behavior by default", func() { + _, err := webRTCSettingEngine(nil) + Expect(err).NotTo(HaveOccurred()) }) It("builds an engine with NAT 1:1 IPs and an interface filter configured", func() { @@ -33,7 +37,36 @@ var _ = Describe("webRTC ICE settings", func() { WebRTCNAT1To1IPs: []string{"192.168.1.10"}, WebRTCICEInterfaces: []string{"eth0"}, } - Expect(func() { webRTCSettingEngine(cfg) }).NotTo(Panic()) + _, err := webRTCSettingEngine(cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It("binds the configured UDP port exclusively for reuse", func() { + probe, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + Expect(err).NotTo(HaveOccurred()) + port := probe.LocalAddr().(*net.UDPAddr).Port + Expect(probe.Close()).To(Succeed()) + + engine, err := webRTCSettingEngine(&config.ApplicationConfig{WebRTCUDPPort: port}) + Expect(err).NotTo(HaveOccurred()) + + duplicate, bindErr := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: port}) + if duplicate != nil { + Expect(duplicate.Close()).To(Succeed()) + } + Expect(bindErr).To(HaveOccurred()) + runtime.KeepAlive(engine) + }) + + It("returns a bind error when the configured UDP port is occupied", func() { + occupied, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(occupied.Close) + + _, err = webRTCSettingEngine(&config.ApplicationConfig{ + WebRTCUDPPort: occupied.LocalAddr().(*net.UDPAddr).Port, + }) + Expect(err).To(MatchError(ContainSubstring("bind WebRTC UDP port"))) }) }) }) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index bf2a50b6f..417fab331 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -323,6 +323,23 @@ container, set `LOCALAI_WEBRTC_NAT_1TO1_IPS` to the host's LAN IP. This is the most reliable fix for WebRTC connections that establish and then drop. {{% /notice %}} +#### Fixed WebRTC UDP port + +By default, each WebRTC peer connection uses an ephemeral UDP port. To route +all realtime WebRTC ICE traffic through one shared port, start LocalAI with +`--web-rtc-udp-port 3478` or set `LOCALAI_WEBRTC_UDP_PORT=3478`. + +When running in a container, publish the same port with the UDP protocol: + +```bash +docker run -p 8080:8080 -p 3478:3478/udp \ + -e LOCALAI_WEBRTC_UDP_PORT=3478 localai/localai:latest +``` + +Allow the selected UDP port through the host and network firewalls. If LocalAI +cannot bind it, WebRTC signaling requests return an HTTP 500 error describing +the bind failure. + ## Protocol The API follows the OpenAI Realtime API protocol for handling sessions, audio buffers, and conversation items.