feat(realtime): add shared WebRTC UDP port (#11436)

* feat(realtime): add shared WebRTC UDP port

Allow realtime WebRTC peer connections to reuse one configurable UDP mux, and surface listener bind failures through signaling.

Assisted-by: Codex:gpt-5

* test(realtime): keep UDP mux alive during bind check

The returned SettingEngine owns the UDP listener. Retain it through the duplicate-bind assertion so macOS cannot finalize the listener early and make the exclusivity check spuriously pass.

Assisted-by: Codex:gpt-5 [systematic-debugging]

* test(realtime): use IPv4 for UDP mux checks

Match the socket family used by the WebRTC UDP mux so macOS does not allocate an IPv6 probe that can coexist with the IPv4 listener.\n\nAssisted-by: Codex:gpt-5

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/cli/run.go

@@ -38,6 +38,7 @@ type RunCMD struct {
 	ExternalBackends             []string      `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"`
 	WebRTCNAT1To1IPs             []string      `env:"LOCALAI_WEBRTC_NAT_1TO1_IPS,WEBRTC_NAT_1TO1_IPS" help:"IPs advertised as the host ICE candidates for /v1/realtime WebRTC instead of every local interface. Set to the reachable host/LAN IP when running under Docker host networking or NAT, where pion otherwise offers unreachable bridge addresses and the connection drops after ICE consent checks fail." group:"api"`
 	WebRTCICEInterfaces          []string      `env:"LOCALAI_WEBRTC_ICE_INTERFACES,WEBRTC_ICE_INTERFACES" help:"Restrict /v1/realtime WebRTC ICE candidate gathering to these network interfaces (e.g. eth0), filtering out docker0/veth noise." group:"api"`
+	WebRTCUDPPort                int           `env:"LOCALAI_WEBRTC_UDP_PORT" help:"Shared UDP port for /v1/realtime WebRTC ICE traffic. Publish this port as UDP and allow it through the firewall." group:"api" name:"web-rtc-udp-port"`
 	BackendsPath                 string        `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
 	BackendsSystemPath           string        `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
 	ModelsPath                   string        `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
@@ -311,6 +312,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithExternalBackends(r.ExternalBackends...),
 		config.WithWebRTCNAT1To1IPs(r.WebRTCNAT1To1IPs...),
 		config.WithWebRTCICEInterfaces(r.WebRTCICEInterfaces...),
+		config.WithWebRTCUDPPort(r.WebRTCUDPPort),
 		config.WithOpaqueErrors(r.OpaqueErrors),
 		config.WithEnforcedPredownloadScans(!r.DisablePredownloadScan),
 		config.WithSubtleKeyComparison(r.UseSubtleKeyComparison),

core/config/application_config.go

@@ -33,7 +33,10 @@ type ApplicationConfig struct {
 	WebRTCNAT1To1IPs []string
 	// WebRTCICEInterfaces, when set, restricts ICE candidate gathering to these
 	// network interfaces (e.g. eth0), filtering out docker0/veth noise.
-	WebRTCICEInterfaces                 []string
+	WebRTCICEInterfaces []string
+	// WebRTCUDPPort, when positive, is the shared UDP port used by all WebRTC
+	// peer connections. Zero keeps pion's default ephemeral-port behavior.
+	WebRTCUDPPort                       int
 	UploadLimitMB, Threads, ContextSize int
 	ArtifactDownloadConcurrency         int
 	F16                                 bool
@@ -367,6 +370,12 @@ func WithWebRTCICEInterfaces(interfaces ...string) AppOption {
 	}
 }
 
+func WithWebRTCUDPPort(port int) AppOption {
+	return func(o *ApplicationConfig) {
+		o.WebRTCUDPPort = port
+	}
+}
+
 func WithMachineTag(tag string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.MachineTag = tag

core/http/endpoints/openai/realtime_webrtc.go

@@ -29,7 +29,15 @@ type RealtimeCallResponse struct {
 
 // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling.
 func RealtimeCalls(application *application.Application) echo.HandlerFunc {
+	se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig())
+	if settingEngineErr != nil {
+		xlog.Error("failed to configure realtime WebRTC UDP listener", "error", settingEngineErr)
+	}
+
 	return func(c echo.Context) error {
+		if settingEngineErr != nil {
+			return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()})
+		}
 		var req RealtimeCallRequest
 		if err := c.Bind(&req); err != nil {
 			return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"})
@@ -48,7 +56,6 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 			return c.JSON(http.StatusInternalServerError, map[string]string{"error": "codec registration failed"})
 		}
 
-		se := webRTCSettingEngine(application.ApplicationConfig())
 		api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithSettingEngine(se))
 
 		pc, err := api.NewPeerConnection(webrtc.Configuration{})

core/http/endpoints/openai/realtime_webrtc_ice.go

@@ -1,6 +1,9 @@
 package openai
 
 import (
+	"fmt"
+	"net"
+
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/xlog"
 	"github.com/pion/webrtc/v4"
@@ -14,10 +17,10 @@ import (
 // connection often establishes on a good pair and then drops once ICE consent
 // checks fail on the unreachable ones. The two opt-in knobs below let an
 // operator advertise only the reachable address.
-func webRTCSettingEngine(cfg *config.ApplicationConfig) webrtc.SettingEngine {
+func webRTCSettingEngine(cfg *config.ApplicationConfig) (webrtc.SettingEngine, error) {
 	s := webrtc.SettingEngine{}
 	if cfg == nil {
-		return s
+		return s, nil
 	}
 	if len(cfg.WebRTCNAT1To1IPs) > 0 {
 		s.SetNAT1To1IPs(cfg.WebRTCNAT1To1IPs, webrtc.ICECandidateTypeHost)
@@ -27,7 +30,14 @@ func webRTCSettingEngine(cfg *config.ApplicationConfig) webrtc.SettingEngine {
 		s.SetInterfaceFilter(filter)
 		xlog.Debug("realtime webrtc: restricting ICE interfaces", "interfaces", cfg.WebRTCICEInterfaces)
 	}
-	return s
+	if cfg.WebRTCUDPPort > 0 {
+		conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: cfg.WebRTCUDPPort})
+		if err != nil {
+			return s, fmt.Errorf("bind WebRTC UDP port %d: %w", cfg.WebRTCUDPPort, err)
+		}
+		s.SetICEUDPMux(webrtc.NewICEUDPMux(nil, conn))
+	}
+	return s, nil
 }
 
 // iceInterfaceFilter returns an interface allow-list predicate for pion, or nil

core/http/endpoints/openai/realtime_webrtc_ice_test.go

@@ -1,6 +1,9 @@
 package openai
 
 import (
+	"net"
+	"runtime"
+
 	"github.com/mudler/LocalAI/core/config"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -24,8 +27,9 @@ var _ = Describe("webRTC ICE settings", func() {
 	})
 
 	Describe("webRTCSettingEngine", func() {
-		It("does not panic on a nil config", func() {
-			Expect(func() { webRTCSettingEngine(nil) }).NotTo(Panic())
+		It("uses pion's ephemeral-port behavior by default", func() {
+			_, err := webRTCSettingEngine(nil)
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("builds an engine with NAT 1:1 IPs and an interface filter configured", func() {
@@ -33,7 +37,36 @@ var _ = Describe("webRTC ICE settings", func() {
 				WebRTCNAT1To1IPs:    []string{"192.168.1.10"},
 				WebRTCICEInterfaces: []string{"eth0"},
 			}
-			Expect(func() { webRTCSettingEngine(cfg) }).NotTo(Panic())
+			_, err := webRTCSettingEngine(cfg)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("binds the configured UDP port exclusively for reuse", func() {
+			probe, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+			Expect(err).NotTo(HaveOccurred())
+			port := probe.LocalAddr().(*net.UDPAddr).Port
+			Expect(probe.Close()).To(Succeed())
+
+			engine, err := webRTCSettingEngine(&config.ApplicationConfig{WebRTCUDPPort: port})
+			Expect(err).NotTo(HaveOccurred())
+
+			duplicate, bindErr := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: port})
+			if duplicate != nil {
+				Expect(duplicate.Close()).To(Succeed())
+			}
+			Expect(bindErr).To(HaveOccurred())
+			runtime.KeepAlive(engine)
+		})
+
+		It("returns a bind error when the configured UDP port is occupied", func() {
+			occupied, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(occupied.Close)
+
+			_, err = webRTCSettingEngine(&config.ApplicationConfig{
+				WebRTCUDPPort: occupied.LocalAddr().(*net.UDPAddr).Port,
+			})
+			Expect(err).To(MatchError(ContainSubstring("bind WebRTC UDP port")))
 		})
 	})
 })

docs/content/features/openai-realtime.md

@@ -323,6 +323,23 @@ container, set `LOCALAI_WEBRTC_NAT_1TO1_IPS` to the host's LAN IP. This is the
 most reliable fix for WebRTC connections that establish and then drop.
 {{% /notice %}}
 
+#### Fixed WebRTC UDP port
+
+By default, each WebRTC peer connection uses an ephemeral UDP port. To route
+all realtime WebRTC ICE traffic through one shared port, start LocalAI with
+`--web-rtc-udp-port 3478` or set `LOCALAI_WEBRTC_UDP_PORT=3478`.
+
+When running in a container, publish the same port with the UDP protocol:
+
+```bash
+docker run -p 8080:8080 -p 3478:3478/udp \
+  -e LOCALAI_WEBRTC_UDP_PORT=3478 localai/localai:latest
+```
+
+Allow the selected UDP port through the host and network firewalls. If LocalAI
+cannot bind it, WebRTC signaling requests return an HTTP 500 error describing
+the bind failure.
+
 ## Protocol
 
 The API follows the OpenAI Realtime API protocol for handling sessions, audio buffers, and conversation items.