diff --git a/core/http/endpoints/openai/realtime_webrtc_ice.go b/core/http/endpoints/openai/realtime_webrtc_ice.go index 1b43bb919..9c71d1769 100644 --- a/core/http/endpoints/openai/realtime_webrtc_ice.go +++ b/core/http/endpoints/openai/realtime_webrtc_ice.go @@ -2,10 +2,10 @@ package openai import ( "fmt" - "net" "github.com/mudler/LocalAI/core/config" "github.com/mudler/xlog" + "github.com/pion/ice/v4" "github.com/pion/webrtc/v4" ) @@ -31,15 +31,48 @@ func webRTCSettingEngine(cfg *config.ApplicationConfig) (webrtc.SettingEngine, e xlog.Debug("realtime webrtc: restricting ICE interfaces", "interfaces", cfg.WebRTCICEInterfaces) } if cfg.WebRTCUDPPort > 0 { - conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: cfg.WebRTCUDPPort}) + mux, err := udpMuxOnPort(cfg.WebRTCUDPPort, cfg.WebRTCICEInterfaces) if err != nil { - return s, fmt.Errorf("bind WebRTC UDP port %d: %w", cfg.WebRTCUDPPort, err) + return s, err } - s.SetICEUDPMux(webrtc.NewICEUDPMux(nil, conn)) + s.SetICEUDPMux(mux) + xlog.Debug("realtime webrtc: sharing a fixed UDP port", "port", cfg.WebRTCUDPPort) } return s, nil } +// udpMuxOnPort binds port on each interface the allow-list admits and returns a +// mux every session shares. +// +// One socket per interface address rather than one wildcard socket: for a mux +// bound to 0.0.0.0 pion derives the host candidates by enumerating interfaces +// itself, with no filter and loopback included, and its muxed gathering path +// never consults SetInterfaceFilter. A wildcard socket therefore silently +// discards WebRTCICEInterfaces and re-advertises exactly the docker0/veth +// addresses that setting exists to suppress. +func udpMuxOnPort(port int, interfaces []string) (ice.UDPMux, error) { + // UDP4 only, matching the socket family this replaces. + opts := []ice.UDPMuxFromPortOption{ice.UDPMuxFromPortWithNetworks(ice.NetworkTypeUDP4)} + if filter := iceInterfaceFilter(interfaces); filter != nil { + opts = append(opts, ice.UDPMuxFromPortWithInterfaceFilter(filter)) + } + + mux, err := ice.NewMultiUDPMuxFromPort(port, opts...) + if err != nil { + return nil, fmt.Errorf("bind WebRTC UDP port %d: %w", port, err) + } + // An empty mux binds nothing and would leave signaling working while no + // candidate is ever advertised, so report the misconfiguration instead. + if len(mux.GetListenAddresses()) == 0 { + _ = mux.Close() + if len(interfaces) > 0 { + return nil, fmt.Errorf("bind WebRTC UDP port %d: no usable IPv4 address on interfaces %v", port, interfaces) + } + return nil, fmt.Errorf("bind WebRTC UDP port %d: no usable IPv4 address on this host", port) + } + return mux, nil +} + // iceInterfaceFilter returns an interface allow-list predicate for pion, or nil // when no interfaces are configured (pion's default: gather from all). func iceInterfaceFilter(allowed []string) func(string) bool { diff --git a/core/http/endpoints/openai/realtime_webrtc_ice_test.go b/core/http/endpoints/openai/realtime_webrtc_ice_test.go index b17870510..1e6c46fea 100644 --- a/core/http/endpoints/openai/realtime_webrtc_ice_test.go +++ b/core/http/endpoints/openai/realtime_webrtc_ice_test.go @@ -3,10 +3,13 @@ package openai import ( "net" "runtime" + "strings" + "time" "github.com/mudler/LocalAI/core/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pion/webrtc/v4" ) var _ = Describe("webRTC ICE settings", func() { @@ -68,5 +71,126 @@ var _ = Describe("webRTC ICE settings", func() { }) Expect(err).To(MatchError(ContainSubstring("bind WebRTC UDP port"))) }) + + // A fixed UDP port and an interface allow-list are the two settings an + // operator behind a firewall reaches for together: one to write the + // firewall rule against, the other to keep unreachable docker0/veth + // addresses out of the candidate list. Pinning the port must not cost + // them the filter. + It("honours the interface allow-list when a UDP port is pinned", func() { + _, err := webRTCSettingEngine(&config.ApplicationConfig{ + WebRTCICEInterfaces: []string{"localai-no-such-interface"}, + WebRTCUDPPort: freeUDPPort(), + }) + Expect(err).To(MatchError(ContainSubstring("localai-no-such-interface"))) + }) + + It("gathers host candidates only on allowed interfaces when a UDP port is pinned", func() { + allowed, others := splitLocalInterfaces() + if allowed == "" || len(others) == 0 { + Skip("needs at least two non-loopback interfaces with IPv4 addresses") + } + + engine, err := webRTCSettingEngine(&config.ApplicationConfig{ + WebRTCICEInterfaces: []string{allowed}, + WebRTCUDPPort: freeUDPPort(), + }) + Expect(err).NotTo(HaveOccurred()) + + gathered := gatheredHostIPs(engine) + Expect(gathered).NotTo(BeEmpty(), "the allowed interface should still produce a candidate") + Expect(gathered).To(ConsistOf(ipsOfInterface(allowed))) + for _, excluded := range others { + for _, ip := range ipsOfInterface(excluded) { + Expect(gathered).NotTo(ContainElement(ip), + "candidate from %s leaked despite the allow-list naming only %s", excluded, allowed) + } + } + }) }) }) + +// freeUDPPort asks the kernel for an unused UDP port and releases it. +func freeUDPPort() int { + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) + Expect(err).NotTo(HaveOccurred()) + port := conn.LocalAddr().(*net.UDPAddr).Port + Expect(conn.Close()).To(Succeed()) + return port +} + +// splitLocalInterfaces returns one interface to allow plus every other +// candidate-bearing interface, so a spec can assert the others stay out. +func splitLocalInterfaces() (allowed string, others []string) { + ifaces, err := net.Interfaces() + Expect(err).NotTo(HaveOccurred()) + var usable []string + for _, iface := range ifaces { + if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { + continue + } + if len(ipsOfInterface(iface.Name)) > 0 { + usable = append(usable, iface.Name) + } + } + if len(usable) == 0 { + return "", nil + } + return usable[0], usable[1:] +} + +// ipsOfInterface returns the IPv4 addresses pion can gather a host candidate +// on for the named interface. +func ipsOfInterface(name string) []string { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil + } + addrs, err := iface.Addrs() + if err != nil { + return nil + } + var ips []string + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok || ipNet.IP.To4() == nil { + continue + } + ips = append(ips, ipNet.IP.String()) + } + return ips +} + +// gatheredHostIPs runs a peer connection through the engine and returns the +// distinct IPs of the host candidates it advertises. +func gatheredHostIPs(engine webrtc.SettingEngine) []string { + pc, err := webrtc.NewAPI(webrtc.WithSettingEngine(engine)).NewPeerConnection(webrtc.Configuration{}) + Expect(err).NotTo(HaveOccurred()) + defer pc.Close() + + _, err = pc.CreateDataChannel("probe", nil) + Expect(err).NotTo(HaveOccurred()) + offer, err := pc.CreateOffer(nil) + Expect(err).NotTo(HaveOccurred()) + gathered := webrtc.GatheringCompletePromise(pc) + Expect(pc.SetLocalDescription(offer)).To(Succeed()) + Eventually(gathered, 10*time.Second).Should(BeClosed()) + + // Candidate lines read "a=candidate: udp + // typ host ...", so the IP is the fifth field. + seen := map[string]struct{}{} + var ips []string + for _, line := range strings.Split(pc.LocalDescription().SDP, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "a=candidate:") || !strings.Contains(line, " typ host") { + continue + } + ip := strings.Fields(line)[4] + if _, dup := seen[ip]; dup { + continue + } + seen[ip] = struct{}{} + ips = append(ips, ip) + } + return ips +} diff --git a/go.mod b/go.mod index 0daf7f612..60ca5b0c5 100644 --- a/go.mod +++ b/go.mod @@ -322,7 +322,7 @@ require ( github.com/otiai10/mint v1.6.3 // indirect github.com/pion/datachannel v1.6.0 // indirect github.com/pion/dtls/v3 v3.1.2 // indirect - github.com/pion/ice/v4 v4.2.2 // indirect + github.com/pion/ice/v4 v4.2.2 github.com/pion/interceptor v0.1.44 // indirect github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect