[httpclient] validate redirect targets and pin dialed addresses for user URLs (#12853)

fix(httpclient): validate redirect targets and pin dialed addresses for user URLs

Clients that fetch user-supplied URLs followed redirects without checking
the target, so a public host could send them to a loopback, link-local
or private address. The host was also resolved twice, once to validate
it and once to connect, so a changing DNS answer could bypass the check.

Add httpclient.WithExternalOnly. It validates every redirect hop against
the external URL policy and, for the default transport, resolves the
host, refuses it if any address is non-public, and dials the validated
address. Environment proxies are not used for such clients.

Move the policy (ValidateExternalURL, IsPublicIP) into pkg/httpclient
so it can be used without an import cycle; pkg/utils keeps delegating
to it. IsPublicIP now also refuses 100.64.0.0/10, 0.0.0.0/8,
192.0.0.0/24, 198.18.0.0/15, 240.0.0.0/4, multicast, and NAT64
addresses that wrap a non-public IPv4 address.

Use the option for the base64, image, audio and video downloads, and
drop the video client's own per-hop check.

Assisted-by: Claude Code:claude-haiku-5-5

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

pkg/utils/base64.go

@@ -13,7 +13,7 @@ import (
 	"github.com/mudler/LocalAI/pkg/httpclient"
 )
 
-var base64DownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects())
+var base64DownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects(), httpclient.WithExternalOnly())
 
 // Match `data:<mime>[;param=value...];base64,` — browser-produced data URIs
 // often carry codec/charset params between the mime type and `;base64,`

pkg/utils/urlfetch.go

@@ -1,81 +1,16 @@
 package utils
 
 import (
-	"fmt"
 	"net"
-	"net/url"
-	"strings"
+
+	"github.com/mudler/LocalAI/pkg/httpclient"
 )
 
 // ValidateExternalURL checks that the given URL does not point to a private,
-// loopback, link-local, or otherwise internal network address. This prevents
-// Server-Side Request Forgery (SSRF) attacks where a user-supplied URL could
-// be used to probe internal services or cloud metadata endpoints.
-func ValidateExternalURL(rawURL string) error {
-	parsed, err := url.Parse(rawURL)
-	if err != nil {
-		return fmt.Errorf("invalid URL: %w", err)
-	}
+// loopback, link-local, or otherwise internal network address. It is the
+// policy implemented in pkg/httpclient, kept here for existing callers.
+func ValidateExternalURL(rawURL string) error { return httpclient.ValidateExternalURL(rawURL) }
 
-	scheme := strings.ToLower(parsed.Scheme)
-	if scheme != "http" && scheme != "https" {
-		return fmt.Errorf("unsupported URL scheme: %s", scheme)
-	}
-
-	hostname := parsed.Hostname()
-	if hostname == "" {
-		return fmt.Errorf("URL has no hostname")
-	}
-
-	// Block well-known internal hostnames
-	lower := strings.ToLower(hostname)
-	if lower == "localhost" || strings.HasSuffix(lower, ".local") {
-		return fmt.Errorf("requests to internal hosts are not allowed")
-	}
-
-	// Block cloud metadata service hostnames
-	if lower == "metadata.google.internal" || lower == "instance-data" {
-		return fmt.Errorf("requests to cloud metadata services are not allowed")
-	}
-
-	ips, err := net.LookupHost(hostname)
-	if err != nil {
-		return fmt.Errorf("failed to resolve hostname: %w", err)
-	}
-
-	for _, ipStr := range ips {
-		ip := net.ParseIP(ipStr)
-		if ip == nil {
-			return fmt.Errorf("unable to parse resolved IP: %s", ipStr)
-		}
-
-		if !IsPublicIP(ip) {
-			return fmt.Errorf("requests to internal network addresses are not allowed")
-		}
-	}
-
-	return nil
-}
-
-// IsPublicIP reports whether ip refers to a host on the public internet, i.e.
-// not loopback, link-local, private (RFC 1918 / RFC 4193), or unspecified.
-// Covers 0.0.0.0/8, ::/128, and IPv4-mapped IPv6 wrapping a private address —
-// holes hand-rolled CIDR lists tend to miss.
-func IsPublicIP(ip net.IP) bool {
-	if ip.IsLoopback() ||
-		ip.IsLinkLocalUnicast() ||
-		ip.IsLinkLocalMulticast() ||
-		ip.IsPrivate() ||
-		ip.IsUnspecified() {
-		return false
-	}
-
-	if ip4 := ip.To4(); ip4 != nil {
-		return !ip4.IsLoopback() &&
-			!ip4.IsLinkLocalUnicast() &&
-			!ip4.IsPrivate() &&
-			!ip4.IsUnspecified()
-	}
-
-	return true
-}
+// IsPublicIP reports whether ip refers to a host on the public internet. See
+// httpclient.IsPublicIP for the ranges that are refused.
+func IsPublicIP(ip net.IP) bool { return httpclient.IsPublicIP(ip) }

pkg/utils/urlfetch_test.go

@@ -90,6 +90,23 @@ var _ = Describe("utils/urlfetch tests", func() {
 			Expect(err.Error()).To(ContainSubstring("internal"))
 		})
 
+		DescribeTable("blocks non-public ranges that net.IP does not flag",
+			func(rawURL string) {
+				err := ValidateExternalURL(rawURL)
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("internal"))
+			},
+			Entry("shared address space (RFC 6598)", "http://100.64.0.1/x"),
+			Entry("shared address space, upper end", "http://100.127.255.254/x"),
+			Entry("this network 0.0.0.0/8", "http://0.1.2.3/x"),
+			Entry("benchmarking range", "http://198.18.0.1/x"),
+			Entry("IPv4-mapped shared address space", "http://[::ffff:100.64.0.1]/x"),
+		)
+
+		It("allows a public address next to the shared address space", func() {
+			Expect(ValidateExternalURL("http://100.128.0.1/x")).To(Succeed())
+		})
+
 		It("blocks IPv6 loopback ::1", func() {
 			err := ValidateExternalURL("http://[::1]/secret")
 			Expect(err).ToNot(BeNil())