[httpclient] validate redirect targets and pin dialed addresses for user URLs (#12853)

fix(httpclient): validate redirect targets and pin dialed addresses for user URLs

Clients that fetch user-supplied URLs followed redirects without checking
the target, so a public host could send them to a loopback, link-local
or private address. The host was also resolved twice, once to validate
it and once to connect, so a changing DNS answer could bypass the check.

Add httpclient.WithExternalOnly. It validates every redirect hop against
the external URL policy and, for the default transport, resolves the
host, refuses it if any address is non-public, and dials the validated
address. Environment proxies are not used for such clients.

Move the policy (ValidateExternalURL, IsPublicIP) into pkg/httpclient
so it can be used without an import cycle; pkg/utils keeps delegating
to it. IsPublicIP now also refuses 100.64.0.0/10, 0.0.0.0/8,
192.0.0.0/24, 198.18.0.0/15, 240.0.0.0/4, multicast, and NAT64
addresses that wrap a non-public IPv4 address.

Use the option for the base64, image, audio and video downloads, and
drop the video client's own per-hop check.

Assisted-by: Claude Code:claude-haiku-5-5

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

pkg/httpclient/client.go

@@ -125,6 +125,7 @@ type options struct {
 	responseHeaderTimeout time.Duration
 	transport             http.RoundTripper
 	followRedirects       bool
+	externalOnly          bool
 }
 
 // Option configures a client built by New.
@@ -159,6 +160,23 @@ func WithResponseHeaderTimeout(d time.Duration) Option {
 // redirects (e.g. some download CDNs) and the request carries a secret.
 func WithFollowRedirects() Option { return func(o *options) { o.followRedirects = true } }
 
+// WithExternalOnly restricts a client to hosts that pass the external URL
+// policy (see IsPublicIP). Use it for every client that fetches a URL supplied
+// by a user. It has two effects:
+//
+//   - Each redirect hop is validated like the original URL, so a public host
+//     cannot bounce the client to a loopback, link-local or private target.
+//     It only matters together with WithFollowRedirects.
+//   - Unless WithTransport is given, the transport resolves the host itself,
+//     rejects it if any resolved address is not public, and dials one of the
+//     validated addresses, so a second DNS lookup cannot return a different
+//     answer. Environment proxies are not used for such a client, since the
+//     proxy address would be what gets validated and dialled.
+//
+// A caller-supplied transport is left untouched and must enforce the policy
+// itself.
+func WithExternalOnly() Option { return func(o *options) { o.externalOnly = true } }
+
 // New returns a hardened *http.Client. By default it refuses redirects, sets a
 // TLS 1.2 floor, bounds connection setup, and imposes no body deadline (safe
 // for streaming). Apply Options to adjust.
@@ -170,7 +188,15 @@ func New(opts ...Option) *http.Client {
 
 	rt := o.transport
 	if rt == nil {
-		rt = HardenedTransport()
+		t := HardenedTransport()
+		if o.externalOnly {
+			t.Proxy = nil
+			t.DialContext = validatedDialContext(&net.Dialer{
+				Timeout:   dialTimeout,
+				KeepAlive: dialKeepAlive,
+			})
+		}
+		rt = t
 	}
 	// Only an *http.Transport carries the knob; a caller-supplied wrapper
 	// (bearer-token RoundTripper, IP-pinned dialer) is left untouched rather
@@ -184,6 +210,14 @@ func New(opts ...Option) *http.Client {
 	check := NoRedirect
 	if o.followRedirects {
 		check = stripAuthOnRedirect
+		if o.externalOnly {
+			check = func(req *http.Request, via []*http.Request) error {
+				if err := validateRedirectTarget(req); err != nil {
+					return err
+				}
+				return stripAuthOnRedirect(req, via)
+			}
+		}
 	}
 
 	return &http.Client{

pkg/httpclient/external.go

@@ -0,0 +1,184 @@
+package httpclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// The external URL policy lives here, below pkg/utils, so the redirect check
+// and the dial path of a client can enforce it without an import cycle.
+// utils.ValidateExternalURL and utils.IsPublicIP delegate to these functions.
+
+// nonPublicNets lists ranges that are not covered by the net.IP helper
+// predicates (IsPrivate, IsLoopback, ...) but are not routable on the public
+// internet either.
+var nonPublicNets = mustParseCIDRs(
+	"0.0.0.0/8",     // "this network", not only the single unspecified address
+	"100.64.0.0/10", // RFC 6598 shared address space (carrier-grade NAT, overlay VPNs)
+	"192.0.0.0/24",  // RFC 6890 IETF protocol assignments
+	"198.18.0.0/15", // RFC 2544 benchmarking
+	"240.0.0.0/4",   // reserved, includes the limited broadcast address
+	"100::/64",      // RFC 6666 discard-only prefix
+	"2001:db8::/32", // documentation prefix
+)
+
+// nat64Prefix is the well-known NAT64 prefix (RFC 6052). The last four bytes
+// are an IPv4 address that must pass the IPv4 rules too.
+var nat64Prefix = mustParseCIDRs("64:ff9b::/96")[0]
+
+func mustParseCIDRs(cidrs ...string) []*net.IPNet {
+	nets := make([]*net.IPNet, 0, len(cidrs))
+	for _, c := range cidrs {
+		_, n, err := net.ParseCIDR(c)
+		if err != nil {
+			panic(err)
+		}
+		nets = append(nets, n)
+	}
+	return nets
+}
+
+// IsPublicIP reports whether ip refers to a host on the public internet, i.e.
+// not loopback, link-local, private (RFC 1918 / RFC 4193), shared address
+// space (RFC 6598), multicast, unspecified or otherwise reserved. An
+// IPv4-mapped or NAT64-embedded IPv4 address is checked as that IPv4 address.
+func IsPublicIP(ip net.IP) bool {
+	if ip == nil {
+		return false
+	}
+	if ip4 := ip.To4(); ip4 != nil {
+		ip = ip4
+	} else if nat64Prefix.Contains(ip) {
+		return IsPublicIP(net.IP(ip.To16()[12:]))
+	}
+
+	if ip.IsLoopback() ||
+		ip.IsLinkLocalUnicast() ||
+		ip.IsLinkLocalMulticast() ||
+		ip.IsInterfaceLocalMulticast() ||
+		ip.IsMulticast() ||
+		ip.IsPrivate() ||
+		ip.IsUnspecified() {
+		return false
+	}
+	for _, n := range nonPublicNets {
+		if n.Contains(ip) {
+			return false
+		}
+	}
+	return true
+}
+
+// ValidateExternalURL checks that the given URL does not point to a private,
+// loopback, link-local, or otherwise internal network address. This prevents
+// Server-Side Request Forgery (SSRF) where a user-supplied URL could be used
+// to probe internal services or cloud metadata endpoints.
+func ValidateExternalURL(rawURL string) error {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return fmt.Errorf("invalid URL: %w", err)
+	}
+
+	scheme := strings.ToLower(parsed.Scheme)
+	if scheme != "http" && scheme != "https" {
+		return fmt.Errorf("unsupported URL scheme: %s", scheme)
+	}
+
+	hostname := parsed.Hostname()
+	if hostname == "" {
+		return fmt.Errorf("URL has no hostname")
+	}
+
+	lower := strings.ToLower(hostname)
+	if lower == "localhost" || strings.HasSuffix(lower, ".local") {
+		return fmt.Errorf("requests to internal hosts are not allowed")
+	}
+	if lower == "metadata.google.internal" || lower == "instance-data" {
+		return fmt.Errorf("requests to cloud metadata services are not allowed")
+	}
+
+	ips, err := net.LookupHost(hostname)
+	if err != nil {
+		return fmt.Errorf("failed to resolve hostname: %w", err)
+	}
+	for _, ipStr := range ips {
+		ip := net.ParseIP(ipStr)
+		if ip == nil {
+			return fmt.Errorf("unable to parse resolved IP: %s", ipStr)
+		}
+		if !IsPublicIP(ip) {
+			return fmt.Errorf("requests to internal network addresses are not allowed")
+		}
+	}
+	return nil
+}
+
+// ErrNonPublicAddress is wrapped by the error a validated dial returns when a
+// host resolves to an address outside the external URL policy.
+var ErrNonPublicAddress = errors.New("httpclient: requests to internal network addresses are not allowed")
+
+// resolver is swapped in tests.
+var resolver interface {
+	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
+} = net.DefaultResolver
+
+// validatedDialContext returns a DialContext that resolves the host itself,
+// rejects the connection if ANY resolved address is not public, and then dials
+// the validated addresses directly. The address that was checked is therefore
+// the address that is connected to, which closes the window in which a second
+// DNS lookup could return a different (internal) answer. net/http keeps the
+// original host for the Host header and TLS server name, because only the dial
+// address is replaced.
+func validatedDialContext(d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var ips []net.IP
+		if lit := net.ParseIP(host); lit != nil {
+			ips = []net.IP{lit}
+		} else {
+			addrs, err := resolver.LookupIPAddr(ctx, host)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve hostname: %w", err)
+			}
+			for _, a := range addrs {
+				ips = append(ips, a.IP)
+			}
+		}
+		if len(ips) == 0 {
+			return nil, fmt.Errorf("no addresses found for %s", host)
+		}
+		for _, ip := range ips {
+			if !IsPublicIP(ip) {
+				return nil, ErrNonPublicAddress
+			}
+		}
+		var lastErr error
+		for _, ip := range ips {
+			conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+			if err == nil {
+				return conn, nil
+			}
+			lastErr = err
+			if ctx.Err() != nil {
+				break
+			}
+		}
+		return nil, lastErr
+	}
+}
+
+// validateRedirectTarget applies the external URL policy to a redirect hop.
+func validateRedirectTarget(req *http.Request) error {
+	if err := ValidateExternalURL(req.URL.String()); err != nil {
+		return fmt.Errorf("redirect URL validation failed: %w", err)
+	}
+	return nil
+}

pkg/httpclient/external_test.go

@@ -0,0 +1,183 @@
+package httpclient
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"syscall"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+type fakeResolver map[string][]string
+
+func (f fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
+	var out []net.IPAddr
+	for _, s := range f[host] {
+		out = append(out, net.IPAddr{IP: net.ParseIP(s)})
+	}
+	if out == nil {
+		return nil, errors.New("no such host")
+	}
+	return out, nil
+}
+
+var _ = Describe("external URL policy", func() {
+	DescribeTable("IsPublicIP",
+		func(addr string, want bool) {
+			Expect(IsPublicIP(net.ParseIP(addr))).To(Equal(want))
+		},
+		Entry("public IPv4", "93.184.216.34", true),
+		Entry("documentation range used by tests", "203.0.113.1", true),
+		Entry("public IPv6", "2606:4700:4700::1111", true),
+		Entry("loopback", "127.0.0.1", false),
+		Entry("IPv6 loopback", "::1", false),
+		Entry("RFC 1918", "10.1.2.3", false),
+		Entry("link-local metadata address", "169.254.169.254", false),
+		Entry("shared address space low edge", "100.64.0.0", false),
+		Entry("shared address space inside", "100.100.100.200", false),
+		Entry("shared address space high edge", "100.127.255.255", false),
+		Entry("just below shared address space", "100.63.255.255", true),
+		Entry("just above shared address space", "100.128.0.1", true),
+		Entry("this network, single address", "0.0.0.0", false),
+		Entry("this network, other address", "0.1.2.3", false),
+		Entry("benchmarking range", "198.19.0.1", false),
+		Entry("multicast", "224.0.0.1", false),
+		Entry("reserved class E", "240.0.0.1", false),
+		Entry("IPv6 unique local", "fd00::1", false),
+		Entry("IPv6 unspecified", "::", false),
+		Entry("IPv4-mapped loopback", "::ffff:127.0.0.1", false),
+		Entry("IPv4-mapped shared address space", "::ffff:100.64.0.1", false),
+		Entry("IPv4-mapped public", "::ffff:93.184.216.34", true),
+		Entry("NAT64 wrapping a private address", "64:ff9b::a00:1", false),
+		Entry("NAT64 wrapping a public address", "64:ff9b::5db8:d822", true),
+	)
+
+	Describe("redirect validation", func() {
+		var (
+			hits     atomic.Int32
+			sink     *httptest.Server
+			redirect func(target string) *http.Client
+		)
+
+		BeforeEach(func() {
+			hits.Store(0)
+			sink = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				hits.Add(1)
+				_, _ = w.Write([]byte("ok"))
+			}))
+			DeferCleanup(sink.Close)
+
+			// The first hop is a TEST-NET literal so no DNS is needed; the
+			// transport serves every request from the in-process server, which
+			// answers with a redirect to target on the first call.
+			redirect = func(target string) *http.Client {
+				var first atomic.Bool
+				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if first.CompareAndSwap(false, true) {
+						http.Redirect(w, r, target, http.StatusFound)
+						return
+					}
+					hits.Add(1)
+					_, _ = w.Write([]byte("ok"))
+				}))
+				DeferCleanup(srv.Close)
+				return New(WithFollowRedirects(), WithExternalOnly(), WithTransport(&http.Transport{
+					DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
+						return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String())
+					},
+				}))
+			}
+		})
+
+		DescribeTable("refuses non-public targets",
+			func(target string) {
+				_, err := redirect(target).Get("http://203.0.113.1/start")
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("redirect URL validation failed"))
+				Expect(hits.Load()).To(BeZero())
+			},
+			Entry("loopback", "http://127.0.0.1:1/x"),
+			Entry("link-local metadata", "http://169.254.169.254/latest/meta-data/"),
+			Entry("private", "http://10.0.0.5/x"),
+			Entry("shared address space", "http://100.64.0.1/x"),
+			Entry("localhost name", "http://localhost/x"),
+			Entry("non-http scheme", "ftp://203.0.113.9/x"),
+		)
+
+		It("follows a redirect to another public target", func() {
+			resp, err := redirect("http://203.0.113.2/next").Get("http://203.0.113.1/start")
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(resp.Body.Close)
+			Expect(resp.StatusCode).To(Equal(http.StatusOK))
+			Expect(hits.Load()).To(Equal(int32(1)))
+		})
+
+		It("keeps the redirect bound", func() {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				http.Redirect(w, r, "http://203.0.113.2/loop", http.StatusFound)
+			}))
+			DeferCleanup(srv.Close)
+			c := New(WithFollowRedirects(), WithExternalOnly(), WithTransport(&http.Transport{
+				DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
+					return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String())
+				},
+			}))
+			_, err := c.Get("http://203.0.113.1/loop")
+			Expect(err).To(MatchError(ContainSubstring("stopped after")))
+		})
+	})
+
+	Describe("validated dial", func() {
+		errStop := errors.New("stop before connecting")
+
+		// capture returns a dial function whose underlying dialer records the
+		// address it was asked to connect to and then aborts, so no packet is
+		// ever sent to the (documentation-range) address.
+		capture := func(got *string) func(context.Context, string, string) (net.Conn, error) {
+			return validatedDialContext(&net.Dialer{Control: func(_, address string, _ syscall.RawConn) error {
+				*got = address
+				return errStop
+			}})
+		}
+
+		BeforeEach(func() {
+			orig := resolver
+			DeferCleanup(func() { resolver = orig })
+		})
+
+		It("refuses a host that resolves only to private addresses", func() {
+			resolver = fakeResolver{"internal.example": {"10.0.0.7"}}
+			var got string
+			_, err := capture(&got)(context.Background(), "tcp", "internal.example:80")
+			Expect(err).To(MatchError(ErrNonPublicAddress))
+			Expect(got).To(BeEmpty())
+		})
+
+		It("refuses a host when any answer is non-public", func() {
+			resolver = fakeResolver{"mixed.example": {"198.51.100.7", "100.64.0.9"}}
+			var got string
+			_, err := capture(&got)(context.Background(), "tcp", "mixed.example:80")
+			Expect(err).To(MatchError(ErrNonPublicAddress))
+			Expect(got).To(BeEmpty())
+		})
+
+		It("refuses a literal internal address", func() {
+			var got string
+			_, err := capture(&got)(context.Background(), "tcp", "127.0.0.1:80")
+			Expect(err).To(MatchError(ErrNonPublicAddress))
+		})
+
+		It("dials the validated address, not a fresh lookup", func() {
+			resolver = fakeResolver{"public.example": {"198.51.100.7"}}
+			var got string
+			_, err := capture(&got)(context.Background(), "tcp", "public.example:8080")
+			Expect(err).To(HaveOccurred())
+			Expect(got).To(Equal("198.51.100.7:8080"))
+		})
+	})
+})