diff --git a/core/http/endpoints/localai/audio.go b/core/http/endpoints/localai/audio.go index da7b96d70..fdcb52db2 100644 --- a/core/http/endpoints/localai/audio.go +++ b/core/http/endpoints/localai/audio.go @@ -25,8 +25,9 @@ import ( var audioDataURIPattern = regexp.MustCompile(`^data:[^,]+?;base64,`) // Downloading user-supplied media URLs legitimately follows redirects (CDNs); -// WithFollowRedirects still strips any credential header on a cross-host hop. -var audioDownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) +// WithFollowRedirects still strips any credential header on a cross-host hop, +// and WithExternalOnly validates each hop and the dialled address. +var audioDownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects(), httpclient.WithExternalOnly()) // decodeAudioInput materialises a URL / data-URI / raw-base64 audio // payload to a temporary file and returns its path plus a cleanup diff --git a/core/http/endpoints/localai/video.go b/core/http/endpoints/localai/video.go index 68ff0418a..0cfd19474 100644 --- a/core/http/endpoints/localai/video.go +++ b/core/http/endpoints/localai/video.go @@ -41,19 +41,11 @@ const ( videoRefFrameNameFormat = "%06d" ) +// Media CDNs commonly redirect; WithExternalOnly validates every hop and the +// dialled address, and the shared redirect policy bounds the chain and strips +// credentials on cross-origin hops. func newVideoDownloadClient() *http.Client { - client := httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) - checkRedirect := client.CheckRedirect - // Media CDNs commonly redirect, so validate every hop rather than trusting - // only the URL supplied by the caller. Keep the shared redirect policy too; - // it bounds the chain and strips credentials on cross-origin hops. - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if err := utils.ValidateExternalURL(req.URL.String()); err != nil { - return fmt.Errorf("redirect URL validation failed: %w", err) - } - return checkRedirect(req, via) - } - return client + return httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects(), httpclient.WithExternalOnly()) } var videoDownloadClient = newVideoDownloadClient() diff --git a/core/http/endpoints/openai/image.go b/core/http/endpoints/openai/image.go index 9d64e8e72..49f907314 100644 --- a/core/http/endpoints/openai/image.go +++ b/core/http/endpoints/openai/image.go @@ -29,13 +29,15 @@ import ( "github.com/mudler/LocalAI/pkg/utils" ) +var imageDownloadClient = httpclient.New(httpclient.WithFollowRedirects(), httpclient.WithExternalOnly()) + func downloadFile(url string) (string, error) { if err := utils.ValidateExternalURL(url); err != nil { return "", fmt.Errorf("URL validation failed: %w", err) } // Get the data - resp, err := httpclient.New(httpclient.WithFollowRedirects()).Get(url) + resp, err := imageDownloadClient.Get(url) if err != nil { return "", err } diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index c18c78185..794cca852 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -125,6 +125,7 @@ type options struct { responseHeaderTimeout time.Duration transport http.RoundTripper followRedirects bool + externalOnly bool } // Option configures a client built by New. @@ -159,6 +160,23 @@ func WithResponseHeaderTimeout(d time.Duration) Option { // redirects (e.g. some download CDNs) and the request carries a secret. func WithFollowRedirects() Option { return func(o *options) { o.followRedirects = true } } +// WithExternalOnly restricts a client to hosts that pass the external URL +// policy (see IsPublicIP). Use it for every client that fetches a URL supplied +// by a user. It has two effects: +// +// - Each redirect hop is validated like the original URL, so a public host +// cannot bounce the client to a loopback, link-local or private target. +// It only matters together with WithFollowRedirects. +// - Unless WithTransport is given, the transport resolves the host itself, +// rejects it if any resolved address is not public, and dials one of the +// validated addresses, so a second DNS lookup cannot return a different +// answer. Environment proxies are not used for such a client, since the +// proxy address would be what gets validated and dialled. +// +// A caller-supplied transport is left untouched and must enforce the policy +// itself. +func WithExternalOnly() Option { return func(o *options) { o.externalOnly = true } } + // New returns a hardened *http.Client. By default it refuses redirects, sets a // TLS 1.2 floor, bounds connection setup, and imposes no body deadline (safe // for streaming). Apply Options to adjust. @@ -170,7 +188,15 @@ func New(opts ...Option) *http.Client { rt := o.transport if rt == nil { - rt = HardenedTransport() + t := HardenedTransport() + if o.externalOnly { + t.Proxy = nil + t.DialContext = validatedDialContext(&net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + }) + } + rt = t } // Only an *http.Transport carries the knob; a caller-supplied wrapper // (bearer-token RoundTripper, IP-pinned dialer) is left untouched rather @@ -184,6 +210,14 @@ func New(opts ...Option) *http.Client { check := NoRedirect if o.followRedirects { check = stripAuthOnRedirect + if o.externalOnly { + check = func(req *http.Request, via []*http.Request) error { + if err := validateRedirectTarget(req); err != nil { + return err + } + return stripAuthOnRedirect(req, via) + } + } } return &http.Client{ diff --git a/pkg/httpclient/external.go b/pkg/httpclient/external.go new file mode 100644 index 000000000..3bd472ca2 --- /dev/null +++ b/pkg/httpclient/external.go @@ -0,0 +1,184 @@ +package httpclient + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" +) + +// The external URL policy lives here, below pkg/utils, so the redirect check +// and the dial path of a client can enforce it without an import cycle. +// utils.ValidateExternalURL and utils.IsPublicIP delegate to these functions. + +// nonPublicNets lists ranges that are not covered by the net.IP helper +// predicates (IsPrivate, IsLoopback, ...) but are not routable on the public +// internet either. +var nonPublicNets = mustParseCIDRs( + "0.0.0.0/8", // "this network", not only the single unspecified address + "100.64.0.0/10", // RFC 6598 shared address space (carrier-grade NAT, overlay VPNs) + "192.0.0.0/24", // RFC 6890 IETF protocol assignments + "198.18.0.0/15", // RFC 2544 benchmarking + "240.0.0.0/4", // reserved, includes the limited broadcast address + "100::/64", // RFC 6666 discard-only prefix + "2001:db8::/32", // documentation prefix +) + +// nat64Prefix is the well-known NAT64 prefix (RFC 6052). The last four bytes +// are an IPv4 address that must pass the IPv4 rules too. +var nat64Prefix = mustParseCIDRs("64:ff9b::/96")[0] + +func mustParseCIDRs(cidrs ...string) []*net.IPNet { + nets := make([]*net.IPNet, 0, len(cidrs)) + for _, c := range cidrs { + _, n, err := net.ParseCIDR(c) + if err != nil { + panic(err) + } + nets = append(nets, n) + } + return nets +} + +// IsPublicIP reports whether ip refers to a host on the public internet, i.e. +// not loopback, link-local, private (RFC 1918 / RFC 4193), shared address +// space (RFC 6598), multicast, unspecified or otherwise reserved. An +// IPv4-mapped or NAT64-embedded IPv4 address is checked as that IPv4 address. +func IsPublicIP(ip net.IP) bool { + if ip == nil { + return false + } + if ip4 := ip.To4(); ip4 != nil { + ip = ip4 + } else if nat64Prefix.Contains(ip) { + return IsPublicIP(net.IP(ip.To16()[12:])) + } + + if ip.IsLoopback() || + ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || + ip.IsInterfaceLocalMulticast() || + ip.IsMulticast() || + ip.IsPrivate() || + ip.IsUnspecified() { + return false + } + for _, n := range nonPublicNets { + if n.Contains(ip) { + return false + } + } + return true +} + +// ValidateExternalURL checks that the given URL does not point to a private, +// loopback, link-local, or otherwise internal network address. This prevents +// Server-Side Request Forgery (SSRF) where a user-supplied URL could be used +// to probe internal services or cloud metadata endpoints. +func ValidateExternalURL(rawURL string) error { + parsed, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("unsupported URL scheme: %s", scheme) + } + + hostname := parsed.Hostname() + if hostname == "" { + return fmt.Errorf("URL has no hostname") + } + + lower := strings.ToLower(hostname) + if lower == "localhost" || strings.HasSuffix(lower, ".local") { + return fmt.Errorf("requests to internal hosts are not allowed") + } + if lower == "metadata.google.internal" || lower == "instance-data" { + return fmt.Errorf("requests to cloud metadata services are not allowed") + } + + ips, err := net.LookupHost(hostname) + if err != nil { + return fmt.Errorf("failed to resolve hostname: %w", err) + } + for _, ipStr := range ips { + ip := net.ParseIP(ipStr) + if ip == nil { + return fmt.Errorf("unable to parse resolved IP: %s", ipStr) + } + if !IsPublicIP(ip) { + return fmt.Errorf("requests to internal network addresses are not allowed") + } + } + return nil +} + +// ErrNonPublicAddress is wrapped by the error a validated dial returns when a +// host resolves to an address outside the external URL policy. +var ErrNonPublicAddress = errors.New("httpclient: requests to internal network addresses are not allowed") + +// resolver is swapped in tests. +var resolver interface { + LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) +} = net.DefaultResolver + +// validatedDialContext returns a DialContext that resolves the host itself, +// rejects the connection if ANY resolved address is not public, and then dials +// the validated addresses directly. The address that was checked is therefore +// the address that is connected to, which closes the window in which a second +// DNS lookup could return a different (internal) answer. net/http keeps the +// original host for the Host header and TLS server name, because only the dial +// address is replaced. +func validatedDialContext(d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + var ips []net.IP + if lit := net.ParseIP(host); lit != nil { + ips = []net.IP{lit} + } else { + addrs, err := resolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("failed to resolve hostname: %w", err) + } + for _, a := range addrs { + ips = append(ips, a.IP) + } + } + if len(ips) == 0 { + return nil, fmt.Errorf("no addresses found for %s", host) + } + for _, ip := range ips { + if !IsPublicIP(ip) { + return nil, ErrNonPublicAddress + } + } + var lastErr error + for _, ip := range ips { + conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + if err == nil { + return conn, nil + } + lastErr = err + if ctx.Err() != nil { + break + } + } + return nil, lastErr + } +} + +// validateRedirectTarget applies the external URL policy to a redirect hop. +func validateRedirectTarget(req *http.Request) error { + if err := ValidateExternalURL(req.URL.String()); err != nil { + return fmt.Errorf("redirect URL validation failed: %w", err) + } + return nil +} diff --git a/pkg/httpclient/external_test.go b/pkg/httpclient/external_test.go new file mode 100644 index 000000000..0c98e83ec --- /dev/null +++ b/pkg/httpclient/external_test.go @@ -0,0 +1,183 @@ +package httpclient + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "syscall" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type fakeResolver map[string][]string + +func (f fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) { + var out []net.IPAddr + for _, s := range f[host] { + out = append(out, net.IPAddr{IP: net.ParseIP(s)}) + } + if out == nil { + return nil, errors.New("no such host") + } + return out, nil +} + +var _ = Describe("external URL policy", func() { + DescribeTable("IsPublicIP", + func(addr string, want bool) { + Expect(IsPublicIP(net.ParseIP(addr))).To(Equal(want)) + }, + Entry("public IPv4", "93.184.216.34", true), + Entry("documentation range used by tests", "203.0.113.1", true), + Entry("public IPv6", "2606:4700:4700::1111", true), + Entry("loopback", "127.0.0.1", false), + Entry("IPv6 loopback", "::1", false), + Entry("RFC 1918", "10.1.2.3", false), + Entry("link-local metadata address", "169.254.169.254", false), + Entry("shared address space low edge", "100.64.0.0", false), + Entry("shared address space inside", "100.100.100.200", false), + Entry("shared address space high edge", "100.127.255.255", false), + Entry("just below shared address space", "100.63.255.255", true), + Entry("just above shared address space", "100.128.0.1", true), + Entry("this network, single address", "0.0.0.0", false), + Entry("this network, other address", "0.1.2.3", false), + Entry("benchmarking range", "198.19.0.1", false), + Entry("multicast", "224.0.0.1", false), + Entry("reserved class E", "240.0.0.1", false), + Entry("IPv6 unique local", "fd00::1", false), + Entry("IPv6 unspecified", "::", false), + Entry("IPv4-mapped loopback", "::ffff:127.0.0.1", false), + Entry("IPv4-mapped shared address space", "::ffff:100.64.0.1", false), + Entry("IPv4-mapped public", "::ffff:93.184.216.34", true), + Entry("NAT64 wrapping a private address", "64:ff9b::a00:1", false), + Entry("NAT64 wrapping a public address", "64:ff9b::5db8:d822", true), + ) + + Describe("redirect validation", func() { + var ( + hits atomic.Int32 + sink *httptest.Server + redirect func(target string) *http.Client + ) + + BeforeEach(func() { + hits.Store(0) + sink = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _, _ = w.Write([]byte("ok")) + })) + DeferCleanup(sink.Close) + + // The first hop is a TEST-NET literal so no DNS is needed; the + // transport serves every request from the in-process server, which + // answers with a redirect to target on the first call. + redirect = func(target string) *http.Client { + var first atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if first.CompareAndSwap(false, true) { + http.Redirect(w, r, target, http.StatusFound) + return + } + hits.Add(1) + _, _ = w.Write([]byte("ok")) + })) + DeferCleanup(srv.Close) + return New(WithFollowRedirects(), WithExternalOnly(), WithTransport(&http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String()) + }, + })) + } + }) + + DescribeTable("refuses non-public targets", + func(target string) { + _, err := redirect(target).Get("http://203.0.113.1/start") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("redirect URL validation failed")) + Expect(hits.Load()).To(BeZero()) + }, + Entry("loopback", "http://127.0.0.1:1/x"), + Entry("link-local metadata", "http://169.254.169.254/latest/meta-data/"), + Entry("private", "http://10.0.0.5/x"), + Entry("shared address space", "http://100.64.0.1/x"), + Entry("localhost name", "http://localhost/x"), + Entry("non-http scheme", "ftp://203.0.113.9/x"), + ) + + It("follows a redirect to another public target", func() { + resp, err := redirect("http://203.0.113.2/next").Get("http://203.0.113.1/start") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(resp.Body.Close) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(hits.Load()).To(Equal(int32(1))) + }) + + It("keeps the redirect bound", func() { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://203.0.113.2/loop", http.StatusFound) + })) + DeferCleanup(srv.Close) + c := New(WithFollowRedirects(), WithExternalOnly(), WithTransport(&http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String()) + }, + })) + _, err := c.Get("http://203.0.113.1/loop") + Expect(err).To(MatchError(ContainSubstring("stopped after"))) + }) + }) + + Describe("validated dial", func() { + errStop := errors.New("stop before connecting") + + // capture returns a dial function whose underlying dialer records the + // address it was asked to connect to and then aborts, so no packet is + // ever sent to the (documentation-range) address. + capture := func(got *string) func(context.Context, string, string) (net.Conn, error) { + return validatedDialContext(&net.Dialer{Control: func(_, address string, _ syscall.RawConn) error { + *got = address + return errStop + }}) + } + + BeforeEach(func() { + orig := resolver + DeferCleanup(func() { resolver = orig }) + }) + + It("refuses a host that resolves only to private addresses", func() { + resolver = fakeResolver{"internal.example": {"10.0.0.7"}} + var got string + _, err := capture(&got)(context.Background(), "tcp", "internal.example:80") + Expect(err).To(MatchError(ErrNonPublicAddress)) + Expect(got).To(BeEmpty()) + }) + + It("refuses a host when any answer is non-public", func() { + resolver = fakeResolver{"mixed.example": {"198.51.100.7", "100.64.0.9"}} + var got string + _, err := capture(&got)(context.Background(), "tcp", "mixed.example:80") + Expect(err).To(MatchError(ErrNonPublicAddress)) + Expect(got).To(BeEmpty()) + }) + + It("refuses a literal internal address", func() { + var got string + _, err := capture(&got)(context.Background(), "tcp", "127.0.0.1:80") + Expect(err).To(MatchError(ErrNonPublicAddress)) + }) + + It("dials the validated address, not a fresh lookup", func() { + resolver = fakeResolver{"public.example": {"198.51.100.7"}} + var got string + _, err := capture(&got)(context.Background(), "tcp", "public.example:8080") + Expect(err).To(HaveOccurred()) + Expect(got).To(Equal("198.51.100.7:8080")) + }) + }) +}) diff --git a/pkg/utils/base64.go b/pkg/utils/base64.go index b0cbdb0d9..80893131e 100644 --- a/pkg/utils/base64.go +++ b/pkg/utils/base64.go @@ -13,7 +13,7 @@ import ( "github.com/mudler/LocalAI/pkg/httpclient" ) -var base64DownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) +var base64DownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects(), httpclient.WithExternalOnly()) // Match `data:[;param=value...];base64,` — browser-produced data URIs // often carry codec/charset params between the mime type and `;base64,` diff --git a/pkg/utils/urlfetch.go b/pkg/utils/urlfetch.go index 072c07b21..9bea94221 100644 --- a/pkg/utils/urlfetch.go +++ b/pkg/utils/urlfetch.go @@ -1,81 +1,16 @@ package utils import ( - "fmt" "net" - "net/url" - "strings" + + "github.com/mudler/LocalAI/pkg/httpclient" ) // ValidateExternalURL checks that the given URL does not point to a private, -// loopback, link-local, or otherwise internal network address. This prevents -// Server-Side Request Forgery (SSRF) attacks where a user-supplied URL could -// be used to probe internal services or cloud metadata endpoints. -func ValidateExternalURL(rawURL string) error { - parsed, err := url.Parse(rawURL) - if err != nil { - return fmt.Errorf("invalid URL: %w", err) - } +// loopback, link-local, or otherwise internal network address. It is the +// policy implemented in pkg/httpclient, kept here for existing callers. +func ValidateExternalURL(rawURL string) error { return httpclient.ValidateExternalURL(rawURL) } - scheme := strings.ToLower(parsed.Scheme) - if scheme != "http" && scheme != "https" { - return fmt.Errorf("unsupported URL scheme: %s", scheme) - } - - hostname := parsed.Hostname() - if hostname == "" { - return fmt.Errorf("URL has no hostname") - } - - // Block well-known internal hostnames - lower := strings.ToLower(hostname) - if lower == "localhost" || strings.HasSuffix(lower, ".local") { - return fmt.Errorf("requests to internal hosts are not allowed") - } - - // Block cloud metadata service hostnames - if lower == "metadata.google.internal" || lower == "instance-data" { - return fmt.Errorf("requests to cloud metadata services are not allowed") - } - - ips, err := net.LookupHost(hostname) - if err != nil { - return fmt.Errorf("failed to resolve hostname: %w", err) - } - - for _, ipStr := range ips { - ip := net.ParseIP(ipStr) - if ip == nil { - return fmt.Errorf("unable to parse resolved IP: %s", ipStr) - } - - if !IsPublicIP(ip) { - return fmt.Errorf("requests to internal network addresses are not allowed") - } - } - - return nil -} - -// IsPublicIP reports whether ip refers to a host on the public internet, i.e. -// not loopback, link-local, private (RFC 1918 / RFC 4193), or unspecified. -// Covers 0.0.0.0/8, ::/128, and IPv4-mapped IPv6 wrapping a private address — -// holes hand-rolled CIDR lists tend to miss. -func IsPublicIP(ip net.IP) bool { - if ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsPrivate() || - ip.IsUnspecified() { - return false - } - - if ip4 := ip.To4(); ip4 != nil { - return !ip4.IsLoopback() && - !ip4.IsLinkLocalUnicast() && - !ip4.IsPrivate() && - !ip4.IsUnspecified() - } - - return true -} +// IsPublicIP reports whether ip refers to a host on the public internet. See +// httpclient.IsPublicIP for the ranges that are refused. +func IsPublicIP(ip net.IP) bool { return httpclient.IsPublicIP(ip) } diff --git a/pkg/utils/urlfetch_test.go b/pkg/utils/urlfetch_test.go index 62c409e5f..468290d7b 100644 --- a/pkg/utils/urlfetch_test.go +++ b/pkg/utils/urlfetch_test.go @@ -90,6 +90,23 @@ var _ = Describe("utils/urlfetch tests", func() { Expect(err.Error()).To(ContainSubstring("internal")) }) + DescribeTable("blocks non-public ranges that net.IP does not flag", + func(rawURL string) { + err := ValidateExternalURL(rawURL) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("internal")) + }, + Entry("shared address space (RFC 6598)", "http://100.64.0.1/x"), + Entry("shared address space, upper end", "http://100.127.255.254/x"), + Entry("this network 0.0.0.0/8", "http://0.1.2.3/x"), + Entry("benchmarking range", "http://198.18.0.1/x"), + Entry("IPv4-mapped shared address space", "http://[::ffff:100.64.0.1]/x"), + ) + + It("allows a public address next to the shared address space", func() { + Expect(ValidateExternalURL("http://100.128.0.1/x")).To(Succeed()) + }) + It("blocks IPv6 loopback ::1", func() { err := ValidateExternalURL("http://[::1]/secret") Expect(err).ToNot(BeNil())