From c91b7188e8b2c0ceba2ca69157883fa30a895ad2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 2 Jun 2026 18:31:36 +0000 Subject: [PATCH] ipn/localapi,tstest/natlab: fix debug derp TLS check for sha256-raw CertName serveDebugDERPRegion built its TLS config with ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), which for a "sha256-raw:" CertName passed the raw fingerprint to Go's stock verifier as a hostname; the handshake always failed with a hostname mismatch. This is the second half of #15579; the first half (tailscaled itself failing with "unexpected multiple certs presented") was fixed in Extract a tlsConfigForNode helper that mirrors derphttp.Client.tlsClient so that sha256-raw and domain-fronting CertName values are dispatched to tlsdial.SetConfigExpectedCertHash and tlsdial.SetConfigExpectedCert respectively, falling back to HostName when CertName is empty. The core fix here was originally written by @imnuke in #19965; that PR also added a unit test in ipn/localapi/debugderp_test.go which is replaced in this commit by a new vmtest that exercises the whole stack: vnet now serves a self-signed cert valid for each fake DERP node's HostName and exposes its SHA-256 fingerprint, and vmtest grows a new SelfSignedDERPCertPinning EnvOption that swaps the test DERP map's nodes to CertName="sha256-raw:" with InsecureForTests cleared. TestSelfSignedDERPHashPinning then stands up two hard-NAT'd nodes, has them communicate over DERP, and calls DebugDERPRegion on each. Before this fix the test fails with the exact x509 hostname-mismatch error from the original bug; after, it passes. Updates #15579 Change-Id: I61f38ffebc7ac5abc962639db1ae88f5cd8633b1 Co-authored-by: Nuke Signed-off-by: Brad Fitzpatrick --- ipn/localapi/debugderp.go | 29 ++++--- tstest/natlab/vmtest/selfsignedderp_test.go | 60 ++++++++++++++ tstest/natlab/vmtest/vmtest.go | 51 +++++++++++- tstest/natlab/vnet/vnet.go | 90 ++++++++++++++++++--- 4 files changed, 206 insertions(+), 24 deletions(-) create mode 100644 tstest/natlab/vmtest/selfsignedderp_test.go diff --git a/ipn/localapi/debugderp.go b/ipn/localapi/debugderp.go index 52987ee0a..fd42d2291 100644 --- a/ipn/localapi/debugderp.go +++ b/ipn/localapi/debugderp.go @@ -15,6 +15,7 @@ "net/http" "net/netip" "strconv" + "strings" "time" "tailscale.com/derp/derphttp" @@ -22,11 +23,28 @@ "tailscale.com/net/netaddr" "tailscale.com/net/netns" "tailscale.com/net/stun" + "tailscale.com/net/tlsdial" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/nettype" ) +// tlsConfigForNode builds a *tls.Config for connecting to a DERP node, +// mirroring the logic in derphttp.Client.tlsClient so that sha256-raw cert +// pinning and domain-fronting CertName values are handled correctly. +func tlsConfigForNode(node *tailcfg.DERPNode) *tls.Config { + conf := tlsdial.Config(nil, nil) + conf.ServerName = node.HostName + if node.CertName != "" { + if suf, ok := strings.CutPrefix(node.CertName, "sha256-raw:"); ok { + tlsdial.SetConfigExpectedCertHash(conf, suf) + } else { + tlsdial.SetConfigExpectedCert(conf, node.CertName) + } + } + return conf +} + func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { if !h.PermitWrite { http.Error(w, "debug access denied", http.StatusForbidden) @@ -100,9 +118,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { defer conn.Close() // Upgrade to TLS and verify that works properly. - tlsConn := tls.Client(conn, &tls.Config{ - ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), - }) + tlsConn := tls.Client(conn, tlsConfigForNode(derpNode)) if err := tlsConn.HandshakeContext(ctx); err != nil { st.Errors = append(st.Errors, fmt.Sprintf("Error upgrading connection to node %q @ %q to TLS over IPv4: %v", derpNode.HostName, addr, err)) } else { @@ -119,12 +135,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) { defer conn.Close() // Upgrade to TLS and verify that works properly. - tlsConn := tls.Client(conn, &tls.Config{ - ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), - // TODO(andrew-d): we should print more - // detailed failure information on if/why TLS - // verification fails - }) + tlsConn := tls.Client(conn, tlsConfigForNode(derpNode)) if err := tlsConn.HandshakeContext(ctx); err != nil { st.Errors = append(st.Errors, fmt.Sprintf("Error upgrading connection to node %q @ %q to TLS over IPv6: %v", derpNode.HostName, addr, err)) } else { diff --git a/tstest/natlab/vmtest/selfsignedderp_test.go b/tstest/natlab/vmtest/selfsignedderp_test.go new file mode 100644 index 000000000..6754eaeaf --- /dev/null +++ b/tstest/natlab/vmtest/selfsignedderp_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest_test + +import ( + "context" + "strings" + "testing" + "time" + + "tailscale.com/tstest/natlab/vmtest" +) + +// TestSelfSignedDERPHashPinning exercises the sha256-raw DERP cert pinning +// code path end-to-end: tailscaled connects to its home DERP whose cert is +// self-signed and pinned via CertName="sha256-raw:" (no separate +// fronting CertName), the two nodes communicate over the resulting tailnet, +// and `tailscale debug derp` against the same region succeeds. +// +// Both nodes sit behind hard NATs with no port mapping available so disco +// cannot punch a direct path and the tailnet ping must traverse DERP, making +// the sha256-raw pinning of the tailscaled→DERP path part of the assertion. +// +// The debug-derp half is the regression test for the bug fixed in PR #19965: +// before that change, [ipn/localapi.serveDebugDERPRegion] passed the raw +// sha256-raw fingerprint as the TLS ServerName and the handshake always +// failed with a hostname mismatch. +func TestSelfSignedDERPHashPinning(t *testing.T) { + env := vmtest.New(t, vmtest.SelfSignedDERPCertPinning()) + n1 := hard(env) + n2 := hard(env) + env.Start() + + if err := env.PingExpect(n1, n2, vmtest.PingRouteDERP, 60*time.Second); err != nil { + t.Fatalf("ping node-0 -> node-1: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + for _, n := range []*vmtest.Node{n1, n2} { + rep, err := n.Agent().DebugDERPRegion(ctx, "1") + if err != nil { + t.Fatalf("[%s] DebugDERPRegion(1): %v", n.Name(), err) + } + t.Logf("[%s] DebugDERPRegion(1): info=%v warnings=%v errors=%v", + n.Name(), rep.Info, rep.Warnings, rep.Errors) + for _, e := range rep.Errors { + // The `hard` builder gives the node only an IPv4 LAN, so the + // DebugDERPRegion IPv6 probe predictably fails with + // "network is unreachable". That's orthogonal to the TLS + // verification path this test exists to cover. + if strings.Contains(e, "over IPv6") { + continue + } + t.Errorf("[%s] DebugDERPRegion(1) error: %s", n.Name(), e) + } + } +} diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index a256f9c10..659e12877 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -89,9 +89,10 @@ type Env struct { qemuProcs []*exec.Cmd // launched QEMU processes - sameTailnetUser bool // all nodes register as the same Tailnet user - allOnline bool // mark every peer as Online=true in MapResponses - peerRelayGrants bool // grant peer-relay capabilities on the wildcard packet filter + sameTailnetUser bool // all nodes register as the same Tailnet user + allOnline bool // mark every peer as Online=true in MapResponses + peerRelayGrants bool // grant peer-relay capabilities on the wildcard packet filter + selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins // Shared resource initialization (sync.Once for things multiple nodes share). vnetOnce sync.Once @@ -384,6 +385,15 @@ func PeerRelayGrants() EnvOption { return envOptFunc(func(e *Env) { e.peerRelayGrants = true }) } +// SelfSignedDERPCertPinning returns an [EnvOption] that makes the test control +// server advertise a DERP map whose nodes use CertName="sha256-raw:" +// pinning against the self-signed certs vnet's fake DERP servers serve. This +// exercises the sha256-raw verification path end-to-end (in tailscaled and in +// `tailscale debug derp`) without involving a real CA. +func SelfSignedDERPCertPinning() EnvOption { + return envOptFunc(func(e *Env) { e.selfSignedDERPCertPinning = true }) +} + // AddNetwork creates a new virtual network. Arguments follow the same pattern as // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values). func (e *Env) AddNetwork(opts ...any) *vnet.Network { @@ -1379,9 +1389,44 @@ func (e *Env) initVnet() { if e.peerRelayGrants { e.server.ControlServer().PeerRelayGrants = true } + if e.selfSignedDERPCertPinning { + e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap() + } }) } +// buildSelfSignedDERPMap returns a DERP map identical in structure to the +// stock test map (same regions, hostnames, virtual IPs) but with each node's +// CertName set to "sha256-raw:" pinning the actual self-signed cert +// served by vnet's fake DERP server, and InsecureForTests cleared so the +// pin is actually exercised. Nodes are matched to certs by HostName. +func (e *Env) buildSelfSignedDERPMap() *tailcfg.DERPMap { + hostToHash := make(map[string]string, 2) + for i := range 2 { + hostToHash[e.server.DERPHostname(i)] = e.server.DERPCertSHA256Hex(i) + } + src := e.server.ControlServer().DERPMap + dm := &tailcfg.DERPMap{ + Regions: make(map[int]*tailcfg.DERPRegion, len(src.Regions)), + } + for id, srcRegion := range src.Regions { + r := *srcRegion + r.Nodes = make([]*tailcfg.DERPNode, len(srcRegion.Nodes)) + for i, srcNode := range srcRegion.Nodes { + n := *srcNode + hash, ok := hostToHash[n.HostName] + if !ok { + e.t.Fatalf("buildSelfSignedDERPMap: no cert hash for HostName %q", n.HostName) + } + n.InsecureForTests = false + n.CertName = "sha256-raw:" + hash + r.Nodes[i] = &n + } + dm.Regions[id] = &r + } + return dm +} + // ensureQEMUSocket creates the Unix stream socket for QEMU VMs. Called once. func (e *Env) ensureQEMUSocket() { e.qemuSockOnce.Do(func() { diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 958da04de..ffe6d3021 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -14,7 +14,13 @@ import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + crand "crypto/rand" + "crypto/sha256" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/binary" "encoding/json" "errors" @@ -23,10 +29,10 @@ "iter" "log" "maps" + "math/big" "math/rand/v2" "net" "net/http" - "net/http/httptest" "net/netip" "os/exec" "strconv" @@ -734,16 +740,23 @@ type derpServer struct { srv *derpserver.Server handler http.Handler tlsConfig *tls.Config + // certSHA256Hex is the SHA-256 hex fingerprint of the leaf certificate + // served by this DERP server. It is the value tests pin against when + // they configure a custom DERP map with CertName="sha256-raw:". + certSHA256Hex string } -func newDERPServer() *derpServer { - // Just to get a self-signed TLS cert: - ts := httptest.NewTLSServer(nil) - ts.Close() - +// newDERPServer returns a derpServer whose TLS cert is a freshly generated +// self-signed ECDSA cert valid for hostname. Tests that use a stock test DERP +// map with InsecureForTests=true ignore the cert content entirely; tests that +// want to exercise sha256-raw cert pinning can read the certSHA256Hex via +// [Server.DERPCertSHA256Hex]. +func newDERPServer(hostname string) *derpServer { + tlsConfig, certHex := selfSignedDERPCert(hostname) ds := &derpServer{ - srv: derpserver.New(key.NewNode(), logger.Discard), - tlsConfig: ts.TLS, // self-signed; test client configure to not check + srv: derpserver.New(key.NewNode(), logger.Discard), + tlsConfig: tlsConfig, + certSHA256Hex: certHex, } var mux http.ServeMux mux.Handle("/derp", derpserver.Handler(ds.srv)) @@ -753,6 +766,40 @@ func newDERPServer() *derpServer { return ds } +// selfSignedDERPCert builds a self-signed ECDSA P-256 cert valid for hostname +// and returns a *tls.Config that serves it, along with the SHA-256 hex digest +// of the cert's DER bytes. +func selfSignedDERPCert(hostname string) (*tls.Config, string) { + key, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader) + if err != nil { + panic(fmt.Sprintf("vnet: generating DERP cert key: %v", err)) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: hostname}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{hostname}, + } + if ip := net.ParseIP(hostname); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + tmpl.DNSNames = nil + } + der, err := x509.CreateCertificate(crand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + panic(fmt.Sprintf("vnet: creating DERP cert: %v", err)) + } + cfg := &tls.Config{ + Certificates: []tls.Certificate{{ + Certificate: [][]byte{der}, + PrivateKey: key, + }}, + } + return cfg, fmt.Sprintf("%x", sha256.Sum256(der)) +} + type Server struct { shutdownCtx context.Context shutdownCancel context.CancelFunc @@ -810,6 +857,11 @@ func (s *Server) SetDHCPCallback(fn func(MAC, int, layers.DHCPMsgType, netip.Add s.onDHCPEvent = fn } +// derpHostnames are the SNI/HostName values vnet's fake DERP servers identify +// as. They are also used to issue the per-DERP self-signed certificate so that +// hostname verification succeeds for tests that pin via sha256-raw. +var derpHostnames = []string{"derp1.tailscale", "derp2.tailscale"} + var derpMap = &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -820,7 +872,7 @@ func (s *Server) SetDHCPCallback(fn func(MAC, int, layers.DHCPMsgType, netip.Add { Name: "1a", RegionID: 1, - HostName: "derp1.tailscale", + HostName: derpHostnames[0], IPv4: fakeDERP1.v4.String(), IPv6: fakeDERP1.v6.String(), InsecureForTests: true, @@ -836,7 +888,7 @@ func (s *Server) SetDHCPCallback(fn func(MAC, int, layers.DHCPMsgType, netip.Add { Name: "2a", RegionID: 2, - HostName: "derp2.tailscale", + HostName: derpHostnames[1], IPv4: fakeDERP2.v4.String(), IPv6: fakeDERP2.v6.String(), InsecureForTests: true, @@ -865,8 +917,8 @@ func New(c *Config) (*Server, error) { networkByWAN: &bart.Table[*network]{}, networks: set.Of[*network](), } - for range 2 { - s.derps = append(s.derps, newDERPServer()) + for _, host := range derpHostnames { + s.derps = append(s.derps, newDERPServer(host)) } if err := s.initFromConfig(c); err != nil { return nil, err @@ -889,6 +941,20 @@ func (s *Server) ControlServer() *testcontrol.Server { return s.control } +// DERPHostname returns the SNI/HostName used by vnet's idx'th fake DERP +// server. idx must be 0 or 1. +func (s *Server) DERPHostname(idx int) string { + return derpHostnames[idx] +} + +// DERPCertSHA256Hex returns the SHA-256 hex fingerprint of the self-signed +// TLS certificate served by vnet's idx'th fake DERP server. It is the value +// to pin against in a [tailcfg.DERPNode.CertName] formatted as +// "sha256-raw:". idx must be 0 or 1. +func (s *Server) DERPCertSHA256Hex(idx int) string { + return s.derps[idx].certSHA256Hex +} + // CloudInitData holds the cloud-init configuration for a node. type CloudInitData struct { MetaData string