diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index b5667dbfd..e8244d5bd 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -821,6 +821,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routecheck from tailscale.com/client/local+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ diff --git a/cmd/tailscaled/depaware-min.txt b/cmd/tailscaled/depaware-min.txt index f9a87a608..56eade3f0 100644 --- a/cmd/tailscaled/depaware-min.txt +++ b/cmd/tailscaled/depaware-min.txt @@ -109,6 +109,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/ping from tailscale.com/net/netcheck+ tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/net/netcheck+ diff --git a/cmd/tailscaled/depaware-minbox.txt b/cmd/tailscaled/depaware-minbox.txt index e5dce3277..3a8756781 100644 --- a/cmd/tailscaled/depaware-minbox.txt +++ b/cmd/tailscaled/depaware-minbox.txt @@ -126,6 +126,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/ping from tailscale.com/net/netcheck+ tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/net/netcheck+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 95b736a6d..cc9bb06a3 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -384,6 +384,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled tailscale.com/net/routecheck from tailscale.com/feature/routecheck+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ tailscale.com/net/routetable from tailscale.com/doctor/routetable 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock+ tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled diff --git a/cmd/tailscaled/netstack.go b/cmd/tailscaled/netstack.go index d896f384f..4e180d36b 100644 --- a/cmd/tailscaled/netstack.go +++ b/cmd/tailscaled/netstack.go @@ -44,11 +44,6 @@ func newNetstack(logf logger.Logf, sys *tsd.System, onlyNetstack bool) (tsd.Nets dialer := sys.Dialer.Get() // must be set by caller already if onlyNetstack { - e := sys.Engine.Get() - dialer.UseNetstackForIP = func(ip netip.Addr) bool { - _, ok := e.PeerForIP(ip) - return ok - } dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { // Note: don't just return ns.DialContextTCP or we'll return // *gonet.TCPConn(nil) instead of a nil interface which trips up diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index f1f278062..325b66182 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -18,6 +18,7 @@ "log" "net" "net/http" + "net/netip" "os" "os/signal" "path/filepath" @@ -694,6 +695,12 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID if err != nil { return nil, fmt.Errorf("ipnlocal.NewLocalBackend: %w", err) } + if onlyNetstack { + dialer.UseNetstackForIP = func(ip netip.Addr) bool { + _, ok := lb.PeerForIP(ip) + return ok + } + } lb.SetVarRoot(opts.VarRoot) if logPol != nil { lb.SetLogFlusher(logPol.Logtail.StartFlush) diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt index 33f885d4a..4b307b6f9 100644 --- a/cmd/tsidp/depaware.txt +++ b/cmd/tsidp/depaware.txt @@ -210,6 +210,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routecheck from tailscale.com/client/local+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index 542426f38..137b8a39b 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -575,13 +575,13 @@ func (mrs mapRoutineState) UpdateUserProfiles(profiles map[tailcfg.UserID]tailcf } } -var _ patchDiscoKeyer = mapRoutineState{} +var _ DiscoKeyUpdater = mapRoutineState{} func (mrs mapRoutineState) PatchDiscoKey(pub key.NodePublic, disco key.DiscoPublic) { c := mrs.c c.mu.Lock() goodState := c.loggedIn && c.inMapPoll - dun, ok := c.observer.(patchDiscoKeyer) + dun, ok := c.observer.(DiscoKeyUpdater) mapCtx := c.mapCtx c.mu.Unlock() diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 062af8dd5..8ac635616 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -286,10 +286,10 @@ type UserProfileUpdater interface { UpdateUserProfiles(profiles map[tailcfg.UserID]tailcfg.UserProfileView) bool } -// patchDiscoKeyer is an optional interface that can be implemented by an [Observer] to be +// DiscoKeyUpdater is an optional interface that can be implemented by an [Observer] to be // notified about node disco keys received out-of-band from control, via // existing connection state. -type patchDiscoKeyer interface { +type DiscoKeyUpdater interface { // PatchDiscoKey reports to the receiver that the specified disco key // for node was obtained out-of-band from control. PatchDiscoKey(key.NodePublic, key.DiscoPublic) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 0236c3fc0..a601c8579 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -362,7 +362,7 @@ func (ms *mapSession) handleNonKeepAliveMapResponse(ctx context.Context, resp *t } func (ms *mapSession) tryMarkDiscoAsLearnedFromTSMP(res *tailcfg.MapResponse) { - dun, ok := ms.netmapUpdater.(patchDiscoKeyer) + dun, ok := ms.netmapUpdater.(DiscoKeyUpdater) if !ok { return } diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index 057e0b4c8..3367f7809 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -34,9 +34,7 @@ "tailscale.com/util/eventbus/eventbustest" "tailscale.com/util/mak" "tailscale.com/util/must" - "tailscale.com/util/usermetric" "tailscale.com/util/zstdframe" - "tailscale.com/wgengine" ) func eps(s ...string) []netip.AddrPort { @@ -1949,20 +1947,6 @@ func TestLearnZstdOfKeepAlive(t *testing.T) { } } -func TestPathDiscokeyerImplementations(t *testing.T) { - bus := eventbustest.NewBus(t) - ht := health.NewTracker(bus) - reg := new(usermetric.Registry) - e, err := wgengine.NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus) - if err != nil { - t.Fatal(err) - } - t.Cleanup(e.Close) - if _, ok := e.(patchDiscoKeyer); !ok { - t.Error("wgengine.userspaceEngine must implement patchDiscoKeyer") - } -} - func TestPeerIDAndKeyByTailscaleIP(t *testing.T) { peerKey1 := key.NewNode().Public() peerKey2 := key.NewNode().Public() diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index dfbfba191..b83ead5ed 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -31,6 +31,7 @@ "sync/atomic" "time" + "github.com/gaissmai/bart" "go4.org/mem" "go4.org/netipx" "golang.org/x/net/dns/dnsmessage" @@ -62,6 +63,7 @@ "tailscale.com/net/netns" "tailscale.com/net/netutil" "tailscale.com/net/packet" + "tailscale.com/net/routemanager" "tailscale.com/net/traffic" "tailscale.com/net/tsaddr" "tailscale.com/net/tsdial" @@ -633,7 +635,8 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo nb.ready() e.SetPeerByIPPacketFunc(b.lookupPeerByIP) - e.SetPeerForIPFunc(b.peerForIP) + e.SetPeerConfigFunc(b.peerAllowedIPs) + e.SetPeerForIPFunc(b.PeerForIP) e.SetPeerSessionStateFunc(b.onPeerWireGuardState) e.SetNetLogSource(netLogNodeSource{b}) e.SetWGPeerLookup(b.lookupPeerWireGuardString) @@ -2129,16 +2132,11 @@ func (b *LocalBackend) setControlClientStatusLocked(c controlclient.Client, st c b.authReconfigLocked() } +// PatchDiscoKey records that a peer's new disco key was learned via TSMP, +// so the netmap update carrying the same change need not reset the peer's +// WireGuard session. It implements [controlclient.DiscoKeyUpdater]. func (b *LocalBackend) PatchDiscoKey(pub key.NodePublic, disco key.DiscoPublic) { - // PatchDiscoKey mirrors the implementation of [controlclient.patchDiscoKeyer]. - // It is implemented here to avoid the dependency edge to controlclient, but must be kept - // in sync with the original implementation. - type patchDiscoKeyer interface { - PatchDiscoKey(key.NodePublic, key.DiscoPublic) - } - if e, ok := b.e.(patchDiscoKeyer); ok { - e.PatchDiscoKey(pub, disco) - } + b.currentNode().recordTSMPLearnedDisco(pub, disco) } type preferencePolicyInfo struct { @@ -2421,6 +2419,7 @@ func (b *LocalBackend) sysPolicyChangedForSession(sess *watchSession) { _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil) _ controlclient.PacketFilterUpdater = (*LocalBackend)(nil) _ controlclient.UserProfileUpdater = (*LocalBackend)(nil) + _ controlclient.DiscoKeyUpdater = (*LocalBackend)(nil) ) // UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater. @@ -2448,7 +2447,8 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // the full-netmap behavior of [tkaFilterNetmapLocked]. muts = b.tkaFilterDeltaMutsLocked(muts) needsAuthReconfig := netmapDeltaNeedsAuthReconfig(cn, muts) - cn.UpdateNetmapDelta(muts) + + _, changedAllowedIPs, discoChanged := cn.UpdateNetmapDelta(muts) if buildfeatures.HasDrive { // Drive's lazy remotes-source caches its rebuild keyed by this // generation, so any delta — peer add/remove, address change, @@ -2490,20 +2490,33 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo } ms.UpdateNetmapDelta(muts) + // Sync the WireGuard device for exactly the peers whose allowed + // source prefixes changed, as computed by the route manager when + // the delta was applied above. Removed peers appear with a nil + // value and are removed from the device; SyncDevicePeer reads + // the live per-peer config, so only the keys matter here. + for k := range changedAllowedIPs { + b.e.SyncDevicePeer(k) + } + b.setDataPlanePeerRoutes() + + // Reset the WireGuard session for peers whose disco key changed in + // a way that indicates a restart, flushing their dead session keys; + // each such peer is lazily re-created on demand with current state. + for _, k := range discoChanged { + b.e.ResetDevicePeer(k) + } + // Force a full authReconfig + SetSelfNode on any peer add or - // remove. netmapDeltaNeedsAuthReconfig only considered - // NodeMutationUpsert of already-known NodeIDs whose - // peerRouteConfigChanged, so brand-new peers and removes left - // e.lastCfgFull and wgdev's PeerLookupFunc closure stale, and - // outbound wgdev encryption missed those peers. authReconfig - // fixes the wireguard side; SetSelfNode refreshes the engine's - // cached self node. PeerForIP / lookupPeerByIP staleness was - // addressed separately in d4f2917c1b, which routes those lookups - // through nodeBackend's live data, and as part of the broader - // netmap.NetworkMap removal effort the engine no longer caches - // the netmap at all (see tailscale/corp#43394). As of 2026-06-24 - // the only remaining staleness this guards against is - // e.lastCfgFull and the wgdev peer set. + // remove. The WireGuard device itself no longer depends on this: + // the SyncDevicePeer calls above keep its peer set delta-correct, + // and its lazy peer creation reads live state via + // [wgengine.Engine.SetPeerConfigFunc]. What still rides + // authReconfig is everything else derived from the full peer set: + // OS routes (router.Config) and the quad-100 resolver's MagicDNS + // hosts map (dnsConfigForNetmap). Once those become delta-aware + // too, this can be gated on the route manager's OS-routes changes + // instead of firing on every peer change. needsAuthReconfig = needsAuthReconfig || peersUpsertedOrRemoved if needsAuthReconfig { if peersUpsertedOrRemoved { @@ -6095,17 +6108,31 @@ func (b *LocalBackend) authReconfigLocked() { } oneCGNATRoute := shouldUseOneCGNATRoute(b.logf, b.sys.NetMon.Get(), b.sys.ControlKnobs(), version.OS()) - rcfg := b.routerConfigLocked(cfg, prefs, nm, oneCGNATRoute) + // Sync the WireGuard device for any peers whose allowed source + // prefixes changed with the new prefs, such as the old and new + // exit node when the selection changes. + for k := range cn.updateRouteManagerPrefs(prefs.ExitNodeID(), flags&netmap.AllowSubnetRoutes != 0, oneCGNATRoute) { + b.e.SyncDevicePeer(k) + } + rcfg := b.routerConfigLocked(cfg, prefs, nm) - // Add these extra Allowed IPs after router configuration, because the expected - // extension (features/conn25), does not want these routes installed on the OS. + // Push each peer's extra WireGuard-only allowed IPs (the conn25 + // extension's Transit IPs) into the route manager, which feeds + // them to WireGuard via the outbound peer lookup and the per-peer + // allowed source prefixes (including for lazily created peers) + // while keeping them out of the OS route set, because the + // expected extension (features/conn25) does not want these routes + // installed on the OS. // See also [Hooks.ExtraWireGuardAllowedIPs]. if extraAllowedIPsFn, ok := b.extHost.hooks.ExtraWireGuardAllowedIPs.GetOk(); ok { - for i := range cfg.Peers { - extras := extraAllowedIPsFn(cfg.Peers[i].PublicKey) - cfg.Peers[i].AllowedIPs = extras.AppendTo(cfg.Peers[i].AllowedIPs) + for k := range cn.updateRouteManagerExtras(extraAllowedIPsFn) { + b.e.SyncDevicePeer(k) } } + // The prefs and extras commits above can both change the outbound + // table (such as installing the selected exit node's /0 routes), + // so refresh the data plane's view of it. + b.setDataPlanePeerRoutes() err = b.e.Reconfig(cfg, rcfg, dcfg) if err == wgengine.ErrNoChanges { @@ -6119,6 +6146,37 @@ func (b *LocalBackend) authReconfigLocked() { } } +// setDataPlanePeerRoutes pushes the route manager's outbound table and +// this node's native Tailscale addresses into the engine's tun-layer +// data plane, which uses them for per-packet NAT rewrites and +// jailed-filter selection. It must be called after every route manager +// commit that can change the outbound table or the set of peers with +// data-plane attributes: netmap updates (full or delta), prefs changes, +// and extra-allowed-IP changes. +// +// When no current peer is jailed or masqueraded, it installs nil, which +// keeps the data plane's per-packet fast path to a nil check. +func (b *LocalBackend) setDataPlanePeerRoutes() { + nb := b.currentNode() + var native4, native6 netip.Addr + var routes *bart.Table[*routemanager.PeerRoute] + if nb.routeMgr.HasDataPlaneAttrs() { + routes = nb.routeMgr.Outbound() + if nm := nb.NetMap(); nm != nil { + for _, pfx := range nm.GetAddresses().All() { + a := pfx.Addr() + switch { + case a.Is4() && !native4.IsValid() && tsaddr.IsTailscaleIP(a): + native4 = a + case a.Is6() && !native6.IsValid() && tsaddr.IsTailscaleIP(a): + native6 = a + } + } + } + } + b.e.SetPeerRoutes(native4, native6, routes) +} + // shouldUseOneCGNATRoute reports whether we should prefer to make one big // CGNAT /10 route rather than a /32 per peer. // @@ -6432,64 +6490,10 @@ func magicDNSRootDomains(nm *netmap.NetworkMap) []dnsname.FQDN { return nil } -// peerRoutes returns the routerConfig.Routes to access peers. -// If there are over cgnatThreshold CGNAT routes, one big CGNAT route -// is used instead. -func peerRoutes(logf logger.Logf, peers []wgcfg.Peer, cgnatThreshold int, routeAll bool) (routes []netip.Prefix) { - tsULA := tsaddr.TailscaleULARange() - cgNAT := tsaddr.CGNATRange() - var didULA bool - var cgNATIPs []netip.Prefix - for _, peer := range peers { - for _, aip := range peer.AllowedIPs { - aip = unmapIPPrefix(aip) - - // Ensure that we're only accepting properly-masked - // prefixes; the control server should be masking - // these, so if we get them, skip. - if mm := aip.Masked(); aip != mm { - // To avoid a DoS where a peer could cause all - // reconfigs to fail by sending a bad prefix, we just - // skip, but don't error, on an unmasked route. - logf("advertised route %s from %s has non-address bits set; expected %s", aip, peer.PublicKey.ShortString(), mm) - continue - } - - // Only add the Tailscale IPv6 ULA once, if we see anybody using part of it. - if aip.Addr().Is6() && aip.IsSingleIP() && tsULA.Contains(aip.Addr()) { - if !didULA { - didULA = true - routes = append(routes, tsULA) - } - continue - } - if aip.IsSingleIP() && cgNAT.Contains(aip.Addr()) { - cgNATIPs = append(cgNATIPs, aip) - } else if routeAll { - routes = append(routes, aip) - } - } - } - if len(cgNATIPs) > cgnatThreshold { - // Probably the hello server. Just append one big route. - routes = append(routes, cgNAT) - } else { - routes = append(routes, cgNATIPs...) - } - - tsaddr.SortPrefixes(routes) - return routes -} - // routerConfig produces a router.Config from a wireguard config and IPN prefs. // // b.mu must be held. -func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView, nm *netmap.NetworkMap, oneCGNATRoute bool) *router.Config { - singleRouteThreshold := 10_000 - if oneCGNATRoute { - singleRouteThreshold = 1 - } - +func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView, nm *netmap.NetworkMap) *router.Config { netfilterKind := b.capForcedNetfilter // protected by b.mu (hence the Locked suffix) if prefs.NetfilterKind() != "" { @@ -6513,7 +6517,7 @@ func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView SNATSubnetRoutes: !prefs.NoSNAT(), StatefulFiltering: doStatefulFiltering, NetfilterMode: prefs.NetfilterMode(), - Routes: peerRoutes(b.logf, cfg.Peers, singleRouteThreshold, prefs.RouteAll()), + Routes: b.currentNode().osRoutes(), NetfilterKind: netfilterKind, RemoveCGNATDropRule: nm.HasCap(tailcfg.NodeAttrDisableLinuxCGNATDropRule), } @@ -7286,7 +7290,8 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { if nm != nil { login = cmp.Or(profileFromView(nm.UserProfiles[nm.User()]).LoginName, "") } - b.currentNode().SetNetMap(nm) + discoChanged, routeChanged := b.currentNode().SetNetMap(nm) + b.setDataPlanePeerRoutes() if ms, ok := b.sys.MagicSock.GetOK(); ok { if nm != nil { if nm.Cached { @@ -7298,6 +7303,18 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { ms.SetNetworkMap(tailcfg.NodeView{}, nil) } } + // Reset the WireGuard session for peers whose disco key changed in + // a way that indicates a restart, flushing their dead session keys; + // each such peer is lazily re-created on demand with current state. + for _, k := range discoChanged { + b.e.ResetDevicePeer(k) + } + // Converge the wireguard-go device for peers whose routes changed + // (or that were removed) in the full-netmap resync above; peers not + // in routeChanged are already up to date. + for k := range routeChanged { + b.e.SyncDevicePeer(k) + } if login != b.activeLogin { b.logf("active login: %v", login) b.activeLogin = login @@ -8435,7 +8452,7 @@ func (b *LocalBackend) ResetAuth() error { } func (b *LocalBackend) GetPeerEndpointChanges(ctx context.Context, ip netip.Addr) ([]magicsock.EndpointChange, error) { - pip, ok := b.e.PeerForIP(ip) + pip, ok := b.PeerForIP(ip) if !ok { return nil, fmt.Errorf("no matching peer") } @@ -9171,6 +9188,11 @@ func maybeUsernameOf(actor ipnauth.Actor) string { metricNetmapDeltaPeerPatched = clientmetric.NewCounter("localbackend_netmap_delta_peer_patched") metricUpdatePacketFilter = clientmetric.NewCounter("localbackend_update_packet_filter") metricUpdateUserProfiles = clientmetric.NewCounter("localbackend_update_user_profiles") + + // metricTSMPLearnedKeyMismatch counts netmap updates carrying a peer + // disco key that doesn't match the one previously learned via TSMP + // for the same peer. See [nodeBackend.discoChangedLocked]. + metricTSMPLearnedKeyMismatch = clientmetric.NewCounter("magicsock_tsmp_learned_key_mismatch") ) func (b *LocalBackend) stateEncrypted() opt.Bool { diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index fc08c7c7b..712b3e65b 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -189,135 +189,6 @@ func TestShrinkDefaultRoute(t *testing.T) { } } -func TestPeerRoutes(t *testing.T) { - pp := netip.MustParsePrefix - tests := []struct { - name string - peers []wgcfg.Peer - want []netip.Prefix - }{ - { - name: "small_v4", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("100.101.102.103/32"), - }, - }, - }, - want: []netip.Prefix{ - pp("100.101.102.103/32"), - }, - }, - { - name: "big_v4", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("100.101.102.103/32"), - pp("100.101.102.104/32"), - pp("100.101.102.105/32"), - }, - }, - }, - want: []netip.Prefix{ - pp("100.64.0.0/10"), - }, - }, - { - name: "has_1_v6", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("fd7a:115c:a1e0:ab12:4843:cd96:6258:b240/128"), - }, - }, - }, - want: []netip.Prefix{ - pp("fd7a:115c:a1e0::/48"), - }, - }, - { - name: "has_2_v6", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("fd7a:115c:a1e0:ab12:4843:cd96:6258:b240/128"), - pp("fd7a:115c:a1e0:ab12:4843:cd96:6258:b241/128"), - }, - }, - }, - want: []netip.Prefix{ - pp("fd7a:115c:a1e0::/48"), - }, - }, - { - name: "big_v4_big_v6", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("100.101.102.103/32"), - pp("100.101.102.104/32"), - pp("100.101.102.105/32"), - pp("fd7a:115c:a1e0:ab12:4843:cd96:6258:b240/128"), - pp("fd7a:115c:a1e0:ab12:4843:cd96:6258:b241/128"), - }, - }, - }, - want: []netip.Prefix{ - pp("100.64.0.0/10"), - pp("fd7a:115c:a1e0::/48"), - }, - }, - { - name: "output-should-be-sorted", - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - pp("100.64.0.2/32"), - pp("10.0.0.0/16"), - }, - }, - { - AllowedIPs: []netip.Prefix{ - pp("100.64.0.1/32"), - pp("10.0.0.0/8"), - }, - }, - }, - want: []netip.Prefix{ - pp("10.0.0.0/8"), - pp("10.0.0.0/16"), - pp("100.64.0.1/32"), - pp("100.64.0.2/32"), - }, - }, - { - name: "skip-unmasked-prefixes", - peers: []wgcfg.Peer{ - { - PublicKey: key.NewNode().Public(), - AllowedIPs: []netip.Prefix{ - pp("100.64.0.2/32"), - pp("10.0.0.100/16"), - }, - }, - }, - want: []netip.Prefix{ - pp("100.64.0.2/32"), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := peerRoutes(t.Logf, tt.peers, 2, true) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("got = %v; want %v", got, tt.want) - } - }) - } -} - func TestPeerAPIBase(t *testing.T) { tests := []struct { name string @@ -2752,7 +2623,7 @@ func TestNotifyForSessionUserProfilesDedupResetsOnSelfChange(t *testing.T) { // tests LocalBackend.updateNetmapDeltaLocked func TestUpdateNetmapDelta(t *testing.T) { b := newTestLocalBackend(t) - if b.currentNode().UpdateNetmapDelta(nil) { + if handled, _, _ := b.currentNode().UpdateNetmapDelta(nil); handled { t.Errorf("updateNetmapDeltaLocked() = true, want false with nil netmap") } @@ -2791,7 +2662,7 @@ func TestUpdateNetmapDelta(t *testing.T) { t.Fatal("netmap.MutationsFromMapResponse failed") } - if !b.currentNode().UpdateNetmapDelta(muts) { + if handled, _, _ := b.currentNode().UpdateNetmapDelta(muts); !handled { t.Fatalf("updateNetmapDeltaLocked() = false, want true with new netmap") } @@ -5497,6 +5368,12 @@ func withAddresses(addresses ...netip.Prefix) peerOptFunc { } } +func withAllowedIPs(prefixes ...netip.Prefix) peerOptFunc { + return func(n *tailcfg.Node) { + n.AllowedIPs = append(n.AllowedIPs, prefixes...) + } +} + func deterministicRegionForTest(t testing.TB, want views.Slice[int], use int) selectRegionFunc { t.Helper() @@ -8774,72 +8651,65 @@ func TestStripKeysFromPrefs(t *testing.T) { func TestRouteAllDisabled(t *testing.T) { pp := netip.MustParsePrefix + peer := &tailcfg.Node{ + ID: 1, + Key: key.NewNode().Public(), + HomeDERP: 1, + Addresses: []netip.Prefix{ + pp("100.80.207.38/32"), + }, + AllowedIPs: []netip.Prefix{ + pp("100.80.207.38/32"), + + // If one IP in the Tailscale ULA range is added, the + // entire range is added to the router config. + pp("fd7a:115c:a1e0::2501:9b83/128"), + + // Other single CGNAT IPs (such as VIP service addresses) + // are added individually regardless of RouteAll. + pp("100.80.207.56/32"), + pp("100.80.207.40/32"), + pp("100.94.122.93/32"), + pp("100.79.141.115/32"), + + // A /28 is a subnet route, added only with RouteAll. + pp("100.64.0.0/28"), + + // Single IPs outside the Tailscale CGNAT/ULA ranges are + // subnet routes too. + pp("192.168.0.45/32"), + pp("fd7a:115c:b1e0::2501:9b83/128"), + pp("fdf8:f966:e27c:0:5:0:0:10/128"), + }, + } + tests := []struct { - name string - peers []wgcfg.Peer - wantEndpoints []netip.Prefix - routeAll bool + name string + routeAll bool + want []netip.Prefix }{ { name: "route_all_disabled", routeAll: false, - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - // if one ip in the Tailscale ULA range is added, the entire range is added to the router config - pp("fd7a:115c:a1e0::2501:9b83/128"), - pp("100.80.207.38/32"), - pp("100.80.207.56/32"), - pp("100.80.207.40/32"), - pp("100.94.122.93/32"), - pp("100.79.141.115/32"), - - // a /28 range will not be added, since this is not a Service IP range (which is always /32, a single IP) - pp("100.64.0.0/28"), - - // ips outside the tailscale cgnat/ula range are not added to the router config - pp("192.168.0.45/32"), - pp("fd7a:115c:b1e0::2501:9b83/128"), - pp("fdf8:f966:e27c:0:5:0:0:10/128"), - }, - }, - }, - wantEndpoints: []netip.Prefix{ - pp("100.80.207.38/32"), - pp("100.80.207.56/32"), - pp("100.80.207.40/32"), - pp("100.94.122.93/32"), + want: []netip.Prefix{ pp("100.79.141.115/32"), + pp("100.80.207.38/32"), + pp("100.80.207.40/32"), + pp("100.80.207.56/32"), + pp("100.94.122.93/32"), pp("fd7a:115c:a1e0::/48"), }, }, { name: "route_all_enabled", routeAll: true, - peers: []wgcfg.Peer{ - { - AllowedIPs: []netip.Prefix{ - // if one ip in the Tailscale ULA range is added, the entire range is added to the router config - pp("fd7a:115c:a1e0::2501:9b83/128"), - pp("100.80.207.38/32"), - pp("100.80.207.56/32"), - pp("100.80.207.40/32"), - pp("100.94.122.93/32"), - pp("100.79.141.115/32"), - - // ips outside the tailscale cgnat/ula range are not added to the router config - pp("192.168.0.45/32"), - pp("fd7a:115c:b1e0::2501:9b83/128"), - pp("fdf8:f966:e27c:0:5:0:0:10/128"), - }, - }, - }, - wantEndpoints: []netip.Prefix{ - pp("100.80.207.38/32"), - pp("100.80.207.56/32"), - pp("100.80.207.40/32"), - pp("100.94.122.93/32"), + want: []netip.Prefix{ + pp("100.64.0.0/28"), pp("100.79.141.115/32"), + pp("100.80.207.38/32"), + pp("100.80.207.40/32"), + pp("100.80.207.56/32"), + pp("100.94.122.93/32"), pp("192.168.0.45/32"), pp("fd7a:115c:a1e0::/48"), pp("fd7a:115c:b1e0::2501:9b83/128"), @@ -8852,9 +8722,6 @@ func TestRouteAllDisabled(t *testing.T) { t.Run(tt.name, func(t *testing.T) { prefs := ipn.Prefs{RouteAll: tt.routeAll} lb := newTestLocalBackend(t) - cfg := &wgcfg.Config{ - Peers: tt.peers, - } ServiceIPMappings := tailcfg.ServiceIPMappings{ "svc:test-service": []netip.Addr{ netip.MustParseAddr("100.64.1.2"), @@ -8877,20 +8744,15 @@ func TestRouteAllDisabled(t *testing.T) { }, }, }).View(), + Peers: []tailcfg.NodeView{peer.View()}, } + cn := lb.currentNode() + cn.SetNetMap(nm) + cn.updateRouteManagerPrefs("", tt.routeAll, false) - rcfg := lb.routerConfigLocked(cfg, prefs.View(), nm, false) - for _, p := range rcfg.Routes { - found := false - for _, r := range tt.wantEndpoints { - if p.Addr() == r.Addr() { - found = true - break - } - } - if !found { - t.Errorf("unexpected prefix %q in router config", p.String()) - } + rcfg := lb.routerConfigLocked(&wgcfg.Config{}, prefs.View(), nm) + if !slices.Equal(rcfg.Routes, tt.want) { + t.Errorf("Routes = %v; want %v", rcfg.Routes, tt.want) } }) } @@ -9324,44 +9186,6 @@ func TestShouldUseOneCGNATRoute(t *testing.T) { } -func TestPeerRoutesCGNATCollapse(t *testing.T) { - pp := netip.MustParsePrefix - - // With cgnatThreshold=1 (oneCGNATRoute), adding a peer should not - // change the route list. Both collapse to a single 100.64.0.0/10. - twoPeers := []wgcfg.Peer{ - {AllowedIPs: []netip.Prefix{pp("100.64.0.1/32")}}, - {AllowedIPs: []netip.Prefix{pp("100.64.0.2/32")}}, - } - threePeers := []wgcfg.Peer{ - {AllowedIPs: []netip.Prefix{pp("100.64.0.1/32")}}, - {AllowedIPs: []netip.Prefix{pp("100.64.0.2/32")}}, - {AllowedIPs: []netip.Prefix{pp("100.64.0.3/32")}}, - } - - routesTwo := peerRoutes(t.Logf, twoPeers, 1, true) - routesThree := peerRoutes(t.Logf, threePeers, 1, true) - - wantCGNAT := []netip.Prefix{pp("100.64.0.0/10")} - if !reflect.DeepEqual(routesTwo, wantCGNAT) { - t.Errorf("two peers: got %v; want %v", routesTwo, wantCGNAT) - } - if !reflect.DeepEqual(routesThree, wantCGNAT) { - t.Errorf("three peers: got %v; want %v", routesThree, wantCGNAT) - } - - // Subnet routes must still appear alongside the collapsed CGNAT route. - peersWithSubnet := []wgcfg.Peer{ - {AllowedIPs: []netip.Prefix{pp("100.64.0.1/32")}}, - {AllowedIPs: []netip.Prefix{pp("100.64.0.2/32"), pp("10.0.0.0/24")}}, - } - got := peerRoutes(t.Logf, peersWithSubnet, 1, true) - want := []netip.Prefix{pp("100.64.0.0/10"), pp("10.0.0.0/24")} - if !reflect.DeepEqual(got, want) { - t.Errorf("with subnet: got %v; want %v", got, want) - } -} - func TestResetAuthClearsMachineKey(t *testing.T) { store := new(mem.Store) @@ -9489,13 +9313,13 @@ func TestEnginePeerForIPAdjustsForPrefs(t *testing.T) { nm := buildNetmapWithPeers(selfNode, exitA, exitB, subnetBig, subnetSmall) - var eng wgengine.Engine + var curLB *LocalBackend var curT *testing.T // active subtest, for test helpers wantPeer := func(ip string, n tailcfg.NodeView) { t := curT t.Helper() - pip, ok := eng.PeerForIP(netip.MustParseAddr(ip)) + pip, ok := curLB.PeerForIP(netip.MustParseAddr(ip)) if !ok { t.Fatalf("PeerForIP(%s): ok=false, want true", ip) } @@ -9509,32 +9333,32 @@ func TestEnginePeerForIPAdjustsForPrefs(t *testing.T) { wantNotPeer := func(ip string) { t := curT t.Helper() - if _, ok := eng.PeerForIP(netip.MustParseAddr(ip)); ok { + if _, ok := curLB.PeerForIP(netip.MustParseAddr(ip)); ok { t.Fatalf("PeerForIP(%s): ok=true, want false", ip) } } wantKey := func(ip string, n tailcfg.NodeView) { t := curT t.Helper() - pk, _, ok := eng.PeerKeyForIP(netip.MustParseAddr(ip)) + pr, ok := curLB.currentNode().routeMgr.Outbound().Lookup(netip.MustParseAddr(ip)) if !ok { - t.Fatalf("PeerKeyForIP(%s): ok=false, want true", ip) + t.Fatalf("routeMgr.Outbound().Lookup(%s): ok=false, want true", ip) } - if pk != n.Key() { - t.Fatalf("PeerKeyForIP(%s): key=%v, want %v", ip, pk, n.Key()) + if pr.Key != n.Key() { + t.Fatalf("routeMgr.Outbound().Lookup(%s): key=%v, want %v", ip, pr.Key, n.Key()) } } wantNotKey := func(ip string) { t := curT t.Helper() - if _, _, ok := eng.PeerKeyForIP(netip.MustParseAddr(ip)); ok { - t.Fatalf("PeerKeyForIP(%s): ok=true, want false", ip) + if _, ok := curLB.currentNode().routeMgr.Outbound().Lookup(netip.MustParseAddr(ip)); ok { + t.Fatalf("routeMgr.Outbound().Lookup(%s): ok=true, want false", ip) } } wantSelf := func(ip string) { t := curT t.Helper() - pip, ok := eng.PeerForIP(netip.MustParseAddr(ip)) + pip, ok := curLB.PeerForIP(netip.MustParseAddr(ip)) if !ok { t.Fatalf("PeerForIP(%s): ok=false, want true", ip) } @@ -9663,7 +9487,7 @@ func TestEnginePeerForIPAdjustsForPrefs(t *testing.T) { LoggedIn: true, }) - eng = lb.sys.Engine.Get() + curLB = lb curT = t tt.check() }) diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index e703f28ed..90ed514d5 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -19,6 +19,7 @@ "tailscale.com/ipn" "tailscale.com/net/dns" "tailscale.com/net/routecheck/peernode" + "tailscale.com/net/routemanager" "tailscale.com/net/tsaddr" "tailscale.com/syncs" "tailscale.com/tailcfg" @@ -158,6 +159,21 @@ type nodeBackend struct { // [nodeBackend.AwaitNodeKeyForTest]. It is populated lazily and remains // nil in production, where no test installs a waiter. keyWaitersForTest map[key.NodePublic]chan struct{} + + // tsmpLearnedDisco records, per node key, a peer disco key that was + // learned via TSMP (that is, over an existing WireGuard session with + // that peer). When a netmap update later reports the same disco key + // change, the peer's WireGuard session does not need to be reset, + // because the change demonstrably arrived over a working session. + // See [nodeBackend.discoChangedLocked]. + tsmpLearnedDisco map[key.NodePublic]key.DiscoPublic + + // routeMgr tracks this node's view of which IPs route to which + // peers and publishes lock-free snapshots for the data plane and + // the OS router. It is initialized once and immutable, but its + // Begin/Commit mutations must be serialized, which we do by only + // mutating it with mu held. + routeMgr *routemanager.RouteManager } func newNodeBackend(ctx context.Context, logf logger.Logf, bus *eventbus.Bus) *nodeBackend { @@ -168,6 +184,7 @@ func newNodeBackend(ctx context.Context, logf logger.Logf, bus *eventbus.Bus) *n ctxCancel: ctxCancel, eventClient: bus.Client("ipnlocal.nodeBackend"), readyCh: make(chan struct{}), + routeMgr: routemanager.New(logf), } // Default filter blocks everything and logs nothing. noneFilter := filter.NewAllowNone(logger.Discard, &netipx.IPSet{}) @@ -252,6 +269,22 @@ func (nb *nodeBackend) NodeByKey(k key.NodePublic) (_ tailcfg.NodeID, ok bool) { return nid, ok } +// PeerAllowedIPs returns the prefixes from which the peer with the +// given public key is currently allowed to originate traffic, or +// ok=false if the key is unknown or the peer currently contributes no +// prefixes. +func (nb *nodeBackend) PeerAllowedIPs(k key.NodePublic) ([]netip.Prefix, bool) { + nb.mu.Lock() + defer nb.mu.Unlock() + id, ok := nb.nodeByKey[k] + if !ok { + return nil, false + } + // Holding nb.mu satisfies routeMgr's serialization requirement: + // all routeMgr mutations also run under nb.mu. + return nb.routeMgr.PeerAllowedIPs(id) +} + // NodeByWireGuardString returns the node ID of the peer whose // [key.NodePublic.WireGuardGoString] form is s (e.g. "peer(IMTB…r7lM)"). // ok is false if no current peer matches. @@ -549,14 +582,14 @@ func (nb *nodeBackend) netMapWithPeers() *netmap.NetworkMap { return nm } -func (nb *nodeBackend) SetNetMap(nm *netmap.NetworkMap) { +func (nb *nodeBackend) SetNetMap(nm *netmap.NetworkMap) (discoChanged []key.NodePublic, routeChanged map[key.NodePublic][]netip.Prefix) { nb.mu.Lock() defer nb.mu.Unlock() nb.netMap = nm nb.updateNodeByAddrLocked() nb.updateNodeByKeyLocked() nb.updateNodeByNameLocked() - nb.updatePeersLocked() + discoChanged, routeChanged = nb.updatePeersLocked() nb.signalKeyWaitersForTestLocked() if nm != nil { nb.userProfiles = maps.Clone(nm.UserProfiles) @@ -569,6 +602,7 @@ func (nb *nodeBackend) SetNetMap(nm *netmap.NetworkMap) { nb.packetFilter = nil nb.derpMapViewPub.Publish(tailcfg.DERPMapView{}) } + return discoChanged, routeChanged } // AwaitNodeKeyForTest returns a channel that is closed once a peer with the @@ -735,18 +769,161 @@ func (nb *nodeBackend) ExtraDNSByName(hostname string) (_ netip.Addr, ok bool) { return ip, ok } -func (nb *nodeBackend) updatePeersLocked() { +func (nb *nodeBackend) updatePeersLocked() (discoChanged []key.NodePublic, routeChanged map[key.NodePublic][]netip.Prefix) { nm := nb.netMap - if nm == nil { - nb.peers = nil - return + oldIDs := slices.Collect(maps.Keys(nb.peers)) + + // Snapshot the previous disco keys (by node key) before the peers + // map is repopulated, to detect restarted peers below. + var prevDisco map[key.NodePublic]key.DiscoPublic + for _, p := range nb.peers { + if !p.DiscoKey().IsZero() { + mak.Set(&prevDisco, p.Key(), p.DiscoKey()) + } } - mapx.RepopulateNonzero(&nb.peers, func() { - for _, p := range nm.Peers { - nb.peers[p.ID()] = p + if nm == nil { + nb.peers = nil + } else { + mapx.RepopulateNonzero(&nb.peers, func() { + for _, p := range nm.Peers { + nb.peers[p.ID()] = p + } + }) + } + + for _, p := range nb.peers { + if prev, ok := prevDisco[p.Key()]; ok && nb.discoChangedLocked(p.Key(), prev, p.DiscoKey()) { + discoChanged = append(discoChanged, p.Key()) } - }) + } + // Drop TSMP-learned disco keys for peers no longer in the netmap. + for k := range nb.tsmpLearnedDisco { + if _, ok := nb.nodeByKey[k]; !ok { + delete(nb.tsmpLearnedDisco, k) + } + } + + // Resync the route manager to the new peer set. Upserts of + // unchanged peers are cheap no-ops; this is a full-netmap event, + // so O(n) work is expected here. + rt := nb.routeMgr.Begin() + for _, id := range oldIDs { + if _, ok := nb.peers[id]; !ok { + rt.RemovePeer(id) + } + } + for _, p := range nb.peers { + rt.UpsertPeer(p) + } + res := rt.Commit() + return discoChanged, res.AllowedIPs +} + +// recordTSMPLearnedDisco notes that a peer's new disco key was learned via +// TSMP, so the netmap update carrying the same change need not reset the +// peer's WireGuard session. See the [nodeBackend.tsmpLearnedDisco] field doc. +func (nb *nodeBackend) recordTSMPLearnedDisco(pub key.NodePublic, disco key.DiscoPublic) { + nb.mu.Lock() + defer nb.mu.Unlock() + mak.Set(&nb.tsmpLearnedDisco, pub, disco) +} + +// discoChangedLocked reports whether a peer's disco key change from prev to +// cur should reset the peer's WireGuard session. A changed disco key means +// the peer restarted, so any existing session key material is dead weight; +// resetting lets the handshake start over immediately. The exception is a +// key change already learned via TSMP: that arrived over a working WireGuard +// session with the peer, so the session is demonstrably fine and is kept. +// +// It consumes any [nodeBackend.tsmpLearnedDisco] entry for pub. +// nb.mu must be held. +func (nb *nodeBackend) discoChangedLocked(pub key.NodePublic, prev, cur key.DiscoPublic) bool { + if prev.IsZero() || cur.IsZero() || prev == cur { + return false + } + if discoTSMP, ok := nb.tsmpLearnedDisco[pub]; ok { + delete(nb.tsmpLearnedDisco, pub) + if discoTSMP == cur { + nb.logf("nodeBackend: skipping WireGuard session reset (TSMP key): %s changed from %q to %q", + pub.ShortString(), prev, cur) + return false + } + // The new disco key does not match what we received via + // TSMP for this peer. This is unexpected, though possible + // if processing a change in a large netmap ends up taking + // longer than the 2 second timeout in + // [controlclient.mapRoutineState.UpdateNetmapDelta], or if + // the context is cancelled mid update. Log the event, and reset + // the session as it is possibly a stale entry in the map + // instead of a TSMP disco key update that led us here. + nb.logf("nodeBackend: [unexpected] using TSMP key for %s (control stale): tsmp=%q control=%q old=%q", + pub.ShortString(), discoTSMP, cur, prev) + metricTSMPLearnedKeyMismatch.Add(1) + } + nb.logf("nodeBackend: peer %s disco key changed from %q to %q", pub.ShortString(), prev, cur) + return true +} + +// updateRouteManagerPrefs pushes the routing-relevant prefs and the +// OneCGNAT decision into the route manager. The exit node arrives as +// a stable ID from prefs and is resolved to the current numeric node +// ID here; it resolves to zero (no exit node) if that peer is not in +// the netmap yet, in which case a later netmap update re-runs this. +// +// It returns the peers whose allowed source prefixes changed as a +// result (for example the old and new exit node when the selection +// changes), as described by [routemanager.Result.AllowedIPs]. +func (nb *nodeBackend) updateRouteManagerPrefs(exitNodeID tailcfg.StableNodeID, routeAll, oneCGNAT bool) (changedAllowedIPs map[key.NodePublic][]netip.Prefix) { + nb.mu.Lock() + defer nb.mu.Unlock() + var exitID tailcfg.NodeID + if !exitNodeID.IsZero() { + for id, p := range nb.peers { + if p.StableID() == exitNodeID { + exitID = id + break + } + } + } + rt := nb.routeMgr.Begin() + rt.SetPrefs(routemanager.Prefs{ExitNodeID: exitID, RouteAll: routeAll}) + rt.SetTailnetConfig(routemanager.TailnetConfig{OneCGNAT: oneCGNAT}) + res := rt.Commit() + return res.AllowedIPs +} + +// updateRouteManagerExtras pushes each peer's extra WireGuard-only +// allowed IPs into the route manager, obtained by calling fn (the +// [ipnext.Hooks.ExtraWireGuardAllowedIPs] hook) with each peer's +// public key. +// +// It returns the peers whose allowed source prefixes changed as a +// result, as described by [routemanager.Result.AllowedIPs]. +func (nb *nodeBackend) updateRouteManagerExtras(fn func(key.NodePublic) views.Slice[netip.Prefix]) (changedAllowedIPs map[key.NodePublic][]netip.Prefix) { + nb.mu.Lock() + defer nb.mu.Unlock() + var extras map[tailcfg.NodeID][]netip.Prefix + for id, p := range nb.peers { + if pfxs := fn(p.Key()); pfxs.Len() > 0 { + mak.Set(&extras, id, pfxs.AsSlice()) + } + } + rt := nb.routeMgr.Begin() + rt.SetExtraAllowedIPs(extras) + res := rt.Commit() + return res.AllowedIPs +} + +// osRoutes returns the sorted set of prefixes that the route manager +// wants programmed into the OS routing table. +func (nb *nodeBackend) osRoutes() []netip.Prefix { + var routes []netip.Prefix + for pfx := range nb.routeMgr.OSRoutes().All() { + routes = append(routes, pfx) + } + tsaddr.SortPrefixes(routes) + return routes } // setPacketFilter stores the live packet filter rules and parsed @@ -778,11 +955,18 @@ func (nb *nodeBackend) mergeUserProfiles(profiles map[tailcfg.UserID]tailcfg.Use } } -func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { +// UpdateNetmapDelta applies the given netmap mutations to the live +// peer state. It reports whether it handled all of them, and returns +// the peers whose allowed source prefixes changed (as described by +// [routemanager.Result.AllowedIPs]) so the caller can sync those +// peers to the WireGuard device, plus the peers whose disco key +// changed in a way that requires a WireGuard session reset (see +// [nodeBackend.discoChangedLocked]). +func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool, changedAllowedIPs map[key.NodePublic][]netip.Prefix, discoChanged []key.NodePublic) { nb.mu.Lock() defer nb.mu.Unlock() if nb.netMap == nil { - return false + return false, nil, nil } // Locally cloned mutable nodes, to avoid calling AsStruct (clone) @@ -790,10 +974,29 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // call (e.g. its endpoints + online status both change) var mutableNodes map[tailcfg.NodeID]*tailcfg.Node + // Mirror peer add/remove mutations into the route manager. Committing + // on all return paths (including the not-handled one) matches the + // peers map above, which is also mutated in place as the loop runs; + // the full-netmap fallback then resyncs both the same way. + rt := nb.routeMgr.Begin() + defer func() { + res := rt.Commit() + changedAllowedIPs = res.AllowedIPs + }() + for _, m := range muts { switch m := m.(type) { case netmap.NodeMutationUpsert: nid := m.Node.ID() + if old, ok := nb.peers[nid]; ok { + if old.Key() == m.Node.Key() { + if nb.discoChangedLocked(m.Node.Key(), old.DiscoKey(), m.Node.DiscoKey()) { + discoChanged = append(discoChanged, m.Node.Key()) + } + } else { + delete(nb.tsmpLearnedDisco, old.Key()) + } + } mak.Set(&nb.peers, nid, m.Node) for _, ipp := range m.Node.Addresses().All() { if ipp.IsSingleIP() { @@ -803,6 +1006,7 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo mak.Set(&nb.nodeByKey, m.Node.Key(), nid) mak.Set(&nb.nodeByWGString, m.Node.Key().WireGuardGoString(), nid) nb.addNodeNameLocked(m.Node.Name(), nid) + rt.UpsertPeer(m.Node) continue case netmap.NodeMutationRemove: nid := m.NodeIDBeingMutated() @@ -814,8 +1018,10 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo } delete(nb.nodeByKey, old.Key()) delete(nb.nodeByWGString, old.Key().WireGuardGoString()) + delete(nb.tsmpLearnedDisco, old.Key()) nb.removeNodeNameLocked(old.Name()) delete(nb.peers, nid) + rt.RemovePeer(nid) } continue } @@ -826,7 +1032,7 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo nv, ok := nb.peers[nid] if !ok { // TODO(bradfitz): unexpected metric? - return false + return false, nil, discoChanged } n = nv.AsStruct() mak.Set(&mutableNodes, nv.ID(), n) @@ -837,7 +1043,7 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo nb.peers[nid] = n.View() } nb.signalKeyWaitersForTestLocked() - return true + return true, nil, discoChanged } // unlockedNodesPermitted reports whether any peer with theUnsignedPeerAPIOnly bool set true has any of its allowed IPs diff --git a/ipn/ipnlocal/node_backend_test.go b/ipn/ipnlocal/node_backend_test.go index 657aadb14..8106bd8e7 100644 --- a/ipn/ipnlocal/node_backend_test.go +++ b/ipn/ipnlocal/node_backend_test.go @@ -7,6 +7,7 @@ "context" "errors" "maps" + "net/netip" "slices" "testing" "time" @@ -14,7 +15,9 @@ "tailscale.com/net/routecheck/peernode" "tailscale.com/tailcfg" "tailscale.com/tstest" + "tailscale.com/types/key" "tailscale.com/types/netmap" + "tailscale.com/types/views" "tailscale.com/util/eventbus" "tailscale.com/util/mak" "tailscale.com/util/set" @@ -271,3 +274,273 @@ func TestNodeBackendReachability(t *testing.T) { func (rp routecheckReport) IsReachable(_ tailcfg.NodeID) peernode.Reachability { return peernode.Reachability(rp) } + +func TestNodeBackendRouteManager(t *testing.T) { + nb := newNodeBackend(t.Context(), tstest.WhileTestRunningLogger(t), eventbus.New()) + + mkPeer := func(id tailcfg.NodeID, stableID tailcfg.StableNodeID, addr4 string, extra ...string) tailcfg.NodeView { + n := &tailcfg.Node{ + ID: id, + StableID: stableID, + Key: key.NewNode().Public(), + HomeDERP: 1, // required by the route manager's reachability filter + Addresses: []netip.Prefix{ + netip.MustParsePrefix(addr4), + }, + } + n.AllowedIPs = append(n.AllowedIPs, n.Addresses...) + for _, s := range extra { + n.AllowedIPs = append(n.AllowedIPs, netip.MustParsePrefix(s)) + } + return n.View() + } + wantPeerFor := func(ip string, want tailcfg.NodeView) { + t.Helper() + got, ok := nb.routeMgr.Outbound().Lookup(netip.MustParseAddr(ip)) + if !want.Valid() { + if ok { + t.Errorf("Outbound lookup %s = %v; want no match", ip, got) + } + return + } + if !ok || got.Key != want.Key() { + t.Errorf("Outbound lookup %s = %v, %v; want %v", ip, got, ok, want.Key()) + } + } + + p1 := mkPeer(1, "stable1", "100.64.0.1/32") + p2 := mkPeer(2, "stable2", "100.64.0.2/32", "0.0.0.0/0", "::/0") + + // A full netmap populates the route manager. + nb.SetNetMap(&netmap.NetworkMap{Peers: []tailcfg.NodeView{p1, p2}}) + wantPeerFor("100.64.0.1", p1) + wantPeerFor("100.64.0.2", p2) + wantPeerFor("8.8.8.8", tailcfg.NodeView{}) // exit node not selected + + // Selecting peer 2 as the exit node resolves its stable ID and + // installs its /0 routes. The commit reports peer 2's allowed + // prefixes as changed. + if changed := nb.updateRouteManagerPrefs("stable2", false, false); len(changed) != 1 || changed[p2.Key()] == nil { + t.Errorf("updateRouteManagerPrefs(exit=stable2) changed = %v; want just %v", changed, p2.Key()) + } + wantPeerFor("8.8.8.8", p2) + if changed := nb.updateRouteManagerPrefs("", false, false); len(changed) != 1 || changed[p2.Key()] == nil { + t.Errorf("updateRouteManagerPrefs(exit=none) changed = %v; want just %v", changed, p2.Key()) + } + wantPeerFor("8.8.8.8", tailcfg.NodeView{}) + + // Incremental deltas: add peer 3, remove peer 1. + p3 := mkPeer(3, "stable3", "100.64.0.3/32") + handled, changed, _ := nb.UpdateNetmapDelta([]netmap.NodeMutation{ + netmap.NodeMutationUpsert{Node: p3}, + netmap.MakeNodeMutationRemove(1), + }) + if !handled { + t.Fatal("UpdateNetmapDelta not handled") + } + if len(changed) != 2 || changed[p3.Key()] == nil { + t.Errorf("UpdateNetmapDelta changed = %v; want entries for %v and %v", changed, p3.Key(), p1.Key()) + } + if v, ok := changed[p1.Key()]; !ok || v != nil { + t.Errorf("UpdateNetmapDelta changed[%v] = %v, %v; want nil, true for removed peer", p1.Key(), v, ok) + } + wantPeerFor("100.64.0.3", p3) + wantPeerFor("100.64.0.1", tailcfg.NodeView{}) + + // A full netmap that drops a peer removes it from the route manager. + nb.SetNetMap(&netmap.NetworkMap{Peers: []tailcfg.NodeView{p2}}) + wantPeerFor("100.64.0.3", tailcfg.NodeView{}) + wantPeerFor("100.64.0.2", p2) +} + +func TestNodeBackendRouteManagerExtras(t *testing.T) { + nb := newNodeBackend(t.Context(), tstest.WhileTestRunningLogger(t), eventbus.New()) + + n := &tailcfg.Node{ + ID: 1, + Key: key.NewNode().Public(), + HomeDERP: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("100.64.0.1/32"), + }, + } + n.AllowedIPs = n.Addresses + p1 := n.View() + nb.SetNetMap(&netmap.NetworkMap{Peers: []tailcfg.NodeView{p1}}) + + transit := netip.MustParsePrefix("fe80::1234/128") + extrasFor := func(k key.NodePublic) views.Slice[netip.Prefix] { + if k == p1.Key() { + return views.SliceOf([]netip.Prefix{transit}) + } + return views.Slice[netip.Prefix]{} + } + + // Installing extras reports the peer's allowed prefixes as + // changed and adds the transit IP to the outbound table. + changed := nb.updateRouteManagerExtras(extrasFor) + if len(changed) != 1 || !slices.Contains(changed[p1.Key()], transit) { + t.Errorf("updateRouteManagerExtras changed = %v; want %v including %v", changed, p1.Key(), transit) + } + if pr, ok := nb.routeMgr.Outbound().Lookup(transit.Addr()); !ok || pr.Key != p1.Key() { + t.Errorf("Outbound lookup %v = %v, %v; want %v", transit.Addr(), pr, ok, p1.Key()) + } + if _, ok := nb.routeMgr.OSRoutes().Get(transit); ok { + t.Errorf("OSRoutes contains %v; extras must not reach the OS route set", transit) + } + + // An unchanged hook result is a no-op. + if changed := nb.updateRouteManagerExtras(extrasFor); changed != nil { + t.Errorf("unchanged extras reported changes: %v", changed) + } + + // A hook that no longer returns extras removes them. + changed = nb.updateRouteManagerExtras(func(key.NodePublic) views.Slice[netip.Prefix] { + return views.Slice[netip.Prefix]{} + }) + if len(changed) != 1 { + t.Errorf("clearing extras changed = %v; want just %v", changed, p1.Key()) + } + if _, ok := nb.routeMgr.Outbound().Lookup(transit.Addr()); ok { + t.Errorf("Outbound still routes %v after extras cleared", transit.Addr()) + } +} + +// TestNodeBackendDiscoChanged exercises the full-netmap disco change +// detection: a peer whose disco key changes has restarted and needs its +// WireGuard session reset, unless the new key was already learned over +// TSMP (that is, over a working WireGuard session with the peer). +func TestNodeBackendDiscoChanged(t *testing.T) { + nb := newNodeBackend(t.Context(), tstest.WhileTestRunningLogger(t), eventbus.New()) + + nk := key.NewNode().Public() + mkNetMap := func(disco key.DiscoPublic) *netmap.NetworkMap { + n := &tailcfg.Node{ + ID: 1, + Key: nk, + DiscoKey: disco, + HomeDERP: 1, + } + return &netmap.NetworkMap{Peers: []tailcfg.NodeView{n.View()}} + } + newDisco := func() key.DiscoPublic { return key.NewDisco().Public() } + + // A brand-new peer is not a disco change. + d1 := newDisco() + if got, _ := nb.SetNetMap(mkNetMap(d1)); len(got) != 0 { + t.Errorf("SetNetMap(new peer) discoChanged = %v; want none", got) + } + + // A changed disco key requires a session reset. + d2 := newDisco() + if got, _ := nb.SetNetMap(mkNetMap(d2)); !slices.Contains(got, nk) { + t.Errorf("SetNetMap(changed disco) discoChanged = %v; want %v", got, nk) + } + + // An unchanged disco key does not. + if got, _ := nb.SetNetMap(mkNetMap(d2)); len(got) != 0 { + t.Errorf("SetNetMap(same disco) discoChanged = %v; want none", got) + } + + // A change already learned via TSMP is suppressed... + d3 := newDisco() + nb.recordTSMPLearnedDisco(nk, d3) + if got, _ := nb.SetNetMap(mkNetMap(d3)); len(got) != 0 { + t.Errorf("SetNetMap(TSMP-learned disco) discoChanged = %v; want none", got) + } + + // ...but the TSMP entry is consumed, so the next change resets again. + d4 := newDisco() + if got, _ := nb.SetNetMap(mkNetMap(d4)); !slices.Contains(got, nk) { + t.Errorf("SetNetMap(after TSMP entry consumed) discoChanged = %v; want %v", got, nk) + } + + // A TSMP-learned key that doesn't match the netmap's new key still + // resets the session and bumps the mismatch metric. + before := metricTSMPLearnedKeyMismatch.Value() + nb.recordTSMPLearnedDisco(nk, newDisco()) + d5 := newDisco() + if got, _ := nb.SetNetMap(mkNetMap(d5)); !slices.Contains(got, nk) { + t.Errorf("SetNetMap(TSMP mismatch) discoChanged = %v; want %v", got, nk) + } + if delta := metricTSMPLearnedKeyMismatch.Value() - before; delta != 1 { + t.Errorf("metricTSMPLearnedKeyMismatch delta = %d; want 1", delta) + } + + // Removing the peer garbage-collects its TSMP entry: after the peer + // comes back, a change to the once-recorded key is a normal reset. + d6 := newDisco() + nb.recordTSMPLearnedDisco(nk, d6) + nb.SetNetMap(&netmap.NetworkMap{}) + nb.SetNetMap(mkNetMap(d5)) + if got, _ := nb.SetNetMap(mkNetMap(d6)); !slices.Contains(got, nk) { + t.Errorf("SetNetMap(after TSMP entry GC) discoChanged = %v; want %v", got, nk) + } + + // Transitions to or from a zero disco key never reset. + if got, _ := nb.SetNetMap(mkNetMap(key.DiscoPublic{})); len(got) != 0 { + t.Errorf("SetNetMap(to zero disco) discoChanged = %v; want none", got) + } + if got, _ := nb.SetNetMap(mkNetMap(d1)); len(got) != 0 { + t.Errorf("SetNetMap(from zero disco) discoChanged = %v; want none", got) + } +} + +// TestNodeBackendDiscoChangedDelta is like TestNodeBackendDiscoChanged +// but for the incremental path: disco changes arriving as +// [netmap.NodeMutationUpsert] deltas. +func TestNodeBackendDiscoChangedDelta(t *testing.T) { + nb := newNodeBackend(t.Context(), tstest.WhileTestRunningLogger(t), eventbus.New()) + + mkNode := func(k key.NodePublic, disco key.DiscoPublic) tailcfg.NodeView { + return (&tailcfg.Node{ID: 1, Key: k, DiscoKey: disco, HomeDERP: 1}).View() + } + newDisco := func() key.DiscoPublic { return key.NewDisco().Public() } + apply := func(muts ...netmap.NodeMutation) []key.NodePublic { + t.Helper() + handled, _, discoChanged := nb.UpdateNetmapDelta(muts) + if !handled { + t.Fatal("UpdateNetmapDelta not handled") + } + return discoChanged + } + + nk := key.NewNode().Public() + d1 := newDisco() + nb.SetNetMap(&netmap.NetworkMap{Peers: []tailcfg.NodeView{mkNode(nk, d1)}}) + + // An upserted peer with a changed disco key needs a session reset. + d2 := newDisco() + if got := apply(netmap.NodeMutationUpsert{Node: mkNode(nk, d2)}); !slices.Contains(got, nk) { + t.Errorf("upsert(changed disco) discoChanged = %v; want %v", got, nk) + } + + // An unchanged disco key does not. + if got := apply(netmap.NodeMutationUpsert{Node: mkNode(nk, d2)}); len(got) != 0 { + t.Errorf("upsert(same disco) discoChanged = %v; want none", got) + } + + // A change already learned via TSMP is suppressed. + d3 := newDisco() + nb.recordTSMPLearnedDisco(nk, d3) + if got := apply(netmap.NodeMutationUpsert{Node: mkNode(nk, d3)}); len(got) != 0 { + t.Errorf("upsert(TSMP-learned disco) discoChanged = %v; want none", got) + } + + // A node key rotation replaces the WireGuard peer outright, so no + // disco-based reset is reported. + nk2 := key.NewNode().Public() + if got := apply(netmap.NodeMutationUpsert{Node: mkNode(nk2, newDisco())}); len(got) != 0 { + t.Errorf("upsert(rotated node key) discoChanged = %v; want none", got) + } + + // Removing the peer garbage-collects its TSMP entry: after the peer + // comes back, a change to the once-recorded key is a normal reset. + d4 := newDisco() + nb.recordTSMPLearnedDisco(nk2, d4) + apply(netmap.MakeNodeMutationRemove(1)) + apply(netmap.NodeMutationUpsert{Node: mkNode(nk2, newDisco())}) + if got := apply(netmap.NodeMutationUpsert{Node: mkNode(nk2, d4)}); !slices.Contains(got, nk2) { + t.Errorf("upsert(after TSMP entry GC) discoChanged = %v; want %v", got, nk2) + } +} diff --git a/ipn/ipnlocal/resolve.go b/ipn/ipnlocal/resolve.go index debc6e09d..c872b7e6d 100644 --- a/ipn/ipnlocal/resolve.go +++ b/ipn/ipnlocal/resolve.go @@ -13,25 +13,38 @@ "tailscale.com/wgengine" ) -// lookupPeerByIP returns the node public key for the peer that owns the -// given IP address. It is the fast path for [Engine.SetPeerByIPPacketFunc], -// handling exact-IP matches against node addresses; subnet routes and exit -// nodes are handled by a BART-based fallback in userspaceEngine that uses -// the wireguard-filtered peer list (see lastCfgFull). +// lookupPeerByIP returns the node public key for the peer that should +// handle traffic to the given IP address. It is installed as the +// [wgengine.Engine.SetPeerByIPPacketFunc] callback: exact node +// addresses hit the nodeByAddr fast path, and subnet routes and +// exit-node default routes fall back to the RouteManager's outbound +// table, so it stays correct under incremental netmap deltas. // -// It is called by wireguard-go on every outbound packet (not cached), so -// it must be fast. +// It is called by wireguard-go on every outbound packet (not cached), +// so it must be fast. func (b *LocalBackend) lookupPeerByIP(ip netip.Addr) (key.NodePublic, bool) { nb := b.currentNode() - nid, ok := nb.NodeByAddr(ip) - if !ok { - return key.NodePublic{}, false + if nid, ok := nb.NodeByAddr(ip); ok { + peer, ok := nb.NodeByID(nid) + if !ok { + return key.NodePublic{}, false + } + return peer.Key(), true } - peer, ok := nb.NodeByID(nid) - if !ok { - return key.NodePublic{}, false + if pr, ok := nb.routeMgr.Outbound().Lookup(ip); ok { + return pr.Key, true } - return peer.Key(), true + return key.NodePublic{}, false +} + +// peerAllowedIPs returns the prefixes from which the peer with the +// given public key is currently allowed to originate traffic, or +// ok=false if the peer is unknown (or currently routable via no +// prefix at all). It is installed as the +// [wgengine.Engine.SetPeerConfigFunc] callback, backing wireguard-go's +// lazy peer creation and per-delta peer sync. +func (b *LocalBackend) peerAllowedIPs(k key.NodePublic) ([]netip.Prefix, bool) { + return b.currentNode().PeerAllowedIPs(k) } // resolveMagicDNS resolves a MagicDNS hostname to the owning node's IP @@ -105,13 +118,15 @@ func addrFamilyMatch(ip netip.Addr, network string) bool { return true } -// peerForIP returns which peer is responsible for a given IP address. +// PeerForIP returns which peer is responsible for a given IP address. // Despite the name, it can also return the self node (with IsSelf set). // It handles both Tailscale IPs (returning the owning peer or self) and // non-Tailscale addresses like subnet-routed IPs or exit-node global // internet IPs (returning whichever peer would route that traffic). -// It is installed as the [wgengine.Engine.SetPeerForIPFunc] callback. -func (b *LocalBackend) peerForIP(ip netip.Addr) (_ wgengine.PeerForIP, ok bool) { +// It is installed as the [wgengine.Engine.SetPeerForIPFunc] callback, +// serving the engine's internal cold-path lookups (Ping, TSMP, pendopen +// diagnostics). +func (b *LocalBackend) PeerForIP(ip netip.Addr) (_ wgengine.PeerForIP, ok bool) { nb := b.currentNode() if tsaddr.IsTailscaleIP(ip) { @@ -130,11 +145,11 @@ func (b *LocalBackend) peerForIP(ip netip.Addr) (_ wgengine.PeerForIP, ok bool) } } - pk, route, ok := b.e.PeerKeyForIP(ip) + route, pr, ok := nb.routeMgr.Outbound().LookupPrefixLPM(netip.PrefixFrom(ip, ip.BitLen())) if !ok { return wgengine.PeerForIP{}, false } - nid, ok := nb.NodeByKey(pk) + nid, ok := nb.NodeByKey(pr.Key) if !ok { return wgengine.PeerForIP{}, false } diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index 7ce4f590c..48243767f 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -17,6 +17,7 @@ "time" qt "github.com/frankban/quicktest" + "github.com/gaissmai/bart" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -29,6 +30,7 @@ "tailscale.com/net/dns" "tailscale.com/net/netmon" "tailscale.com/net/packet" + "tailscale.com/net/routemanager" "tailscale.com/net/tsdial" "tailscale.com/tailcfg" "tailscale.com/tsd" @@ -1205,10 +1207,10 @@ func TestEngineReconfigOnStateChange(t *testing.T) { connect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: true}, WantRunningSet: true} disconnect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: false}, WantRunningSet: true} node1 := buildNetmapWithPeers( - makePeer(1, withName("node-1"), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))), + makePeer(1, withName("node-1"), withAddresses(netip.MustParsePrefix("100.64.1.1/32")), withAllowedIPs(netip.MustParsePrefix("100.64.1.1/32"))), ) node2 := buildNetmapWithPeers( - makePeer(2, withName("node-2"), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))), + makePeer(2, withName("node-2"), withAddresses(netip.MustParsePrefix("100.64.1.2/32")), withAllowedIPs(netip.MustParsePrefix("100.64.1.2/32"))), ) node3 := buildNetmapWithPeers( makePeer(3, withName("node-3"), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))), @@ -1241,6 +1243,7 @@ func TestEngineReconfigOnStateChange(t *testing.T) { steps func(*testing.T, *LocalBackend, func() *mockControl) wantState ipn.State wantCfg *wgcfg.Config + wantPeers []key.NodePublic wantRouterCfg *router.Config wantDNSCfg *dns.Config }{ @@ -1285,7 +1288,6 @@ func TestEngineReconfigOnStateChange(t *testing.T) { // After the auth is completed, the configs must be updated to reflect the node's netmap. wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{}, Addresses: node1.SelfNode.Addresses().AsSlice(), }, wantRouterCfg: &router.Config{ @@ -1342,7 +1344,6 @@ func TestEngineReconfigOnStateChange(t *testing.T) { // Once the auth is completed, the configs must be updated to reflect the node's netmap. wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{}, Addresses: node2.SelfNode.Addresses().AsSlice(), }, wantRouterCfg: &router.Config{ @@ -1391,7 +1392,6 @@ func TestEngineReconfigOnStateChange(t *testing.T) { // must be updated to reflect the node's netmap. wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{}, Addresses: node1.SelfNode.Addresses().AsSlice(), }, wantRouterCfg: &router.Config{ @@ -1415,23 +1415,17 @@ func TestEngineReconfigOnStateChange(t *testing.T) { }, wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: node1.SelfNode.Key(), - DiscoKey: node1.SelfNode.DiscoKey(), - }, - { - PublicKey: node2.SelfNode.Key(), - DiscoKey: node2.SelfNode.DiscoKey(), - }, - }, Addresses: node3.SelfNode.Addresses().AsSlice(), }, + wantPeers: []key.NodePublic{ + node1.SelfNode.Key(), + node2.SelfNode.Key(), + }, wantRouterCfg: &router.Config{ SNATSubnetRoutes: true, NetfilterMode: preftype.NetfilterOn, LocalAddrs: node3.SelfNode.Addresses().AsSlice(), - Routes: routesWithQuad100(), + Routes: routesWithQuad100(netip.MustParsePrefix("100.64.1.1/32"), netip.MustParsePrefix("100.64.1.2/32")), }, wantDNSCfg: &dns.Config{ AcceptDNS: true, @@ -1470,7 +1464,6 @@ func TestEngineReconfigOnStateChange(t *testing.T) { // Starting a reauth should leave everything up: wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{}, Addresses: node1.SelfNode.Addresses().AsSlice(), }, wantRouterCfg: &router.Config{ @@ -1501,7 +1494,6 @@ func TestEngineReconfigOnStateChange(t *testing.T) { }, wantState: ipn.Starting, wantCfg: &wgcfg.Config{ - Peers: []wgcfg.Peer{}, Addresses: node1.SelfNode.Addresses().AsSlice(), }, wantRouterCfg: &router.Config{ @@ -1548,13 +1540,12 @@ func TestEngineReconfigOnStateChange(t *testing.T) { t.Errorf("State: got %v; want %v", gotState, tt.wantState) } - if engine.Config() != nil { - for _, p := range engine.Config().Peers { - pKey := p.PublicKey.UntypedHexString() - _, err := lb.MagicConn().ParseEndpoint(pKey) - if err != nil { - t.Errorf("ParseEndpoint(%q) failed: %v", pKey, err) - } + // Peers are not part of wgcfg.Config; the engine learns + // them from the config source installed by LocalBackend + // via SetPeerConfigFunc. + for _, k := range tt.wantPeers { + if _, ok := engine.PeerAllowedIPs(k); !ok { + t.Errorf("PeerAllowedIPs(%v) = false; want peer known", k.ShortString()) } } @@ -1574,7 +1565,10 @@ func TestEngineReconfigOnStateChange(t *testing.T) { } } -func TestEngineReconfigOnPeerRouteDelta(t *testing.T) { +// TestPeerConfigUpdatedOnPeerRouteDelta tests that a netmap delta that +// changes a peer's allowed IPs is visible through the live per-peer +// config source that LocalBackend installs on the engine. +func TestPeerConfigUpdatedOnPeerRouteDelta(t *testing.T) { connect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: true}, WantRunningSet: true} peerAddr := netip.MustParsePrefix("100.64.1.1/32") vipAddr := netip.MustParsePrefix("100.99.99.99/32") @@ -1600,20 +1594,13 @@ func TestEngineReconfigOnPeerRouteDelta(t *testing.T) { t.Fatal("UpdateNetmapDelta = false, want true") } - cfg := engine.Config() - if cfg == nil { - t.Fatal("engine config is nil") + ips, ok := engine.PeerAllowedIPs(replacement.Key) + if !ok { + t.Fatalf("peer config source missing peer %v", replacement.Key.ShortString()) } - for _, peer := range cfg.Peers { - if peer.PublicKey != replacement.Key { - continue - } - if !slices.Contains(peer.AllowedIPs, vipAddr) { - t.Fatalf("peer AllowedIPs = %v; want %v", peer.AllowedIPs, vipAddr) - } - return + if !slices.Contains(ips, vipAddr) { + t.Fatalf("peer AllowedIPs = %v; want %v", ips, vipAddr) } - t.Fatalf("engine config missing peer %v", replacement.Key.ShortString()) } // TestSendPreservesAuthURL tests that wgengine updates arriving in the middle of @@ -1877,6 +1864,8 @@ type mockEngine struct { filter, jailedFilter *filter.Filter + peerConfigFn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool) + statusCb wgengine.StatusCallback } @@ -1916,10 +1905,6 @@ func (e *mockEngine) DNSConfig() *dns.Config { return e.dnsCfg } -func (e *mockEngine) PeerForIP(netip.Addr) (_ wgengine.PeerForIP, ok bool) { - return wgengine.PeerForIP{}, false -} - func (e *mockEngine) GetFilter() *filter.Filter { e.mu.Lock() defer e.mu.Unlock() @@ -1944,6 +1929,9 @@ func (e *mockEngine) SetJailedFilter(f *filter.Filter) { e.mu.Unlock() } +func (e *mockEngine) SetPeerRoutes(native4, native6 netip.Addr, routes *bart.Table[*routemanager.PeerRoute]) { +} + func (e *mockEngine) SetStatusCallback(cb wgengine.StatusCallback) { e.mu.Lock() e.statusCb = cb @@ -1983,9 +1971,26 @@ func (e *mockEngine) InstallCaptureHook(packet.CaptureCallback) {} func (e *mockEngine) SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool)) {} func (e *mockEngine) SetPeerForIPFunc(func(netip.Addr) (_ wgengine.PeerForIP, ok bool)) {} -func (e *mockEngine) PeerKeyForIP(netip.Addr) (_ key.NodePublic, _ netip.Prefix, ok bool) { - return key.NodePublic{}, netip.Prefix{}, false +func (e *mockEngine) SetPeerConfigFunc(fn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)) { + e.mu.Lock() + defer e.mu.Unlock() + e.peerConfigFn = fn } + +// PeerAllowedIPs looks up a peer's allowed IPs via the live per-peer +// config source installed by LocalBackend with SetPeerConfigFunc. +func (e *mockEngine) PeerAllowedIPs(k key.NodePublic) (_ []netip.Prefix, ok bool) { + e.mu.Lock() + fn := e.peerConfigFn + e.mu.Unlock() + if fn == nil { + return nil, false + } + return fn(k) +} + +func (e *mockEngine) SyncDevicePeer(key.NodePublic) {} +func (e *mockEngine) ResetDevicePeer(key.NodePublic) {} func (e *mockEngine) SetPeerSessionStateFunc(func(key.NodePublic, wgengine.PeerWireGuardState)) { } func (e *mockEngine) SetNetLogSource(wgengine.NetLogSource) {} diff --git a/net/routemanager/fortest.go b/net/routemanager/fortest.go new file mode 100644 index 000000000..79f4db8ac --- /dev/null +++ b/net/routemanager/fortest.go @@ -0,0 +1,23 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package routemanager + +import "tailscale.com/util/testenv" + +// forTest is an unexported type to hide the test-only methods on +// [RouteManager] from godoc. +type forTest struct{ rm *RouteManager } + +// ForTest returns a handle to test-only methods on rm. The resulting +// type is unexported to make it very obvious in godoc that this is +// not stable API. This method panics if called outside of tests, +// which also centralizes all must-be-in-tests validation. +func (rm *RouteManager) ForTest() forTest { + testenv.AssertInTest() + return forTest{rm} +} + +// PeerCount returns the number of peers currently tracked. Callers +// must serialize it with mutations like any other write-path access. +func (f forTest) PeerCount() int { return len(f.rm.peers) } diff --git a/net/routemanager/routemanager.go b/net/routemanager/routemanager.go new file mode 100644 index 000000000..4f774b7d2 --- /dev/null +++ b/net/routemanager/routemanager.go @@ -0,0 +1,996 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package routemanager tracks which peers own which IP prefixes and +// incrementally derives the routing data structures used by the rest +// of the system: a table mapping destination IP to the outbound peer, +// and the set of routes to program into the operating system's +// routing table. +// +// Updates are transactional: callers open a [Mutation] with +// [RouteManager.Begin], stage operations, and call [Mutation.Commit] +// to publish new snapshots. The published snapshots are immutable +// bart tables that share memory with their predecessors, so readers +// (notably the wireguard-go data plane) can hold and read them +// without locks. +package routemanager + +import ( + "net/netip" + "slices" + "sync/atomic" + + "github.com/gaissmai/bart" + "tailscale.com/net/tsaddr" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + "tailscale.com/types/logger" + "tailscale.com/types/views" + "tailscale.com/util/mak" + "tailscale.com/util/set" +) + +// peerView is the subset of a peer's netmap state that affects +// routing. It is intentionally much narrower than tailcfg.NodeView so +// the RouteManager can be driven and tested without full netmap +// nodes; UpsertPeer converts from tailcfg.NodeView. +type peerView struct { + // ID is the peer's node ID. It is the peer's identity within + // the RouteManager. + ID tailcfg.NodeID + + // Key is the peer's WireGuard public key. It is what gets + // published into the data-path tables, so per-packet readers + // need no NodeID-to-key translation. + Key key.NodePublic + + // Jailed is whether the peer is restricted from initiating + // connections to this node, in which case the data plane + // applies its more restrictive jailed packet filter to the + // peer's traffic. + Jailed bool + + // MasqAddr4 and MasqAddr6 are the addresses the peer knows + // this node as, per address family, if they differ from this + // node's native addresses. The data plane masquerades (NATs) + // traffic to and from the peer accordingly. An invalid address + // means no masquerading for that family. + MasqAddr4, MasqAddr6 netip.Addr + + // SelfAddrs are the peer's own addresses (its CGNAT IPv4 /32, + // if any, and its Tailscale ULA IPv6 /128) plus any other + // single Tailscale IPs it routes for, such as VIP service + // addresses. They are routable regardless of Prefs.RouteAll. + SelfAddrs []netip.Prefix + + // Routes are the peer's advertised routes: subnet routes and, + // for exit-node candidates, the 0.0.0.0/0 and ::/0 exit routes. + Routes []netip.Prefix +} + +// PeerRoute is the payload of the outbound table: the attributes the +// data plane needs when handling a packet to or from one of the +// peer's prefixes. All prefixes contributed by a peer share a single +// interned *PeerRoute, and a new one is allocated whenever the +// attributes change, so published snapshots stay immutable and +// pointer identity doubles as a change check. +type PeerRoute struct { + // Key is the peer's WireGuard public key. + Key key.NodePublic + + // Jailed, MasqAddr4, and MasqAddr6 mirror the fields of the + // same names in peerView. + Jailed bool + MasqAddr4, MasqAddr6 netip.Addr +} + +// routeAttrs returns the peer's attributes as published in the +// outbound table. +func (p peerView) routeAttrs() PeerRoute { + return PeerRoute{ + Key: p.Key, + Jailed: p.Jailed, + MasqAddr4: p.MasqAddr4, + MasqAddr6: p.MasqAddr6, + } +} + +// hasDataPlaneAttrs reports whether the peer has any attributes that +// require per-packet attention from the tun-layer data plane. +func (p peerView) hasDataPlaneAttrs() bool { + return p.Jailed || p.MasqAddr4.IsValid() || p.MasqAddr6.IsValid() +} + +// Prefs is the subset of ipn.Prefs that affects routing. +type Prefs struct { + // ExitNodeID is the node ID of the peer selected as this + // node's exit node, or zero if no exit node is selected. + // (Callers resolve ipn.Prefs's stable node ID to a NodeID.) + ExitNodeID tailcfg.NodeID + + // RouteAll is whether advertised subnet routes (non-exit + // routes) from peers are accepted. + RouteAll bool +} + +// TailnetConfig is tailnet-global and environment-derived +// configuration that affects routing. +type TailnetConfig struct { + // DisableIPv4 is whether the tailnet has disabled IPv4 self + // addresses. If set, peers' IPv4 self addresses are ignored. + DisableIPv4 bool + + // OneCGNAT is whether the OS route set should collapse peers' + // CGNAT IPv4 self addresses into the single CGNAT /10 route + // rather than per-peer /32s. The decision is made by the + // caller (it depends on platform and interface state); the + // RouteManager just applies it. + OneCGNAT bool +} + +// cgnatThreshold is the number of distinct CGNAT self-address routes +// above which the OS route set collapses them into the single CGNAT +// /10, even without TailnetConfig.OneCGNAT. +const cgnatThreshold = 10_000 + +// contribKind describes how a peer contributes a prefix: as one of +// its own addresses, as an advertised route, or both. +type contribKind uint8 + +const ( + kindSelf contribKind = 1 << iota + kindRoute + + // kindExtra marks a prefix from the extra allowed IPs set via + // [Mutation.SetExtraAllowedIPs]. Extra prefixes appear in the + // outbound table and in [RouteManager.PeerAllowedIPs], but + // never in the OS route set. + kindExtra +) + +// scoreKey identifies a per-(node, prefix) score. +type scoreKey struct { + id tailcfg.NodeID + pfx netip.Prefix +} + +// RouteManager tracks peers' addresses and advertised routes and +// derives the routing snapshots. +// +// The snapshot accessors (Outbound, OSRoutes) may be called +// concurrently from any goroutine. Mutations must be serialized by +// the caller: Begin, then stage operations, then Commit (or Discard), +// before the next Begin. Commit panics if it detects overlapping +// mutations. +type RouteManager struct { + logf logger.Logf + txGen atomic.Uint64 + + outbound atomic.Pointer[bart.Table[*PeerRoute]] + osRoutes atomic.Pointer[bart.Table[struct{}]] + + // attrPeers counts the current peers with data-plane attributes + // (jailed or masquerade addresses). It backs + // [RouteManager.HasDataPlaneAttrs]. + attrPeers atomic.Int64 + + // The fields below are working state, accessed only during + // Commit, which callers must serialize. + + peers map[tailcfg.NodeID]peerView + routes map[tailcfg.NodeID]*PeerRoute + byPrefix map[netip.Prefix]map[tailcfg.NodeID]contribKind + scores map[scoreKey]int + extras map[tailcfg.NodeID][]netip.Prefix + prefs Prefs + cfg TailnetConfig + + // cgnatPfxs and ulaPfxs are the prefixes currently eligible + // for the OS route set that are single CGNAT IPv4 addresses or + // single Tailscale ULA IPv6 addresses, respectively. They feed + // the coarse-route decisions. + cgnatPfxs set.Set[netip.Prefix] + ulaPfxs set.Set[netip.Prefix] + + // coarseCGNAT is whether the OS route set currently contains + // the single CGNAT /10 instead of per-peer /32s. + coarseCGNAT bool +} + +// New returns a new RouteManager with empty snapshots. +func New(logf logger.Logf) *RouteManager { + if logf == nil { + logf = logger.Discard + } + rm := &RouteManager{ + logf: logf, + peers: make(map[tailcfg.NodeID]peerView), + routes: make(map[tailcfg.NodeID]*PeerRoute), + byPrefix: make(map[netip.Prefix]map[tailcfg.NodeID]contribKind), + scores: make(map[scoreKey]int), + cgnatPfxs: make(set.Set[netip.Prefix]), + ulaPfxs: make(set.Set[netip.Prefix]), + } + rm.outbound.Store(&bart.Table[*PeerRoute]{}) + rm.osRoutes.Store(&bart.Table[struct{}]{}) + return rm +} + +// Outbound returns the current destination-IP-to-peer table. The +// returned table and the PeerRoutes it points to are immutable; +// callers must not modify them. +func (rm *RouteManager) Outbound() *bart.Table[*PeerRoute] { + return rm.outbound.Load() +} + +// OSRoutes returns the current set of prefixes to program into the +// OS routing table. The returned table is immutable; callers must not +// modify it. +func (rm *RouteManager) OSRoutes() *bart.Table[struct{}] { + return rm.osRoutes.Load() +} + +// HasDataPlaneAttrs reports whether any current peer has data-plane +// attributes (jailed or masquerade addresses), that is, whether the +// tun-layer data plane needs the outbound table for per-packet NAT +// rewrites and jailed-filter selection. When it reports false, the +// data plane can skip those per-packet lookups entirely. +func (rm *RouteManager) HasDataPlaneAttrs() bool { + return rm.attrPeers.Load() > 0 +} + +// PeerAllowedIPs returns the prefixes from which the given peer is +// currently allowed to originate traffic: its self addresses, its +// advertised subnet routes when Prefs.RouteAll is set, the exit +// routes when it is the selected exit node, and its extra allowed +// IPs (see [Mutation.SetExtraAllowedIPs]). It returns ok=false if +// the peer is unknown or currently contributes no prefixes; such a +// peer should not exist in the WireGuard device at all. +// +// The result is sorted, so identical routing state yields identical +// slices (letting callers cheaply detect no-op updates). +// +// Unlike the snapshot accessors, PeerAllowedIPs reads the working +// state, so callers must serialize calls with mutations, the same +// requirement as Begin/Commit. +func (rm *RouteManager) PeerAllowedIPs(id tailcfg.NodeID) (pfxs []netip.Prefix, ok bool) { + p, ok := rm.peers[id] + if !ok { + return nil, false + } + for pfx, kind := range rm.contribs(p) { + if rm.eligible(id, pfx, kind) { + pfxs = append(pfxs, pfx) + } + } + if len(pfxs) == 0 { + return nil, false + } + tsaddr.SortPrefixes(pfxs) + return pfxs, true +} + +// Result describes what a Commit changed. +type Result struct { + // PeersUpserted is the number of UpsertPeer operations applied. + PeersUpserted int + // PeersRemoved is the number of RemovePeer operations that + // removed a known peer. + PeersRemoved int + // ScoresChanged is the number of SetScore operations that + // changed a stored score. + ScoresChanged int + // ExtrasChanged is whether the commit changed the extra + // allowed IPs. + ExtrasChanged bool + // PrefsChanged is whether the commit changed the prefs. + PrefsChanged bool + // TailnetCfgChanged is whether the commit changed the tailnet config. + TailnetCfgChanged bool + + // OutboundChanged and OSRoutesChanged report whether each + // output snapshot actually changed contents. + OutboundChanged bool + OSRoutesChanged bool + + // Outbound and OSRoutes are the fresh snapshot pointers, or + // nil for any that didn't change. + Outbound *bart.Table[*PeerRoute] + OSRoutes *bart.Table[struct{}] + + // AllowedIPs maps the public key of each peer whose allowed + // source prefixes changed in this commit to its new sorted + // prefix list, as [RouteManager.PeerAllowedIPs] would now + // return it. A nil value means the peer no longer has any + // allowed prefixes, because it was removed or now contributes + // nothing. When a peer's key changes, the old key maps to nil + // and the new key to the peer's prefixes. The map is nil when + // no peer's allowed prefixes changed. + AllowedIPs map[key.NodePublic][]netip.Prefix +} + +type opKind uint8 + +const ( + opUpsert opKind = iota + opRemove + opPrefs + opCfg + opScore + opExtras +) + +type stagedOp struct { + kind opKind + peer peerView // for opUpsert + id tailcfg.NodeID // for opRemove, opScore + pfx netip.Prefix // for opScore + score int // for opScore + prefs Prefs // for opPrefs + cfg TailnetConfig // for opCfg + extras map[tailcfg.NodeID][]netip.Prefix // for opExtras +} + +// Mutation is an open transaction against a RouteManager. Operations +// are staged in order and applied atomically by Commit. A Mutation +// must not be used after Commit or Discard. +type Mutation struct { + rm *RouteManager + gen uint64 + done bool + ops []stagedOp +} + +// Begin starts a mutation. Callers must serialize Begin/Commit pairs; +// overlapping mutations cause Commit to panic. +func (rm *RouteManager) Begin() *Mutation { + return &Mutation{rm: rm, gen: rm.txGen.Load()} +} + +func (m *Mutation) checkOpen() { + if m.done { + panic("routemanager: use of finished Mutation") + } +} + +// UpsertPeer stages an add or update of a peer. The peer's prefixes +// come solely from n.AllowedIPs: entries that are single Tailscale +// IPs (the peer's own addresses, or VIP service addresses it hosts) +// or that appear in n.Addresses count as self addresses and are +// always routable; the rest are treated as its advertised routes +// (subnet routes and, for exit-node candidates, the /0 exit routes). +func (m *Mutation) UpsertPeer(n tailcfg.NodeView) { + m.upsertPeer(peerViewOf(n)) +} + +// peerViewOf reduces a tailcfg.NodeView to the routing-relevant +// peerView. +// +// It mirrors the peer and prefix filtering in nmcfg.WGCfg: peers we +// cannot communicate with (expired, or predating both DERP and disco) +// contribute no prefixes. They remain tracked by ID and key so that a +// later update can make them routable again. AllowedIPs is the sole +// source of prefixes; an address absent from AllowedIPs is not +// routable. The self-vs-route split mirrors nmcfg's cidrIsSubnet: +// single Tailscale IPs are never subnets, so a VIP service address +// hosted by the peer lands in SelfAddrs and stays routable without +// Prefs.RouteAll. +func peerViewOf(n tailcfg.NodeView) peerView { + pv := peerView{ + ID: n.ID(), + Key: n.Key(), + Jailed: n.IsJailed(), + MasqAddr4: n.SelfNodeV4MasqAddrForThisPeer().GetOr(netip.Addr{}), + MasqAddr6: n.SelfNodeV6MasqAddrForThisPeer().GetOr(netip.Addr{}), + } + if n.Expired() { + return pv + } + if n.DiscoKey().IsZero() && n.HomeDERP() == 0 && !n.IsWireGuardOnly() { + return pv + } + for _, aip := range n.AllowedIPs().All() { + isSelf := aip.IsSingleIP() && tsaddr.IsTailscaleIP(aip.Addr()) || + views.SliceContains(n.Addresses(), aip) + if isSelf { + pv.SelfAddrs = append(pv.SelfAddrs, aip) + } else { + pv.Routes = append(pv.Routes, aip) + } + } + return pv +} + +// upsertPeer stages an add or update of a peer from an +// already-reduced view. +func (m *Mutation) upsertPeer(p peerView) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opUpsert, peer: p}) +} + +// RemovePeer stages the removal of a peer. +func (m *Mutation) RemovePeer(id tailcfg.NodeID) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opRemove, id: id}) +} + +// SetPrefs stages a prefs update. +func (m *Mutation) SetPrefs(p Prefs) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opPrefs, prefs: p}) +} + +// SetTailnetConfig stages a tailnet config update. +func (m *Mutation) SetTailnetConfig(c TailnetConfig) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opCfg, cfg: c}) +} + +// SetScore stages a score update for the given peer and prefix. +// Scores are used to pick the outbound peer when multiple peers +// advertise the same prefix: highest score wins, with ties broken by +// lowest node ID. The default score is zero. Scores for a peer are +// dropped when the peer is removed. +func (m *Mutation) SetScore(id tailcfg.NodeID, pfx netip.Prefix, score int) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opScore, id: id, pfx: normalizePrefix(pfx), score: score}) +} + +// SetExtraAllowedIPs stages a wholesale replacement of the extra +// allowed IPs: additional prefixes, keyed by node ID, that each peer +// may originate traffic from and that outbound traffic to should be +// sent to that peer. Extra prefixes appear in the outbound table and +// in [RouteManager.PeerAllowedIPs], but never in the OS route set. +// (In Tailscale they carry the conn25 extension's Transit IPs, which +// must reach WireGuard but not the OS routing table.) +// +// An entry for an unknown node ID is retained and takes effect if a +// peer with that ID is later upserted. Entries are dropped only by a +// later SetExtraAllowedIPs that omits them, not by peer removal. +// +// The caller must not mutate extras or its values after the call, and +// each prefix list must be in a stable order across calls so that +// unchanged entries are detected as no-ops. +func (m *Mutation) SetExtraAllowedIPs(extras map[tailcfg.NodeID][]netip.Prefix) { + m.checkOpen() + m.ops = append(m.ops, stagedOp{kind: opExtras, extras: extras}) +} + +// Discard abandons the mutation without applying any staged +// operations. +func (m *Mutation) Discard() { + m.checkOpen() + m.done = true + m.ops = nil +} + +// Commit applies the staged operations, publishes any changed +// snapshots, and reports what changed. +// +// Commit panics if this Mutation overlapped another Begin/Commit, +// which indicates a caller bug: callers are required to serialize +// mutations. +func (m *Mutation) Commit() Result { + m.checkOpen() + m.done = true + rm := m.rm + if !rm.txGen.CompareAndSwap(m.gen, m.gen+1) { + panic("routemanager: concurrent Begin/Commit detected: caller must serialize mutations") + } + + if len(m.ops) == 0 { + // Nothing was staged, so nothing can have changed; return + // before allocating any of the diff-tracking state below. + return Result{} + } + + var res Result + dirty := make(set.Set[netip.Prefix]) + fullRebuild := false + + // before records each affected peer's key and allowed prefixes + // as of the start of the commit, so Result.AllowedIPs can be + // computed by comparison afterwards. Each peer is snapshotted + // the first time an op touches it (or on the first prefs or + // config change, which can affect every peer), which is always + // before any of its state has been mutated. + type peerBefore struct { + key key.NodePublic + pfxs []netip.Prefix // nil if the peer had no allowed prefixes + existed bool + } + var before map[tailcfg.NodeID]peerBefore + snapshot := func(id tailcfg.NodeID) { + if _, ok := before[id]; ok { + return + } + var pb peerBefore + if p, ok := rm.peers[id]; ok { + pb.existed = true + pb.key = p.Key + pb.pfxs, _ = rm.PeerAllowedIPs(id) + } + mak.Set(&before, id, pb) + } + // TODO(bradfitz): snapshotting all peers on any prefs or tailnet + // config change is a temporary lazy hack that makes those commits + // O(all peers). Each such change only affects a knowable subset: + // an exit node change affects only the old and new exit node, and + // a RouteAll change affects only the peers currently contributing + // non-self routes, which we could track incrementally. We + // shouldn't need O(all peers) work here except once at startup. + snapshotAll := func() { + for id := range rm.peers { + snapshot(id) + } + } + + for _, op := range m.ops { + switch op.kind { + case opUpsert: + snapshot(op.peer.ID) + rm.applyUpsert(op.peer, dirty) + res.PeersUpserted++ + case opRemove: + snapshot(op.id) + if rm.applyRemove(op.id, dirty) { + res.PeersRemoved++ + } + case opPrefs: + if rm.prefs != op.prefs { + snapshotAll() + rm.prefs = op.prefs + res.PrefsChanged = true + fullRebuild = true + } + case opCfg: + if rm.cfg != op.cfg { + snapshotAll() + rm.cfg = op.cfg + res.TailnetCfgChanged = true + fullRebuild = true + } + case opScore: + sk := scoreKey{op.id, op.pfx} + old, had := rm.scores[sk] + if op.score == 0 { + if had { + delete(rm.scores, sk) + res.ScoresChanged++ + dirty.Add(op.pfx) + } + } else if !had || old != op.score { + rm.scores[sk] = op.score + res.ScoresChanged++ + dirty.Add(op.pfx) + } + case opExtras: + for id := range rm.extras { + snapshot(id) + } + for id := range op.extras { + snapshot(id) + } + if rm.applyExtras(op.extras, dirty) { + res.ExtrasChanged = true + } + } + } + + if fullRebuild { + rm.rebuildAll(&res) + } else if len(dirty) > 0 { + rm.applyDirty(dirty, &res) + } + + // Diff each snapshotted peer's allowed prefixes against the + // final working state to populate Result.AllowedIPs. Both + // sides are sorted, so slices.Equal detects no-ops. + for id, was := range before { + now, _ := rm.PeerAllowedIPs(id) + p, exists := rm.peers[id] + switch { + case !exists: + if was.pfxs != nil { + mak.Set(&res.AllowedIPs, was.key, nil) + } + case was.existed && was.key != p.Key: + if was.pfxs != nil { + mak.Set(&res.AllowedIPs, was.key, nil) + } + if now != nil { + mak.Set(&res.AllowedIPs, p.Key, now) + } + default: + if !slices.Equal(was.pfxs, now) { + mak.Set(&res.AllowedIPs, p.Key, now) + } + } + } + return res +} + +// normalizePrefix unmaps a 4-in-6 prefix address and masks off any +// non-address bits. +func normalizePrefix(p netip.Prefix) netip.Prefix { + return netip.PrefixFrom(p.Addr().Unmap(), p.Bits()).Masked() +} + +// contribs returns the normalized per-prefix contributions of p, +// skipping (with a log message) any prefix that arrives with +// non-address bits set, mirroring the defensive check in +// ipnlocal.peerRoutes. It includes the peer's extra allowed IPs, +// except for peers that contribute no addresses or routes of their +// own (expired or otherwise non-communicable peers, which mirrors +// nmcfg.WGCfg dropping such peers entirely). +func (rm *RouteManager) contribs(p peerView) map[netip.Prefix]contribKind { + c := make(map[netip.Prefix]contribKind, len(p.SelfAddrs)+len(p.Routes)) + add := func(pfx netip.Prefix, kind contribKind) { + pfx = netip.PrefixFrom(pfx.Addr().Unmap(), pfx.Bits()) + if mm := pfx.Masked(); pfx != mm { + rm.logf("routemanager: prefix %s from %s has non-address bits set; expected %s; skipping", pfx, p.Key.ShortString(), mm) + return + } + c[pfx] |= kind + } + for _, pfx := range p.SelfAddrs { + add(pfx, kindSelf) + } + for _, pfx := range p.Routes { + add(pfx, kindRoute) + } + if len(c) > 0 { + for _, pfx := range rm.extras[p.ID] { + add(pfx, kindExtra) + } + } + return c +} + +// applyExtras replaces the extra allowed IPs with newExtras, updating +// working state and dirty for every peer whose extras changed. It +// reports whether anything changed. +func (rm *RouteManager) applyExtras(newExtras map[tailcfg.NodeID][]netip.Prefix, dirty set.Set[netip.Prefix]) (changed bool) { + apply := func(id tailcfg.NodeID, pfxs []netip.Prefix) { + if slices.Equal(rm.extras[id], pfxs) { + return + } + changed = true + p, exists := rm.peers[id] + var oldC map[netip.Prefix]contribKind + if exists { + oldC = rm.contribs(p) + } + if len(pfxs) == 0 { + delete(rm.extras, id) + } else { + mak.Set(&rm.extras, id, pfxs) + } + if !exists { + return + } + newC := rm.contribs(p) + for pfx, kind := range oldC { + if newC[pfx] != kind { + rm.dropContrib(id, pfx) + dirty.Add(pfx) + } + } + for pfx, kind := range newC { + if oldC[pfx] != kind { + rm.addContrib(id, pfx, kind) + dirty.Add(pfx) + } + } + } + for id := range rm.extras { + if _, ok := newExtras[id]; !ok { + apply(id, nil) + } + } + for id, pfxs := range newExtras { + apply(id, pfxs) + } + return changed +} + +// applyUpsert updates working state for an upserted peer, adding any +// affected prefixes to dirty. +func (rm *RouteManager) applyUpsert(p peerView, dirty set.Set[netip.Prefix]) { + newC := rm.contribs(p) + old, had := rm.peers[p.ID] + attrsChanged := had && old.routeAttrs() != p.routeAttrs() + if !had || attrsChanged { + // Intern a fresh PeerRoute rather than mutating the old + // one, which published snapshots may still reference. + pr := p.routeAttrs() + rm.routes[p.ID] = &pr + } + if had && old.hasDataPlaneAttrs() { + rm.attrPeers.Add(-1) + } + if p.hasDataPlaneAttrs() { + rm.attrPeers.Add(1) + } + if had { + oldC := rm.contribs(old) + for pfx, kind := range oldC { + if newC[pfx] != kind { + rm.dropContrib(p.ID, pfx) + dirty.Add(pfx) + } else if attrsChanged { + dirty.Add(pfx) + } + } + for pfx, kind := range newC { + if oldC[pfx] != kind { + rm.addContrib(p.ID, pfx, kind) + dirty.Add(pfx) + } + } + } else { + for pfx, kind := range newC { + rm.addContrib(p.ID, pfx, kind) + dirty.Add(pfx) + } + } + rm.peers[p.ID] = p +} + +// applyRemove updates working state for a removed peer, adding its +// prefixes to dirty. It reports whether the peer was known. +func (rm *RouteManager) applyRemove(id tailcfg.NodeID, dirty set.Set[netip.Prefix]) bool { + old, had := rm.peers[id] + if !had { + return false + } + for pfx := range rm.contribs(old) { + rm.dropContrib(id, pfx) + dirty.Add(pfx) + } + delete(rm.peers, id) + delete(rm.routes, id) + if old.hasDataPlaneAttrs() { + rm.attrPeers.Add(-1) + } + for sk := range rm.scores { + if sk.id == id { + delete(rm.scores, sk) + } + } + return true +} + +func (rm *RouteManager) addContrib(id tailcfg.NodeID, pfx netip.Prefix, kind contribKind) { + advs := rm.byPrefix[pfx] + if advs == nil { + advs = make(map[tailcfg.NodeID]contribKind) + rm.byPrefix[pfx] = advs + } + advs[id] = kind +} + +func (rm *RouteManager) dropContrib(id tailcfg.NodeID, pfx netip.Prefix) { + advs := rm.byPrefix[pfx] + delete(advs, id) + if len(advs) == 0 { + delete(rm.byPrefix, pfx) + } +} + +// eligible reports whether id's contribution of pfx (of the given +// kind) should be reflected in the outbound table and in the peer's +// allowed source prefixes, per current prefs and tailnet config. +// Extra allowed IPs are always eligible here, but are excluded from +// the OS route set by [RouteManager.desiredFor]. +func (rm *RouteManager) eligible(id tailcfg.NodeID, pfx netip.Prefix, kind contribKind) bool { + if kind&kindSelf != 0 { + if !(pfx.Addr().Is4() && rm.cfg.DisableIPv4) { + return true + } + } + if kind&kindRoute != 0 { + if tsaddr.IsExitRoute(pfx) { + return rm.prefs.ExitNodeID != 0 && id == rm.prefs.ExitNodeID + } + return rm.prefs.RouteAll + } + return kind&kindExtra != 0 +} + +// desired is the computed per-prefix output state: the outbound +// winner, if any, and whether the prefix belongs in the OS route set. +type desired struct { + out *PeerRoute // nil if the prefix has no outbound winner + os bool +} + +// desiredFor computes the desired output state for pfx from the +// current working state. +func (rm *RouteManager) desiredFor(pfx netip.Prefix) desired { + var d desired + var bestID tailcfg.NodeID + var bestScore int + for id, kind := range rm.byPrefix[pfx] { + if !rm.eligible(id, pfx, kind) { + continue + } + if rm.eligible(id, pfx, kind&^kindExtra) { + d.os = true + } + sc := rm.scores[scoreKey{id, pfx}] + if d.out == nil || sc > bestScore || (sc == bestScore && id < bestID) { + bestID, bestScore = id, sc + d.out = rm.routes[id] + } + } + return d +} + +// osClass classifies a prefix for the OS route set. +type osClass uint8 + +const ( + osPlain osClass = iota // installed as-is when eligible + osCGNAT // single CGNAT IPv4 addr; subject to /10 coarsening + osULA // single Tailscale ULA IPv6 addr; always coarsened +) + +func classify(pfx netip.Prefix) osClass { + if !pfx.IsSingleIP() { + return osPlain + } + if pfx.Addr().Is4() && tsaddr.CGNATRange().Contains(pfx.Addr()) { + return osCGNAT + } + if pfx.Addr().Is6() && tsaddr.TailscaleULARange().Contains(pfx.Addr()) { + return osULA + } + return osPlain +} + +func (rm *RouteManager) cgnatThreshold() int { + if rm.cfg.OneCGNAT { + return 1 + } + return cgnatThreshold +} + +// tableSet returns a table where pfx's presence matches want, +// reporting whether it differed. +func tableSet(t *bart.Table[struct{}], pfx netip.Prefix, want bool) (*bart.Table[struct{}], bool) { + _, has := t.Get(pfx) + if has == want { + return t, false + } + if want { + return t.InsertPersist(pfx, struct{}{}), true + } + return t.DeletePersist(pfx), true +} + +// applyDirty incrementally updates the output snapshots for the dirty +// prefixes and publishes any that changed. +func (rm *RouteManager) applyDirty(dirty set.Set[netip.Prefix], res *Result) { + out := rm.outbound.Load() + osr := rm.osRoutes.Load() + var outChanged, osChanged bool + + var cgnatDirty []netip.Prefix + for pfx := range dirty { + d := rm.desiredFor(pfx) + + if cur, ok := out.Get(pfx); (d.out != nil) != ok || (ok && cur != d.out) { + if d.out != nil { + out = out.InsertPersist(pfx, d.out) + } else { + out = out.DeletePersist(pfx) + } + outChanged = true + } + + switch classify(pfx) { + case osULA: + if d.os { + rm.ulaPfxs.Add(pfx) + } else { + rm.ulaPfxs.Delete(pfx) + } + case osCGNAT: + if d.os { + rm.cgnatPfxs.Add(pfx) + } else { + rm.cgnatPfxs.Delete(pfx) + } + cgnatDirty = append(cgnatDirty, pfx) + case osPlain: + var ch bool + osr, ch = tableSet(osr, pfx, d.os) + osChanged = osChanged || ch + } + } + + var ch bool + osr, ch = tableSet(osr, tsaddr.TailscaleULARange(), len(rm.ulaPfxs) > 0) + osChanged = osChanged || ch + + wantCoarse := len(rm.cgnatPfxs) > rm.cgnatThreshold() + if wantCoarse != rm.coarseCGNAT { + rm.coarseCGNAT = wantCoarse + for pfx := range rm.cgnatPfxs { + osr, ch = tableSet(osr, pfx, !wantCoarse) + osChanged = osChanged || ch + } + osr, ch = tableSet(osr, tsaddr.CGNATRange(), wantCoarse) + osChanged = osChanged || ch + } + for _, pfx := range cgnatDirty { + osr, ch = tableSet(osr, pfx, !rm.coarseCGNAT && rm.cgnatPfxs.Contains(pfx)) + osChanged = osChanged || ch + } + + rm.publish(out, outChanged, osr, osChanged, res) +} + +// rebuildAll recomputes the output snapshots from scratch and +// publishes those that changed. It runs on prefs or tailnet config +// changes, where O(number of prefixes) work is acceptable. +func (rm *RouteManager) rebuildAll(res *Result) { + out := &bart.Table[*PeerRoute]{} + osr := &bart.Table[struct{}]{} + clear(rm.cgnatPfxs) + clear(rm.ulaPfxs) + + var plain []netip.Prefix + for pfx := range rm.byPrefix { + d := rm.desiredFor(pfx) + if d.out == nil { + continue + } + out.Insert(pfx, d.out) + if !d.os { + continue + } + switch classify(pfx) { + case osULA: + rm.ulaPfxs.Add(pfx) + case osCGNAT: + rm.cgnatPfxs.Add(pfx) + case osPlain: + plain = append(plain, pfx) + } + } + + for _, pfx := range plain { + osr.Insert(pfx, struct{}{}) + } + if len(rm.ulaPfxs) > 0 { + osr.Insert(tsaddr.TailscaleULARange(), struct{}{}) + } + rm.coarseCGNAT = len(rm.cgnatPfxs) > rm.cgnatThreshold() + if rm.coarseCGNAT { + osr.Insert(tsaddr.CGNATRange(), struct{}{}) + } else { + for pfx := range rm.cgnatPfxs { + osr.Insert(pfx, struct{}{}) + } + } + + rm.publish(out, !out.Equal(rm.outbound.Load()), + osr, !osr.Equal(rm.osRoutes.Load()), res) +} + +// publish stores the changed snapshots and records them in res. +func (rm *RouteManager) publish(out *bart.Table[*PeerRoute], outChanged bool, + osr *bart.Table[struct{}], osChanged bool, res *Result) { + if outChanged { + rm.outbound.Store(out) + res.OutboundChanged = true + res.Outbound = out + } + if osChanged { + rm.osRoutes.Store(osr) + res.OSRoutesChanged = true + res.OSRoutes = osr + } +} diff --git a/net/routemanager/routemanager_test.go b/net/routemanager/routemanager_test.go new file mode 100644 index 000000000..4d1148f44 --- /dev/null +++ b/net/routemanager/routemanager_test.go @@ -0,0 +1,897 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package routemanager + +import ( + "net/netip" + "reflect" + "slices" + "testing" + + "tailscale.com/net/tsaddr" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + "tailscale.com/util/set" +) + +var ( + k1 = key.NewNode().Public() + k2 = key.NewNode().Public() + k3 = key.NewNode().Public() +) + +func pfx(s string) netip.Prefix { return netip.MustParsePrefix(s) } +func addr(s string) netip.Addr { return netip.MustParseAddr(s) } + +// peer1 is a basic peer with self addresses only. +func peer1() peerView { + return peerView{ + ID: 1, + Key: k1, + SelfAddrs: []netip.Prefix{ + pfx("100.64.0.1/32"), + pfx("fd7a:115c:a1e0::1/128"), + }, + } +} + +func peer2() peerView { + return peerView{ + ID: 2, + Key: k2, + SelfAddrs: []netip.Prefix{ + pfx("100.64.0.2/32"), + pfx("fd7a:115c:a1e0::2/128"), + }, + } +} + +// commit applies fn within a single mutation and returns the Result. +func commit(rm *RouteManager, fn func(*Mutation)) Result { + m := rm.Begin() + fn(m) + return m.Commit() +} + +func wantOutbound(t *testing.T, rm *RouteManager, ip string, want key.NodePublic, wantOK bool) { + t.Helper() + got, ok := rm.Outbound().Lookup(addr(ip)) + if ok != wantOK || (ok && got.Key != want) { + var gotKey key.NodePublic + if got != nil { + gotKey = got.Key + } + t.Errorf("Outbound lookup %s = %v, %v; want %v, %v", ip, gotKey.ShortString(), ok, want.ShortString(), wantOK) + } +} + +func wantOSRoutes(t *testing.T, rm *RouteManager, want ...string) { + t.Helper() + wantSet := make(set.Set[netip.Prefix]) + for _, s := range want { + wantSet.Add(pfx(s)) + } + got := make(set.Set[netip.Prefix]) + for p := range rm.OSRoutes().All() { + got.Add(p) + } + if !got.Equal(wantSet) { + t.Errorf("OSRoutes = %v; want %v", got.Slice(), wantSet.Slice()) + } +} + +func TestSelfAddrs(t *testing.T) { + rm := New(t.Logf) + res := commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + + if res.PeersUpserted != 1 { + t.Errorf("PeersUpserted = %d; want 1", res.PeersUpserted) + } + if !res.OutboundChanged || !res.OSRoutesChanged { + t.Errorf("changed flags = %v/%v; want all true", res.OutboundChanged, res.OSRoutesChanged) + } + if res.Outbound == nil || res.OSRoutes == nil { + t.Error("Result snapshot pointers should be non-nil when changed") + } + + wantOutbound(t, rm, "100.64.0.1", k1, true) + wantOutbound(t, rm, "fd7a:115c:a1e0::1", k1, true) + wantOutbound(t, rm, "100.64.0.2", key.NodePublic{}, false) + // Individual CGNAT /32 (below threshold) plus the coarse ULA range. + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") +} + +func TestRemovePeer(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + res := commit(rm, func(m *Mutation) { m.RemovePeer(1) }) + + if res.PeersRemoved != 1 { + t.Errorf("PeersRemoved = %d; want 1", res.PeersRemoved) + } + wantOutbound(t, rm, "100.64.0.1", key.NodePublic{}, false) + wantOSRoutes(t, rm) + if rm.ForTest().PeerCount() != 0 { + t.Errorf("PeerCount = %d; want 0", rm.ForTest().PeerCount()) + } + + // Removing an unknown peer is a no-op. + res = commit(rm, func(m *Mutation) { m.RemovePeer(42) }) + if res.PeersRemoved != 0 || res.OutboundChanged { + t.Errorf("remove of unknown peer: %+v", res) + } +} + +func TestExitNodeGating(t *testing.T) { + rm := New(t.Logf) + exitPeer := peer1() + exitPeer.Routes = tsaddr.ExitRoutes() + commit(rm, func(m *Mutation) { m.upsertPeer(exitPeer) }) + + // Not selected: no /0 entries anywhere. + wantOutbound(t, rm, "8.8.8.8", key.NodePublic{}, false) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") + + // Select it as exit node. + res := commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{ExitNodeID: 1}) }) + if !res.PrefsChanged || !res.OutboundChanged || !res.OSRoutesChanged { + t.Errorf("select exit node: %+v", res) + } + wantOutbound(t, rm, "8.8.8.8", k1, true) + wantOutbound(t, rm, "2001:db8::1", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48", "0.0.0.0/0", "::/0") + + // Deselect: /0s vanish. + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{}) }) + wantOutbound(t, rm, "8.8.8.8", key.NodePublic{}, false) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") + + // Selecting a peer that doesn't advertise /0s adds nothing. + commit(rm, func(m *Mutation) { + m.upsertPeer(peer2()) + m.SetPrefs(Prefs{ExitNodeID: 2}) + }) + wantOutbound(t, rm, "8.8.8.8", key.NodePublic{}, false) +} + +func TestRouteAll(t *testing.T) { + rm := New(t.Logf) + p := peer1() + p.Routes = []netip.Prefix{pfx("10.0.0.0/24")} + commit(rm, func(m *Mutation) { m.upsertPeer(p) }) + + // Subnet routes ignored until RouteAll. + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") + + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantOutbound(t, rm, "10.0.0.5", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48", "10.0.0.0/24") + + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{}) }) + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) +} + +func TestSharedSubnetRoute(t *testing.T) { + rm := New(t.Logf) + route := pfx("10.0.0.0/24") + pa, pb := peer1(), peer2() + pa.Routes = []netip.Prefix{route} + pb.Routes = []netip.Prefix{route} + commit(rm, func(m *Mutation) { + m.upsertPeer(pa) + m.upsertPeer(pb) + m.SetPrefs(Prefs{RouteAll: true}) + }) + + // Tie on score: lowest NodeID wins outbound. + wantOutbound(t, rm, "10.0.0.5", k1, true) + + // Raise peer 2's score: outbound flips, OS routes unchanged. + res := commit(rm, func(m *Mutation) { m.SetScore(2, route, 100) }) + if res.ScoresChanged != 1 || !res.OutboundChanged || res.OSRoutesChanged { + t.Errorf("score change: %+v", res) + } + wantOutbound(t, rm, "10.0.0.5", k2, true) + + // Setting the same score again is a no-op. + res = commit(rm, func(m *Mutation) { m.SetScore(2, route, 100) }) + if res.ScoresChanged != 0 || res.OutboundChanged { + t.Errorf("same score: %+v", res) + } + + // Resetting the score to zero restores the NodeID tie-break. + res = commit(rm, func(m *Mutation) { m.SetScore(2, route, 0) }) + if res.ScoresChanged != 1 || !res.OutboundChanged { + t.Errorf("score reset: %+v", res) + } + wantOutbound(t, rm, "10.0.0.5", k1, true) + + // Resetting an already-absent score is a no-op. + res = commit(rm, func(m *Mutation) { m.SetScore(2, route, 0) }) + if res.ScoresChanged != 0 { + t.Errorf("absent score reset: %+v", res) + } + + // Restore peer 2 as winner for the removal test below. + commit(rm, func(m *Mutation) { m.SetScore(2, route, 100) }) + wantOutbound(t, rm, "10.0.0.5", k2, true) + + // Removing the winner falls back to the other advertiser. + commit(rm, func(m *Mutation) { m.RemovePeer(2) }) + wantOutbound(t, rm, "10.0.0.5", k1, true) + + // Peer 2's score was dropped on removal: re-adding it ties again + // and peer 1 wins by lower NodeID. + commit(rm, func(m *Mutation) { m.upsertPeer(pb) }) + wantOutbound(t, rm, "10.0.0.5", k1, true) +} + +func TestOneCGNAT(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { + m.SetTailnetConfig(TailnetConfig{OneCGNAT: true}) + m.upsertPeer(peer1()) + }) + // One CGNAT addr: not above threshold (1 > 1 is false), so + // still individual, mirroring ipnlocal.peerRoutes. + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") + + // Second CGNAT addr crosses the threshold: collapse to /10. + res := commit(rm, func(m *Mutation) { m.upsertPeer(peer2()) }) + if !res.OSRoutesChanged { + t.Errorf("crossing threshold: %+v", res) + } + wantOSRoutes(t, rm, "100.64.0.0/10", "fd7a:115c:a1e0::/48") + + // Hot-path tables stay exact regardless of OS coarsening. + wantOutbound(t, rm, "100.64.0.2", k2, true) + wantOutbound(t, rm, "100.64.0.3", key.NodePublic{}, false) + + // Dropping back below the threshold un-coarsens. + commit(rm, func(m *Mutation) { m.RemovePeer(2) }) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") +} + +// TestFullRebuildWhileCoarse exercises the full-rebuild path (a prefs +// change) while CGNAT coarsening is already active. +func TestFullRebuildWhileCoarse(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { + m.SetTailnetConfig(TailnetConfig{OneCGNAT: true}) + m.upsertPeer(peer1()) + m.upsertPeer(peer2()) + }) + wantOSRoutes(t, rm, "100.64.0.0/10", "fd7a:115c:a1e0::/48") + + res := commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + if !res.PrefsChanged || res.OSRoutesChanged { + t.Errorf("prefs change while coarse: %+v", res) + } + wantOSRoutes(t, rm, "100.64.0.0/10", "fd7a:115c:a1e0::/48") + wantOutbound(t, rm, "100.64.0.2", k2, true) +} + +// TestSingleIPPlainRoute checks that a single-IP route outside the +// CGNAT and ULA ranges is programmed individually, not coarsened. +func TestSingleIPPlainRoute(t *testing.T) { + rm := New(t.Logf) + p := peer1() + p.Routes = []netip.Prefix{pfx("192.0.2.7/32")} + commit(rm, func(m *Mutation) { + m.upsertPeer(p) + m.SetPrefs(Prefs{RouteAll: true}) + }) + wantOutbound(t, rm, "192.0.2.7", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48", "192.0.2.7/32") +} + +func TestDisableIPv4(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { + m.SetTailnetConfig(TailnetConfig{DisableIPv4: true}) + m.upsertPeer(peer1()) + }) + wantOutbound(t, rm, "100.64.0.1", key.NodePublic{}, false) + wantOutbound(t, rm, "fd7a:115c:a1e0::1", k1, true) + wantOSRoutes(t, rm, "fd7a:115c:a1e0::/48") +} + +func TestKeyRotation(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + + rotated := peer1() + rotated.Key = k3 + res := commit(rm, func(m *Mutation) { m.upsertPeer(rotated) }) + if !res.OutboundChanged || res.OSRoutesChanged { + t.Errorf("key rotation: %+v", res) + } + wantOutbound(t, rm, "100.64.0.1", k3, true) +} + +// TestPeerRouteAttrs checks that the data-plane attributes (jailed, +// masquerade addresses) are published in the outbound table, that +// changing only an attribute republishes the peer's prefixes, and +// that previously published snapshots keep the old attributes. +func TestPeerRouteAttrs(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + + pr, ok := rm.Outbound().Lookup(addr("100.64.0.1")) + if !ok || pr.Jailed || pr.MasqAddr4.IsValid() || pr.MasqAddr6.IsValid() { + t.Fatalf("initial attrs = %+v, %v; want unjailed, no masq", pr, ok) + } + if rm.HasDataPlaneAttrs() { + t.Error("HasDataPlaneAttrs = true with no jailed or masqueraded peers") + } + oldOut := rm.Outbound() + + jailed := peer1() + jailed.Jailed = true + jailed.MasqAddr4 = addr("100.99.0.1") + res := commit(rm, func(m *Mutation) { m.upsertPeer(jailed) }) + if !res.OutboundChanged || res.OSRoutesChanged { + t.Errorf("attr change: %+v", res) + } + if res.AllowedIPs != nil { + t.Errorf("attr change reported AllowedIPs %v; attrs must not affect allowed prefixes", res.AllowedIPs) + } + pr, ok = rm.Outbound().Lookup(addr("100.64.0.1")) + if !ok || !pr.Jailed || pr.MasqAddr4 != addr("100.99.0.1") || pr.Key != k1 { + t.Fatalf("updated attrs = %+v, %v", pr, ok) + } + if !rm.HasDataPlaneAttrs() { + t.Error("HasDataPlaneAttrs = false with a jailed peer") + } + if pr, ok := oldOut.Lookup(addr("100.64.0.1")); !ok || pr.Jailed { + t.Errorf("old snapshot attrs = %+v, %v; want original unjailed entry", pr, ok) + } + + // An identical re-upsert is a no-op. + res = commit(rm, func(m *Mutation) { m.upsertPeer(jailed) }) + if res.OutboundChanged { + t.Errorf("identical attrs re-upsert changed outbound: %+v", res) + } + if !rm.HasDataPlaneAttrs() { + t.Error("HasDataPlaneAttrs = false after identical re-upsert of a jailed peer") + } + + // Removing the peer drops its data-plane attributes. + commit(rm, func(m *Mutation) { m.RemovePeer(1) }) + if rm.HasDataPlaneAttrs() { + t.Error("HasDataPlaneAttrs = true after removing the only jailed peer") + } +} + +func TestNoopCommit(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + + res := commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + if res.OutboundChanged || res.OSRoutesChanged { + t.Errorf("identical re-upsert changed tables: %+v", res) + } + if res.Outbound != nil || res.OSRoutes != nil { + t.Error("Result snapshot pointers should be nil when unchanged") + } + if res.PeersUpserted != 1 { + t.Errorf("PeersUpserted = %d; want 1", res.PeersUpserted) + } + + // Same prefs again: no rebuild reported. + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + res = commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + if res.PrefsChanged { + t.Errorf("identical prefs reported changed: %+v", res) + } + + // An empty mutation commits fine. + res = commit(rm, func(m *Mutation) {}) + if !reflect.DeepEqual(res, Result{}) { + t.Errorf("empty commit: %+v", res) + } +} + +func TestUnmaskedPrefixSkipped(t *testing.T) { + rm := New(t.Logf) + p := peer1() + p.Routes = []netip.Prefix{netip.PrefixFrom(addr("10.0.0.5"), 24)} // non-address bits set + commit(rm, func(m *Mutation) { + m.upsertPeer(p) + m.SetPrefs(Prefs{RouteAll: true}) + }) + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) +} + +func TestSnapshotImmutability(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + oldOut := rm.Outbound() + + commit(rm, func(m *Mutation) { m.RemovePeer(1) }) + + // The old snapshot still sees the removed peer. + if _, ok := oldOut.Lookup(addr("100.64.0.1")); !ok { + t.Error("old outbound snapshot lost entry after later commit") + } + // And the new one doesn't. + wantOutbound(t, rm, "100.64.0.1", key.NodePublic{}, false) +} + +func TestDiscard(t *testing.T) { + rm := New(t.Logf) + m := rm.Begin() + m.upsertPeer(peer1()) + m.Discard() + if rm.ForTest().PeerCount() != 0 { + t.Error("discarded mutation was applied") + } + // A new mutation works after a discard. + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + if rm.ForTest().PeerCount() != 1 { + t.Error("commit after discard failed") + } +} + +func TestConcurrentMutationPanics(t *testing.T) { + rm := New(t.Logf) + m1 := rm.Begin() + m2 := rm.Begin() + m1.upsertPeer(peer1()) + m1.Commit() + + defer func() { + if recover() == nil { + t.Error("overlapping Commit did not panic") + } + }() + m2.Commit() +} + +func TestFinishedMutationPanics(t *testing.T) { + rm := New(t.Logf) + m := rm.Begin() + m.Commit() + defer func() { + if recover() == nil { + t.Error("op on finished Mutation did not panic") + } + }() + m.upsertPeer(peer1()) +} + +func TestPeerModifyRoutes(t *testing.T) { + rm := New(t.Logf) + p := peer1() + p.Routes = []netip.Prefix{pfx("10.0.0.0/24")} + commit(rm, func(m *Mutation) { + m.upsertPeer(p) + m.SetPrefs(Prefs{RouteAll: true}) + }) + wantOutbound(t, rm, "10.0.0.5", k1, true) + + // Swap the advertised route for another. + p.Routes = []netip.Prefix{pfx("192.168.1.0/24")} + res := commit(rm, func(m *Mutation) { m.upsertPeer(p) }) + if !res.OutboundChanged { + t.Errorf("route swap: %+v", res) + } + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) + wantOutbound(t, rm, "192.168.1.5", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48", "192.168.1.0/24") +} + +// BenchmarkUpsertOnePeer measures the cost of a single-peer mutation +// against a manager already tracking many peers, the case the +// incremental design exists for. +func BenchmarkUpsertOnePeer(b *testing.B) { + rm := New(nil) + m := rm.Begin() + for i := range 10_000 { + id := tailcfg.NodeID(i + 1) + m.upsertPeer(peerView{ + ID: id, + Key: key.NewNode().Public(), + SelfAddrs: []netip.Prefix{ + netip.PrefixFrom(netip.AddrFrom4([4]byte{100, 64, byte(i >> 8), byte(i)}), 32), + }, + }) + } + m.Commit() + + p := peerView{ + ID: 99_999, + Key: key.NewNode().Public(), + SelfAddrs: []netip.Prefix{pfx("100.100.1.1/32")}, + } + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + p.SelfAddrs[0] = netip.PrefixFrom(netip.AddrFrom4([4]byte{100, 100, byte(i >> 8), byte(i)}), 32) + m := rm.Begin() + m.upsertPeer(p) + m.Commit() + } +} + +func TestUpsertPeerNodeView(t *testing.T) { + rm := New(t.Logf) + n := &tailcfg.Node{ + ID: 1, + Key: k1, + HomeDERP: 1, + Addresses: []netip.Prefix{ + pfx("100.64.0.1/32"), + pfx("fd7a:115c:a1e0::1/128"), + }, + AllowedIPs: []netip.Prefix{ + pfx("100.64.0.1/32"), + pfx("fd7a:115c:a1e0::1/128"), + pfx("10.0.0.0/24"), + pfx("0.0.0.0/0"), + pfx("::/0"), + }, + IsJailed: true, + SelfNodeV4MasqAddrForThisPeer: new(addr("100.99.0.5")), + } + commit(rm, func(m *Mutation) { m.UpsertPeer(n.View()) }) + + // The data-plane attributes are carried through from the node. + if pr, ok := rm.Outbound().Lookup(addr("100.64.0.1")); !ok || !pr.Jailed || pr.MasqAddr4 != addr("100.99.0.5") || pr.MasqAddr6.IsValid() { + t.Errorf("attrs = %+v, %v; want jailed with v4 masq only", pr, ok) + } + + // Self addresses are live immediately; the subnet route and the + // exit routes were classified as routes and stay gated by prefs. + wantOutbound(t, rm, "100.64.0.1", k1, true) + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) + wantOutbound(t, rm, "8.8.8.8", key.NodePublic{}, false) + + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true, ExitNodeID: 1}) }) + wantOutbound(t, rm, "10.0.0.5", k1, true) + wantOutbound(t, rm, "8.8.8.8", k1, true) +} + +// TestUpsertPeerNodeViewVIPService checks that a single Tailscale IP +// in a peer's AllowedIPs but not in its Addresses (such as a VIP +// service address hosted by the peer) is classified as a self address +// and stays routable without Prefs.RouteAll, mirroring nmcfg's +// cidrIsSubnet. +func TestUpsertPeerNodeViewVIPService(t *testing.T) { + rm := New(t.Logf) + n := &tailcfg.Node{ + ID: 1, + Key: k1, + HomeDERP: 1, + Addresses: []netip.Prefix{ + pfx("100.64.0.1/32"), + }, + AllowedIPs: []netip.Prefix{ + pfx("100.64.0.1/32"), + pfx("100.100.5.5/32"), // VIP service address + pfx("192.168.1.99/32"), // single non-Tailscale IP: a subnet route + pfx("10.0.0.0/24"), + }, + } + commit(rm, func(m *Mutation) { m.UpsertPeer(n.View()) }) + + // With RouteAll off, the node address and the VIP service address + // are routable, but the subnet routes are not. + wantOutbound(t, rm, "100.64.0.1", k1, true) + wantOutbound(t, rm, "100.100.5.5", k1, true) + wantOutbound(t, rm, "192.168.1.99", key.NodePublic{}, false) + wantOutbound(t, rm, "10.0.0.5", key.NodePublic{}, false) + wantOSRoutes(t, rm, "100.64.0.1/32", "100.100.5.5/32") + + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantOutbound(t, rm, "192.168.1.99", k1, true) + wantOutbound(t, rm, "10.0.0.5", k1, true) +} + +// TestUpsertPeerNodeViewEmptyAllowedIPs checks that AllowedIPs is +// the sole source of prefixes: a peer whose AllowedIPs is empty is +// not routable even if it has addresses, matching nmcfg.WGCfg. +func TestUpsertPeerNodeViewEmptyAllowedIPs(t *testing.T) { + rm := New(t.Logf) + n := &tailcfg.Node{ + ID: 1, + Key: k1, + HomeDERP: 1, + Addresses: []netip.Prefix{pfx("100.64.0.1/32")}, + } + commit(rm, func(m *Mutation) { m.UpsertPeer(n.View()) }) + wantOutbound(t, rm, "100.64.0.1", key.NodePublic{}, false) + wantOSRoutes(t, rm) + if _, ok := rm.PeerAllowedIPs(1); ok { + t.Error("PeerAllowedIPs = ok; want !ok for peer with no AllowedIPs") + } +} + +// TestUpsertPeerNodeViewIneligible checks that peers we cannot +// communicate with (expired, or predating both DERP and disco) are +// tracked but contribute no prefixes, mirroring nmcfg.WGCfg's peer +// filtering, and that a later update restores them. +func TestUpsertPeerNodeViewIneligible(t *testing.T) { + base := func() *tailcfg.Node { + return &tailcfg.Node{ + ID: 1, + Key: k1, + HomeDERP: 1, + Addresses: []netip.Prefix{pfx("100.64.0.1/32")}, + AllowedIPs: []netip.Prefix{ + pfx("100.64.0.1/32"), + }, + } + } + + for _, tc := range []struct { + name string + mod func(*tailcfg.Node) + }{ + {"expired", func(n *tailcfg.Node) { n.Expired = true }}, + {"noDERPOrDisco", func(n *tailcfg.Node) { n.HomeDERP = 0 }}, + } { + t.Run(tc.name, func(t *testing.T) { + rm := New(t.Logf) + n := base() + tc.mod(n) + commit(rm, func(m *Mutation) { m.UpsertPeer(n.View()) }) + wantOutbound(t, rm, "100.64.0.1", key.NodePublic{}, false) + wantOSRoutes(t, rm) + if _, ok := rm.PeerAllowedIPs(1); ok { + t.Error("PeerAllowedIPs = ok; want !ok for ineligible peer") + } + + // An update that clears the condition restores the peer. + commit(rm, func(m *Mutation) { m.UpsertPeer(base().View()) }) + wantOutbound(t, rm, "100.64.0.1", k1, true) + }) + } +} + +// wantChangedAllowedIPs checks res.AllowedIPs against want, where +// want maps each expected key to its expected new prefixes (nil for +// "no allowed prefixes"). +func wantChangedAllowedIPs(t *testing.T, res Result, want map[key.NodePublic][]string) { + t.Helper() + wantMap := make(map[key.NodePublic][]netip.Prefix) + for k, ss := range want { + var pfxs []netip.Prefix + for _, s := range ss { + pfxs = append(pfxs, pfx(s)) + } + tsaddr.SortPrefixes(pfxs) + wantMap[k] = pfxs + } + if len(want) == 0 { + if res.AllowedIPs != nil { + t.Errorf("Result.AllowedIPs = %v; want nil", res.AllowedIPs) + } + return + } + if !reflect.DeepEqual(res.AllowedIPs, wantMap) { + t.Errorf("Result.AllowedIPs = %v; want %v", res.AllowedIPs, wantMap) + } +} + +func TestResultAllowedIPs(t *testing.T) { + rm := New(t.Logf) + + // A new peer appears in the map with its prefixes. + res := commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128"}, + }) + + // An identical re-upsert reports nothing. + res = commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + wantChangedAllowedIPs(t, res, nil) + + // Add a second peer advertising a subnet route and the exit + // routes; with default prefs only its self addresses count. + p2 := peer2() + p2.Routes = []netip.Prefix{pfx("10.0.0.0/24"), pfx("0.0.0.0/0"), pfx("::/0")} + res = commit(rm, func(m *Mutation) { m.upsertPeer(p2) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k2: {"100.64.0.2/32", "fd7a:115c:a1e0::2/128"}, + }) + + // RouteAll adds the subnet route to peer 2 only; peer 1 has no + // routes, so it does not appear. + res = commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k2: {"100.64.0.2/32", "fd7a:115c:a1e0::2/128", "10.0.0.0/24"}, + }) + + // Selecting peer 2 as exit node adds the /0s to it only. + res = commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true, ExitNodeID: 2}) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k2: {"100.64.0.2/32", "fd7a:115c:a1e0::2/128", "10.0.0.0/24", "0.0.0.0/0", "::/0"}, + }) + + // Deselecting removes them again. + res = commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k2: {"100.64.0.2/32", "fd7a:115c:a1e0::2/128", "10.0.0.0/24"}, + }) + + // A score change affects only the outbound winner, not any + // peer's allowed prefixes. + res = commit(rm, func(m *Mutation) { m.SetScore(2, pfx("10.0.0.0/24"), 100) }) + wantChangedAllowedIPs(t, res, nil) + + // A key rotation reports the old key with no prefixes and the + // new key with the peer's prefixes. + rotated := peer1() + rotated.Key = k3 + res = commit(rm, func(m *Mutation) { m.upsertPeer(rotated) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: nil, + k3: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128"}, + }) + + // A removed peer reports its key with no prefixes. + res = commit(rm, func(m *Mutation) { m.RemovePeer(2) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k2: nil, + }) + + // A prefs change combined with an upsert of a brand-new peer in + // the same commit reports both correctly: peer 1 gains nothing + // from RouteAll going away (it has no routes), and the new peer + // appears with its prefixes. + p4 := peerView{ + ID: 4, + Key: key.NewNode().Public(), + SelfAddrs: []netip.Prefix{pfx("100.64.0.4/32")}, + } + res = commit(rm, func(m *Mutation) { + m.SetPrefs(Prefs{}) + m.upsertPeer(p4) + }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + p4.Key: {"100.64.0.4/32"}, + }) +} + +func TestPeerAllowedIPs(t *testing.T) { + rm := New(t.Logf) + p := peer1() + p.Routes = []netip.Prefix{pfx("10.0.0.0/24"), pfx("0.0.0.0/0"), pfx("::/0")} + commit(rm, func(m *Mutation) { m.upsertPeer(p) }) + + wantAllowed := func(want ...string) { + t.Helper() + var wantPfxs []netip.Prefix + for _, s := range want { + wantPfxs = append(wantPfxs, pfx(s)) + } + tsaddr.SortPrefixes(wantPfxs) + got, ok := rm.PeerAllowedIPs(1) + if len(want) == 0 { + if ok { + t.Errorf("PeerAllowedIPs = %v; want !ok", got) + } + return + } + if !ok || !slices.Equal(got, wantPfxs) { + t.Errorf("PeerAllowedIPs = %v, %v; want %v", got, ok, wantPfxs) + } + } + + // Default prefs: self addresses only. + wantAllowed("100.64.0.1/32", "fd7a:115c:a1e0::1/128") + + // RouteAll adds the subnet route but not the exit routes. + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantAllowed("100.64.0.1/32", "fd7a:115c:a1e0::1/128", "10.0.0.0/24") + + // Selecting the peer as exit node adds the /0s. + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true, ExitNodeID: 1}) }) + wantAllowed("100.64.0.1/32", "fd7a:115c:a1e0::1/128", "10.0.0.0/24", "0.0.0.0/0", "::/0") + + // Unknown peers report !ok. + if _, ok := rm.PeerAllowedIPs(42); ok { + t.Error("PeerAllowedIPs(42) = ok; want !ok") + } + + // A removed peer reports !ok. + commit(rm, func(m *Mutation) { m.RemovePeer(1) }) + wantAllowed() +} + +func TestExtraAllowedIPs(t *testing.T) { + rm := New(t.Logf) + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + + // Extras appear in the outbound table and in PeerAllowedIPs, + // but not in the OS route set, even for prefixes (like the + // CGNAT one here) whose class would otherwise be coarsened. + res := commit(rm, func(m *Mutation) { + m.SetExtraAllowedIPs(map[tailcfg.NodeID][]netip.Prefix{ + 1: {pfx("fe80::1234/128"), pfx("100.100.100.100/32")}, + }) + }) + if !res.ExtrasChanged { + t.Error("ExtrasChanged = false; want true") + } + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128", "fe80::1234/128", "100.100.100.100/32"}, + }) + wantOutbound(t, rm, "fe80::1234", k1, true) + wantOutbound(t, rm, "100.100.100.100", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") + + // An identical re-set is a no-op. + res = commit(rm, func(m *Mutation) { + m.SetExtraAllowedIPs(map[tailcfg.NodeID][]netip.Prefix{ + 1: {pfx("fe80::1234/128"), pfx("100.100.100.100/32")}, + }) + }) + if res.ExtrasChanged || res.OutboundChanged || res.AllowedIPs != nil { + t.Errorf("no-op re-set changed something: %+v", res) + } + + // Replacing the set drops the old prefixes and adds the new one. + res = commit(rm, func(m *Mutation) { + m.SetExtraAllowedIPs(map[tailcfg.NodeID][]netip.Prefix{ + 1: {pfx("fe80::5678/128")}, + }) + }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128", "fe80::5678/128"}, + }) + wantOutbound(t, rm, "fe80::1234", key.NodePublic{}, false) + wantOutbound(t, rm, "fe80::5678", k1, true) + + // Clearing the set removes the remaining extra. + res = commit(rm, func(m *Mutation) { m.SetExtraAllowedIPs(nil) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128"}, + }) + wantOutbound(t, rm, "fe80::5678", key.NodePublic{}, false) +} + +func TestExtraAllowedIPsPeerLifecycle(t *testing.T) { + rm := New(t.Logf) + + // Extras for a peer that doesn't exist yet have no visible + // effect until the peer is upserted. + res := commit(rm, func(m *Mutation) { + m.SetExtraAllowedIPs(map[tailcfg.NodeID][]netip.Prefix{ + 1: {pfx("fe80::1234/128")}, + }) + }) + if !res.ExtrasChanged || res.OutboundChanged || res.AllowedIPs != nil { + t.Errorf("extras for unknown peer: %+v", res) + } + wantOutbound(t, rm, "fe80::1234", key.NodePublic{}, false) + + res = commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{ + k1: {"100.64.0.1/32", "fd7a:115c:a1e0::1/128", "fe80::1234/128"}, + }) + wantOutbound(t, rm, "fe80::1234", k1, true) + + // Removing the peer removes its extras from the tables, but the + // stored entry survives and re-applies if the peer comes back. + res = commit(rm, func(m *Mutation) { m.RemovePeer(1) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{k1: nil}) + wantOutbound(t, rm, "fe80::1234", key.NodePublic{}, false) + + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + wantOutbound(t, rm, "fe80::1234", k1, true) + + // A non-communicable peer (no addresses or routes) contributes + // no prefixes at all, including its extras. + res = commit(rm, func(m *Mutation) { m.upsertPeer(peerView{ID: 1, Key: k1}) }) + wantChangedAllowedIPs(t, res, map[key.NodePublic][]string{k1: nil}) + wantOutbound(t, rm, "fe80::1234", key.NodePublic{}, false) + + // Extras survive a prefs-driven full rebuild. + commit(rm, func(m *Mutation) { m.upsertPeer(peer1()) }) + commit(rm, func(m *Mutation) { m.SetPrefs(Prefs{RouteAll: true}) }) + wantOutbound(t, rm, "fe80::1234", k1, true) + wantOSRoutes(t, rm, "100.64.0.1/32", "fd7a:115c:a1e0::/48") +} diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index dbaabdc0b..7107fa592 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -9,9 +9,7 @@ "io" "net/netip" "os" - "reflect" "runtime" - "slices" "strings" "sync" "sync/atomic" @@ -28,6 +26,7 @@ "tailscale.com/feature/buildfeatures" "tailscale.com/net/packet" "tailscale.com/net/packet/checksum" + "tailscale.com/net/routemanager" "tailscale.com/net/tsaddr" "tailscale.com/syncs" "tailscale.com/tstime/mono" @@ -587,9 +586,9 @@ func findV6(addrs []netip.Prefix) netip.Addr { return netip.Addr{} } -// peerConfigTable contains configuration for individual peers and related -// information necessary to perform peer-specific operations. It should be -// treated as immutable. +// peerConfigTable contains the per-peer route attributes and related +// information necessary to perform peer-specific operations. It should +// be treated as immutable. // // The nil value is a valid configuration. type peerConfigTable struct { @@ -602,57 +601,17 @@ type peerConfigTable struct { // inbound packet is IPv6. nativeAddr4, nativeAddr6 netip.Addr - // byIP contains configuration for each peer, indexed by a peer's IP - // address(es). - byIP bart.Table[*peerConfig] - - // masqAddrCounts is a count of peers by MasqueradeAsIP. - // TODO? for logging - masqAddrCounts map[netip.Addr]int -} - -// peerConfig is the configuration for a single peer. -type peerConfig struct { - // dstMasqAddr{4,6} are the addresses that should be used as the - // source address when masquerading packets to this peer (i.e. - // SNAT). If an address is not valid, the packet should not be - // masqueraded for that address family. - dstMasqAddr4 netip.Addr - dstMasqAddr6 netip.Addr - - // jailed is whether this peer is "jailed" (i.e. is restricted from being - // able to initiate connections to this node). This is the case for shared - // nodes. - jailed bool + // byIP maps a peer's IP addresses and routed prefixes to the + // peer's route attributes. It is a shared immutable snapshot + // from [routemanager.RouteManager.Outbound]. + byIP *bart.Table[*routemanager.PeerRoute] } func (c *peerConfigTable) String() string { if c == nil { return "peerConfigTable(nil)" } - var b strings.Builder - b.WriteString("peerConfigTable{") - fmt.Fprintf(&b, "nativeAddr4: %v, ", c.nativeAddr4) - fmt.Fprintf(&b, "nativeAddr6: %v, ", c.nativeAddr6) - - // TODO: figure out how to iterate/debug/print c.byIP - - b.WriteString("}") - - return b.String() -} - -func (c *peerConfig) String() string { - if c == nil { - return "peerConfig(nil)" - } - var b strings.Builder - b.WriteString("peerConfig{") - fmt.Fprintf(&b, "dstMasqAddr4: %v, ", c.dstMasqAddr4) - fmt.Fprintf(&b, "dstMasqAddr6: %v, ", c.dstMasqAddr6) - fmt.Fprintf(&b, "jailed: %v}", c.jailed) - - return b.String() + return fmt.Sprintf("peerConfigTable{nativeAddr4: %v, nativeAddr6: %v}", c.nativeAddr4, c.nativeAddr6) } // mapDstIP returns the destination IP to use for a packet to dst. @@ -675,10 +634,10 @@ func (pc *peerConfigTable) mapDstIP(src, oldDst netip.Addr) netip.Addr { return oldDst } - if oldDst.Is4() && pc.nativeAddr4.IsValid() && c.dstMasqAddr4 == oldDst { + if oldDst.Is4() && pc.nativeAddr4.IsValid() && c.MasqAddr4 == oldDst { return pc.nativeAddr4 } - if oldDst.Is6() && pc.nativeAddr6.IsValid() && c.dstMasqAddr6 == oldDst { + if oldDst.Is6() && pc.nativeAddr6.IsValid() && c.MasqAddr6 == oldDst { return pc.nativeAddr6 } return oldDst @@ -708,105 +667,17 @@ func (pc *peerConfigTable) selectSrcIP(oldSrc, dst netip.Addr) netip.Addr { // Perform SNAT based on the address family and whether we have a valid // addr. - if oldSrc.Is4() && c.dstMasqAddr4.IsValid() { - return c.dstMasqAddr4 + if oldSrc.Is4() && c.MasqAddr4.IsValid() { + return c.MasqAddr4 } - if oldSrc.Is6() && c.dstMasqAddr6.IsValid() { - return c.dstMasqAddr6 + if oldSrc.Is6() && c.MasqAddr6.IsValid() { + return c.MasqAddr6 } // No SNAT; use old src return oldSrc } -// peerConfigTableFromWGConfig generates a peerConfigTable from nm. If NAT is -// not required, and no additional configuration is present, it returns nil. -func peerConfigTableFromWGConfig(wcfg *wgcfg.Config) *peerConfigTable { - if wcfg == nil { - return nil - } - - nativeAddr4 := findV4(wcfg.Addresses) - nativeAddr6 := findV6(wcfg.Addresses) - if !nativeAddr4.IsValid() && !nativeAddr6.IsValid() { - return nil - } - - ret := &peerConfigTable{ - nativeAddr4: nativeAddr4, - nativeAddr6: nativeAddr6, - masqAddrCounts: make(map[netip.Addr]int), - } - - // When using an exit node that requires masquerading, we need to - // fill out the routing table with all peers not just the ones that - // require masquerading. - exitNodeRequiresMasq := false // true if using an exit node and it requires masquerading - for _, p := range wcfg.Peers { - isExitNode := slices.Contains(p.AllowedIPs, tsaddr.AllIPv4()) || slices.Contains(p.AllowedIPs, tsaddr.AllIPv6()) - if isExitNode { - hasMasqAddr := false || - (p.V4MasqAddr != nil && p.V4MasqAddr.IsValid()) || - (p.V6MasqAddr != nil && p.V6MasqAddr.IsValid()) - if hasMasqAddr { - exitNodeRequiresMasq = true - } - break - } - } - - byIPSize := 0 - for i := range wcfg.Peers { - p := &wcfg.Peers[i] - - // Build a routing table that configures DNAT (i.e. changing - // the V4MasqAddr/V6MasqAddr for a given peer to the current - // peer's v4/v6 IP). - var addrToUse4, addrToUse6 netip.Addr - if p.V4MasqAddr != nil && p.V4MasqAddr.IsValid() { - addrToUse4 = *p.V4MasqAddr - ret.masqAddrCounts[addrToUse4]++ - } - if p.V6MasqAddr != nil && p.V6MasqAddr.IsValid() { - addrToUse6 = *p.V6MasqAddr - ret.masqAddrCounts[addrToUse6]++ - } - - // If the exit node requires masquerading, set the masquerade - // addresses to our native addresses. - if exitNodeRequiresMasq { - if !addrToUse4.IsValid() && nativeAddr4.IsValid() { - addrToUse4 = nativeAddr4 - } - if !addrToUse6.IsValid() && nativeAddr6.IsValid() { - addrToUse6 = nativeAddr6 - } - } - - if !addrToUse4.IsValid() && !addrToUse6.IsValid() && !p.IsJailed { - // NAT not required for this peer. - continue - } - - // Use the same peer configuration for each address of the peer. - pc := &peerConfig{ - dstMasqAddr4: addrToUse4, - dstMasqAddr6: addrToUse6, - jailed: p.IsJailed, - } - - // Insert an entry into our routing table for each allowed IP. - for _, ip := range p.AllowedIPs { - ret.byIP.Insert(ip, pc) - byIPSize++ - } - } - if byIPSize == 0 && len(ret.masqAddrCounts) == 0 { - return nil - } - return ret -} - func (pc *peerConfigTable) inboundPacketIsJailed(p *packet.Parsed) bool { if pc == nil { return false @@ -815,7 +686,7 @@ func (pc *peerConfigTable) inboundPacketIsJailed(p *packet.Parsed) bool { if !ok { return false } - return c.jailed + return c.Jailed } func (pc *peerConfigTable) outboundPacketIsJailed(p *packet.Parsed) bool { @@ -826,7 +697,7 @@ func (pc *peerConfigTable) outboundPacketIsJailed(p *packet.Parsed) bool { if !ok { return false } - return c.jailed + return c.Jailed } // SetIPer is the interface expected to be implemented by the TAP implementation @@ -836,16 +707,40 @@ type SetIPer interface { SetIP(ipV4, ipV6 netip.Addr) error } -// SetWGConfig is called when a new NetworkMap is received. +// SetWGConfig is called when a new NetworkMap is received. Its only +// remaining job is updating the TAP device's IP addresses; the +// per-peer route attributes arrive via [Wrapper.SetPeerRoutes]. func (t *Wrapper) SetWGConfig(wcfg *wgcfg.Config) { if t.isTAP { if sip, ok := t.tdev.(SetIPer); ok { sip.SetIP(findV4(wcfg.Addresses), findV6(wcfg.Addresses)) } } - cfg := peerConfigTableFromWGConfig(wcfg) +} + +// SetPeerRoutes is called whenever this node's Tailscale addresses or +// the route manager's outbound table change. native4 and native6 are +// this node's own Tailscale addresses, and routes maps each peer's +// addresses and routed prefixes to its route attributes. +// +// A nil routes table disables all per-packet peer processing (NAT +// rewrites and jailed-filter selection); callers pass nil when no +// current peer has any such attributes, which keeps the common +// per-packet path to a nil check. +func (t *Wrapper) SetPeerRoutes(native4, native6 netip.Addr, routes *bart.Table[*routemanager.PeerRoute]) { + var cfg *peerConfigTable + if routes != nil && (native4.IsValid() || native6.IsValid()) { + cfg = &peerConfigTable{ + nativeAddr4: native4, + nativeAddr6: native6, + byIP: routes, + } + } old := t.peerConfig.Swap(cfg) - if !reflect.DeepEqual(old, cfg) { + // Log only on nil-ness or native address transitions; the routes + // table changes with every routing update and is too chatty to log. + if (old == nil) != (cfg == nil) || + (old != nil && (old.nativeAddr4 != cfg.nativeAddr4 || old.nativeAddr6 != cfg.nativeAddr6)) { t.logf("peer config: %v", cfg) } } diff --git a/net/tstun/wrap_test.go b/net/tstun/wrap_test.go index 631dc083e..7fd4f2ce9 100644 --- a/net/tstun/wrap_test.go +++ b/net/tstun/wrap_test.go @@ -18,6 +18,7 @@ "unicode" "unsafe" + "github.com/gaissmai/bart" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/tailscale/wireguard-go/tun/tuntest" @@ -29,6 +30,7 @@ "tailscale.com/net/netaddr" "tailscale.com/net/packet" "tailscale.com/net/packet/checksum" + "tailscale.com/net/routemanager" "tailscale.com/tstest" "tailscale.com/tstime/mono" "tailscale.com/types/ipproto" @@ -42,7 +44,6 @@ "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/netstack/gro" - "tailscale.com/wgengine/wgcfg" ) func udp4(src, dst string, sport, dport uint16) []byte { @@ -710,22 +711,23 @@ func TestFilterDiscoLoop(t *testing.T) { } // TODO(andrew-d): refactor this test to no longer use addrFam, after #11945 -// removed it in peerConfigFromWGConfig func TestPeerCfg_NAT(t *testing.T) { - node := func(ip, masqIP netip.Addr, otherAllowedIPs ...netip.Prefix) wgcfg.Peer { - p := wgcfg.Peer{ - PublicKey: key.NewNode().Public(), - AllowedIPs: []netip.Prefix{ - netip.PrefixFrom(ip, ip.BitLen()), - }, + type testPeer struct { + pr *routemanager.PeerRoute + pfxs []netip.Prefix + } + node := func(ip, masqIP netip.Addr, otherAllowedIPs ...netip.Prefix) testPeer { + pr := &routemanager.PeerRoute{Key: key.NewNode().Public()} + switch { + case masqIP.Is4(): + pr.MasqAddr4 = masqIP + case masqIP.Is6(): + pr.MasqAddr6 = masqIP } - if masqIP.Is4() { - p.V4MasqAddr = new(masqIP) - } else { - p.V6MasqAddr = new(masqIP) + return testPeer{ + pr: pr, + pfxs: append([]netip.Prefix{netip.PrefixFrom(ip, ip.BitLen())}, otherAllowedIPs...), } - p.AllowedIPs = append(p.AllowedIPs, otherAllowedIPs...) - return p } test := func(addrFam ipproto.Version) { var ( @@ -734,7 +736,6 @@ func TestPeerCfg_NAT(t *testing.T) { selfNativeIP = netip.MustParseAddr("100.64.0.1") selfEIP1 = netip.MustParseAddr("100.64.1.1") selfEIP2 = netip.MustParseAddr("100.64.1.2") - selfAddrs = []netip.Prefix{netip.PrefixFrom(selfNativeIP, selfNativeIP.BitLen())} peer1IP = netip.MustParseAddr("100.64.0.2") peer2IP = netip.MustParseAddr("100.64.0.3") @@ -749,7 +750,6 @@ func TestPeerCfg_NAT(t *testing.T) { selfNativeIP = netip.MustParseAddr("fd7a:115c:a1e0::a") selfEIP1 = netip.MustParseAddr("fd7a:115c:a1e0::1a") selfEIP2 = netip.MustParseAddr("fd7a:115c:a1e0::1b") - selfAddrs = []netip.Prefix{netip.PrefixFrom(selfNativeIP, selfNativeIP.BitLen())} peer1IP = netip.MustParseAddr("fd7a:115c:a1e0::b") peer2IP = netip.MustParseAddr("fd7a:115c:a1e0::c") @@ -769,13 +769,13 @@ type dnatTest struct { tests := []struct { name string - wcfg *wgcfg.Config + peers []testPeer // nil means no config at all snatMap map[netip.Addr]netip.Addr // dst -> src dnat []dnatTest }{ { - name: "no-cfg", - wcfg: nil, + name: "no-cfg", + peers: nil, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfNativeIP, peer2IP: selfNativeIP, @@ -789,12 +789,9 @@ type dnatTest struct { }, { name: "single-peer-requires-nat", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, noIP), - node(peer2IP, selfEIP2), - }, + peers: []testPeer{ + node(peer1IP, noIP), + node(peer2IP, selfEIP2), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfNativeIP, @@ -810,12 +807,9 @@ type dnatTest struct { }, { name: "multiple-peers-require-nat", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, selfEIP1), - node(peer2IP, selfEIP2), - }, + peers: []testPeer{ + node(peer1IP, selfEIP1), + node(peer2IP, selfEIP2), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfEIP1, @@ -831,12 +825,9 @@ type dnatTest struct { }, { name: "multiple-peers-require-nat-with-subnet", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, selfEIP1), - node(peer2IP, selfEIP2, subnet), - }, + peers: []testPeer{ + node(peer1IP, selfEIP1), + node(peer2IP, selfEIP2, subnet), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfEIP1, @@ -852,12 +843,9 @@ type dnatTest struct { }, { name: "multiple-peers-require-nat-with-default-route", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, selfEIP1), - node(peer2IP, selfEIP2, exitRoute), - }, + peers: []testPeer{ + node(peer1IP, selfEIP1), + node(peer2IP, selfEIP2, exitRoute), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfEIP1, @@ -873,12 +861,9 @@ type dnatTest struct { }, { name: "no-nat", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, noIP), - node(peer2IP, noIP), - }, + peers: []testPeer{ + node(peer1IP, noIP), + node(peer2IP, noIP), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfNativeIP, @@ -894,12 +879,9 @@ type dnatTest struct { }, { name: "exit-node-require-nat-peer-doesnt", - wcfg: &wgcfg.Config{ - Addresses: selfAddrs, - Peers: []wgcfg.Peer{ - node(peer1IP, noIP), - node(peer2IP, selfEIP2, exitRoute), - }, + peers: []testPeer{ + node(peer1IP, noIP), + node(peer2IP, selfEIP2, exitRoute), }, snatMap: map[netip.Addr]netip.Addr{ peer1IP: selfNativeIP, @@ -916,7 +898,20 @@ type dnatTest struct { for _, tc := range tests { t.Run(fmt.Sprintf("%v/%v", addrFam, tc.name), func(t *testing.T) { - pcfg := peerConfigTableFromWGConfig(tc.wcfg) + var pcfg *peerConfigTable + if tc.peers != nil { + pcfg = &peerConfigTable{byIP: &bart.Table[*routemanager.PeerRoute]{}} + if selfNativeIP.Is4() { + pcfg.nativeAddr4 = selfNativeIP + } else { + pcfg.nativeAddr6 = selfNativeIP + } + for _, p := range tc.peers { + for _, pfx := range p.pfxs { + pcfg.byIP.Insert(pfx, p.pr) + } + } + } for peer, want := range tc.snatMap { if got := pcfg.selectSrcIP(selfNativeIP, peer); got != want { t.Errorf("selectSrcIP[%v]: got %v; want %v", peer, got, want) diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 7162bb616..53304f583 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -206,6 +206,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routecheck from tailscale.com/client/local+ tailscale.com/net/routecheck/peernode from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/routemanager from tailscale.com/ipn/ipnlocal+ 💣 tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 50e75417d..5ca840c5b 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -886,7 +886,8 @@ func (s *Server) start() (reterr error) { ns.GetUDPHandlerForFlow = s.getUDPHandlerForFlow s.netstack = ns s.dialer.UseNetstackForIP = func(ip netip.Addr) bool { - _, ok := eng.PeerForIP(ip) + // s.lb is assigned below, before any dials can happen. + _, ok := s.lb.PeerForIP(ip) return ok } s.dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { diff --git a/tsnet/tsnet_test.go b/tsnet/tsnet_test.go index 6a8aff4f2..6b54060bf 100644 --- a/tsnet/tsnet_test.go +++ b/tsnet/tsnet_test.go @@ -1746,7 +1746,7 @@ func TestFallbackTCPHandler(t *testing.T) { // Before aa5da2e5f22a, every peer change rebuilt a full netmap on // the engine, so [wgengine.Engine.SetNetworkMap] kept the engine's // cached netmap fresh and [wgengine.Engine.Reconfig] kept wgdev and -// e.lastCfgFull fresh. After that refactor, peer adds and removes +// e.lastCfg fresh. After that refactor, peer adds and removes // ride the delta path and only mutate [nodeBackend]; the engine's // cached netmap and wireguard config stayed stale, and // [wgengine.Engine.PeerForIP] / [wgengine.Engine.Ping] / wgdev's @@ -1925,7 +1925,7 @@ func TestPingSubnetRouteOfDeltaPeer(t *testing.T) { // outbound wgdev encryption, magicsock transport. The receiving // side (s2's tstun.Wrapper) intercepts TSMP regardless of dst // IP and replies with a pong. So this catches both halves of - // the bug: a stale BART / lastCfgFull (PeerForIP / lookupPeerByIP + // the bug: a stale BART / lastCfg (PeerForIP / lookupPeerByIP // miss) AND a stale wgdev PeerLookupFunc closure (peer's noise // key not yet registered for outbound encryption). pingCtx, cancelPing := pingTimeout(ctx) diff --git a/util/deephash/tailscale_types_test.go b/util/deephash/tailscale_types_test.go index 7e803c841..7cff5335c 100644 --- a/util/deephash/tailscale_types_test.go +++ b/util/deephash/tailscale_types_test.go @@ -86,11 +86,6 @@ func getVal() *tailscaleTypes { return &tailscaleTypes{ &wgcfg.Config{ Addresses: []netip.Prefix{netip.PrefixFrom(netip.AddrFrom16([16]byte{3: 3}).Unmap(), 5)}, - Peers: []wgcfg.Peer{ - { - PublicKey: key.NodePublic{}, - }, - }, }, &router.Config{ Routes: []netip.Prefix{ diff --git a/wgengine/bench/wg.go b/wgengine/bench/wg.go index 4264ae448..0e96a7b61 100644 --- a/wgengine/bench/wg.go +++ b/wgengine/bench/wg.go @@ -82,6 +82,29 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netip. e1.SetFilter(filter.NewAllowAllForTest(l1)) e2.SetFilter(filter.NewAllowAllForTest(l2)) + // There is no LocalBackend in this benchmark, so install trivial + // outbound peer lookups and per-peer config sources; without them, + // outbound packets can't lazily create their WireGuard peer. + k1pub, k2pub := k1.Public(), k2.Public() + e1.SetPeerByIPPacketFunc(func(dst netip.Addr) (_ key.NodePublic, ok bool) { + return k2pub, a2.Contains(dst) + }) + e2.SetPeerByIPPacketFunc(func(dst netip.Addr) (_ key.NodePublic, ok bool) { + return k1pub, a1.Contains(dst) + }) + e1.SetPeerConfigFunc(func(pubk key.NodePublic) (_ []netip.Prefix, ok bool) { + if pubk == k2pub { + return []netip.Prefix{a2}, true + } + return nil, false + }) + e2.SetPeerConfigFunc(func(pubk key.NodePublic) (_ []netip.Prefix, ok bool) { + if pubk == k1pub { + return []netip.Prefix{a1}, true + } + return nil, false + }) + var wait sync.WaitGroup wait.Add(2) @@ -96,12 +119,6 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netip. logf("e1 status: %v", *st) e2.SetSelfNode(tailcfg.NodeView{}) - - p := wgcfg.Peer{ - PublicKey: c1.PrivateKey.Public(), - AllowedIPs: []netip.Prefix{a1}, - } - c2.Peers = []wgcfg.Peer{p} e2.Reconfig(&c2, &router.Config{}, new(dns.Config)) e1waitDoneOnce.Do(wait.Done) }) @@ -117,12 +134,6 @@ func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netip. logf("e2 status: %v", *st) e1.SetSelfNode(tailcfg.NodeView{}) - - p := wgcfg.Peer{ - PublicKey: c2.PrivateKey.Public(), - AllowedIPs: []netip.Prefix{a2}, - } - c1.Peers = []wgcfg.Peer{p} e1.Reconfig(&c1, &router.Config{}, new(dns.Config)) e2waitDoneOnce.Do(wait.Done) }) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 8e4c6bba7..6b759682e 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -327,12 +327,6 @@ type Conn struct { // for a period of time before withdrawing them. endpointTracker endpointTracker - // peerSet is the set of peers that are currently configured in - // WireGuard. These are not used to filter inbound or outbound - // traffic at all, but only to track what state can be cleaned up - // in other maps below that are keyed by peer public key. - peerSet set.Set[key.NodePublic] - // peerMap tracks the networkmap Node entity for each peer // by node key, node ID, and discovery key. peerMap peerMap @@ -2801,31 +2795,6 @@ func (c *Conn) SetPrivateKey(privateKey key.NodePrivate) error { return nil } -// UpdatePeers is called when the set of WireGuard peers changes. It -// then removes any state for old peers. -// -// The caller passes ownership of newPeers map to UpdatePeers. -func (c *Conn) UpdatePeers(newPeers set.Set[key.NodePublic]) { - c.mu.Lock() - defer c.mu.Unlock() - - oldPeers := c.peerSet - c.peerSet = newPeers - - // Clean up any key.NodePublic-keyed maps for peers that no longer - // exist. - for peer := range oldPeers { - if !newPeers.Contains(peer) { - delete(c.derpRoute, peer) - delete(c.peerLastDerp, peer) - } - } - - if len(oldPeers) == 0 && len(newPeers) > 0 { - go c.ReSTUN("non-zero-peers") - } -} - // debugRingBufferSize returns a maximum size for our set of endpoint ring // buffers by assuming that a single large update is ~500 bytes, and that we // want to not use more than 1MiB of memory on phones / 4MiB on other devices. @@ -3168,7 +3137,21 @@ func (c *Conn) updateNodes(self tailcfg.NodeView, peers []tailcfg.NodeView) (pee // Duplicate NodeIDs in the input shouldn't happen, but log if so. c.logf("[unexpected] magicsock.updateNodes: %d peers input but %d unique IDs", len(peers), len(newPeers)) } + // Clean up node-key-keyed state for peers that are gone or have + // rotated their node key. + for id, prev := range c.peersByID { + if n, ok := newPeers[id]; !ok || n.Key() != prev.Key() { + delete(c.derpRoute, prev.Key()) + delete(c.peerLastDerp, prev.Key()) + } + } + hadPeers := len(c.peersByID) > 0 c.peersByID = newPeers + // A key-less conn can't STUN; the ReSTUN("set-private-key") on + // first key covers peers that arrived before the key. + if !hadPeers && len(newPeers) > 0 && !c.privateKey.IsZero() { + go c.ReSTUN("non-zero-peers") + } // If the upsert pass left stale endpoints in peerMap (peers removed // relative to before), clean them up. @@ -3322,7 +3305,16 @@ func (c *Conn) UpsertPeer(n tailcfg.NodeView) { return } flags := c.debugFlagsLocked() + if prev, ok := c.peersByID[n.ID()]; ok && prev.Key() != n.Key() { + delete(c.derpRoute, prev.Key()) + delete(c.peerLastDerp, prev.Key()) + } + hadPeers := len(c.peersByID) > 0 mak.Set(&c.peersByID, n.ID(), n) + // See the matching trigger in updateNodes for why the key check. + if !hadPeers && !c.privateKey.IsZero() { + go c.ReSTUN("non-zero-peers") + } c.upsertPeerLocked(n, flags, debugRingBufferSize(len(c.peersByID))) var relayUpsert candidatePeerRelay @@ -3360,6 +3352,8 @@ func (c *Conn) RemovePeer(nid tailcfg.NodeID) { return } delete(c.peersByID, nid) + delete(c.derpRoute, prev.Key()) + delete(c.peerLastDerp, prev.Key()) if ep, ok := c.peerMap.endpointForNodeID(nid); ok { c.peerMap.deleteEndpoint(ep) } @@ -3629,7 +3623,7 @@ func (c *Conn) shouldDoPeriodicReSTUNLocked() bool { if c.networkDown() || c.homeless { return false } - if len(c.peerSet) == 0 || c.privateKey.IsZero() { + if len(c.peersByID) == 0 || c.privateKey.IsZero() { // If no peers, not worth doing. // Also don't if there's no key (not running). return false diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 84b4e999e..009ab72d2 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -32,6 +32,7 @@ "unsafe" qt "github.com/frankban/quicktest" + "github.com/gaissmai/bart" "github.com/google/go-cmp/cmp" wgconn "github.com/tailscale/wireguard-go/conn" "github.com/tailscale/wireguard-go/device" @@ -53,6 +54,7 @@ "tailscale.com/net/netmon" "tailscale.com/net/packet" "tailscale.com/net/ping" + "tailscale.com/net/routemanager" "tailscale.com/net/stun" "tailscale.com/net/stun/stuntest" "tailscale.com/net/tstun" @@ -242,28 +244,101 @@ func newMagicStackWithKey(t testing.TB, logf logger.Logf, ln nettype.PacketListe } } -func (s *magicStack) Reconfig(cfg *wgcfg.Config) error { +// wgPeer is the per-peer configuration a test hands to +// [magicStack.Reconfig], standing in for what LocalBackend's route +// manager provides to the engine in production. +type wgPeer struct { + publicKey key.NodePublic + allowedIPs []netip.Prefix + jailed bool + masqAddr4 netip.Addr + masqAddr6 netip.Addr +} + +// wgPeersOfNetmap converts nm's peers into the per-peer configuration +// that [magicStack.Reconfig] takes. +func wgPeersOfNetmap(nm *netmap.NetworkMap) []wgPeer { + peers := make([]wgPeer, 0, len(nm.Peers)) + for _, p := range nm.Peers { + peers = append(peers, wgPeer{ + publicKey: p.Key(), + allowedIPs: p.AllowedIPs().AsSlice(), + jailed: p.IsJailed(), + masqAddr4: p.SelfNodeV4MasqAddrForThisPeer().GetOr(netip.Addr{}), + masqAddr6: p.SelfNodeV6MasqAddrForThisPeer().GetOr(netip.Addr{}), + }) + } + return peers +} + +// Reconfig applies cfg and peers to the stack's WireGuard device and +// tun-layer data plane. In production these flow from LocalBackend +// (via [tailscale.com/wgengine.Engine.Reconfig] and the live per-peer +// config source installed with Engine.SetPeerConfigFunc); tests that +// bypass LocalBackend replicate that wiring here. +func (s *magicStack) Reconfig(cfg *wgcfg.Config, peers []wgPeer) error { s.tsTun.SetWGConfig(cfg) - // In production, LocalBackend installs a PeerByIPPacketFunc via - // Engine.SetPeerByIPPacketFunc. Tests that bypass LocalBackend need - // to install one here for outbound packet routing. - ipToPeer := make(map[netip.Addr]device.NoisePublicKey, len(cfg.Peers)) - for _, p := range cfg.Peers { - pk := p.PublicKey.Raw32() - for _, pfx := range p.AllowedIPs { + // Build the tun-layer per-peer route attribute table so that + // masquerade rewrites and jailed classification work, and the + // destination IP => peer map for outbound packet routing. + tbl := &bart.Table[*routemanager.PeerRoute]{} + byKey := make(map[device.NoisePublicKey]wgPeer, len(peers)) + ipToPeer := make(map[netip.Addr]device.NoisePublicKey) + for _, p := range peers { + pr := &routemanager.PeerRoute{ + Key: p.publicKey, + Jailed: p.jailed, + MasqAddr4: p.masqAddr4, + MasqAddr6: p.masqAddr6, + } + pk := p.publicKey.Raw32() + byKey[pk] = p + for _, pfx := range p.allowedIPs { + tbl.Insert(pfx, pr) if pfx.IsSingleIP() { ipToPeer[pfx.Addr()] = pk } } } + var native4, native6 netip.Addr + for _, pfx := range cfg.Addresses { + a := pfx.Addr() + switch { + case a.Is4() && !native4.IsValid(): + native4 = a + case a.Is6() && !native6.IsValid(): + native6 = a + } + } + s.tsTun.SetPeerRoutes(native4, native6, tbl) + s.dev.SetPeerByIPPacketFunc(func(_, dst netip.Addr, _ []byte) (device.NoisePublicKey, bool) { pk, ok := ipToPeer[dst] return pk, ok }) + // Install the live per-peer config source (peers are created + // lazily on first traffic) and converge existing device peers with + // it, mirroring what Engine.SyncDevicePeer does per peer. + s.dev.SetPeerLookupFunc(wgcfg.NewPeerLookupFunc(s.conn.Bind(), s.conn.logf, func(pubk device.NoisePublicKey) ([]netip.Prefix, bool) { + p, ok := byKey[pubk] + if !ok { + return nil, false + } + return p.allowedIPs, true + })) s.dev.SetPrivateKey(key.NodePrivateAs[device.NoisePrivateKey](cfg.PrivateKey)) - return wgcfg.ReconfigDevice(s.dev, cfg, s.conn.logf) + s.dev.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool { + _, ok := byKey[pk] + return !ok + }) + for pk, p := range byKey { + if peer, ok := s.dev.LookupActivePeer(pk); ok { + peer.SetAllowedIPs(p.allowedIPs) + } + } + return nil } func (s *magicStack) String() string { @@ -365,18 +440,13 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM for i, m := range ms { nm := buildNetmapLocked(i) m.conn.SetNetworkMap(nm.SelfNode, nm.Peers) - peerSet := make(set.Set[key.NodePublic], len(nm.Peers)) - for _, peer := range nm.Peers { - peerSet.Add(peer.Key()) - } - m.conn.UpdatePeers(peerSet) wg, err := nmcfg.WGCfg(ms[i].privateKey, nm, logf, 0, "") if err != nil { // We're too far from the *testing.T to be graceful, // blow up. Shouldn't happen anyway. panic(fmt.Sprintf("failed to construct wgcfg from netmap: %v", err)) } - if err := m.Reconfig(wg); err != nil { + if err := m.Reconfig(wg, wgPeersOfNetmap(nm)); err != nil { if ctx.Err() != nil || errors.Is(err, errConnClosed) { // shutdown race, don't care. return @@ -1182,30 +1252,28 @@ func testTwoDevicePing(t *testing.T, d *devices) { m1cfg := &wgcfg.Config{ PrivateKey: m1.privateKey, Addresses: []netip.Prefix{netip.MustParsePrefix("1.0.0.1/32")}, - Peers: []wgcfg.Peer{ - { - PublicKey: m2.privateKey.Public(), - DiscoKey: m2.conn.DiscoPublicKey(), - AllowedIPs: []netip.Prefix{netip.MustParsePrefix("1.0.0.2/32")}, - }, + } + m1peers := []wgPeer{ + { + publicKey: m2.privateKey.Public(), + allowedIPs: []netip.Prefix{netip.MustParsePrefix("1.0.0.2/32")}, }, } m2cfg := &wgcfg.Config{ PrivateKey: m2.privateKey, Addresses: []netip.Prefix{netip.MustParsePrefix("1.0.0.2/32")}, - Peers: []wgcfg.Peer{ - { - PublicKey: m1.privateKey.Public(), - DiscoKey: m1.conn.DiscoPublicKey(), - AllowedIPs: []netip.Prefix{netip.MustParsePrefix("1.0.0.1/32")}, - }, + } + m2peers := []wgPeer{ + { + publicKey: m1.privateKey.Public(), + allowedIPs: []netip.Prefix{netip.MustParsePrefix("1.0.0.1/32")}, }, } - if err := m1.Reconfig(m1cfg); err != nil { + if err := m1.Reconfig(m1cfg, m1peers); err != nil { t.Fatal(err) } - if err := m2.Reconfig(m2cfg); err != nil { + if err := m2.Reconfig(m2cfg, m2peers); err != nil { t.Fatal(err) } @@ -1329,7 +1397,7 @@ func testTwoDevicePing(t *testing.T, d *devices) { t.Run("no-op-dev1-reconfig", func(t *testing.T) { setT(t) defer setT(outerT) - if err := m1.Reconfig(m1cfg); err != nil { + if err := m1.Reconfig(m1cfg, m1peers); err != nil { t.Fatal(err) } ping1(t) @@ -2418,7 +2486,7 @@ func TestIsWireGuardOnlyPeer(t *testing.T) { if err != nil { t.Fatal(err) } - m.Reconfig(cfg) + m.Reconfig(cfg, wgPeersOfNetmap(nm)) pbuf := tuntest.Ping(wgaip.Addr(), tsaip.Addr()) m.tun.Outbound <- pbuf @@ -2479,7 +2547,7 @@ func TestIsWireGuardOnlyPeerWithMasquerade(t *testing.T) { if err != nil { t.Fatal(err) } - m.Reconfig(cfg) + m.Reconfig(cfg, wgPeersOfNetmap(nm)) pbuf := tuntest.Ping(wgaip.Addr(), tsaip.Addr()) m.tun.Outbound <- pbuf @@ -2520,7 +2588,7 @@ func applyNetworkMap(t *testing.T, m *magicStack, nm *netmap.NetworkMap) { t.Fatal(err) } // Apply the wireguard config to the tailscale internal wireguard device. - if err := m.Reconfig(cfg); err != nil { + if err := m.Reconfig(cfg, wgPeersOfNetmap(nm)); err != nil { t.Fatal(err) } } diff --git a/wgengine/pendopen.go b/wgengine/pendopen.go index 4c830bf68..52055b174 100644 --- a/wgengine/pendopen.go +++ b/wgengine/pendopen.go @@ -138,14 +138,14 @@ func (e *userspaceEngine) isOSNetworkProbe(dst netip.AddrPort) bool { // iOS had log spam like: // open-conn-track: timeout opening (100.115.73.60:52501 => 17.125.252.5:443); no associated peer node if runtime.GOOS == "ios" && dst.Port() == 443 && appleIPRange.Contains(dst.Addr()) { - if _, ok := e.PeerForIP(dst.Addr()); !ok { + if _, ok := e.peerForIP(dst.Addr()); !ok { return true } } // NetworkManager; https://github.com/tailscale/tailscale/issues/13687 // open-conn-track: timeout opening (TCP 100.96.229.119:42798 => 185.125.190.49:80); no associated peer node if runtime.GOOS == "linux" && dst.Port() == 80 && canonicalIPs()(dst.Addr()) { - if _, ok := e.PeerForIP(dst.Addr()); !ok { + if _, ok := e.peerForIP(dst.Addr()); !ok { return true } } @@ -199,7 +199,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) { } // Diagnose why it might've timed out. - pip, ok := e.PeerForIP(flow.DstAddr()) + pip, ok := e.peerForIP(flow.DstAddr()) if !ok { e.logf("open-conn-track: timeout opening %v; no associated peer node", flow) return diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 32683e3c7..c546bcc1f 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -34,6 +34,7 @@ "tailscale.com/net/ipset" "tailscale.com/net/netmon" "tailscale.com/net/packet" + "tailscale.com/net/routemanager" "tailscale.com/net/sockstats" "tailscale.com/net/tsdial" "tailscale.com/net/tstun" @@ -51,7 +52,6 @@ "tailscale.com/util/eventbus" "tailscale.com/util/execqueue" "tailscale.com/util/mak" - "tailscale.com/util/set" "tailscale.com/util/singleflight" "tailscale.com/util/testenv" "tailscale.com/util/usermetric" @@ -90,9 +90,6 @@ type userspaceEngine struct { birdClient BIRDClient // or nil controlKnobs *controlknobs.Knobs // or nil - testMaybeReconfigHook func() // for tests; if non-nil, fires if maybeReconfigWireguardLocked called - testDiscoChangedHook func(map[key.NodePublic]bool) // for tests; if non-nil, fires after assembling discoChanged map - // isLocalAddr reports the whether an IP is assigned to the local // tunnel interface. It's used to reflect local packets // incorrectly sent to us. @@ -104,25 +101,18 @@ type userspaceEngine struct { wgLock sync.Mutex // serializes all wgdev operations; see lock order comment below - // peerByIPRoute is a longest-prefix-match table built from - // lastCfgFull.Peers AllowedIPs. It's the slow path for - // SetPeerByIPPacketFunc, used when LocalBackend's exact-IP fast path - // (nodeByAddr) misses — i.e. for subnet routes and exit-node default - // routes. Built from lastCfgFull (the wireguard-filtered peer list) - // rather than the netmap so that exit-node selection is honored: the - // netmap has 0.0.0.0/0 in AllowedIPs for every exit-capable peer, but - // lastCfgFull only has it for the currently-selected exit node. - // - // Replaced (not mutated) by maybeReconfigWireguardLocked. Read by - // the per-packet wgdev callback without locking. - peerByIPRoute atomic.Pointer[bart.Table[key.NodePublic]] - - // peerForIP, if non-nil, is the callback installed via - // [userspaceEngine.SetPeerForIPFunc]. PeerForIP delegates to it + // peerForIPFn, if non-nil, is the callback installed via + // [userspaceEngine.SetPeerForIPFunc]. peerForIP delegates to it // for the cold-path control lookups (Ping, TSMP, pendopen, etc). - peerForIP atomic.Pointer[func(netip.Addr) (_ PeerForIP, ok bool)] + peerForIPFn atomic.Pointer[func(netip.Addr) (_ PeerForIP, ok bool)] - lastCfgFull wgcfg.Config + // peerConfigFn, if non-nil, is the live per-peer allowed-IPs + // source installed via [userspaceEngine.SetPeerConfigFunc]. When + // set, wgdev's PeerLookupFunc queries it directly, so reconfigs + // no longer install per-config lookup closures. + peerConfigFn atomic.Pointer[func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)] + + lastCfg wgcfg.Config lastRouter *router.Config lastDNSConfig dns.ConfigView // or invalid if none lastIsSubnetRouter bool // was the node a primary subnet router in the last run. @@ -169,10 +159,6 @@ type userspaceEngine struct { // rewritten. wgPeerLookup syncs.AtomicValue[func(wgString string) (tsString string, ok bool)] - // tsmpLearnedDisco tracks per node key if a peer disco key was learned via TSMP. - // wgLock must be held when using this map. - tsmpLearnedDisco map[key.NodePublic]key.DiscoPublic - // Lock ordering: magicsock.Conn.mu, wgLock, then mu. } @@ -519,16 +505,6 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) e.logf("Creating WireGuard device...") e.wgdev = wgcfg.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger) closePool.addFunc(e.wgdev.Close) - - // Install a default outbound-packet peer lookup callback. It uses only - // the engine's BART table, which is rebuilt from the wireguard-filtered - // peer list on every Reconfig. Consumers (e.g. LocalBackend) may later - // call SetPeerByIPPacketFunc to additionally install a fast path for - // exact node-address matches; the BART remains the slow-path fallback. - // Without this default, callers that don't run a LocalBackend would - // have no way to route outbound packets to peers, since peers are - // created lazily from inbound packets only via SetPeerLookupFunc. - e.SetPeerByIPPacketFunc(nil) closePool.addFunc(func() { if err := e.magicConn.Close(); err != nil { e.logf("error closing magicconn: %v", err) @@ -623,7 +599,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) pkt := packet.TSMPDiscoKeyAdvertisement{ Key: update.Key, } - peer, ok := e.PeerForIP(update.Src) + peer, ok := e.peerForIP(update.Src) if !ok { e.logf("wgengine: no peer found for %v", update.Src) return @@ -699,62 +675,66 @@ func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper) return filter.Accept } -// maybeReconfigWireguardLocked reconfigures wireguard-go with the current -// full config, installing a PeerLookupFunc for on-demand peer creation. -// -// e.wgLock must be held. -func (e *userspaceEngine) maybeReconfigWireguardLocked() error { - if hook := e.testMaybeReconfigHook; hook != nil { - hook() - return nil - } +// SetPeerConfigFunc implements [Engine.SetPeerConfigFunc]. It stores +// fn and installs a single wgdev PeerLookupFunc wrapping it, so +// lazily-created peers always get current allowed IPs and the lookup +// func never needs to be reinstalled as the peer set changes. +func (e *userspaceEngine) SetPeerConfigFunc(fn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)) { + e.peerConfigFn.Store(&fn) + e.wgdev.SetPeerLookupFunc(wgcfg.NewPeerLookupFunc(e.wgdev.Bind(), e.logf, func(pubk device.NoisePublicKey) ([]netip.Prefix, bool) { + return fn(key.NodePublicFromRaw32(mem.B(pubk[:]))) + })) +} - full := e.lastCfgFull - // The wireguard-go peer set may have changed; drop the cached - // peer-string rewrites so the next log line re-resolves them - // against the current lookup. +// SyncDevicePeer implements [Engine.SyncDevicePeer]. +func (e *userspaceEngine) SyncDevicePeer(k key.NodePublic) { + fn := e.peerConfigFn.Load() + if fn == nil { + return + } + e.wgLock.Lock() + defer e.wgLock.Unlock() + // The peer set may be about to change; drop the wgLogger's cached + // peer-string rewrites so the next log line re-resolves them. e.wgLogger.Invalidate() - - // Rebuild the prefix-match peer routing table from the current - // (wireguard-filtered) peer list and publish it atomically. - rt := &bart.Table[key.NodePublic]{} - for _, p := range full.Peers { - for _, pfx := range p.AllowedIPs { - rt.Insert(pfx, p.PublicKey) - } + allowedIPs, ok := (*fn)(k) + if !ok { + e.wgdev.RemovePeer(k.Raw32()) + return } - e.peerByIPRoute.Store(rt) - - e.logf("wgengine: Reconfig: configuring userspace WireGuard config (with %d peers)", len(full.Peers)) - if err := wgcfg.ReconfigDevice(e.wgdev, &full, e.logf); err != nil { - e.logf("wgdev.Reconfig: %v", err) - return err + if peer, ok := e.wgdev.LookupActivePeer(k.Raw32()); ok { + peer.SetAllowedIPs(allowedIPs) } - return nil +} + +// ResetDevicePeer implements [Engine.ResetDevicePeer]. +func (e *userspaceEngine) ResetDevicePeer(k key.NodePublic) { + e.wgLock.Lock() + defer e.wgLock.Unlock() + e.wgLogger.Invalidate() + e.wgdev.RemovePeer(k.Raw32()) } // SetPeerByIPPacketFunc installs a callback used by wireguard-go to look up // which peer should handle an outbound packet by destination IP. // -// fn is an optional fast path for exact node-address matches (e.g. dst is a -// Tailscale IP). On miss (or if fn is nil), the engine's own BART table -// ([userspaceEngine.peerByIPRoute], built from the wireguard-filtered peer -// list) is consulted to handle subnet routes and exit-node default routes. +// LocalBackend's implementation consults both the exact node-address fast +// path and the RouteManager's outbound table (covering subnet routes and +// exit-node default routes), and stays correct under incremental netmap +// deltas. // -// [NewUserspaceEngine] installs a BART-only default at engine creation time, -// so callers that don't call SetPeerByIPPacketFunc (e.g. those not running -// a LocalBackend) still get working outbound packet routing. +// A nil fn uninstalls the callback, reverting the device to its standard +// WireGuard AllowedIPs trie, which only contains peers that already exist +// in the device. Callers without a LocalBackend that need outbound packets +// to lazily create peers must install their own callback. func (e *userspaceEngine) SetPeerByIPPacketFunc(fn func(netip.Addr) (_ key.NodePublic, ok bool)) { + if fn == nil { + e.wgdev.SetPeerByIPPacketFunc(nil) + return + } e.wgdev.SetPeerByIPPacketFunc(func(_, dst netip.Addr, _ []byte) (device.NoisePublicKey, bool) { - if fn != nil { - if pk, ok := fn(dst); ok { - return pk.Raw32(), true - } - } - if rt := e.peerByIPRoute.Load(); rt != nil { - if pk, ok := rt.Lookup(dst); ok { - return pk.Raw32(), true - } + if pk, ok := fn(dst); ok { + return pk.Raw32(), true } return device.NoisePublicKey{}, false }) @@ -820,12 +800,6 @@ func (e *userspaceEngine) ResetAndStop() (*Status, error) { return e.getStatus() } -func (e *userspaceEngine) PatchDiscoKey(pub key.NodePublic, disco key.DiscoPublic) { - e.wgLock.Lock() - defer e.wgLock.Unlock() - mak.Set(&e.tsmpLearnedDisco, pub, disco) -} - func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, dnsCfg *dns.Config) error { if routerCfg == nil { panic("routerCfg must not be nil") @@ -840,11 +814,6 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, defer e.wgLock.Unlock() e.tundev.SetWGConfig(cfg) - peerSet := make(set.Set[key.NodePublic], len(cfg.Peers)) - for _, p := range cfg.Peers { - peerSet.Add(p.PublicKey) - } - e.mu.Lock() self := e.selfNode e.mu.Unlock() @@ -865,7 +834,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, } isSubnetRouterChanged := buildfeatures.HasAdvertiseRoutes && isSubnetRouter != e.lastIsSubnetRouter - engineChanged := !e.lastCfgFull.Equal(cfg) + engineChanged := !e.lastCfg.Equal(cfg) routerChanged := checkchange.Update(&e.lastRouter, routerCfg) dnsChanged := buildfeatures.HasDNS && !e.lastDNSConfig.Equal(dnsCfg.View()) if dnsChanged { @@ -897,64 +866,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, e.isDNSIPOverTailscale.Store(ipset.NewContainsIPFunc(views.SliceOf(dnsIPsOverTailscale(dnsCfg, routerCfg)))) } - // See if any peers have changed disco keys, which means they've restarted. - // If so, remove the peer from wireguard-go to flush its session key, - // then let the PeerLookupFunc re-create it on demand. - discoChanged := make(map[key.NodePublic]bool) - if engineChanged { - prevEP := make(map[key.NodePublic]key.DiscoPublic) - for i := range e.lastCfgFull.Peers { - if p := &e.lastCfgFull.Peers[i]; !p.DiscoKey.IsZero() { - prevEP[p.PublicKey] = p.DiscoKey - } - } - for i := range cfg.Peers { - p := &cfg.Peers[i] - if p.DiscoKey.IsZero() { - continue - } - - pub := p.PublicKey - - if old, ok := prevEP[pub]; ok && old != p.DiscoKey { - // If the disco key was learned via TSMP, we do not need to reset the - // wireguard config as the new key was received over an existing wireguard - // connection. - if discoTSMP, okTSMP := e.tsmpLearnedDisco[p.PublicKey]; okTSMP { - // Key matches, remove entry from map. - delete(e.tsmpLearnedDisco, p.PublicKey) - if discoTSMP == p.DiscoKey { - e.logf("wgengine: Skipping reconfig (TSMP key): %s changed from %q to %q", - pub.ShortString(), old, p.DiscoKey) - // Skip session clear. - continue - } - - // The new disco key does not match what we received via - // TSMP for this peer. This is unexpected, though possible - // if processing a change in a large netmap ends up taking - // longer than the 2 second timeout in - // [controlClient.mapRoutineState.UpdateNetmapDelta], or if - // the context is cancelled mid update. Log the event, and reset - // the connection as it is possibly a stale entry in the map - // instead of a TSMP disco key update that led us here. - e.logf("wgengine: [unexpected] Reconfig: using TSMP key for %s (control stale): tsmp=%q control=%q old=%q", - pub.ShortString(), discoTSMP, p.DiscoKey, old) - metricTSMPLearnedKeyMismatch.Add(1) - } - - discoChanged[pub] = true - e.logf("wgengine: Reconfig: %s changed from %q to %q", pub.ShortString(), old, p.DiscoKey) - } - } - } - - // For tests, what disco connections needs to be changed. - if e.testDiscoChangedHook != nil { - e.testDiscoChangedHook(discoChanged) - } - - if !e.lastCfgFull.PrivateKey.Equal(cfg.PrivateKey) { + if !e.lastCfg.PrivateKey.Equal(cfg.PrivateKey) { // Tell magicsock about the new (or initial) private key // (which is needed by DERP) before wgdev gets it, as wgdev // will start trying to handshake, which we want to be able to @@ -968,30 +880,16 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, } } - e.lastCfgFull = *cfg.Clone() + e.lastCfg = *cfg.Clone() - e.magicConn.UpdatePeers(peerSet) e.magicConn.SetPreferredPort(listenPort) e.magicConn.UpdatePMTUD() - if engineChanged { - if err := e.maybeReconfigWireguardLocked(); err != nil { - return err - } - // Now that we've reconfigured wireguard-go, remove any peers with - // changed disco keys to flush their session keys, and let them be - // re-created on demand by the PeerLookupFunc. - for pub := range discoChanged { - e.wgdev.RemovePeer(pub.Raw32()) - } - } - - // Cleanup map of tsmp marks for peers that no longer exists in config. - for nodeKey := range e.tsmpLearnedDisco { - if !peerSet.Contains(nodeKey) { - delete(e.tsmpLearnedDisco, nodeKey) - } - } + // Note: no wireguard-go device reconfig happens here. The device + // learns its peer set from the live config source installed via + // [Engine.SetPeerConfigFunc] (peers are lazily created and synced + // per peer by [Engine.SyncDevicePeer]), and its private key is set + // above when it changes. if routerChanged { e.logf("wgengine: Reconfig: configuring router") @@ -1076,6 +974,10 @@ func (e *userspaceEngine) SetJailedFilter(filt *filter.Filter) { e.tundev.SetJailedFilter(filt) } +func (e *userspaceEngine) SetPeerRoutes(native4, native6 netip.Addr, routes *bart.Table[*routemanager.PeerRoute]) { + e.tundev.SetPeerRoutes(native4, native6, routes) +} + func (e *userspaceEngine) SetStatusCallback(cb StatusCallback) { e.mu.Lock() defer e.mu.Unlock() @@ -1358,7 +1260,7 @@ func (e *userspaceEngine) UpdateStatus(sb *ipnstate.StatusBuilder) { func (e *userspaceEngine) Ping(ip netip.Addr, pingType tailcfg.PingType, size int, cb func(*ipnstate.PingResult)) { res := &ipnstate.PingResult{IP: ip.String()} - pip, ok := e.PeerForIP(ip) + pip, ok := e.peerForIP(ip) if !ok { e.logf("ping(%v): no matching peer", ip) res.Err = "no matching peer" @@ -1555,38 +1457,24 @@ func (e *userspaceEngine) ProbeLocks() { e.wgLock.Unlock() } -// SetPeerForIPFunc installs the callback used by [userspaceEngine.PeerForIP]. +// SetPeerForIPFunc installs the callback used by [userspaceEngine.peerForIP]. // See [Engine.SetPeerForIPFunc]. func (e *userspaceEngine) SetPeerForIPFunc(fn func(netip.Addr) (PeerForIP, bool)) { if fn == nil { - e.peerForIP.Store(nil) + e.peerForIPFn.Store(nil) return } - e.peerForIP.Store(&fn) + e.peerForIPFn.Store(&fn) } -// PeerKeyForIP looks up ip in the engine's AllowedIPs table -// ([userspaceEngine.peerByIPRoute]). See [Engine.PeerKeyForIP]. -func (e *userspaceEngine) PeerKeyForIP(ip netip.Addr) (pk key.NodePublic, route netip.Prefix, ok bool) { - if !ip.IsValid() { - return pk, route, false - } - rt := e.peerByIPRoute.Load() - if rt == nil { - return pk, route, false - } - route, pk, ok = rt.LookupPrefixLPM(netip.PrefixFrom(ip, ip.BitLen())) - return pk, route, ok -} - -// PeerForIP returns the node responsible for handling the given IP. -// It delegates to the callback installed via [SetPeerForIPFunc]; engines -// without an installed callback return (zero, false). -func (e *userspaceEngine) PeerForIP(ip netip.Addr) (ret PeerForIP, ok bool) { +// peerForIP returns the node responsible for handling the given IP. +// It delegates to the callback installed via [Engine.SetPeerForIPFunc]; +// engines without an installed callback return (zero, false). +func (e *userspaceEngine) peerForIP(ip netip.Addr) (ret PeerForIP, ok bool) { if !ip.IsValid() { return ret, false } - if fn := e.peerForIP.Load(); fn != nil { + if fn := e.peerForIPFn.Load(); fn != nil { return (*fn)(ip) } return ret, false @@ -1676,8 +1564,6 @@ func (ls fwdDNSLinkSelector) PickLink(ip netip.Addr) (linkName string) { metricTSMPDiscoKeyAdvertisementSent = clientmetric.NewCounter("magicsock_tsmp_disco_key_advertisement_sent") metricTSMPDiscoKeyAdvertisementError = clientmetric.NewCounter("magicsock_tsmp_disco_key_advertisement_error") - - metricTSMPLearnedKeyMismatch = clientmetric.NewCounter("magicsock_tsmp_learned_key_mismatch") ) func (e *userspaceEngine) InstallCaptureHook(cb packet.CaptureCallback) { diff --git a/wgengine/userspace_test.go b/wgengine/userspace_test.go index 95fbf5a75..fc1c89403 100644 --- a/wgengine/userspace_test.go +++ b/wgengine/userspace_test.go @@ -90,7 +90,7 @@ func TestUserspaceEngineReconfig(t *testing.T) { routerCfg := &router.Config{} - for _, nodeHex := range []string{ + for i, nodeHex := range []string{ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", } { @@ -102,18 +102,9 @@ func TestUserspaceEngineReconfig(t *testing.T) { }, }), } - nk, err := key.ParseNodePublicUntyped(mem.S(nodeHex)) - if err != nil { - t.Fatal(err) - } cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nk, - AllowedIPs: []netip.Prefix{ - netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), - }, - }, + Addresses: []netip.Prefix{ + netip.PrefixFrom(netaddr.IPv4(100, 100, 99, byte(1+i)), 32), }, } @@ -125,167 +116,6 @@ func TestUserspaceEngineReconfig(t *testing.T) { } } -func TestUserspaceEngineTSMPLearned(t *testing.T) { - bus := eventbustest.NewBus(t) - - ht := health.NewTracker(bus) - reg := new(usermetric.Registry) - e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus) - if err != nil { - t.Fatal(err) - } - t.Cleanup(e.Close) - ue := e.(*userspaceEngine) - - discoChangedChan := make(chan map[key.NodePublic]bool, 1) - ue.testDiscoChangedHook = func(m map[key.NodePublic]bool) { - discoChangedChan <- m - } - - routerCfg := &router.Config{} - - keyChanges := []struct { - tsmp bool - inMap bool - }{ - {tsmp: false, inMap: false}, - {tsmp: true, inMap: false}, - {tsmp: false, inMap: true}, - } - - nkHex := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - for _, change := range keyChanges { - oldDisco := key.NewDisco() - nm := &netmap.NetworkMap{ - Peers: nodeViews([]*tailcfg.Node{ - { - ID: 1, - Key: nkFromHex(nkHex), - DiscoKey: oldDisco.Public(), - }, - }), - } - nk, err := key.ParseNodePublicUntyped(mem.S(nkHex)) - if err != nil { - t.Fatal(err) - } - e.SetSelfNode(nm.SelfNode) - - newDisco := key.NewDisco() - cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nk, - DiscoKey: newDisco.Public(), - }, - }, - } - - if change.tsmp { - ue.PatchDiscoKey(nk, newDisco.Public()) - } - err = e.Reconfig(cfg, routerCfg, &dns.Config{}) - if err != nil { - t.Fatal(err) - } - - changeMap := <-discoChangedChan - - if _, ok := changeMap[nk]; ok != change.inMap { - t.Fatalf("expect key %v in map %v to be %t, got %t", nk, changeMap, - change.inMap, ok) - } - } -} - -func TestUserspaceEngineTSMPLearnedMismatch(t *testing.T) { - bus := eventbustest.NewBus(t) - - ht := health.NewTracker(bus) - reg := new(usermetric.Registry) - e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus) - if err != nil { - t.Fatal(err) - } - t.Cleanup(e.Close) - ue := e.(*userspaceEngine) - - discoChangedChan := make(chan map[key.NodePublic]bool, 1) - ue.testDiscoChangedHook = func(m map[key.NodePublic]bool) { - discoChangedChan <- m - } - - routerCfg := &router.Config{} - var metricValue int64 = 0 - - keyChanges := []struct { - tsmp bool - inMap bool - wrongKey bool - }{ - {tsmp: false, inMap: false, wrongKey: false}, - {tsmp: true, inMap: false, wrongKey: false}, - {tsmp: true, inMap: true, wrongKey: true}, - {tsmp: false, inMap: true, wrongKey: false}, - } - - nkHex := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - for _, change := range keyChanges { - oldDisco := key.NewDisco() - nm := &netmap.NetworkMap{ - Peers: nodeViews([]*tailcfg.Node{ - { - ID: 1, - Key: nkFromHex(nkHex), - DiscoKey: oldDisco.Public(), - }, - }), - } - nk, err := key.ParseNodePublicUntyped(mem.S(nkHex)) - if err != nil { - t.Fatal(err) - } - e.SetSelfNode(nm.SelfNode) - - newDisco := key.NewDisco() - cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nk, - DiscoKey: newDisco.Public(), - }, - }, - } - - tsmpKey := newDisco.Public() - if change.tsmp { - if change.wrongKey { - tsmpKey = key.NewDisco().Public() - } - ue.PatchDiscoKey(nk, tsmpKey) - } - err = e.Reconfig(cfg, routerCfg, &dns.Config{}) - if err != nil { - t.Fatal(err) - } - - changeMap := <-discoChangedChan - - if _, ok := changeMap[nk]; ok != change.inMap { - t.Fatalf("expect key %v in map %v to be %t, got %t", nk, changeMap, - change.inMap, ok) - } - - metric := metricTSMPLearnedKeyMismatch.Value() - delta := metric - metricValue - metricValue = metric - - if change.wrongKey && delta != 1 { - t.Fatalf("expected a delta of 1, got %d", delta) - } - } -} - func TestUserspaceEnginePortReconfig(t *testing.T) { flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/2855") const defaultPort = 49983 @@ -317,18 +147,9 @@ func TestUserspaceEnginePortReconfig(t *testing.T) { t.Cleanup(ue.Close) startingPort := ue.magicConn.LocalPort() - nodeKey, err := key.ParseNodePublicUntyped(mem.S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) - if err != nil { - t.Fatal(err) - } cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nodeKey, - AllowedIPs: []netip.Prefix{ - netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), - }, - }, + Addresses: []netip.Prefix{ + netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), }, } routerCfg := &router.Config{} @@ -399,18 +220,9 @@ func TestUserspaceEnginePeerMTUReconfig(t *testing.T) { t.Logf("Info: OS default don't fragment bit(s) setting: %v", osDefaultDF) // Build a set of configs to use as we change the peer MTU settings. - nodeKey, err := key.ParseNodePublicUntyped(mem.S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) - if err != nil { - t.Fatal(err) - } cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nodeKey, - AllowedIPs: []netip.Prefix{ - netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), - }, - }, + Addresses: []netip.Prefix{ + netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), }, } routerCfg := &router.Config{} @@ -476,14 +288,7 @@ func TestTSMPKeyAdvertisement(t *testing.T) { }).View(), } cfg := &wgcfg.Config{ - Peers: []wgcfg.Peer{ - { - PublicKey: nodeKey, - AllowedIPs: []netip.Prefix{ - netip.PrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32), - }, - }, - }, + Addresses: nm.SelfNode.Addresses().AsSlice(), } ue.SetSelfNode(nm.SelfNode) diff --git a/wgengine/wgcfg/config.go b/wgengine/wgcfg/config.go index bd37c19f3..d613f0fc5 100644 --- a/wgengine/wgcfg/config.go +++ b/wgengine/wgcfg/config.go @@ -11,14 +11,17 @@ "tailscale.com/types/key" ) -//go:generate go run tailscale.com/cmd/cloner -type=Config,Peer +//go:generate go run tailscale.com/cmd/cloner -type=Config // Config is a WireGuard configuration. // It only supports the set of things Tailscale uses. +// +// Peers are not part of the config: wireguard-go learns the peer set +// and each peer's allowed IPs from the live per-peer config source +// installed via [tailscale.com/wgengine.Engine.SetPeerConfigFunc]. type Config struct { PrivateKey key.NodePrivate Addresses []netip.Prefix - Peers []Peer } func (c *Config) Equal(o *Config) bool { @@ -26,43 +29,5 @@ func (c *Config) Equal(o *Config) bool { return c == o } return c.PrivateKey.Equal(o.PrivateKey) && - slices.Equal(c.Addresses, o.Addresses) && - slices.EqualFunc(c.Peers, o.Peers, Peer.Equal) -} - -type Peer struct { - PublicKey key.NodePublic - DiscoKey key.DiscoPublic // present only so we can handle restarts within wgengine, not passed to WireGuard - AllowedIPs []netip.Prefix - V4MasqAddr *netip.Addr // if non-nil, masquerade IPv4 traffic to this peer using this address - V6MasqAddr *netip.Addr // if non-nil, masquerade IPv6 traffic to this peer using this address - IsJailed bool // if true, this peer is jailed and cannot initiate connections - PersistentKeepalive uint16 // in seconds between keep-alives; 0 to disable -} - -func addrPtrEq(a, b *netip.Addr) bool { - if a == nil || b == nil { - return a == b - } - return *a == *b -} - -func (p Peer) Equal(o Peer) bool { - return p.PublicKey == o.PublicKey && - p.DiscoKey == o.DiscoKey && - slices.Equal(p.AllowedIPs, o.AllowedIPs) && - p.IsJailed == o.IsJailed && - p.PersistentKeepalive == o.PersistentKeepalive && - addrPtrEq(p.V4MasqAddr, o.V4MasqAddr) && - addrPtrEq(p.V6MasqAddr, o.V6MasqAddr) -} - -// PeerWithKey returns the Peer with key k and reports whether it was found. -func (config Config) PeerWithKey(k key.NodePublic) (Peer, bool) { - for _, p := range config.Peers { - if p.PublicKey == k { - return p, true - } - } - return Peer{}, false + slices.Equal(c.Addresses, o.Addresses) } diff --git a/wgengine/wgcfg/config_test.go b/wgengine/wgcfg/config_test.go index fc8fdcbbb..64ffc8041 100644 --- a/wgengine/wgcfg/config_test.go +++ b/wgengine/wgcfg/config_test.go @@ -14,25 +14,10 @@ func TestConfigEqual(t *testing.T) { rt := reflect.TypeFor[Config]() for sf := range rt.Fields() { switch sf.Name { - case "Name", "NodeID", "PrivateKey", "Addresses", "Peers": + case "Name", "NodeID", "PrivateKey", "Addresses": // These are compared in [Config.Equal]. default: t.Errorf("Have you added field %q to Config.Equal? Do so if not, and then update TestConfigEqual", sf.Name) } } } - -// Tests that [Peer.Equal] tests all fields of [Peer], even ones -// that might get added in the future. -func TestPeerEqual(t *testing.T) { - rt := reflect.TypeFor[Peer]() - for sf := range rt.Fields() { - switch sf.Name { - case "PublicKey", "DiscoKey", "AllowedIPs", "IsJailed", - "PersistentKeepalive", "V4MasqAddr", "V6MasqAddr": - // These are compared in [Peer.Equal]. - default: - t.Errorf("Have you added field %q to Peer.Equal? Do so if not, and then update TestPeerEqual", sf.Name) - } - } -} diff --git a/wgengine/wgcfg/device.go b/wgengine/wgcfg/device.go index 2963c344d..c5d39ea3d 100644 --- a/wgengine/wgcfg/device.go +++ b/wgengine/wgcfg/device.go @@ -18,47 +18,12 @@ func NewDevice(tunDev tun.Device, bind conn.Bind, logger *device.Logger) *device return device.NewDevice(tunDev, bind, logger) } -// ReconfigDevice replaces the existing device configuration with cfg. -// -// Instead of using the UAPI text protocol, it uses the wireguard-go direct API -// to install a [device.PeerLookupFunc] callback that creates peers on demand. -// -// The caller is responsible for: -// - calling [device.Device.SetPrivateKey] when the key changes -// - installing a [device.PeerByIPPacketFunc] on the device for outbound -// packet routing (e.g. via [tailscale.com/wgengine.Engine.SetPeerByIPPacketFunc]) -func ReconfigDevice(d *device.Device, cfg *Config, logf logger.Logf) (err error) { - defer func() { - if err != nil { - logf("wgcfg.Reconfig failed: %v", err) - } - }() - - // Build peer map: public key → allowed IPs. - peers := make(map[device.NoisePublicKey][]netip.Prefix, len(cfg.Peers)) - for _, p := range cfg.Peers { - peers[p.PublicKey.Raw32()] = p.AllowedIPs - } - - // Remove peers not in the new config. - d.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool { - _, exists := peers[pk] - return !exists - }) - - // Update AllowedIPs on any already-active peers whose config may have - // changed. Peers that don't exist yet will get the correct AllowedIPs - // from PeerLookupFunc when they are lazily created. - for pk, allowedIPs := range peers { - if peer, ok := d.LookupActivePeer(pk); ok { - peer.SetAllowedIPs(allowedIPs) - } - } - - // Install callback for lazy peer creation (incoming packets). - bind := d.Bind() - d.SetPeerLookupFunc(func(pubk device.NoisePublicKey) (_ *device.NewPeerConfig, ok bool) { - allowedIPs, ok := peers[pubk] +// NewPeerLookupFunc returns a [device.PeerLookupFunc] that lazily +// creates peers using allowedIPs as the source of each peer's allowed +// IPs. The peer's endpoint is derived from its public key via bind. +func NewPeerLookupFunc(bind conn.Bind, logf logger.Logf, allowedIPs func(device.NoisePublicKey) ([]netip.Prefix, bool)) device.PeerLookupFunc { + return func(pubk device.NoisePublicKey) (_ *device.NewPeerConfig, ok bool) { + ips, ok := allowedIPs(pubk) if !ok { return nil, false } @@ -68,18 +33,8 @@ func ReconfigDevice(d *device.Device, cfg *Config, logf logger.Logf) (err error) return nil, false } return &device.NewPeerConfig{ - AllowedIPs: allowedIPs, + AllowedIPs: ips, Endpoint: ep, }, true - }) - - // RemoveMatchingPeers _again_, now that SetPeerLookupFunc is installed, - // lest any removed peers got re-created before the new SetPeerLookupFunc - // func was installed. - d.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool { - _, exists := peers[pk] - return !exists - }) - - return nil + } } diff --git a/wgengine/wgcfg/device_test.go b/wgengine/wgcfg/device_test.go index 07eb41adb..c07e14702 100644 --- a/wgengine/wgcfg/device_test.go +++ b/wgengine/wgcfg/device_test.go @@ -15,76 +15,47 @@ "tailscale.com/types/key" ) -func TestReconfigDevice(t *testing.T) { - k1, pk1 := newK() - ip1 := netip.MustParsePrefix("10.0.0.1/32") +func TestNewPeerLookupFunc(t *testing.T) { + k1, _ := newK() k2, _ := newK() ip2 := netip.MustParsePrefix("10.0.0.2/32") k3, _ := newK() - ip3 := netip.MustParsePrefix("10.0.0.3/32") - - cfg1 := &Config{ - PrivateKey: pk1, - Peers: []Peer{ - {PublicKey: k2, AllowedIPs: []netip.Prefix{ip2}}, - }, - } dev := NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "test")) defer dev.Close() - t.Run("initial-config", func(t *testing.T) { - if err := ReconfigDevice(dev, cfg1, t.Logf); err != nil { - t.Fatal(err) - } - // Peer should be creatable on demand via LookupPeer. - peer := dev.LookupPeer(k2.Raw32()) - if peer == nil { + // peers is the live per-peer config source, standing in for what + // LocalBackend provides via wgengine.Engine.SetPeerConfigFunc. + peers := map[device.NoisePublicKey][]netip.Prefix{ + k2.Raw32(): {ip2}, + } + dev.SetPeerLookupFunc(NewPeerLookupFunc(dev.Bind(), t.Logf, func(pubk device.NoisePublicKey) ([]netip.Prefix, bool) { + ips, ok := peers[pubk] + return ips, ok + })) + + t.Run("lazy-creation", func(t *testing.T) { + // A peer known to the config source should be creatable on + // demand via LookupPeer. + if p := dev.LookupPeer(k2.Raw32()); p == nil { t.Fatal("expected peer k2 to exist via LookupPeer") } - // Unknown peer should not be found. - peer = dev.LookupPeer(k3.Raw32()) - if peer != nil { + // An unknown peer should not be found. + if p := dev.LookupPeer(k3.Raw32()); p != nil { t.Fatal("expected unknown peer k3 to not exist") } }) - t.Run("add-peer", func(t *testing.T) { - cfg1.Peers = append(cfg1.Peers, Peer{ - PublicKey: k3, - AllowedIPs: []netip.Prefix{ip3}, - }) - if err := ReconfigDevice(dev, cfg1, t.Logf); err != nil { - t.Fatal(err) - } - // Both peers should now be discoverable. - if p := dev.LookupPeer(k2.Raw32()); p == nil { - t.Fatal("expected peer k2 to exist") - } - if p := dev.LookupPeer(k3.Raw32()); p == nil { - t.Fatal("expected peer k3 to exist") - } - }) - t.Run("remove-peer", func(t *testing.T) { - cfg2 := &Config{ - PrivateKey: pk1, - Peers: []Peer{ - {PublicKey: k2, AllowedIPs: []netip.Prefix{ip2}}, - }, - } - if err := ReconfigDevice(dev, cfg2, t.Logf); err != nil { - t.Fatal(err) - } - // k2 should still be discoverable. - if p := dev.LookupPeer(k2.Raw32()); p == nil { - t.Fatal("expected peer k2 to exist") - } - // k3 should no longer be discoverable. - if p := dev.LookupPeer(k3.Raw32()); p != nil { - t.Fatal("expected peer k3 to not exist after removal") + delete(peers, k2.Raw32()) + dev.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool { + _, ok := peers[pk] + return !ok + }) + if p := dev.LookupPeer(k2.Raw32()); p != nil { + t.Fatal("expected peer k2 to not exist after removal") } }) @@ -94,8 +65,6 @@ func TestReconfigDevice(t *testing.T) { t.Fatal("expected own key to not be a peer") } }) - - _ = ip1 // suppress unused } func newK() (key.NodePublic, key.NodePrivate) { diff --git a/wgengine/wgcfg/nmcfg/nmcfg.go b/wgengine/wgcfg/nmcfg/nmcfg.go index 7f633e4b5..da4d038db 100644 --- a/wgengine/wgcfg/nmcfg/nmcfg.go +++ b/wgengine/wgcfg/nmcfg/nmcfg.go @@ -45,11 +45,16 @@ func cidrIsSubnet(node tailcfg.NodeView, cidr netip.Prefix) bool { } // WGCfg returns the NetworkMaps's WireGuard configuration. +// +// The config does not include peers; wireguard-go gets those from the +// live per-peer config source installed via +// [tailscale.com/wgengine.Engine.SetPeerConfigFunc], fed by the route +// manager. WGCfg still walks the peers to log which ones are not +// routable and why, mirroring the route manager's filtering. func WGCfg(pk key.NodePrivate, nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags, exitNode tailcfg.StableNodeID) (*wgcfg.Config, error) { cfg := &wgcfg.Config{ PrivateKey: pk, Addresses: nm.GetAddresses().AsSlice(), - Peers: make([]wgcfg.Peer, 0, len(nm.Peers)), } var skippedExitNode, skippedSubnetRouter, skippedExpired []tailcfg.NodeView @@ -69,16 +74,7 @@ func WGCfg(pk key.NodePrivate, nm *netmap.NetworkMap, logf logger.Logf, flags ne continue } - cfg.Peers = append(cfg.Peers, wgcfg.Peer{ - PublicKey: peer.Key(), - DiscoKey: peer.DiscoKey(), - }) - cpeer := &cfg.Peers[len(cfg.Peers)-1] - didExitNodeLog := false - cpeer.V4MasqAddr = peer.SelfNodeV4MasqAddrForThisPeer().Clone() - cpeer.V6MasqAddr = peer.SelfNodeV6MasqAddrForThisPeer().Clone() - cpeer.IsJailed = peer.IsJailed() for _, allowedIP := range peer.AllowedIPs().All() { if allowedIP.Bits() == 0 && peer.StableID() != exitNode { if didExitNodeLog { @@ -87,14 +83,11 @@ func WGCfg(pk key.NodePrivate, nm *netmap.NetworkMap, logf logger.Logf, flags ne } didExitNodeLog = true skippedExitNode = append(skippedExitNode, peer) - continue } else if cidrIsSubnet(peer, allowedIP) { if (flags & netmap.AllowSubnetRoutes) == 0 { skippedSubnetRouter = append(skippedSubnetRouter, peer) - continue } } - cpeer.AllowedIPs = append(cpeer.AllowedIPs, allowedIP) } } diff --git a/wgengine/wgcfg/wgcfg_clone.go b/wgengine/wgcfg/wgcfg_clone.go index 32e7e31e5..7b56729c4 100644 --- a/wgengine/wgcfg/wgcfg_clone.go +++ b/wgengine/wgcfg/wgcfg_clone.go @@ -20,12 +20,6 @@ func (src *Config) Clone() *Config { dst := new(Config) *dst = *src dst.Addresses = append(src.Addresses[:0:0], src.Addresses...) - if src.Peers != nil { - dst.Peers = make([]Peer, len(src.Peers)) - for i := range dst.Peers { - dst.Peers[i] = *src.Peers[i].Clone() - } - } return dst } @@ -33,34 +27,4 @@ func (src *Config) Clone() *Config { var _ConfigCloneNeedsRegeneration = Config(struct { PrivateKey key.NodePrivate Addresses []netip.Prefix - Peers []Peer -}{}) - -// Clone makes a deep copy of Peer. -// The result aliases no memory with the original. -func (src *Peer) Clone() *Peer { - if src == nil { - return nil - } - dst := new(Peer) - *dst = *src - dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...) - if dst.V4MasqAddr != nil { - dst.V4MasqAddr = new(*src.V4MasqAddr) - } - if dst.V6MasqAddr != nil { - dst.V6MasqAddr = new(*src.V6MasqAddr) - } - return dst -} - -// A compilation failure here means this code must be regenerated, with the command at the top of this file. -var _PeerCloneNeedsRegeneration = Peer(struct { - PublicKey key.NodePublic - DiscoKey key.DiscoPublic - AllowedIPs []netip.Prefix - V4MasqAddr *netip.Addr - V6MasqAddr *netip.Addr - IsJailed bool - PersistentKeepalive uint16 }{}) diff --git a/wgengine/wgengine.go b/wgengine/wgengine.go index 7d389d64d..652774756 100644 --- a/wgengine/wgengine.go +++ b/wgengine/wgengine.go @@ -9,9 +9,11 @@ "net/netip" "time" + "github.com/gaissmai/bart" "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" "tailscale.com/net/packet" + "tailscale.com/net/routemanager" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/netmap" @@ -98,46 +100,20 @@ type Engine interface { // Unlike Reconfig, it does not return ErrNoChanges. ResetAndStop() (*Status, error) - // PeerForIP returns the node to which the provided IP routes, - // if any. If none is found, (zero, false) is returned. + // SetPeerForIPFunc installs the IP-to-node lookup used by the + // engine's internal cold paths (Ping, TSMP, pendopen diagnostics). + // It parallels [Engine.SetPeerByIPPacketFunc] but returns richer + // data (a full NodeView, the matched route prefix, and the IsSelf + // flag). // - // Despite the name, it can return the self node (with - // PeerForIP.IsSelf set). It handles Tailscale IPs, subnet-routed - // IPs, and exit-node global internet IPs, returning whichever - // node would handle that traffic. - // - // This is the cold path used by Ping, TSMP, pendopen diagnostics, - // and debug endpoints. It uses the same underlying data structures - // as the wireguard-go outbound packet path - // ([Engine.SetPeerByIPPacketFunc]), but is slower because it - // returns richer data (a full NodeView, the matched route prefix, - // and the IsSelf flag) requiring extra lookups. - // - // In production, the lookup is implemented by LocalBackend and - // plumbed in via [Engine.SetPeerForIPFunc]; the engine itself holds - // no peer-lookup state on this path. - PeerForIP(netip.Addr) (_ PeerForIP, ok bool) - - // SetPeerForIPFunc installs a callback used by [Engine.PeerForIP]. - // It parallels [Engine.SetPeerByIPPacketFunc] but serves the - // cold-path control lookups (Ping, TSMP, pendopen diagnostics, - // [tsdial.Dialer.UseNetstackForIP], debug endpoints). - // - // If fn is nil, PeerForIP returns (zero, false) for every IP. + // If fn is nil, those lookups fail for every IP. // // LocalBackend installs a func backed by the live nodeBackend for - // exact-match and self addresses, with [Engine.PeerKeyForIP] - // supplying the subnet-route / exit-node fallback. + // exact-match and self addresses, with the RouteManager's outbound + // table supplying the subnet-route / exit-node fallback; the engine + // itself holds no peer-lookup state on this path. SetPeerForIPFunc(fn func(netip.Addr) (_ PeerForIP, ok bool)) - // PeerKeyForIP returns the peer's NodePublic and the matched prefix - // for the longest-prefix match of ip in the engine's AllowedIPs - // table (the wireguard config most recently installed via - // [Engine.Reconfig]). Exit-node selection is honored: an unselected - // exit node's 0.0.0.0/0 is not matched. It is the same table the - // outbound packet hot path consults via [Engine.SetPeerByIPPacketFunc]. - PeerKeyForIP(netip.Addr) (_ key.NodePublic, _ netip.Prefix, ok bool) - // GetFilter returns the current packet filter, if any. GetFilter() *filter.Filter @@ -151,6 +127,17 @@ type Engine interface { // SetJailedFilter updates the packet filter for jailed nodes. SetJailedFilter(*filter.Filter) + // SetPeerRoutes updates the per-peer route attributes used by the + // tun-layer data plane for per-packet NAT rewrites and + // jailed-filter selection. native4 and native6 are this node's own + // Tailscale addresses, and routes maps each peer's addresses and + // routed prefixes to its attributes; it is a shared immutable + // snapshot from [routemanager.RouteManager.Outbound]. + // + // A nil routes table disables all per-packet peer processing; + // callers pass nil when no current peer has any such attributes. + SetPeerRoutes(native4, native6 netip.Addr, routes *bart.Table[*routemanager.PeerRoute]) + // SetStatusCallback sets the function to call when the // WireGuard status changes. SetStatusCallback(StatusCallback) @@ -197,6 +184,43 @@ type Engine interface { // look up which peer should handle an outbound packet by destination IP. SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool)) + // SetPeerConfigFunc installs the live source of per-peer WireGuard + // configuration: given a peer's public key, fn returns the prefixes + // the peer is currently allowed to originate traffic from, or + // ok=false if the peer is unknown (in which case it must not exist + // in the WireGuard device). The engine installs a single + // [device.PeerLookupFunc] wrapping fn, so lazily-created peers + // always see current state and the lookup func never needs to be + // reinstalled as peers come and go. + // + // It is expected to be called once during LocalBackend construction, + // before the first [Engine.Reconfig]. fn is called rarely (when + // wireguard-go first hears from a peer it doesn't have) and may + // acquire locks. + SetPeerConfigFunc(fn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)) + + // SyncDevicePeer synchronizes the WireGuard device's state for a + // single peer with the config source installed via + // [Engine.SetPeerConfigFunc]: if the source no longer knows the + // peer, it is removed from the device; if the peer is active in the + // device, its allowed IPs are updated. It does O(1) work (plus the + // config source lookup) and is intended to be called for each peer + // added, updated, or removed by an incremental netmap delta, + // avoiding a full [Engine.Reconfig]. + // + // It is a no-op if no config source is installed. + SyncDevicePeer(key.NodePublic) + + // ResetDevicePeer removes the peer from the WireGuard device, + // discarding any session key material and in-flight handshake + // state. If the peer is still known to the config source installed + // via [Engine.SetPeerConfigFunc], it is lazily re-created on demand + // with fresh state. + // + // LocalBackend calls it when a peer's disco key changes, which + // means the peer restarted and its old sessions are dead. + ResetDevicePeer(key.NodePublic) + // SetNetLogSource installs the [NetLogSource] consulted by the // engine's network flow logger for node lookups and the current // audit logging identity.