diff --git a/feature/conn25/datapath.go b/feature/conn25/datapath.go index b5cdd5155..ef143cdd6 100644 --- a/feature/conn25/datapath.go +++ b/feature/conn25/datapath.go @@ -108,17 +108,17 @@ func (dh *datapathHandler) HandlePacketFromWireGuard(p *packet.Parsed) filter.Re // Check if this is an existing (return) flow on a client. // If found, perform the action for the existing client flow and return. - existing, ok := dh.clientFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) + action, ok := dh.clientFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) if ok { - existing.Action(p) + action(p) return filter.Accept } // Check if this is an existing connector outbound flow. // If found, perform the action for the existing connector outbound flow and return. - existing, ok = dh.connectorFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) + action, ok = dh.connectorFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) if ok { - existing.Action(p) + action(p) return filter.Accept } @@ -145,15 +145,18 @@ func (dh *datapathHandler) HandlePacketFromWireGuard(p *packet.Parsed) filter.Re // This is a new outbound flow on a connector. Install a DNAT TransitIP-to-RealIP action // for the outgoing direction, and an SNAT RealIP-to-TransitIP action for the // return direction. - outgoing := FlowData{ + outgoing := TupleAndAction{ Tuple: flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst), Action: dh.dnatAction(realIP), } - incoming := FlowData{ + incoming := TupleAndAction{ Tuple: flowtrack.MakeTuple(p.IPProto, netip.AddrPortFrom(realIP, p.Dst.Port()), p.Src), Action: dh.snatAction(transitIP), } - if err := dh.connectorFlowTable.NewFlowFromWireGuard(outgoing, incoming); err != nil { + if err := dh.connectorFlowTable.NewFlow(FlowData{ + FromTun: incoming, + FromWG: outgoing, + }); err != nil { dh.debugLogf("error installing flow, passing packet unmodified: %v", err) return filter.Accept } @@ -174,17 +177,17 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed) filter.Re // Check if this is an existing client outbound flow. // If found, perform the action for the existing client flow and return. - existing, ok := dh.clientFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) + action, ok := dh.clientFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) if ok { - existing.Action(p) + action(p) return filter.Accept } // Check if this is an existing connector return flow. // If found, perform the action for the existing connector return flow and return. - existing, ok = dh.connectorFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) + action, ok = dh.connectorFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst)) if ok { - existing.Action(p) + action(p) return filter.Accept } @@ -211,15 +214,18 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed) filter.Re // This is a new outbound client flow. Install a DNAT MagicIP-to-TransitIP action // for the outgoing direction, and an SNAT TransitIP-to-MagicIP action for the // return direction. - outgoing := FlowData{ + outgoing := TupleAndAction{ Tuple: flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst), Action: dh.dnatAction(transitIP), } - incoming := FlowData{ + incoming := TupleAndAction{ Tuple: flowtrack.MakeTuple(p.IPProto, netip.AddrPortFrom(transitIP, p.Dst.Port()), p.Src), Action: dh.snatAction(magicIP), } - if err := dh.clientFlowTable.NewFlowFromTunDevice(outgoing, incoming); err != nil { + if err := dh.clientFlowTable.NewFlow(FlowData{ + FromTun: outgoing, + FromWG: incoming, + }); err != nil { dh.debugLogf("error installing flow from tun device, passing packet unmodified: %v", err) return filter.Accept } diff --git a/feature/conn25/flowtable.go b/feature/conn25/flowtable.go index 27486ded9..bf07064f2 100644 --- a/feature/conn25/flowtable.go +++ b/feature/conn25/flowtable.go @@ -4,7 +4,7 @@ package conn25 import ( - "errors" + "container/list" "sync" "tailscale.com/net/flowtrack" @@ -14,12 +14,23 @@ // PacketAction may modify the packet. type PacketAction func(*packet.Parsed) -// FlowData is an entry stored in the [FlowTable]. -type FlowData struct { +// TupleAndAction wraps the [flowtrack.Tuple] and +// the [PacketAction] to return on lookups to that +// tuple. +type TupleAndAction struct { Tuple flowtrack.Tuple Action PacketAction } +// FlowData is an entry stored in the [FlowTable] +// constructed by the consumer of the table. +// It specifies tuples and actions for each direction +// of the flow. +type FlowData struct { + FromTun TupleAndAction + FromWG TupleAndAction +} + // Origin is used to track the direction of a flow. type Origin uint8 @@ -31,119 +42,131 @@ type FlowData struct { FromWireGuard ) +// cachedFlow is the main unit of storage in the table. +// It wraps the [FlowData] passed in by the consumer, as well +// as internal metadata and callbacks. type cachedFlow struct { - flow FlowData - paired flowtrack.Tuple // tuple for the other direction + data FlowData // user-defined tuples and actions for both directions + + // lastSeen time.Time // tracks when the flow was last hit for expiration management + // onRemove func() // fires on removal/expiration (e.g. update watchers, send RST to client) } // FlowTable stores and retrieves [FlowData] that can be looked up -// by 5-tuple. New entries specify the tuple to use for both directions +// by 5-tuple [flowtrack.Tuple] and direction. +// New entries specify the tuple to use for both directions // of traffic flow. The underlying cache is LRU, and the maximum number // of entries is specified in calls to [NewFlowTable]. FlowTable has // its own mutex and is safe for concurrent use. type FlowTable struct { mu sync.Mutex - fromTunCache *flowtrack.Cache[cachedFlow] // guarded by mu - fromWGCache *flowtrack.Cache[cachedFlow] // guarded by mu + fromTunCache map[flowtrack.Tuple]*list.Element + fromWGCache map[flowtrack.Tuple]*list.Element + lru *list.List + maxEntries int } -// NewFlowTable returns a [FlowTable] maxEntries maximum entries. +// NewFlowTable returns a [FlowTable] with maxEntries maximum entries. // A maxEntries of 0 indicates no maximum. See also [FlowTable]. func NewFlowTable(maxEntries int) *FlowTable { return &FlowTable{ - fromTunCache: &flowtrack.Cache[cachedFlow]{ - MaxEntries: maxEntries, - }, - fromWGCache: &flowtrack.Cache[cachedFlow]{ - MaxEntries: maxEntries, - }, + fromTunCache: make(map[flowtrack.Tuple]*list.Element, maxEntries), + fromWGCache: make(map[flowtrack.Tuple]*list.Element, maxEntries), + lru: list.New(), + maxEntries: maxEntries, } } -// LookupFromTunDevice looks up a [FlowData] entry that is valid to run for packets +// LookupFromTunDevice looks up a [PacketAction] that is valid to run on packets // observed as coming from the tun device. The tuple must match the direction it was // stored with. -func (t *FlowTable) LookupFromTunDevice(k flowtrack.Tuple) (FlowData, bool) { +func (t *FlowTable) LookupFromTunDevice(k flowtrack.Tuple) (PacketAction, bool) { return t.lookup(k, FromTun) } -// LookupFromWireGuard looks up a [FlowData] entry that is valid to run for packets +// LookupFromWireGuard looks up a [PacketAction] that is valid to run for packets // observed as coming from the WireGuard tunnel. The tuple must match the direction it was // stored with. -func (t *FlowTable) LookupFromWireGuard(k flowtrack.Tuple) (FlowData, bool) { +func (t *FlowTable) LookupFromWireGuard(k flowtrack.Tuple) (PacketAction, bool) { return t.lookup(k, FromWireGuard) } -func (t *FlowTable) lookup(k flowtrack.Tuple, want Origin) (FlowData, bool) { - var cache *flowtrack.Cache[cachedFlow] - switch want { +func (t *FlowTable) lookup(k flowtrack.Tuple, dir Origin) (PacketAction, bool) { + var cache map[flowtrack.Tuple]*list.Element + switch dir { case FromTun: cache = t.fromTunCache case FromWireGuard: cache = t.fromWGCache default: - return FlowData{}, false + return nil, false } t.mu.Lock() defer t.mu.Unlock() - v, ok := cache.Get(k) + ele, ok := cache[k] if !ok { - return FlowData{}, false - } - return v.flow, true -} - -// NewFlowFromTunDevice installs (or overwrites) both the forward and return entries. -// The forward tuple is tagged as FromTun, and the return tuple is tagged as FromWireGuard. -// If overwriting, it removes the old paired tuple for the forward key to avoid stale reverse mappings. -func (t *FlowTable) NewFlowFromTunDevice(fwd, rev FlowData) error { - return t.newFlow(FromTun, fwd, rev) -} - -// NewFlowFromWireGuard installs (or overwrites) both the forward and return entries. -// The forward tuple is tagged as FromWireGuard, and the return tuple is tagged as FromTun. -// If overwriting, it removes the old paired tuple for the forward key to avoid stale reverse mappings. -func (t *FlowTable) NewFlowFromWireGuard(fwd, rev FlowData) error { - return t.newFlow(FromWireGuard, fwd, rev) -} - -func (t *FlowTable) newFlow(fwdOrigin Origin, fwd, rev FlowData) error { - if fwd.Action == nil || rev.Action == nil { - return errors.New("nil action received for flow") + return nil, false } - var fwdCache, revCache *flowtrack.Cache[cachedFlow] - switch fwdOrigin { + flow := ele.Value.(*cachedFlow) + + var action PacketAction + switch dir { case FromTun: - fwdCache, revCache = t.fromTunCache, t.fromWGCache + action = flow.data.FromTun.Action case FromWireGuard: - fwdCache, revCache = t.fromWGCache, t.fromTunCache - default: - return errors.New("newFlow called with unknown direction") + action = flow.data.FromWG.Action } + // Support LRU. + t.lru.MoveToFront(ele) + + // TODO(mzb): Update flow.lastSeen. + + return action, true +} + +// NewFlow installs data as an flow in the table, and evicts any flow that +// either tuple already points at. This can result in two flows being evicted +// if each of the new tuples point at distinct existing flows. If the new flow +// would cause the table to exceed its maximum size, the least recently used +// (looked-up or created) flow is evicted. data is not validated, the caller must +// supply non-nil packet actions. +func (t *FlowTable) NewFlow(data FlowData) error { t.mu.Lock() defer t.mu.Unlock() - // If overwriting an existing entry, remove its previously-paired mapping so - // we don't leave stale tuples around. - if old, ok := fwdCache.Get(fwd.Tuple); ok { - revCache.Remove(old.paired) - } - if old, ok := revCache.Get(rev.Tuple); ok { - fwdCache.Remove(old.paired) + // If either tuple leads to anything existing, remove it. + t.removeFlowLocked(t.fromTunCache[data.FromTun.Tuple]) + t.removeFlowLocked(t.fromWGCache[data.FromWG.Tuple]) + + flow := &cachedFlow{ + data: data, + // Populate lastSeen + // Populate onRemove() } - fwdCache.Add(fwd.Tuple, cachedFlow{ - flow: fwd, - paired: rev.Tuple, - }) - revCache.Add(rev.Tuple, cachedFlow{ - flow: rev, - paired: fwd.Tuple, - }) + ele := t.lru.PushFront(flow) + if t.maxEntries > 0 && t.lru.Len() > t.maxEntries { + t.removeFlowLocked(t.lru.Back()) + } + + t.fromTunCache[data.FromTun.Tuple] = ele + t.fromWGCache[data.FromWG.Tuple] = ele return nil } + +func (t *FlowTable) removeFlowLocked(ele *list.Element) { + if ele == nil { + return + } + + flow := t.lru.Remove(ele).(*cachedFlow) + delete(t.fromTunCache, flow.data.FromTun.Tuple) + delete(t.fromWGCache, flow.data.FromWG.Tuple) + + // TODO(mzb): run flow.onRemove() +} diff --git a/feature/conn25/flowtable_test.go b/feature/conn25/flowtable_test.go index 8c3cd63a2..ae3d27144 100644 --- a/feature/conn25/flowtable_test.go +++ b/feature/conn25/flowtable_test.go @@ -12,114 +12,220 @@ "tailscale.com/types/ipproto" ) -func TestFlowTable(t *testing.T) { - ft := NewFlowTable(0) +var nilPacket *packet.Parsed // nil packet to perform actions against - fwdTuple := flowtrack.MakeTuple( +// mkTuple wraps flowtrack.MakeTuple with the UDP proto. +func mkTuple(src, dst string) flowtrack.Tuple { + return flowtrack.MakeTuple( ipproto.UDP, - netip.MustParseAddrPort("1.2.3.4:1000"), - netip.MustParseAddrPort("4.3.2.1:80"), - ) - // Reverse tuple is defined by caller. Doesn't have to be mirror image of fwd. - // To account for intentional modifications, like NAT. - revTuple := flowtrack.MakeTuple( - ipproto.UDP, - netip.MustParseAddrPort("4.3.2.2:80"), - netip.MustParseAddrPort("1.2.3.4:1000"), + netip.MustParseAddrPort(src), + netip.MustParseAddrPort(dst), ) +} - fwdAction, revAction := 0, 0 - fwdData := FlowData{ - Tuple: fwdTuple, - Action: func(_ *packet.Parsed) { fwdAction++ }, - } - revData := FlowData{ - Tuple: revTuple, - Action: func(_ *packet.Parsed) { revAction++ }, - } +func mkFlowWithActions(fromTun, fromWG flowtrack.Tuple) (FlowData, *int, *int) { + var tunCalls, wgCalls int + fd := mkFlow(fromTun, fromWG) + fd.FromTun.Action = func(_ *packet.Parsed) { tunCalls++ } + fd.FromWG.Action = func(_ *packet.Parsed) { wgCalls++ } + return fd, &tunCalls, &wgCalls +} - // For this test setup, from the tun device will be "forward", - // and from WG will be "reverse". - if err := ft.NewFlowFromTunDevice(fwdData, revData); err != nil { - t.Fatalf("got non-nil error for new flow from tun device") - } - - // Test basic lookups. - lookupFwd, ok := ft.LookupFromTunDevice(fwdTuple) - if !ok { - t.Fatalf("got not found on first lookup from tun device") - } - lookupFwd.Action(nil) - if fwdAction != 1 { - t.Errorf("action for fwd tuple key was not executed") - } - - lookupRev, ok := ft.LookupFromWireGuard(revTuple) - if !ok { - t.Fatalf("got not found on first lookup from WireGuard") - } - lookupRev.Action(nil) - if revAction != 1 { - t.Errorf("action for rev tuple key was not executed") - } - - // Test not found error. - notFoundTuple := flowtrack.MakeTuple( - ipproto.UDP, - netip.MustParseAddrPort("1.2.3.4:1000"), - netip.MustParseAddrPort("4.0.4.4:80"), - ) - if _, ok := ft.LookupFromTunDevice(notFoundTuple); ok { - t.Errorf("expected not found for foreign tuple") - } - - // Wrong direction is also not found. - if _, ok := ft.LookupFromWireGuard(fwdTuple); ok { - t.Errorf("expected not found for wrong direction tuple") - } - - // Overwriting forward tuple removes its reverse pair as well. - newRevData := FlowData{ - Tuple: flowtrack.MakeTuple( - ipproto.UDP, - netip.MustParseAddrPort("9.9.9.9:99"), - netip.MustParseAddrPort("8.8.8.8:88"), - ), - Action: func(_ *packet.Parsed) {}, - } - if err := ft.NewFlowFromTunDevice( - fwdData, - newRevData, - ); err != nil { - t.Fatalf("got non-nil error for new flow from tun device") - } - if _, ok := ft.LookupFromWireGuard(revTuple); ok { - t.Errorf("expected not found for removed reverse tuple") - } - - // Overwriting reverse tuple removes its forward pair as well. - if err := ft.NewFlowFromTunDevice( - FlowData{ - Tuple: flowtrack.MakeTuple( - ipproto.UDP, - netip.MustParseAddrPort("8.8.8.8:88"), - netip.MustParseAddrPort("9.9.9.9:99"), - ), +func mkFlow(fromTun, fromWG flowtrack.Tuple) FlowData { + return FlowData{ + FromTun: TupleAndAction{ + Tuple: fromTun, + Action: func(_ *packet.Parsed) {}, + }, + FromWG: TupleAndAction{ + Tuple: fromWG, Action: func(_ *packet.Parsed) {}, }, - newRevData, // This is the same "reverse" data installed in previous test. - ); err != nil { - t.Fatalf("got non-nil error for new flow from tun device") - } - if _, ok := ft.LookupFromTunDevice(fwdTuple); ok { - t.Errorf("expected not found for removed forward tuple") - } - - // Nil action returns an error. - if err := ft.NewFlowFromTunDevice( - FlowData{}, - FlowData{}, - ); err == nil { - t.Errorf("expected non-nil error for nil data") } } + +func mustInstallFlow(t *testing.T, ft *FlowTable, flow FlowData) { + t.Helper() + if err := ft.NewFlow(flow); err != nil { + t.Fatalf("error installing flow: %v", err) + } +} + +func assertFlowHit(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple) PacketAction { + t.Helper() + return assertFlowLookup(t, ft, dir, tuple, true) +} + +func assertFlowMiss(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple) { + t.Helper() + assertFlowLookup(t, ft, dir, tuple, false) +} + +func assertFlowLookup(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple, wantHit bool) PacketAction { + t.Helper() + var action PacketAction + var ok bool + + switch dir { + case FromTun: + action, ok = ft.LookupFromTunDevice(tuple) + case FromWireGuard: + action, ok = ft.LookupFromWireGuard(tuple) + default: + t.Fatalf("invalid direction: %v", dir) + } + + if wantHit && !ok { + t.Fatalf("expected flow hit for tuple: %v, dir: %v", tuple, dir) + } + if !wantHit && ok { + t.Fatalf("expected flow miss for tuple: %v, dir: %v", tuple, dir) + } + + if wantHit { + return action + } + return nil +} + +func TestFlowTable_NewFlow_Lookup(t *testing.T) { + ft := NewFlowTable(0) + + // The tuples in both directions are defined by the caller. + // The don't have to be mirror images of each other, + // to account for intentional modifications, like NAT. + fromTunTuple := mkTuple("1.2.3.4:1000", "4.3.2.1:80") + fromWGTuple := mkTuple("4.3.2.2:80", "1.2.3.4:1000") + + flow1, tunCount1, wgCount1 := mkFlowWithActions(fromTunTuple, fromWGTuple) + mustInstallFlow(t, ft, flow1) + + // Test basic lookups, and perform actions on packet. + assertFlowHit(t, ft, FromTun, fromTunTuple)(nilPacket) + assertFlowHit(t, ft, FromWireGuard, fromWGTuple)(nilPacket) + + if *tunCount1 != 1 { + t.Fatal("action for from-tun tuple key was not executed") + } + if *wgCount1 != 1 { + t.Fatal("action for from-wg tuple key was not executed") + } + + // Test tuple not found. + notFoundTuple := mkTuple("1.2.3.4:1000", "4.0.4.4:80") + assertFlowMiss(t, ft, FromTun, notFoundTuple) + + // Wrong direction is also not found. + assertFlowMiss(t, ft, FromWireGuard, fromTunTuple) + + // Overwriting from-tun tuple removes the from-wg tuple as well. + newFromWGTuple := mkTuple("9.9.9.9:99", "8.8.8.8:88") + flow2 := mkFlow(fromTunTuple, newFromWGTuple) + mustInstallFlow(t, ft, flow2) + assertFlowMiss(t, ft, FromWireGuard, fromWGTuple) + + // Overwriting the from-wg tuple removes the from-tun tuple as well. + newFromTunTuple := mkTuple("8.8.8.8:88", "9.9.9.9:99") + flow3 := mkFlow(newFromTunTuple, newFromWGTuple) + mustInstallFlow(t, ft, flow3) + assertFlowMiss(t, ft, FromTun, fromTunTuple) +} + +// TestFlowTable_OneReplacesTwo targets a specific case +// in which a single new flow replaces two existing flows +// because each tuple of the new flow matches one tuple +// of an existing flow. +func TestFlowTable_OneReplacesTwo(t *testing.T) { + ft := NewFlowTable(0) + + tunTuple1 := mkTuple("1.2.3.4:1000", "4.3.2.1:80") + wgTuple1 := mkTuple("4.3.2.2:80", "1.2.3.4:1000") + flow1, tunCount1, wgCount1 := mkFlowWithActions(tunTuple1, wgTuple1) + + tunTuple2 := mkTuple("8.8.8.8:88", "9.9.9.9:99") + wgTuple2 := mkTuple("9.9.9.9:99", "8.8.8.8:88") + flow2, tunCount2, wgCount2 := mkFlowWithActions(tunTuple2, wgTuple2) + + // Install the first two flows. + mustInstallFlow(t, ft, flow1) + mustInstallFlow(t, ft, flow2) + + // Confirm they are properly installed through lookups. + assertFlowHit(t, ft, FromTun, tunTuple1) + assertFlowHit(t, ft, FromWireGuard, wgTuple1) + assertFlowHit(t, ft, FromTun, tunTuple2) + assertFlowHit(t, ft, FromWireGuard, wgTuple2) + + // flow3 tuples overlap with flow1 and flow2. + flow3, tunCount3, wgCount3 := mkFlowWithActions(tunTuple1, wgTuple2) + mustInstallFlow(t, ft, flow3) + + // flow3 lookups hit on both of their tuples. + tunAction3 := assertFlowHit(t, ft, FromTun, tunTuple1) + wgAction3 := assertFlowHit(t, ft, FromWireGuard, wgTuple2) + + // The non-overlapping tuples from flow1 and flow2 should now miss. + assertFlowMiss(t, ft, FromTun, tunTuple2) + assertFlowMiss(t, ft, FromWireGuard, wgTuple1) + + // Perform both actions on a nil packet to bump counters. + tunAction3(nilPacket) + wgAction3(nilPacket) + + // Only flow3 counters should have been bumped. + if *tunCount1 != 0 || *wgCount1 != 0 { + t.Fatalf("flow1 counters (tun, wg), want: (0,0), got: (%d,%d)", *tunCount1, *wgCount1) + } + if *tunCount2 != 0 || *wgCount2 != 0 { + t.Fatalf("flow2 counters (tun, wg), want: (0,0), got: (%d,%d)", *tunCount2, *wgCount2) + } + if *tunCount3 != 1 || *wgCount3 != 1 { + t.Fatalf("flow3 counters (tun, wg), want: (1,1), got: (%d,%d)", *tunCount3, *wgCount3) + } +} + +func TestFlowTable_Eviction(t *testing.T) { + // Table only has two spots. + ft := NewFlowTable(2) + aTun, aWG := mkTuple("3.0.0.1:1000", "3.0.0.2:80"), mkTuple("3.0.0.2:80", "3.0.0.1:1000") + bTun, bWG := mkTuple("3.0.0.3:1000", "3.0.0.4:80"), mkTuple("3.0.0.4:80", "3.0.0.3:1000") + cTun, cWG := mkTuple("3.0.0.5:1000", "3.0.0.6:80"), mkTuple("3.0.0.6:80", "3.0.0.5:1000") + dTun, dWG := mkTuple("3.0.0.7:1000", "3.0.0.8:80"), mkTuple("3.0.0.8:80", "3.0.0.7:1000") + + a := mkFlow(aTun, aWG) + b := mkFlow(bTun, bWG) + c := mkFlow(cTun, cWG) + d := mkFlow(dTun, dWG) + + // Install a and b. + mustInstallFlow(t, ft, a) + mustInstallFlow(t, ft, b) + + // Move a to the front from tun side, b is ready for eviction. + assertFlowHit(t, ft, FromTun, aTun) + + // Install c. + mustInstallFlow(t, ft, c) + + // Check b is out. + assertFlowMiss(t, ft, FromTun, bTun) + assertFlowMiss(t, ft, FromWireGuard, bWG) + + // Check c is in. + assertFlowHit(t, ft, FromTun, cTun) + assertFlowHit(t, ft, FromWireGuard, cWG) + + // Move a to the front again, now from WG side. + assertFlowHit(t, ft, FromWireGuard, aWG) + + // Install d. + mustInstallFlow(t, ft, d) + + // Check c is out. + assertFlowMiss(t, ft, FromTun, cTun) + assertFlowMiss(t, ft, FromWireGuard, cWG) + + // Check d is in. + assertFlowHit(t, ft, FromTun, dTun) + assertFlowHit(t, ft, FromWireGuard, dWG) +}