Files
tailscale/tka/sync_test.go
Alex Chan aec6f8bf13 tka/sync: improve the signature of SeedAUMs
I wrote this function two hours ago, tried to use it in corp, and
immediately found myself confused about the meaning of the arguments.
Time for named parameters!

Updates tailscale/corp#40404

Change-Id: Ic2866e052ccc9f6361b8d529233df54d63abbaa1
Signed-off-by: Alex Chan <alexc@tailscale.com>
2026-07-24 19:16:46 +01:00

623 lines
20 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package tka
import (
"bytes"
"fmt"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"tailscale.com/tstest"
"tailscale.com/util/must"
)
// getSyncOffer returns a SyncOffer for the given Chonk.
func getSyncOffer(t *testing.T, storage Chonk) SyncOffer {
t.Helper()
a, err := Open(storage)
if err != nil {
t.Fatal(err)
}
offer, err := a.SyncOffer(storage)
if err != nil {
t.Fatal(err)
}
return offer
}
func TestSyncOffer(t *testing.T) {
fakeState := &State{
Keys: []Key{{Kind: Key25519, Votes: 1}},
DisablementValues: [][]byte{bytes.Repeat([]byte{1}, 32)},
}
checkpointTemplate := optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})
// If we have a small chain with just a handful of AUMs, the SyncOffer
// contains the current HEAD and the first checkpoint.
t.Run("short-chain", func(t *testing.T) {
c := newTestchain(t, `A1 -> A2 -> A3 -> A4 -> A5`)
got := getSyncOffer(t, c.Chonk())
// A SyncOffer includes the first checkpoint.
want := SyncOffer{
Head: c.AUMHashes["A5"],
Ancestors: []AUMHash{
c.AUMHashes["A1"],
},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("SyncOffer diff (-want, +got):\n%s", diff)
}
})
// If the chain contains multiple checkpoints, the SyncOffer includes
// all of them.
t.Run("chain-with-multiple-checkpoints", func(t *testing.T) {
c := newTestchain(t, `
A1 -> A2 -> A3 -> A4 -> A5 -> A6 -> A7 -> A8 -> A9 -> A10
A10 -> A11 -> A12 -> A13 -> A14 -> A15 -> A16 -> A17 -> A18
A18 -> A19 -> A20 -> A21 -> A22 -> A23 -> A24 -> A25 -> A26
A26 -> A27 -> A28 -> A29 -> A30 -> A31 -> A32 -> A33 -> A34
A34 -> A35 -> A36 -> A37 -> A38 -> A39 -> A40 -> A41 -> A42
A42 -> A43 -> A45 -> A46 -> A47 -> A48 -> A49 -> A50 -> A51
A51 -> A52 -> A53 -> A54 -> A55
A1.template = checkpoint
A11.template = checkpoint
A21.template = checkpoint
A31.template = checkpoint
A41.template = checkpoint
A51.template = checkpoint
`, checkpointTemplate)
got := getSyncOffer(t, c.Chonk())
// A SyncOffer includes the first checkpoint.
want := SyncOffer{
Head: c.AUMHashes["A55"],
Ancestors: []AUMHash{
c.AUMHashes["A51"],
c.AUMHashes["A41"],
c.AUMHashes["A31"],
c.AUMHashes["A21"],
c.AUMHashes["A11"],
c.AUMHashes["A1"],
},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("SyncOffer diff (-want, +got):\n%s", diff)
}
})
// The size of a SyncOffer does not grow without bound as the number of AUMs increases.
t.Run("long-chain-size-is-bounded", func(t *testing.T) {
size := 1800
// Build a template string with a checkpoint every 50 AUMs.
var sb strings.Builder
sb.WriteString("A")
for i := range size {
sb.WriteString(fmt.Sprintf(" -> A%d", i))
}
for i := range size {
if i%50 == 0 {
sb.WriteString(fmt.Sprintf("\nA%d.template = checkpoint", i))
}
}
c := newTestchain(t, sb.String(), checkpointTemplate)
got := getSyncOffer(t, c.Chonk())
// We expect the SyncOffer to include:
//
// - the latest AUM as the HEAD
// - the checkpoints from the last 1000 AUMs (maxSyncHeadIntersectionIter)
// - the oldest AUM in storage
//
want := SyncOffer{
Head: c.AUMHashes["A1799"],
Ancestors: []AUMHash{
c.AUMHashes["A1750"], c.AUMHashes["A1700"], c.AUMHashes["A1650"],
c.AUMHashes["A1600"], c.AUMHashes["A1550"], c.AUMHashes["A1500"],
c.AUMHashes["A1450"], c.AUMHashes["A1400"], c.AUMHashes["A1350"],
c.AUMHashes["A1300"], c.AUMHashes["A1250"], c.AUMHashes["A1200"],
c.AUMHashes["A1150"], c.AUMHashes["A1100"], c.AUMHashes["A1050"],
c.AUMHashes["A1000"], c.AUMHashes["A950"], c.AUMHashes["A900"],
c.AUMHashes["A850"], c.AUMHashes["A800"], c.AUMHashes["A"],
},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("SyncOffer diff (-want, +got):\n%s", diff)
}
})
}
func TestComputeSyncIntersection_FastForward(t *testing.T) {
// Node 1 has: A1 -> A2
// Node 2 has: A1 -> A2 -> A3 -> A4
c := newTestchain(t, `
A1 -> A2 -> A3 -> A4
`)
a1H, a2H := c.AUMHashes["A1"], c.AUMHashes["A2"]
chonk1 := c.ChonkWith("A1", "A2")
offer1 := getSyncOffer(t, chonk1)
chonk2 := c.Chonk() // All AUMs
offer2 := getSyncOffer(t, chonk2)
// Node 1 only knows about the first two nodes, so the head of n2 is
// alien to it.
t.Run("n1", func(t *testing.T) {
got, err := computeSyncIntersection(chonk1, offer1, offer2)
if err != nil {
t.Fatalf("computeSyncIntersection() failed: %v", err)
}
want := &intersection{
tailIntersection: &a1H,
}
if diff := cmp.Diff(want, got, cmp.AllowUnexported(intersection{})); diff != "" {
t.Errorf("intersection diff (-want, +got):\n%s", diff)
}
})
// Node 2 knows about the full chain, so it can see that the head of n1
// intersects with a subset of its chain (a Head Intersection).
t.Run("n2", func(t *testing.T) {
got, err := computeSyncIntersection(chonk2, offer2, offer1)
if err != nil {
t.Fatalf("computeSyncIntersection() failed: %v", err)
}
want := &intersection{
headIntersection: &a2H,
}
if diff := cmp.Diff(want, got, cmp.AllowUnexported(intersection{})); diff != "" {
t.Errorf("intersection diff (-want, +got):\n%s", diff)
}
})
}
func TestComputeSyncIntersection_ForkSmallDiff(t *testing.T) {
// The number of nodes in the chain is longer than ancestorSkipStart,
// so that during sync both nodes are able to find a common ancestor
// which was later than A1.
c := newTestchain(t, `
A1 -> A2 -> A3 -> A4 -> A5 -> A6 -> A7 -> A8 -> A9 -> A10
| -> F1
// Make F1 different to A9.
// hashSeed is chosen such that the hash is higher than A9.
F1.hashSeed = 7
`)
// Node 1 has: A1 -> A2 -> A3 -> A4 -> A5 -> A6 -> A7 -> A8 -> F1
// Node 2 has: A1 -> A2 -> A3 -> A4 -> A5 -> A6 -> A7 -> A8 -> A9 -> A10
f1H, a9H := c.AUMHashes["F1"], c.AUMHashes["A9"]
if bytes.Compare(f1H[:], a9H[:]) < 0 {
t.Fatal("failed assert: h(a9) > h(f1H)\nTweak hashSeed till this passes")
}
chonk1 := c.ChonkWith("A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "F1")
offer1 := getSyncOffer(t, chonk1)
chonk2 := c.ChonkWith("A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "A9", "A10")
offer2 := getSyncOffer(t, chonk2)
// Node 1 only knows about the first eight nodes, so the head of n2 is
// alien to it.
t.Run("n1", func(t *testing.T) {
// n2 has 10 nodes, so the first common ancestor is the genesis AUM
wantIntersection := c.AUMHashes["A1"]
got, err := computeSyncIntersection(chonk1, offer1, offer2)
if err != nil {
t.Fatalf("computeSyncIntersection() failed: %v", err)
}
want := &intersection{
tailIntersection: &wantIntersection,
}
if diff := cmp.Diff(want, got, cmp.AllowUnexported(intersection{})); diff != "" {
t.Errorf("intersection diff (-want, +got):\n%s", diff)
}
})
// Node 2 knows about the full chain but doesn't recognize the head.
t.Run("n2", func(t *testing.T) {
// n1 has 9 nodes, so the first common ancestor is the genesis AUM
wantIntersection := c.AUMHashes["A1"]
got, err := computeSyncIntersection(chonk2, offer2, offer1)
if err != nil {
t.Fatalf("computeSyncIntersection() failed: %v", err)
}
want := &intersection{
tailIntersection: &wantIntersection,
}
if diff := cmp.Diff(want, got, cmp.AllowUnexported(intersection{})); diff != "" {
t.Errorf("intersection diff (-want, +got):\n%s", diff)
}
})
}
func TestMissingAUMs_FastForward(t *testing.T) {
// Node 1 has: A1 -> A2
// Node 2 has: A1 -> A2 -> A3 -> A4
c := newTestchain(t, `
A1 -> A2 -> A3 -> A4
A1.hashSeed = 1
A2.hashSeed = 2
A3.hashSeed = 3
A4.hashSeed = 4
`)
chonk1 := c.ChonkWith("A1", "A2")
n1, err := Open(chonk1)
if err != nil {
t.Fatal(err)
}
offer1, err := n1.SyncOffer(chonk1)
if err != nil {
t.Fatal(err)
}
chonk2 := c.Chonk() // All AUMs
n2, err := Open(chonk2)
if err != nil {
t.Fatal(err)
}
offer2, err := n2.SyncOffer(chonk2)
if err != nil {
t.Fatal(err)
}
// Node 1 only knows about the first two nodes, so the head of n2 is
// alien to it. As such, it should send history from the newest ancestor,
// A1 (if the chain was longer there would be one in the middle).
t.Run("n1", func(t *testing.T) {
got, err := n1.MissingAUMs(chonk1, offer2)
if err != nil {
t.Fatalf("MissingAUMs() failed: %v", err)
}
// Both sides have A1, so the only AUM that n2 might not have is
// A2.
want := []AUM{c.AUMs["A2"]}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("MissingAUMs diff (-want, +got):\n%s", diff)
}
})
// Node 2 knows about the full chain, so it can see that the head of n1
// intersects with a subset of its chain (a Head Intersection).
t.Run("n2", func(t *testing.T) {
got, err := n2.MissingAUMs(chonk2, offer1)
if err != nil {
t.Fatalf("MissingAUMs() failed: %v", err)
}
want := []AUM{
c.AUMs["A3"],
c.AUMs["A4"],
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("MissingAUMs diff (-want, +got):\n%s", diff)
}
})
}
func TestMissingAUMs_Fork(t *testing.T) {
// Node 1 has: A1 -> A2 -> A3 -> F1
// Node 2 has: A1 -> A2 -> A3 -> A4
c := newTestchain(t, `
A1 -> A2 -> A3 -> A4
| -> F1
A1.hashSeed = 1
A2.hashSeed = 2
A3.hashSeed = 3
A4.hashSeed = 4
`)
chonk1 := c.ChonkWith("A1", "A2", "A3", "F1")
n1, err := Open(chonk1)
if err != nil {
t.Fatal(err)
}
offer1, err := n1.SyncOffer(chonk1)
if err != nil {
t.Fatal(err)
}
chonk2 := c.ChonkWith("A1", "A2", "A3", "A4")
n2, err := Open(chonk2)
if err != nil {
t.Fatal(err)
}
offer2, err := n2.SyncOffer(chonk2)
if err != nil {
t.Fatal(err)
}
t.Run("n1", func(t *testing.T) {
got, err := n1.MissingAUMs(chonk1, offer2)
if err != nil {
t.Fatalf("MissingAUMs() failed: %v", err)
}
// Both sides have A1, so n1 will send everything it knows from
// there to head.
want := []AUM{
c.AUMs["A2"],
c.AUMs["A3"],
c.AUMs["F1"],
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("MissingAUMs diff (-want, +got):\n%s", diff)
}
})
t.Run("n2", func(t *testing.T) {
got, err := n2.MissingAUMs(chonk2, offer1)
if err != nil {
t.Fatalf("MissingAUMs() failed: %v", err)
}
// Both sides have A1, so n2 will send everything it knows from
// there to head.
want := []AUM{
c.AUMs["A2"],
c.AUMs["A3"],
c.AUMs["A4"],
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("MissingAUMs diff (-want, +got):\n%s", diff)
}
})
}
func TestSyncSimpleE2E(t *testing.T) {
pub, priv := testingKey25519(t, 1)
key := Key{Kind: Key25519, Public: pub, Votes: 2}
c := newTestchain(t, `
G1 -> L1 -> L2 -> L3
G1.template = genesis
`,
genesisTemplate(key),
optKey("key", key, priv),
optSignAllUsing("key"))
nodeStorage := ChonkMem()
node, err := Bootstrap(nodeStorage, c.AUMs["G1"])
if err != nil {
t.Fatalf("node Bootstrap() failed: %v", err)
}
controlStorage := c.Chonk()
control, err := Open(controlStorage)
if err != nil {
t.Fatalf("control Open() failed: %v", err)
}
// Control knows the full chain, node only knows the genesis. Let's see
// if they can sync.
nodeOffer, err := node.SyncOffer(nodeStorage)
if err != nil {
t.Fatal(err)
}
controlAUMs, err := control.MissingAUMs(controlStorage, nodeOffer)
if err != nil {
t.Fatalf("control.MissingAUMs(%v) failed: %v", nodeOffer, err)
}
if err := node.Inform(nodeStorage, controlAUMs); err != nil {
t.Fatalf("node.Inform(%v) failed: %v", controlAUMs, err)
}
if cHash, nHash := control.Head(), node.Head(); cHash != nHash {
t.Errorf("node & control are not synced: c=%x, n=%x", cHash, nHash)
}
}
// TestSyncFromFarBehind checks that nodes with compacted state can still find
// a common ancestor when the remote is significantly ahead.
//
// We simulate a node that has compacted its early history and is now ~500 AUMs
// behind the control plane, a distance that previously caused exponential sampling
// in SyncOffer to skip the node's entire local history.
//
// Regression test for http://go/corp/40404
func TestSyncFromFarBehind(t *testing.T) {
pub1, priv1 := testingKey25519(t, 1)
signer1 := signer25519(priv1)
key1 := Key{Kind: Key25519, Public: pub1, Votes: 2}
// Setup: persistentAuthority (control plane) vs compactingAuthority (client node).
state := State{
Keys: []Key{key1},
DisablementValues: [][]byte{DisablementKDF([]byte{1, 2, 3})},
}
persistentStorage, compactingStorage := ChonkMem(), ChonkMem()
persistentSize := func() int { return len(must.Get(persistentStorage.AllAUMs())) }
compactingSize := func() int { return len(must.Get(compactingStorage.AllAUMs())) }
// Backdate the clock on the compactingStorage so all AUMs will be old enough
// to be considered for compacting.
clock := tstest.NewClock(tstest.ClockOpts{
Start: time.Now().Add(-(CompactionDefaults.MinAge + 24*time.Hour)),
})
compactingStorage.SetClock(clock)
persistentAuthority, genesisAUM := must.Get2(Create(persistentStorage, state, signer1))
compactingAuthority := must.Get(Bootstrap(compactingStorage, genesisAUM))
// 1. Generate enough history to trigger checkpoints.
persistentNode := CreateSeedNode(t, persistentAuthority, persistentStorage)
compactingNode := CreateSeedNode(t, compactingAuthority, compactingStorage)
SeedAUMs(t, SeedAUMConfig{
Count: checkpointEvery * 2,
Signer: signer1,
Nodes: []SeedNode{persistentNode, compactingNode},
})
t.Logf("genesis and first batch of AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize())
// 2. Compact the node state.
//
// It now has a different 'oldestAncestor' than the control plane.
beforeCompacting := compactingSize()
must.Do(compactingAuthority.Compact(compactingStorage, CompactionDefaults))
afterCompacting := compactingSize()
if beforeCompacting == afterCompacting {
t.Errorf("expected Compact to reduce the number of AUMs, but unchanged: size = %d", afterCompacting)
}
// 3. Advance the control plane far beyond the node.
//
// As of 2026-04-17, the largest TKA has ~750 AUMs.
//
// If you keep increasing this number, eventually the sync will fail because you
// hit the hard-coded limits on iteration during the sync process.
SeedAUMs(t, SeedAUMConfig{
Count: compactingSize() - persistentSize() + 800,
Signer: signer1, Nodes: []SeedNode{persistentNode},
})
t.Logf("post-compacting and extra AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize())
// 4. Verify Intersection.
// The node should find an intersection even with a 500-AUM gap.
persistentOffer := must.Get(persistentAuthority.SyncOffer(persistentStorage))
compactingOffer := must.Get(compactingAuthority.SyncOffer(compactingStorage))
if _, err := compactingAuthority.MissingAUMs(compactingStorage, persistentOffer); err != nil {
t.Errorf("node failed to find intersection with far-ahead control plane: %v", err)
}
// 5. Check that the persistent authority can find an intersection with the
// compacting authority, and has missing AUMs to send it.
missing, err := persistentAuthority.MissingAUMs(persistentStorage, compactingOffer)
if len(missing) == 0 {
t.Errorf("control plane did not find any missing AUMs for node")
}
if err != nil {
t.Errorf("control plane failed to find missing AUMs for node: %v", err)
}
}
// TestSyncFromFarBehindFork checks that nodes with compacted state that have
// also branched from the active chain can still find a common ancestor when
// the remote is significantly ahead of the inersection point.
//
// We simulate a node that has compacted its early history and is now ~500 AUMs
// behind the control plane, plus a few AUMs extra, a distance that previously
// caused exponential sampling in SyncOffer to skip the node's entire local history.
//
// Regression test for http://go/corp/40404
func TestSyncFromFarBehindFork(t *testing.T) {
// Set up two signing keys. They have a different number of votes, so if there's
// a fork, the chain with the winning key will take precedence.
majorityPub, majorityPriv := testingKey25519(t, 1)
losingPub, losingPriv := testingKey25519(t, 2)
winningSigner := signer25519(majorityPriv)
losingSigner := signer25519(losingPriv)
losingKey := Key{Kind: Key25519, Public: majorityPub, Votes: 1}
winningKey := Key{Kind: Key25519, Public: losingPub, Votes: 2}
// Setup: persistentAuthority (control plane) vs compactingAuthority (client node).
state := State{
Keys: []Key{losingKey, winningKey},
DisablementValues: [][]byte{DisablementKDF([]byte{1, 2, 3})},
}
persistentStorage, compactingStorage := ChonkMem(), ChonkMem()
persistentSize := func() int { return len(must.Get(persistentStorage.AllAUMs())) }
compactingSize := func() int { return len(must.Get(compactingStorage.AllAUMs())) }
// Backdate the clock on the compactingStorage so all AUMs will be old enough
// to be considered for compacting.
clock := tstest.NewClock(tstest.ClockOpts{
Start: time.Now().Add(-(CompactionDefaults.MinAge + 24*time.Hour)),
})
compactingStorage.SetClock(clock)
persistentAuthority, genesisAUM := must.Get2(Create(persistentStorage, state, winningSigner))
compactingAuthority := must.Get(Bootstrap(compactingStorage, genesisAUM))
// 1. Generate enough history to trigger checkpoints.
persistentNode := CreateSeedNode(t, persistentAuthority, persistentStorage)
compactingNode := CreateSeedNode(t, compactingAuthority, compactingStorage)
SeedAUMs(t, SeedAUMConfig{
Count: checkpointEvery * 2,
Signer: winningSigner,
Nodes: []SeedNode{persistentNode, compactingNode},
})
t.Logf("genesis and first batch of AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize())
// 2. Compact the node state.
//
// It now has a different 'oldestAncestor' than the control plane.
beforeCompacting := compactingSize()
must.Do(compactingAuthority.Compact(compactingStorage, CompactionDefaults))
afterCompacting := compactingSize()
if beforeCompacting == afterCompacting {
t.Errorf("expected Compact to reduce the number of AUMs, but unchanged: size = %d", afterCompacting)
}
// 2. Advance the node state slightly beyond the control plane, using the
// losing signer.
SeedAUMs(t, SeedAUMConfig{
Count: 1,
Signer: losingSigner,
Nodes: []SeedNode{compactingNode},
})
// 3. Advance the control plane far beyond the node, using the winning signer.
//
// Now the node is forked from the control plane state, and the control plane's
// chain will win because its chain was signed by a key with more votes.
//
// As of 2026-04-17, the largest TKA has ~750 AUMs.
//
// If you keep increasing this number, eventually the sync will fail because you
// hit the hard-coded limits on iteration during the sync process.
SeedAUMs(t, SeedAUMConfig{
Count: compactingSize() - persistentSize() + 800,
Signer: winningSigner,
Nodes: []SeedNode{persistentNode},
})
t.Logf("post-compacting and extra AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize())
// 4. Verify Intersection.
// The node should find an intersection even with a 500-AUM gap.
persistentOffer := must.Get(persistentAuthority.SyncOffer(persistentStorage))
compactingOffer := must.Get(compactingAuthority.SyncOffer(compactingStorage))
if _, err := compactingAuthority.MissingAUMs(compactingStorage, persistentOffer); err != nil {
t.Errorf("node failed to find intersection with far-ahead control plane: %v", err)
}
// 5. Check that the persistent authority can find an intersection with the
// compacting authority, and has missing AUMs to send it.
missing, err := persistentAuthority.MissingAUMs(persistentStorage, compactingOffer)
if len(missing) == 0 {
t.Errorf("control plane did not find any missing AUMs for node")
}
if err != nil {
t.Errorf("control plane failed to find missing AUMs for node: %v", err)
}
}