.github/workflows: cache the Go build and tsgo toolchain in natlab CI (#20815)

The natlab workflows cached only the VM images, so every job re-downloaded
the tsgo toolchain and built Go from scratch. Cache the toolchain, module
cache, and build cache in both workflows. natlab-test's prepare job is the
only writer of the build cache; everything else, including its 54 matrix
jobs and natlab-basic, restores it.

The warm step compiles with both toolchains on purpose: ./tool/go builds the
test binaries, while the bare `go` that vmtest.go and gokrazy/build shell out
to is the runner's stock Go. GOCACHE keys embed the compiler's build ID, so
those entries don't interchange, but both land in ~/.cache/go-build.

Updates #13038

Signed-off-by: Brendan Creane <bcreane@gmail.com>

.github/workflows/natlab-basic.yml

@@ -23,6 +23,26 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      # Go caches, keyed the same as natlab-test.yml's. Only that workflow's
+      # prepare job writes the build cache. A miss here just means a cold build.
+      - name: Cache tsgo toolchain
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # zizmor: ignore[cache-poisoning] v5.0.4
+        with:
+          path: ~/.cache/tsgo
+          key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}
+      - name: Cache Go modules
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # zizmor: ignore[cache-poisoning] v5.0.4
+        with:
+          path: ~/go/pkg/mod
+          key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}
+      - name: Restore Go build cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/go-build
+          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
+          restore-keys: |
+            natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
+            natlab-gobuild-${{ runner.os }}-
       - name: Enable KVM
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules

.github/workflows/natlab-test.yml

@@ -32,10 +32,10 @@ on:
     - cron: "23 3,15 * * *"
 
 jobs:
-  # prepare warms the per-workflow-run caches (gokrazy image, cloud VM
-  # images) and emits the dynamic matrix of test names. By doing the work
-  # once here, the matrix test jobs never race to rebuild or re-download
-  # the same artifacts on a cold cache.
+  # prepare warms the shared caches (gokrazy image, cloud VM images, tsgo
+  # toolchain, Go module and build caches) and emits the dynamic matrix of
+  # test names. By doing the work once here, the matrix test jobs never race
+  # to rebuild or re-download the same artifacts on a cold cache.
   prepare:
     if: |
       github.event_name == 'workflow_dispatch' ||
@@ -49,6 +49,31 @@ jobs:
       - name: Check out code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
+      # ./tool/go downloads and extracts the tsgo toolchain on a cold runner.
+      # go.toolchain.rev names an immutable release, so an exact hit is enough.
+      - name: Cache tsgo toolchain
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/tsgo
+          key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}
+
+      - name: Cache Go modules
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/go/pkg/mod
+          key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}
+
+      # Saved at the end of the job. The run_id suffix makes each save unique,
+      # and restore-keys finds an earlier one.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/go-build
+          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
+          restore-keys: |
+            natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
+            natlab-gobuild-${{ runner.os }}-
+
       # The cloud VM image cache is keyed only on images.go (image URLs and
       # SHAs), so it survives across workflow runs and is invalidated only
       # when a new image source is added.
@@ -90,6 +115,24 @@ jobs:
         run: |
           make -C gokrazy natlab
 
+      # No Go source imports this module, so nothing above fetches it. Pulling
+      # it here puts it in the module cache the matrix jobs restore.
+      - name: Download kernel.amd64 module
+        run: |
+          ./tool/go mod download github.com/gokrazy/kernel.amd64
+
+      # Warm the build cache for the matrix jobs. Two toolchains are involved:
+      # ./tool/go (tsgo) compiles the test binaries, and both vmtest.go and
+      # gokrazy/build shell out to the runner's stock `go`. GOCACHE entries
+      # embed the compiler's build ID, so one toolchain's entries are useless
+      # to the other. The env below mirrors compileBinariesForOS in vmtest.go.
+      - name: Warm Go build cache
+        run: |
+          ./tool/go test -c -o /dev/null ./tstest/natlab/vmtest ./tstest/integration/nat
+          for pkg in tta tailscale tailscaled; do
+            GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /dev/null ./cmd/"$pkg"
+          done
+
       - name: Discover tests
         id: list
         # Grep the test files directly rather than invoking `go test -list`
@@ -122,6 +165,21 @@ jobs:
           echo "Discovered tests:"
           jq . "$tmp"
 
+      - name: Tidy Go build cache
+        run: |
+          find $(./tool/go env GOCACHE) -type f -mmin +90 -delete
+
+      # Saved on every run, not just main. The matrix jobs read this entry by
+      # exact key within the same run. Gating on main would leave every one of
+      # them recompiling the gokrazy image from scratch. Saves from main also
+      # seed later runs and natlab-basic.
+      - name: Save Go build cache
+        if: always()
+        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/go-build
+          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
+
   test:
     needs: prepare
     runs-on: ubuntu-latest
@@ -135,6 +193,30 @@ jobs:
       - name: Check out code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
+      # Restore the Go caches before anything invokes ./tool/go.
+      - name: Restore tsgo toolchain
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/tsgo
+          key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}
+
+      - name: Restore Go modules
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/go/pkg/mod
+          key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}
+
+      # An exact hit means prepare saved during this run. Otherwise
+      # restore-keys falls back to the most recent earlier entry.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: ~/.cache/go-build
+          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
+          restore-keys: |
+            natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
+            natlab-gobuild-${{ runner.os }}-
+
       - name: Enable KVM
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
@@ -168,7 +250,8 @@ jobs:
       # vmlinuz that ships in the gokrazy/kernel.amd64 module.
       # Tests look it up under GOMODCACHE via findKernelPath, so the
       # module has to be present even though no Go source imports it
-      # in the test package itself.
+      # in the test package itself. This is normally a no-op, since prepare
+      # already fetched it into the module cache restored above.
       - name: Download kernel.amd64 module
         run: |
           ./tool/go mod download github.com/gokrazy/kernel.amd64