tailscale/.github/workflows/natlab-test.yml

# Run the full natlab/vmtest opt-in test suite. These tests boot QEMU VMs
# (gokrazy, Ubuntu, FreeBSD) and exercise vnet-driven networking scenarios.
# They are gated behind --run-vm-tests because they need KVM and are slow.
#
# This workflow runs:
#   - on demand (workflow_dispatch)
#   - on PRs that carry the "run-natlab-tests" label
#   - on main, every 12 hours, via cron
#
# Layout:
#   - "prepare" builds the gokrazy VM image, downloads the cloud images
#     (Ubuntu, FreeBSD), and discovers every Test* function in the two
#     opt-in packages.
#   - "test" is a per-TestFoo matrix that depends on prepare. Each matrix
#     job restores the shared caches and runs a single test. Adding a new
#     TestFoo automatically gets its own job — no workflow edits needed.
#
# A separate workflow (.github/workflows/natlab-basic.yml) runs a single
# canary natlab test on every PR; this one runs the full suite.
name: "natlab-test"

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
    types: [labeled, synchronize, reopened]
  schedule:
    # Every 12 hours, off-the-hour to avoid GitHub's :00 cron-stampede window.
    - cron: "23 3,15 * * *"

jobs:
  # prepare warms the shared caches (gokrazy image, cloud VM images, tsgo
  # toolchain, Go module and build caches) and emits the dynamic matrix of
  # test names. By doing the work once here, the matrix test jobs never race
  # to rebuild or re-download the same artifacts on a cold cache.
  prepare:
    if: |
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-natlab-tests'))
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      matrix: ${{ steps.list.outputs.matrix }}
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # ./tool/go downloads and extracts the tsgo toolchain on a cold runner.
      # go.toolchain.rev names an immutable release, so an exact hit is enough.
      - name: Cache tsgo toolchain
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/tsgo
          key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}

      - name: Cache Go modules
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/go/pkg/mod
          key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}

      # Saved at the end of the job. The run_id suffix makes each save unique,
      # and restore-keys finds an earlier one.
      - name: Restore Go build cache
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/go-build
          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
          restore-keys: |
            natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
            natlab-gobuild-${{ runner.os }}-

      # The cloud VM image cache is keyed only on images.go (image URLs and
      # SHAs), so it survives across workflow runs and is invalidated only
      # when a new image source is added.
      - name: Cache cloud VM images
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/tailscale/vmtest/images
          key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }}

      # The gokrazy VM image is keyed by github.sha. That means we rebuild
      # it once per commit but matrix test jobs in the same run all share
      # the result. Per-PR re-runs of the same sha (e.g. a rerun-failed)
      # also get the cache.
      - name: Cache gokrazy VM image
        id: gokrazy-cache
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: gokrazy/natlabapp.qcow2
          key: natlab-gokrazy-${{ github.sha }}

      # qemu-utils provides qemu-img, which the gokrazy Makefile uses to
      # convert natlabapp.img to qcow2. Only install if we need it (cache
      # miss); the test matrix jobs install qemu separately for the runtime.
      - name: Install qemu-utils
        if: steps.gokrazy-cache.outputs.cache-hit != 'true'
        run: |
          sudo rm -f /var/lib/man-db/auto-update
          sudo apt-get -y update
          sudo apt-get -y remove man-db
          sudo apt-get install -y qemu-utils

      - name: Download cloud VM images
        # natlabprep is idempotent: it checks the cache before downloading.
        run: |
          ./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep

      - name: Build gokrazy VM image
        if: steps.gokrazy-cache.outputs.cache-hit != 'true'
        run: |
          make -C gokrazy natlab

      # No Go source imports this module, so nothing above fetches it. Pulling
      # it here puts it in the module cache the matrix jobs restore.
      - name: Download kernel.amd64 module
        run: |
          ./tool/go mod download github.com/gokrazy/kernel.amd64

      # Warm the build cache for the matrix jobs. Everything must be built
      # with ./tool/go (tsgo): it compiles the test binaries, and the go
      # command puts its own $GOROOT/bin first on the test binary's $PATH,
      # so the "go" that vmtest.go and gokrazy/build shell out to from
      # inside the test is tsgo as well. GOCACHE entries embed the
      # compiler's build ID, so anything built here with the runner's
      # stock Go would be useless to the tests. The env below mirrors
      # compileBinariesForOS in vmtest.go.
      - name: Warm Go build cache
        run: |
          ./tool/go test -c -o /dev/null ./tstest/natlab/vmtest ./tstest/integration/nat
          for pkg in tta tailscale tailscaled; do
            GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o /dev/null ./cmd/"$pkg"
          done

      - name: Discover tests
        id: list
        # Grep the test files directly rather than invoking `go test -list`
        # so we don't pay the cost of compiling the test binaries here. The
        # only test functions in these packages use the canonical
        # `func TestFoo(t *testing.T)` signature.
        #
        # exclude is the set of tests that need special invocation
        # (extra flags, a specific environment) and don't fit the
        # single-test-per-matrix-job model. They stay runnable locally.
        run: |
          set -euo pipefail
          exclude='^(TestGrid|TestVnetPerf.*)$'
          tmp=$(mktemp)
          for pkg_dir in tstest/natlab/vmtest tstest/integration/nat; do
            pkg="./${pkg_dir}/"
            for f in "${pkg_dir}"/*_test.go; do
              [ -e "$f" ] || continue
              { grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(t \*testing\.T\)' "$f" || true; } \
                | sed -E 's/^func (Test[A-Za-z0-9_]+).*/\1/' \
                | { grep -vE "$exclude" || true; } \
                | while read -r t; do
                    jq -nc --arg pkg "$pkg" --arg test "$t" \
                      '{pkg: $pkg, test: $test}' >> "$tmp"
                  done
            done
          done
          matrix=$(jq -s -c . "$tmp")
          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
          echo "Discovered tests:"
          jq . "$tmp"

      - name: Tidy Go build cache
        run: |
          find $(./tool/go env GOCACHE) -type f -mmin +90 -delete

      # Saved on every run, not just main. The matrix jobs read this entry by
      # exact key within the same run. Gating on main would leave every one of
      # them recompiling the gokrazy image from scratch. Saves from main also
      # seed later runs and natlab-basic.
      - name: Save Go build cache
        if: always()
        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/go-build
          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}

  test:
    needs: prepare
    runs-on: ubuntu-latest
    timeout-minutes: 20
    name: "${{ matrix.test }}"
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.prepare.outputs.matrix) }}
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # Restore the Go caches before anything invokes ./tool/go.
      - name: Restore tsgo toolchain
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/tsgo
          key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}

      - name: Restore Go modules
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/go/pkg/mod
          key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}

      # An exact hit means prepare saved during this run. Otherwise
      # restore-keys falls back to the most recent earlier entry.
      - name: Restore Go build cache
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/go-build
          key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
          restore-keys: |
            natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
            natlab-gobuild-${{ runner.os }}-

      - name: Enable KVM
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: Install qemu
        run: |
          sudo rm -f /var/lib/man-db/auto-update
          sudo apt-get -y update
          sudo apt-get -y remove man-db
          sudo apt-get install -y qemu-system-x86 qemu-utils

      # restore-only: prepare is the single writer of these caches, so
      # matrix jobs don't write back. fail-on-cache-miss would be too
      # strict for the gokrazy cache (e.g. a non-fatal cache eviction
      # between prepare and us); we just rebuild on miss instead.
      - name: Restore cloud VM images
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: ~/.cache/tailscale/vmtest/images
          key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }}

      - name: Restore gokrazy VM image
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        with:
          path: gokrazy/natlabapp.qcow2
          key: natlab-gokrazy-${{ github.sha }}

      # The gokrazy-based tests boot the kernel directly from
      # vmlinuz that ships in the gokrazy/kernel.amd64 module.
      # Tests look it up under GOMODCACHE via findKernelPath, so the
      # module has to be present even though no Go source imports it
      # in the test package itself. This is normally a no-op, since prepare
      # already fetched it into the module cache restored above.
      - name: Download kernel.amd64 module
        run: |
          ./tool/go mod download github.com/gokrazy/kernel.amd64

      - name: Run ${{ matrix.test }}
        # Per-test timeout is well above the few-minute typical runtime
        # but small enough that a stuck test fails fast instead of holding
        # the runner for the job's 20-minute budget.
        run: |
          ./tool/go test -v -timeout=15m -count=1 ${{ matrix.pkg }} \
            -run='^${{ matrix.test }}$' --run-vm-tests